chore: import upstream snapshot with attribution

tests/async/test_0.4.2_browser_manager.py

@@ -0,0 +1,160 @@
+import os
+import sys
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+
+# Assuming that the changes made allow different configurations
+# for managed browser, persistent context, and so forth.
+
+
+async def test_default_headless():
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "mobile", "os_type": "android"},
+        use_managed_browser=False,
+        use_persistent_context=False,
+        ignore_https_errors=True,
+        # Testing normal ephemeral context
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_default_headless] success:", result.success)
+        print("HTML length:", len(result.html if result.html else ""))
+
+
+async def test_managed_browser_persistent():
+    # Treating use_persistent_context=True as managed_browser scenario.
+    async with AsyncWebCrawler(
+        headless=False,
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "desktop", "os_type": "mac"},
+        use_managed_browser=True,
+        use_persistent_context=True,  # now should behave same as managed browser
+        user_data_dir="./outpu/test_profile",
+        # This should store and reuse profile data across runs
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.google.com",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_managed_browser_persistent] success:", result.success)
+        print("HTML length:", len(result.html if result.html else ""))
+
+
+async def test_session_reuse():
+    # Test creating a session, using it for multiple calls
+    session_id = "my_session"
+    async with AsyncWebCrawler(
+        headless=False,
+        verbose=True,
+        user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        # Fixed user-agent for consistency
+        use_managed_browser=False,
+        use_persistent_context=False,
+    ) as crawler:
+        # First call: create session
+        result1 = await crawler.arun(
+            url="https://www.example.com",
+            cache_mode=CacheMode.BYPASS,
+            session_id=session_id,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_session_reuse first call] success:", result1.success)
+
+        # Second call: same session, possibly cookie retained
+        result2 = await crawler.arun(
+            url="https://www.example.com/about",
+            cache_mode=CacheMode.BYPASS,
+            session_id=session_id,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_session_reuse second call] success:", result2.success)
+
+
+async def test_magic_mode():
+    # Test magic mode with override_navigator and simulate_user
+    async with AsyncWebCrawler(
+        headless=False,
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "desktop", "os_type": "windows"},
+        use_managed_browser=False,
+        use_persistent_context=False,
+        magic=True,
+        override_navigator=True,
+        simulate_user=True,
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/business",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_magic_mode] success:", result.success)
+        print("HTML length:", len(result.html if result.html else ""))
+
+
+async def test_proxy_settings():
+    # Test with a proxy (if available) to ensure code runs with proxy
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=False,
+        user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        proxy_config={"server": "http://127.0.0.1:8080"},  # Assuming local proxy server for test
+        use_managed_browser=False,
+        use_persistent_context=False,
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://httpbin.org/ip",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_proxy_settings] success:", result.success)
+        if result.success:
+            print("HTML preview:", result.html[:200] if result.html else "")
+
+
+async def test_ignore_https_errors():
+    # Test ignore HTTPS errors with a self-signed or invalid cert domain
+    # This is just conceptual, the domain should be one that triggers SSL error.
+    # Using a hypothetical URL that fails SSL:
+    async with AsyncWebCrawler(
+        headless=True,
+        verbose=True,
+        user_agent="Mozilla/5.0",
+        ignore_https_errors=True,
+        use_managed_browser=False,
+        use_persistent_context=False,
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://self-signed.badssl.com/",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+        )
+        print("[test_ignore_https_errors] success:", result.success)
+
+
+async def main():
+    print("Running tests...")
+    # await test_default_headless()
+    # await test_managed_browser_persistent()
+    # await test_session_reuse()
+    # await test_magic_mode()
+    # await test_proxy_settings()
+    await test_ignore_https_errors()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/async/test_0.4.2_config_params.py

@@ -0,0 +1,211 @@
+import os, sys
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai import JsonCssExtractionStrategy
+from crawl4ai.chunking_strategy import RegexChunking
+
+
+# Category 1: Browser Configuration Tests
+async def test_browser_config_object():
+    """Test the new BrowserConfig object with various browser settings"""
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=False,
+        viewport_width=1920,
+        viewport_height=1080,
+        use_managed_browser=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "desktop", "os_type": "windows"},
+    )
+
+    async with AsyncWebCrawler(config=browser_config, verbose=True) as crawler:
+        result = await crawler.arun("https://example.com", cache_mode=CacheMode.BYPASS)
+        assert result.success, "Browser config crawl failed"
+        assert len(result.html) > 0, "No HTML content retrieved"
+
+
+async def test_browser_performance_config():
+    """Test browser configurations focused on performance"""
+    browser_config = BrowserConfig(
+        text_mode=True,
+        light_mode=True,
+        extra_args=["--disable-gpu", "--disable-software-rasterizer"],
+        ignore_https_errors=True,
+        java_script_enabled=False,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("https://example.com")
+        assert result.success, "Performance optimized crawl failed"
+        assert result.status_code == 200, "Unexpected status code"
+
+
+# Category 2: Content Processing Tests
+async def test_content_extraction_config():
+    """Test content extraction with various strategies"""
+    crawler_config = CrawlerRunConfig(
+        word_count_threshold=300,
+        extraction_strategy=JsonCssExtractionStrategy(
+            schema={
+                "name": "article",
+                "baseSelector": "div",
+                "fields": [{"name": "title", "selector": "h1", "type": "text"}],
+            }
+        ),
+        chunking_strategy=RegexChunking(),
+        content_filter=PruningContentFilter(),
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://example.com/article", config=crawler_config
+        )
+        assert result.extracted_content is not None, "Content extraction failed"
+        assert "title" in result.extracted_content, "Missing expected content field"
+
+
+# Category 3: Cache and Session Management Tests
+async def test_cache_and_session_management():
+    """Test different cache modes and session handling"""
+    browser_config = BrowserConfig(use_persistent_context=True)
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.WRITE_ONLY,
+        process_iframes=True,
+        remove_overlay_elements=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # First request - should write to cache
+        result1 = await crawler.arun("https://example.com", config=crawler_config)
+
+        # Second request - should use fresh fetch due to WRITE_ONLY mode
+        result2 = await crawler.arun("https://example.com", config=crawler_config)
+
+        assert result1.success and result2.success, "Cache mode crawl failed"
+        assert result1.html == result2.html, "Inconsistent results between requests"
+
+
+# Category 4: Media Handling Tests
+async def test_media_handling_config():
+    """Test configurations related to media handling"""
+    # Get the base path for home directroy ~/.crawl4ai/downloads, make sure it exists
+    os.makedirs(os.path.expanduser("~/.crawl4ai/downloads"), exist_ok=True)
+    browser_config = BrowserConfig(
+        viewport_width=1920,
+        viewport_height=1080,
+        accept_downloads=True,
+        downloads_path=os.path.expanduser("~/.crawl4ai/downloads"),
+    )
+    crawler_config = CrawlerRunConfig(
+        screenshot=True,
+        pdf=True,
+        adjust_viewport_to_content=True,
+        wait_for_images=True,
+        screenshot_height_threshold=20000,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun("https://example.com", config=crawler_config)
+        assert result.screenshot is not None, "Screenshot capture failed"
+        assert result.pdf is not None, "PDF generation failed"
+
+
+# Category 5: Anti-Bot and Site Interaction Tests
+async def test_antibot_config():
+    """Test configurations for handling anti-bot measures"""
+    crawler_config = CrawlerRunConfig(
+        simulate_user=True,
+        override_navigator=True,
+        magic=True,
+        wait_for="js:()=>document.querySelector('body')",
+        delay_before_return_html=1.0,
+        log_console=True,
+        cache_mode=CacheMode.BYPASS,
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun("https://example.com", config=crawler_config)
+        assert result.success, "Anti-bot measure handling failed"
+
+
+# Category 6: Parallel Processing Tests
+async def test_parallel_processing():
+    """Test parallel processing capabilities"""
+    crawler_config = CrawlerRunConfig(mean_delay=0.5, max_range=1.0, semaphore_count=5)
+
+    urls = ["https://example.com/1", "https://example.com/2", "https://example.com/3"]
+
+    async with AsyncWebCrawler() as crawler:
+        results = await crawler.arun_many(urls, config=crawler_config)
+        assert len(results) == len(urls), "Not all URLs were processed"
+        assert all(r.success for r in results), "Some parallel requests failed"
+
+
+# Category 7: Backwards Compatibility Tests
+async def test_legacy_parameter_support():
+    """Test that legacy parameters still work"""
+    async with AsyncWebCrawler(
+        headless=True, browser_type="chromium", viewport_width=1024, viewport_height=768
+    ) as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            screenshot=True,
+            word_count_threshold=200,
+            bypass_cache=True,
+            css_selector=".main-content",
+        )
+        assert result.success, "Legacy parameter support failed"
+
+
+# Category 8: Mixed Configuration Tests
+async def test_mixed_config_usage():
+    """Test mixing new config objects with legacy parameters"""
+    browser_config = BrowserConfig(headless=True)
+    crawler_config = CrawlerRunConfig(screenshot=True)
+
+    async with AsyncWebCrawler(
+        config=browser_config,
+        verbose=True,  # legacy parameter
+    ) as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            config=crawler_config,
+            cache_mode=CacheMode.BYPASS,  # legacy parameter
+            css_selector="body",  # legacy parameter
+        )
+        assert result.success, "Mixed configuration usage failed"
+
+
+if __name__ == "__main__":
+
+    async def run_tests():
+        test_functions = [
+            test_browser_config_object,
+            # test_browser_performance_config,
+            # test_content_extraction_config,
+            # test_cache_and_session_management,
+            # test_media_handling_config,
+            # test_antibot_config,
+            # test_parallel_processing,
+            # test_legacy_parameter_support,
+            # test_mixed_config_usage
+        ]
+
+        for test in test_functions:
+            print(f"\nRunning {test.__name__}...")
+            try:
+                await test()
+                print(f"✓ {test.__name__} passed")
+            except AssertionError as e:
+                print(f"✗ {test.__name__} failed: {str(e)}")
+            except Exception as e:
+                print(f"✗ {test.__name__} error: {str(e)}")
+
+    asyncio.run(run_tests())

tests/async/test_async_doanloader.py

@@ -0,0 +1,247 @@
+import os
+import sys
+import asyncio
+import shutil
+from typing import List
+import tempfile
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+class TestDownloads:
+    def __init__(self):
+        self.temp_dir = tempfile.mkdtemp(prefix="crawl4ai_test_")
+        self.download_dir = os.path.join(self.temp_dir, "downloads")
+        os.makedirs(self.download_dir, exist_ok=True)
+        self.results: List[str] = []
+
+    def cleanup(self):
+        shutil.rmtree(self.temp_dir)
+
+    def log_result(self, test_name: str, success: bool, message: str = ""):
+        result = f"{'✅' if success else '❌'} {test_name}: {message}"
+        self.results.append(result)
+        print(result)
+
+    async def test_basic_download(self):
+        """Test basic file download functionality"""
+        try:
+            async with AsyncWebCrawler(
+                accept_downloads=True, downloads_path=self.download_dir, verbose=True
+            ) as crawler:
+                # Python.org downloads page typically has stable download links
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="""
+                    // Click first download link
+                    const downloadLink = document.querySelector('a[href$=".exe"]');
+                    if (downloadLink) downloadLink.click();
+                    """,
+                )
+
+                success = (
+                    result.downloaded_files is not None
+                    and len(result.downloaded_files) > 0
+                )
+                self.log_result(
+                    "Basic Download",
+                    success,
+                    f"Downloaded {len(result.downloaded_files or [])} files"
+                    if success
+                    else "No files downloaded",
+                )
+        except Exception as e:
+            self.log_result("Basic Download", False, str(e))
+
+    async def test_persistent_context_download(self):
+        """Test downloads with persistent context"""
+        try:
+            user_data_dir = os.path.join(self.temp_dir, "user_data")
+            os.makedirs(user_data_dir, exist_ok=True)
+
+            async with AsyncWebCrawler(
+                accept_downloads=True,
+                downloads_path=self.download_dir,
+                use_persistent_context=True,
+                user_data_dir=user_data_dir,
+                verbose=True,
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="""
+                    const downloadLink = document.querySelector('a[href$=".exe"]');
+                    if (downloadLink) downloadLink.click();
+                    """,
+                )
+
+                success = (
+                    result.downloaded_files is not None
+                    and len(result.downloaded_files) > 0
+                )
+                self.log_result(
+                    "Persistent Context Download",
+                    success,
+                    f"Downloaded {len(result.downloaded_files or [])} files"
+                    if success
+                    else "No files downloaded",
+                )
+        except Exception as e:
+            self.log_result("Persistent Context Download", False, str(e))
+
+    async def test_multiple_downloads(self):
+        """Test multiple simultaneous downloads"""
+        try:
+            async with AsyncWebCrawler(
+                accept_downloads=True, downloads_path=self.download_dir, verbose=True
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="""
+                    // Click multiple download links
+                    const downloadLinks = document.querySelectorAll('a[href$=".exe"]');
+                    downloadLinks.forEach(link => link.click());
+                    """,
+                )
+
+                success = (
+                    result.downloaded_files is not None
+                    and len(result.downloaded_files) > 1
+                )
+                self.log_result(
+                    "Multiple Downloads",
+                    success,
+                    f"Downloaded {len(result.downloaded_files or [])} files"
+                    if success
+                    else "Not enough files downloaded",
+                )
+        except Exception as e:
+            self.log_result("Multiple Downloads", False, str(e))
+
+    async def test_different_browsers(self):
+        """Test downloads across different browser types"""
+        browsers = ["chromium", "firefox", "webkit"]
+
+        for browser_type in browsers:
+            try:
+                async with AsyncWebCrawler(
+                    accept_downloads=True,
+                    downloads_path=self.download_dir,
+                    browser_type=browser_type,
+                    verbose=True,
+                ) as crawler:
+                    result = await crawler.arun(
+                        url="https://www.python.org/downloads/",
+                        js_code="""
+                        const downloadLink = document.querySelector('a[href$=".exe"]');
+                        if (downloadLink) downloadLink.click();
+                        """,
+                    )
+
+                    success = (
+                        result.downloaded_files is not None
+                        and len(result.downloaded_files) > 0
+                    )
+                    self.log_result(
+                        f"{browser_type.title()} Download",
+                        success,
+                        f"Downloaded {len(result.downloaded_files or [])} files"
+                        if success
+                        else "No files downloaded",
+                    )
+            except Exception as e:
+                self.log_result(f"{browser_type.title()} Download", False, str(e))
+
+    async def test_edge_cases(self):
+        """Test various edge cases"""
+
+        # Test 1: Downloads without specifying download path
+        try:
+            async with AsyncWebCrawler(accept_downloads=True, verbose=True) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="document.querySelector('a[href$=\".exe\"]').click()",
+                )
+                self.log_result(
+                    "Default Download Path",
+                    True,
+                    f"Downloaded to default path: {result.downloaded_files[0] if result.downloaded_files else 'None'}",
+                )
+        except Exception as e:
+            self.log_result("Default Download Path", False, str(e))
+
+        # Test 2: Downloads with invalid path
+        try:
+            async with AsyncWebCrawler(
+                accept_downloads=True,
+                downloads_path="/invalid/path/that/doesnt/exist",
+                verbose=True,
+            ) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="document.querySelector('a[href$=\".exe\"]').click()",
+                )
+                self.log_result(
+                    "Invalid Download Path", False, "Should have raised an error"
+                )
+        except Exception:
+            self.log_result(
+                "Invalid Download Path", True, "Correctly handled invalid path"
+            )
+
+        # Test 3: Download with accept_downloads=False
+        try:
+            async with AsyncWebCrawler(accept_downloads=False, verbose=True) as crawler:
+                result = await crawler.arun(
+                    url="https://www.python.org/downloads/",
+                    js_code="document.querySelector('a[href$=\".exe\"]').click()",
+                )
+                success = result.downloaded_files is None
+                self.log_result(
+                    "Disabled Downloads",
+                    success,
+                    "Correctly ignored downloads"
+                    if success
+                    else "Unexpectedly downloaded files",
+                )
+        except Exception as e:
+            self.log_result("Disabled Downloads", False, str(e))
+
+    async def run_all_tests(self):
+        """Run all test cases"""
+        print("\n🧪 Running Download Tests...\n")
+
+        test_methods = [
+            self.test_basic_download,
+            self.test_persistent_context_download,
+            self.test_multiple_downloads,
+            self.test_different_browsers,
+            self.test_edge_cases,
+        ]
+
+        for test in test_methods:
+            print(f"\n📝 Running {test.__doc__}...")
+            await test()
+            await asyncio.sleep(2)  # Brief pause between tests
+
+        print("\n📊 Test Results Summary:")
+        for result in self.results:
+            print(result)
+
+        successes = len([r for r in self.results if "✅" in r])
+        total = len(self.results)
+        print(f"\nTotal: {successes}/{total} tests passed")
+
+        self.cleanup()
+
+
+async def main():
+    tester = TestDownloads()
+    await tester.run_all_tests()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/async/test_basic_crawling.py

@@ -0,0 +1,90 @@
+import os
+import sys
+import pytest
+import time
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_successful_crawl():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.url == url
+        assert result.html
+        assert result.markdown
+        assert result.cleaned_html
+
+
+@pytest.mark.asyncio
+async def test_invalid_url():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.invalidurl12345.com"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert not result.success
+        assert result.error_message
+
+
+@pytest.mark.asyncio
+async def test_multiple_urls():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        urls = [
+            "https://www.nbcnews.com/business",
+            "https://www.example.com",
+            "https://www.python.org",
+        ]
+        results = await crawler.arun_many(urls=urls, bypass_cache=True)
+        assert len(results) == len(urls)
+        assert all(result.success for result in results)
+        assert all(result.html for result in results)
+
+
+@pytest.mark.asyncio
+async def test_javascript_execution():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        js_code = "document.body.innerHTML = '<h1>Modified by JS</h1>';"
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True, js_code=js_code)
+        assert result.success
+        assert "<h1>Modified by JS</h1>" in result.html
+
+
+@pytest.mark.asyncio
+async def test_concurrent_crawling_performance():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        urls = [
+            "https://www.nbcnews.com/business",
+            "https://www.example.com",
+            "https://www.python.org",
+            "https://www.github.com",
+            "https://www.stackoverflow.com",
+        ]
+
+        start_time = time.time()
+        results = await crawler.arun_many(urls=urls, bypass_cache=True)
+        end_time = time.time()
+
+        total_time = end_time - start_time
+        print(f"Total time for concurrent crawling: {total_time:.2f} seconds")
+
+        assert all(result.success for result in results)
+        assert len(results) == len(urls)
+
+        # Assert that concurrent crawling is faster than sequential
+        # This multiplier may need adjustment based on the number of URLs and their complexity
+        assert (
+            total_time < len(urls) * 5
+        ), f"Concurrent crawling not significantly faster: {total_time:.2f} seconds"
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_browser_lifecycle.py

@@ -0,0 +1,972 @@
+"""
+Browser lifecycle & concurrency tests.
+
+Covers all the browser launch paths and lock interactions:
+  - Standalone (playwright.launch)
+  - Managed browser (subprocess + CDP connect)
+  - Managed browser with create_isolated_context
+  - Page reuse on shared default context
+  - Context caching / LRU eviction
+  - Session lifecycle across all modes
+  - Concurrent crawls racing for pages / contexts
+  - Recycle interacting with managed browser
+  - Multiple crawlers sharing a managed browser via CDP
+"""
+
+import asyncio
+import time
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Local test server
+# ---------------------------------------------------------------------------
+
+PAGES = {}
+for i in range(100):
+    PAGES[f"/page{i}"] = (
+        f"<!DOCTYPE html><html><head><title>Page {i}</title></head>"
+        f"<body><h1>Page {i}</h1><p>Content for page {i}.</p>"
+        f"<a href='/page{(i+1)%100}'>next</a></body></html>"
+    ).encode()
+
+# Login/dashboard for session tests
+PAGES["/login"] = (
+    b"<!DOCTYPE html><html><head><title>Login</title></head>"
+    b"<body><h1>Login</h1><p>Logged in.</p></body></html>"
+)
+PAGES["/dashboard"] = (
+    b"<!DOCTYPE html><html><head><title>Dashboard</title></head>"
+    b"<body><h1>Dashboard</h1><p>Dashboard content.</p></body></html>"
+)
+
+
+class Handler(SimpleHTTPRequestHandler):
+    def log_message(self, *a):
+        pass
+
+    def do_GET(self):
+        body = PAGES.get(self.path, PAGES["/page0"])
+        self.send_response(200)
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(body)
+
+
+class _Server(HTTPServer):
+    allow_reuse_address = True
+
+
+@pytest.fixture(scope="module")
+def srv():
+    s = _Server(("127.0.0.1", 0), Handler)
+    port = s.server_address[1]
+    t = threading.Thread(target=s.serve_forever, daemon=True)
+    t.start()
+    yield f"http://127.0.0.1:{port}"
+    s.shutdown()
+
+
+def _u(base, i):
+    return f"{base}/page{i}"
+
+
+def _bm(c):
+    return c.crawler_strategy.browser_manager
+
+
+# ===================================================================
+# SECTION A — Standalone browser (no CDP, no managed browser)
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_standalone_basic_crawl(srv):
+    """Standalone browser: launch, crawl, close. Baseline correctness."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+        assert "Page 0" in r.html
+
+
+@pytest.mark.asyncio
+async def test_standalone_sequential_crawls(srv):
+    """10 sequential pages — each gets its own page, context reused by config sig."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+            assert f"Page {i}" in r.html
+
+
+@pytest.mark.asyncio
+async def test_standalone_concurrent_crawls(srv):
+    """10 concurrent crawls on standalone browser — no crashes,
+    context lock prevents race conditions."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(10)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        assert all(r.success for r in results if not isinstance(r, Exception))
+
+
+@pytest.mark.asyncio
+async def test_standalone_context_reuse(srv):
+    """Two crawls with identical config should reuse the same context.
+    Two crawls with different configs should create different contexts."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        run_a = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        r1 = await c.arun(url=_u(srv, 0), config=run_a)
+        assert r1.success
+        ctx_count_after_first = len(bm.contexts_by_config)
+
+        # Same config → same context
+        r2 = await c.arun(url=_u(srv, 1), config=run_a)
+        assert r2.success
+        assert len(bm.contexts_by_config) == ctx_count_after_first, (
+            "Same config should reuse context"
+        )
+
+        # Different config → new context
+        run_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, verbose=False,
+            override_navigator=True,
+        )
+        r3 = await c.arun(url=_u(srv, 2), config=run_b)
+        assert r3.success
+        assert len(bm.contexts_by_config) == ctx_count_after_first + 1, (
+            "Different config should create new context"
+        )
+
+
+@pytest.mark.asyncio
+async def test_standalone_session_multistep(srv):
+    """Session across 3 pages on standalone browser."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="standalone_sess", verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(3):
+            r = await c.arun(url=_u(srv, i), config=sess)
+            assert r.success
+            assert "standalone_sess" in bm.sessions
+
+        # Refcount should be exactly 1
+        _, page, _ = bm.sessions["standalone_sess"]
+        sig = bm._page_to_sig.get(page)
+        if sig:
+            assert bm._context_refcounts.get(sig, 0) == 1
+
+        # Kill session and verify cleanup
+        await c.crawler_strategy.kill_session("standalone_sess")
+        assert "standalone_sess" not in bm.sessions
+        if sig:
+            assert bm._context_refcounts.get(sig, 0) == 0
+
+
+@pytest.mark.asyncio
+async def test_standalone_recycle(srv):
+    """Recycling on standalone browser — close/start cycle."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False, max_pages_before_recycle=5,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+        for i in range(8):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        # Recycle happened at page 5, pages 6-8 after → counter = 3
+        assert bm._pages_served == 3
+
+
+@pytest.mark.asyncio
+async def test_standalone_recycle_with_concurrent_crawls(srv):
+    """15 concurrent crawls straddling a recycle boundary on standalone."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False, max_pages_before_recycle=5,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(15)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 15
+
+
+# ===================================================================
+# SECTION B — Managed browser (subprocess + CDP)
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_managed_basic_crawl(srv):
+    """Managed browser: start subprocess, connect via CDP, crawl, close."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+        assert "Page 0" in r.html
+
+
+@pytest.mark.asyncio
+async def test_managed_sequential_crawls(srv):
+    """Sequential crawls on managed browser — pages reused from default context."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        for i in range(8):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+
+@pytest.mark.asyncio
+async def test_managed_concurrent_crawls(srv):
+    """Concurrent crawls on managed browser — _global_pages_lock prevents
+    two tasks from grabbing the same page."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(8)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 8
+
+
+@pytest.mark.asyncio
+async def test_managed_page_reuse(srv):
+    """On managed browser (non-isolated), pages should be reused when
+    released back to the pool."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Crawl 3 pages sequentially — page should be reused each time
+        for i in range(3):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # On managed browser, total pages created should be small
+        # (pages reused, not new ones for each crawl)
+        default_ctx = bm.default_context
+        total_pages = len(default_ctx.pages)
+        assert total_pages <= 3, (
+            f"Expected page reuse, but {total_pages} pages exist"
+        )
+
+
+@pytest.mark.asyncio
+async def test_managed_session_multistep(srv):
+    """Multi-step session on managed browser — session page stays alive."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="managed_sess", verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        r = await c.arun(url=f"{srv}/login", config=sess)
+        assert r.success
+
+        r = await c.arun(url=f"{srv}/dashboard", config=sess)
+        assert r.success
+
+        assert "managed_sess" in bm.sessions
+
+
+@pytest.mark.asyncio
+async def test_managed_recycle(srv):
+    """Recycling on managed browser — kills subprocess, restarts, crawls resume."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        max_pages_before_recycle=4,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(7):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed after managed recycle"
+
+        # Recycled at 4 → pages 5,6,7 after → counter = 3
+        assert bm._pages_served == 3
+
+
+# ===================================================================
+# SECTION C — Managed browser with create_isolated_context
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_isolated_context_basic(srv):
+    """Isolated context mode: each config gets its own browser context."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_concurrent(srv):
+    """Concurrent crawls with isolated contexts — _contexts_lock prevents
+    race conditions in context creation."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(10)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 10
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_caching(srv):
+    """Same config signature → same context. Different config → different context."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        run_a = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        await c.arun(url=_u(srv, 0), config=run_a)
+        count_after_a = len(bm.contexts_by_config)
+
+        # Same config → reuse
+        await c.arun(url=_u(srv, 1), config=run_a)
+        assert len(bm.contexts_by_config) == count_after_a
+
+        # Different config → new context
+        run_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, verbose=False,
+            override_navigator=True,
+        )
+        await c.arun(url=_u(srv, 2), config=run_b)
+        assert len(bm.contexts_by_config) == count_after_a + 1
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_refcount(srv):
+    """Refcount increases with concurrent crawls and decreases on release."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # After a single sequential crawl (page released), refcount should be 0
+        r = await c.arun(url=_u(srv, 0), config=run)
+        assert r.success
+
+        # All contexts should have refcount 0 (page was released)
+        for sig, rc in bm._context_refcounts.items():
+            assert rc == 0, f"Refcount for {sig[:8]}... should be 0, got {rc}"
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_session_with_interleaved(srv):
+    """Session on isolated context + non-session crawls interleaved."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        sess = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="iso_sess", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess)
+        assert r.success
+
+        # Non-session crawls
+        run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        for i in range(5):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # Session still alive
+        assert "iso_sess" in bm.sessions
+        r = await c.arun(url=f"{srv}/dashboard", config=sess)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_isolated_context_recycle(srv):
+    """Recycling with isolated contexts — all contexts cleared, new ones
+    created fresh on the new browser."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+        max_pages_before_recycle=4,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(6):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        # Recycled at 4 → 5,6 after → counter = 2
+        assert bm._pages_served == 2
+        # Contexts dict should only have entries from after recycle
+        assert all(rc == 0 for rc in bm._context_refcounts.values()), (
+            "All refcounts should be 0 after sequential crawls"
+        )
+
+
+# ===================================================================
+# SECTION D — Two crawlers sharing one managed browser via CDP URL
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_two_crawlers_share_managed_browser(srv):
+    """Two AsyncWebCrawler instances connect to the same managed browser
+    via its CDP URL. Both should crawl successfully without interfering."""
+    # First crawler owns the managed browser
+    cfg1 = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg1) as c1:
+        bm1 = _bm(c1)
+        # Grab the CDP URL from the managed browser
+        cdp_url = f"http://{bm1.managed_browser.host}:{bm1.managed_browser.debugging_port}"
+
+        # Second crawler connects to the same browser via CDP
+        cfg2 = BrowserConfig(
+            headless=True, verbose=False,
+            cdp_url=cdp_url,
+            cdp_cleanup_on_close=True,
+        )
+        async with AsyncWebCrawler(config=cfg2) as c2:
+            run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+            # Crawl sequentially to avoid page contention on shared context
+            r1 = await c1.arun(url=_u(srv, 0), config=run)
+            r2 = await c2.arun(url=_u(srv, 1), config=run)
+
+            assert r1.success, f"Crawler 1 failed: {r1.error_message}"
+            assert r2.success, f"Crawler 2 failed: {r2.error_message}"
+            assert "Page 0" in r1.html
+            assert "Page 1" in r2.html
+
+
+@pytest.mark.asyncio
+async def test_two_crawlers_concurrent_heavy(srv):
+    """Two crawlers sharing one managed browser, each doing 5 concurrent crawls."""
+    cfg1 = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+
+    async with AsyncWebCrawler(config=cfg1) as c1:
+        bm1 = _bm(c1)
+        cdp_url = f"http://{bm1.managed_browser.host}:{bm1.managed_browser.debugging_port}"
+
+        cfg2 = BrowserConfig(
+            headless=True, verbose=False,
+            cdp_url=cdp_url,
+            cdp_cleanup_on_close=True,
+        )
+        async with AsyncWebCrawler(config=cfg2) as c2:
+            run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+            # Each crawler does 5 sequential crawls while both are connected
+            for i in range(5):
+                r1 = await c1.arun(url=_u(srv, i), config=run)
+                assert r1.success, f"Crawler 1 page {i} failed: {r1.error_message}"
+                r2 = await c2.arun(url=_u(srv, i + 50), config=run)
+                assert r2.success, f"Crawler 2 page {i} failed: {r2.error_message}"
+
+
+# ===================================================================
+# SECTION E — Session lifecycle edge cases
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_session_then_nonsession_then_session(srv):
+    """session crawl → non-session crawl → session crawl.
+    The session should persist across non-session activity."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="interleave_sess", verbose=False,
+    )
+    no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        r = await c.arun(url=_u(srv, 0), config=sess)
+        assert r.success
+
+        # Non-session crawls
+        for i in range(3):
+            r = await c.arun(url=_u(srv, 10 + i), config=no_sess)
+            assert r.success
+
+        # Session should still exist and work
+        assert "interleave_sess" in bm.sessions
+        r = await c.arun(url=_u(srv, 99), config=sess)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_multiple_sessions_simultaneous(srv):
+    """3 independent sessions open at the same time, each navigating
+    different pages. They should not interfere."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        sessions = [
+            CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS, session_id=f"sess_{j}", verbose=False,
+            )
+            for j in range(3)
+        ]
+
+        # Step 1: open all sessions
+        for j, s in enumerate(sessions):
+            r = await c.arun(url=_u(srv, j * 10), config=s)
+            assert r.success, f"Session {j} open failed"
+
+        assert len(bm.sessions) == 3
+
+        # Step 2: navigate each session to a second page
+        for j, s in enumerate(sessions):
+            r = await c.arun(url=_u(srv, j * 10 + 1), config=s)
+            assert r.success, f"Session {j} step 2 failed"
+
+        # Step 3: kill sessions one by one, verify others unaffected
+        await c.crawler_strategy.kill_session("sess_0")
+        assert "sess_0" not in bm.sessions
+        assert "sess_1" in bm.sessions
+        assert "sess_2" in bm.sessions
+
+        # Remaining sessions still work
+        r = await c.arun(url=_u(srv, 99), config=sessions[1])
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_session_kill_then_recreate(srv):
+    """Kill a session, then create a new session with the same ID.
+    The new session should work on a fresh page."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    sess = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, session_id="reuse_id", verbose=False,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        r = await c.arun(url=_u(srv, 0), config=sess)
+        assert r.success
+        _, page_v1, _ = bm.sessions["reuse_id"]
+
+        await c.crawler_strategy.kill_session("reuse_id")
+        assert "reuse_id" not in bm.sessions
+
+        # Re-create with same ID
+        r = await c.arun(url=_u(srv, 50), config=sess)
+        assert r.success
+        assert "reuse_id" in bm.sessions
+        _, page_v2, _ = bm.sessions["reuse_id"]
+
+        # Should be a different page object
+        assert page_v1 is not page_v2, "Re-created session should have a new page"
+
+
+# ===================================================================
+# SECTION F — Concurrent recycle + session stress tests
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_recycle_concurrent_sessions_and_nonsessions(srv):
+    """Open 2 sessions + fire 10 non-session crawls concurrently with
+    recycle threshold=5. Sessions should block recycle until they're
+    done or killed. All crawls should succeed."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=5,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Open sessions first
+        sess_a = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="stress_a", verbose=False,
+        )
+        sess_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="stress_b", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess_a)
+        assert r.success
+        r = await c.arun(url=f"{srv}/login", config=sess_b)
+        assert r.success
+
+        # Fire 10 concurrent non-session crawls
+        no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        tasks = [c.arun(url=_u(srv, i), config=no_sess) for i in range(10)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        # Sessions should still be alive (blocking recycle)
+        assert "stress_a" in bm.sessions
+        assert "stress_b" in bm.sessions
+
+        # Use sessions again — should work
+        r = await c.arun(url=f"{srv}/dashboard", config=sess_a)
+        assert r.success
+        r = await c.arun(url=f"{srv}/dashboard", config=sess_b)
+        assert r.success
+
+
+@pytest.mark.asyncio
+async def test_arun_many_with_session_open(srv):
+    """Session open while arun_many batch runs with recycle enabled.
+    Session survives the batch."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=5,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        sess = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="batch_guard", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess)
+        assert r.success
+
+        no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        urls = [_u(srv, i) for i in range(12)]
+        results = await c.arun_many(urls, config=no_sess)
+        assert all(r.success for r in results)
+
+        # Session still alive
+        assert "batch_guard" in bm.sessions
+
+
+@pytest.mark.asyncio
+async def test_rapid_recycle_stress(srv):
+    """Recycle threshold=2 with 20 sequential crawls → 10 recycle cycles.
+    Every crawl must succeed. Proves recycle is stable under rapid cycling."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        for i in range(20):
+            r = await c.arun(url=_u(srv, i % 100), config=run)
+            assert r.success, f"Page {i} failed during rapid recycle"
+
+
+@pytest.mark.asyncio
+async def test_rapid_recycle_concurrent(srv):
+    """Recycle threshold=3 with 12 concurrent crawls. Concurrency +
+    rapid recycling together."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(12)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 12
+
+
+# ===================================================================
+# SECTION G — Lock correctness under contention
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_context_lock_no_duplicate_contexts(srv):
+    """Fire 20 concurrent crawls with the same config on isolated context mode.
+    Despite concurrency, only 1 context should be created (all share the
+    same config signature)."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(20)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        # All had the same config → only 1 context should exist
+        assert len(bm.contexts_by_config) == 1, (
+            f"Expected 1 context, got {len(bm.contexts_by_config)} — "
+            f"lock failed to prevent duplicate creation"
+        )
+
+
+@pytest.mark.asyncio
+async def test_page_lock_no_duplicate_pages_managed(srv):
+    """On managed browser (shared default context), concurrent crawls should
+    never get the same page. After all complete, pages_in_use should be empty."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(8)]
+        await asyncio.gather(*tasks)
+
+        # After all crawls complete, no pages should be marked in use
+        piu = bm._get_pages_in_use()
+        assert len(piu) == 0, (
+            f"After all crawls complete, {len(piu)} pages still marked in use"
+        )
+
+
+@pytest.mark.asyncio
+async def test_refcount_correctness_under_concurrency(srv):
+    """Fire 15 concurrent crawls with isolated context. After all complete,
+    all refcounts should be 0."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(15)]
+        await asyncio.gather(*tasks)
+
+        for sig, rc in bm._context_refcounts.items():
+            assert rc == 0, (
+                f"Refcount for context {sig[:8]}... is {rc}, expected 0 "
+                f"after all crawls complete"
+            )
+
+
+# ===================================================================
+# SECTION H — Close / cleanup correctness
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_close_cleans_up_standalone(srv):
+    """After closing standalone crawler, browser and playwright are None."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    c = AsyncWebCrawler(config=cfg)
+    await c.start()
+    bm = _bm(c)
+
+    r = await c.arun(url=_u(srv, 0), config=run)
+    assert r.success
+
+    await c.close()
+    assert bm.browser is None
+    assert bm.playwright is None
+
+
+@pytest.mark.asyncio
+async def test_close_cleans_up_managed(srv):
+    """After closing managed crawler, managed_browser is cleaned up."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    c = AsyncWebCrawler(config=cfg)
+    await c.start()
+    bm = _bm(c)
+
+    r = await c.arun(url=_u(srv, 0), config=run)
+    assert r.success
+
+    await c.close()
+    assert bm.browser is None
+    assert bm.managed_browser is None
+
+
+@pytest.mark.asyncio
+async def test_double_close_safe(srv):
+    """Calling close() twice should not raise."""
+    cfg = BrowserConfig(headless=True, verbose=False)
+
+    c = AsyncWebCrawler(config=cfg)
+    await c.start()
+    r = await c.arun(url=_u(srv, 0), config=CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS, verbose=False,
+    ))
+    assert r.success
+
+    await c.close()
+    # Second close should be safe
+    await c.close()
+
+
+# ===================================================================
+# SECTION I — Mixed modes: session + recycle + managed + concurrent
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_managed_isolated_session_recycle_concurrent(srv):
+    """The ultimate stress test: managed browser + isolated contexts +
+    sessions + recycle + concurrent crawls.
+
+    Flow:
+    1. Open session A
+    2. Fire 8 concurrent non-session crawls (threshold=5, but session blocks)
+    3. Kill session A
+    4. Fire 3 more non-session crawls to trigger recycle
+    5. Open session B on the fresh browser
+    6. Verify session B works
+    """
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+        max_pages_before_recycle=5,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+        no_sess = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+        # Step 1: open session
+        sess_a = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="ultimate_a", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess_a)
+        assert r.success
+
+        # Step 2: concurrent non-session crawls
+        tasks = [c.arun(url=_u(srv, i), config=no_sess) for i in range(8)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions in step 2: {excs[:3]}"
+
+        # Session blocks recycle
+        assert "ultimate_a" in bm.sessions
+
+        # Step 3: kill session
+        await c.crawler_strategy.kill_session("ultimate_a")
+
+        # Step 4: trigger recycle
+        for i in range(3):
+            r = await c.arun(url=_u(srv, 80 + i), config=no_sess)
+            assert r.success
+
+        await asyncio.sleep(0.5)
+
+        # Step 5: new session on fresh browser
+        sess_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, session_id="ultimate_b", verbose=False,
+        )
+        r = await c.arun(url=f"{srv}/login", config=sess_b)
+        assert r.success
+        assert "ultimate_b" in bm.sessions
+
+        # Step 6: verify it works
+        r = await c.arun(url=f"{srv}/dashboard", config=sess_b)
+        assert r.success

tests/async/test_browser_recycle_v2.py

@@ -0,0 +1,386 @@
+"""
+Tests for version-based browser recycling.
+
+The new recycle approach:
+1. When pages_served hits threshold, bump _browser_version
+2. Old signatures go to _pending_cleanup
+3. New requests get new contexts (different version = different signature)
+4. When old context's refcount hits 0, it gets cleaned up
+5. No blocking — old and new browsers coexist during transition
+
+These tests use small thresholds (3-5 pages) to verify the mechanics.
+"""
+
+import asyncio
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+
+# ---------------------------------------------------------------------------
+# Local test server
+# ---------------------------------------------------------------------------
+
+PAGES = {}
+for i in range(100):
+    PAGES[f"/page{i}"] = (
+        f"<!DOCTYPE html><html><head><title>Page {i}</title></head>"
+        f"<body><h1>Page {i}</h1><p>Content for page {i}.</p></body></html>"
+    ).encode()
+
+
+class Handler(SimpleHTTPRequestHandler):
+    def log_message(self, *a):
+        pass
+
+    def do_GET(self):
+        body = PAGES.get(self.path, PAGES["/page0"])
+        self.send_response(200)
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(body)
+
+
+class _Server(HTTPServer):
+    allow_reuse_address = True
+
+
+@pytest.fixture(scope="module")
+def srv():
+    s = _Server(("127.0.0.1", 0), Handler)
+    port = s.server_address[1]
+    t = threading.Thread(target=s.serve_forever, daemon=True)
+    t.start()
+    yield f"http://127.0.0.1:{port}"
+    s.shutdown()
+
+
+def _u(base, i):
+    return f"{base}/page{i}"
+
+
+def _bm(c):
+    return c.crawler_strategy.browser_manager
+
+
+# ===================================================================
+# SECTION A — Version bump mechanics
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_version_bump_on_threshold(srv):
+    """Browser version should bump when threshold is reached."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        assert bm._browser_version == 1
+
+        # Crawl 2 pages — no bump yet
+        for i in range(2):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        assert bm._browser_version == 1, "Version should still be 1 after 2 pages"
+        assert bm._pages_served == 2
+
+        # 3rd page hits threshold (3) and triggers bump AFTER the page is served
+        r = await c.arun(url=_u(srv, 2), config=run)
+        assert r.success
+        assert bm._browser_version == 2, "Version should bump after 3rd page"
+        assert bm._pages_served == 0, "Counter resets after bump"
+
+        # 4th page is first page of version 2
+        r = await c.arun(url=_u(srv, 3), config=run)
+        assert r.success
+        assert bm._pages_served == 1
+
+
+@pytest.mark.asyncio
+async def test_signature_changes_after_version_bump(srv):
+    """Same CrawlerRunConfig should produce different signatures after version bump."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Get signature before bump
+        sig_v1 = bm._make_config_signature(run)
+
+        # Crawl 2 pages
+        for i in range(2):
+            await c.arun(url=_u(srv, i), config=run)
+
+        # 3rd request triggers bump
+        await c.arun(url=_u(srv, 2), config=run)
+
+        # Signature should be different now
+        sig_v2 = bm._make_config_signature(run)
+        assert sig_v1 != sig_v2, "Signature should change after version bump"
+
+
+@pytest.mark.asyncio
+async def test_no_version_bump_when_disabled(srv):
+    """Version should stay at 1 when max_pages_before_recycle=0."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=0,  # Disabled
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(20):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        assert bm._browser_version == 1, "Version should not bump when disabled"
+        assert bm._pages_served == 20
+
+
+# ===================================================================
+# SECTION B — Pending cleanup mechanics
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_old_signature_goes_to_pending_cleanup(srv):
+    """Version bump works and old contexts get cleaned up."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Crawl 2 pages — creates signature for version 1, bumps on 2nd
+        for i in range(2):
+            await c.arun(url=_u(srv, i), config=run)
+
+        # After 2 pages with threshold=2, version should have bumped
+        assert bm._browser_version == 2
+
+        # Since sequential crawls release pages immediately (refcount=0),
+        # old contexts get cleaned up right away. Pending cleanup should be empty.
+        # This is correct behavior — cleanup is eager when possible.
+        assert len(bm._pending_cleanup) == 0
+
+
+@pytest.mark.asyncio
+async def test_cleanup_happens_when_refcount_hits_zero(srv):
+    """Old context should be closed when its refcount drops to 0."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Sequential crawls: each page is released before next request
+        # So refcount is always 0 between requests, and cleanup happens immediately
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # Should have bumped twice (at 3 and 6) with version now at 3
+        # But since refcount=0 immediately, pending_cleanup should be empty
+        assert len(bm._pending_cleanup) == 0, "All old contexts should be cleaned up"
+
+
+# ===================================================================
+# SECTION C — Concurrent crawls with recycling
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_concurrent_crawls_dont_block_on_recycle(srv):
+    """Concurrent crawls should not block — old browser drains while new one serves."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=5,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # Launch 20 concurrent crawls
+        tasks = [c.arun(url=_u(srv, i), config=run) for i in range(20)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # All should succeed — no blocking, no errors
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 20, f"Only {len(successes)} succeeded"
+
+        # Version should have bumped multiple times
+        assert bm._browser_version >= 2, "Should have recycled at least once"
+
+
+@pytest.mark.asyncio
+async def test_high_concurrency_with_small_threshold(srv):
+    """Stress test: 50 concurrent crawls with threshold=3."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        # 50 concurrent crawls with threshold of 3 — many version bumps
+        tasks = [c.arun(url=_u(srv, i % 100), config=run) for i in range(50)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        excs = [r for r in results if isinstance(r, Exception)]
+        assert len(excs) == 0, f"Exceptions: {excs[:3]}"
+
+        successes = [r for r in results if not isinstance(r, Exception) and r.success]
+        assert len(successes) == 50
+
+
+# ===================================================================
+# SECTION D — Safety cap (max pending browsers)
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_safety_cap_limits_pending_browsers(srv):
+    """Should not exceed _max_pending_browsers old browsers draining."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+        bm._max_pending_browsers = 2  # Lower cap for testing
+
+        # Run enough crawls to potentially exceed the cap
+        for i in range(15):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # Pending cleanup should never have exceeded the cap
+        # (We can't directly test this during execution, but if it works without
+        # deadlock/timeout, the cap logic is functioning)
+        assert len(bm._pending_cleanup) <= bm._max_pending_browsers
+
+
+# ===================================================================
+# SECTION E — Managed browser mode
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_managed_browser_recycle(srv):
+    """Recycling should work with managed browser mode."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        # Version should have bumped
+        assert bm._browser_version >= 2
+
+
+@pytest.mark.asyncio
+async def test_managed_browser_isolated_context_recycle(srv):
+    """Recycling with managed browser + isolated contexts."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        use_managed_browser=True,
+        create_isolated_context=True,
+        max_pages_before_recycle=3,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(10):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success, f"Page {i} failed"
+
+        assert bm._browser_version >= 2
+
+
+# ===================================================================
+# SECTION F — Edge cases
+# ===================================================================
+
+@pytest.mark.asyncio
+async def test_threshold_of_one(srv):
+    """Edge case: threshold=1 means version bump after every page."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=1,
+    )
+    run = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        for i in range(5):
+            r = await c.arun(url=_u(srv, i), config=run)
+            assert r.success
+
+        # With threshold=1, each page triggers a bump after being served:
+        # Page 0: served, counter=1 >= 1, bump -> version=2, counter=0
+        # Page 1: served, counter=1 >= 1, bump -> version=3, counter=0
+        # ... etc.
+        # After 5 pages, should have bumped 5 times
+        assert bm._browser_version == 6  # Started at 1, bumped 5 times
+
+
+@pytest.mark.asyncio
+async def test_different_configs_get_separate_cleanup_tracking(srv):
+    """Different CrawlerRunConfigs should track separately in pending cleanup."""
+    cfg = BrowserConfig(
+        headless=True, verbose=False,
+        max_pages_before_recycle=2,
+    )
+
+    async with AsyncWebCrawler(config=cfg) as c:
+        bm = _bm(c)
+
+        run_a = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+        run_b = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS, verbose=False,
+            override_navigator=True,  # Different config
+        )
+
+        # Alternate between configs
+        for i in range(6):
+            cfg_to_use = run_a if i % 2 == 0 else run_b
+            r = await c.arun(url=_u(srv, i), config=cfg_to_use)
+            assert r.success
+
+        # Both configs should work fine
+        assert bm._browser_version >= 2

tests/async/test_caching.py

@@ -0,0 +1,87 @@
+import os
+import sys
+import pytest
+import asyncio
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_caching():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # First crawl (should not use cache)
+        start_time = asyncio.get_event_loop().time()
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        end_time = asyncio.get_event_loop().time()
+        time_taken1 = end_time - start_time
+
+        assert result1.success
+
+        # Second crawl (should use cache)
+        start_time = asyncio.get_event_loop().time()
+        result2 = await crawler.arun(url=url, bypass_cache=False)
+        end_time = asyncio.get_event_loop().time()
+        time_taken2 = end_time - start_time
+
+        assert result2.success
+        assert time_taken2 < time_taken1  # Cached result should be faster
+
+
+@pytest.mark.asyncio
+async def test_bypass_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # First crawl
+        result1 = await crawler.arun(url=url, bypass_cache=False)
+        assert result1.success
+
+        # Second crawl with bypass_cache=True
+        result2 = await crawler.arun(url=url, bypass_cache=True)
+        assert result2.success
+
+        # Content should be different (or at least, not guaranteed to be the same)
+        assert result1.html != result2.html or result1.markdown != result2.markdown
+
+
+@pytest.mark.asyncio
+async def test_clear_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # Crawl and cache
+        await crawler.arun(url=url, bypass_cache=False)
+
+        # Clear cache
+        await crawler.aclear_cache()
+
+        # Check cache size
+        cache_size = await crawler.aget_cache_size()
+        assert cache_size == 0
+
+
+@pytest.mark.asyncio
+async def test_flush_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # Crawl and cache
+        await crawler.arun(url=url, bypass_cache=False)
+
+        # Flush cache
+        await crawler.aflush_cache()
+
+        # Check cache size
+        cache_size = await crawler.aget_cache_size()
+        assert cache_size == 0
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_chunking_and_extraction_strategies.py

@@ -0,0 +1,86 @@
+import os
+import sys
+import pytest
+import json
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai import LLMConfig
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+from crawl4ai.chunking_strategy import RegexChunking
+from crawl4ai import LLMExtractionStrategy
+
+
+@pytest.mark.asyncio
+async def test_regex_chunking():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        chunking_strategy = RegexChunking(patterns=["\n\n"])
+        result = await crawler.arun(
+            url=url, chunking_strategy=chunking_strategy, bypass_cache=True
+        )
+        assert result.success
+        assert result.extracted_content
+        chunks = json.loads(result.extracted_content)
+        assert len(chunks) > 1  # Ensure multiple chunks were created
+
+
+# @pytest.mark.asyncio
+# async def test_cosine_strategy():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://www.nbcnews.com/business"
+#         extraction_strategy = CosineStrategy(word_count_threshold=10, max_dist=0.2, linkage_method="ward", top_k=3, sim_threshold=0.3)
+#         result = await crawler.arun(
+#             url=url,
+#             extraction_strategy=extraction_strategy,
+#             bypass_cache=True
+#         )
+#         assert result.success
+#         assert result.extracted_content
+#         extracted_data = json.loads(result.extracted_content)
+#         assert len(extracted_data) > 0
+#         assert all('tags' in item for item in extracted_data)
+
+
+@pytest.mark.asyncio
+async def test_llm_extraction_strategy():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        extraction_strategy = LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o-mini",api_token=os.getenv("OPENAI_API_KEY")),
+            instruction="Extract only content related to technology",
+        )
+        result = await crawler.arun(
+            url=url, extraction_strategy=extraction_strategy, bypass_cache=True
+        )
+        assert result.success
+        assert result.extracted_content
+        extracted_data = json.loads(result.extracted_content)
+        assert len(extracted_data) > 0
+        assert all("content" in item for item in extracted_data)
+
+
+# @pytest.mark.asyncio
+# async def test_combined_chunking_and_extraction():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://www.nbcnews.com/business"
+#         chunking_strategy = RegexChunking(patterns=["\n\n"])
+#         extraction_strategy = CosineStrategy(word_count_threshold=10, max_dist=0.2, linkage_method="ward", top_k=3, sim_threshold=0.3)
+#         result = await crawler.arun(
+#             url=url,
+#             chunking_strategy=chunking_strategy,
+#             extraction_strategy=extraction_strategy,
+#             bypass_cache=True
+#         )
+#         assert result.success
+#         assert result.extracted_content
+#         extracted_data = json.loads(result.extracted_content)
+#         assert len(extracted_data) > 0
+#         assert all('tags' in item for item in extracted_data)
+#         assert all('content' in item for item in extracted_data)
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_content_extraction.py

@@ -0,0 +1,108 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_extract_markdown():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.markdown
+        assert isinstance(result.markdown, str)
+        assert len(result.markdown) > 0
+
+
+@pytest.mark.asyncio
+async def test_extract_cleaned_html():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.cleaned_html
+        assert isinstance(result.cleaned_html, str)
+        assert len(result.cleaned_html) > 0
+
+
+@pytest.mark.asyncio
+async def test_extract_media():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.media
+        media = result.media
+        assert isinstance(media, dict)
+        assert "images" in media
+        assert isinstance(media["images"], list)
+        for image in media["images"]:
+            assert "src" in image
+            assert "alt" in image
+            assert "type" in image
+
+
+@pytest.mark.asyncio
+async def test_extract_links():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.links
+        links = result.links
+        assert isinstance(links, dict)
+        assert "internal" in links
+        assert "external" in links
+        assert isinstance(links["internal"], list)
+        assert isinstance(links["external"], list)
+        for link in links["internal"] + links["external"]:
+            assert "href" in link
+            assert "text" in link
+
+
+@pytest.mark.asyncio
+async def test_extract_metadata():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert result.metadata
+        metadata = result.metadata
+        assert isinstance(metadata, dict)
+        assert "title" in metadata
+        assert isinstance(metadata["title"], str)
+
+
+@pytest.mark.asyncio
+async def test_css_selector_extraction():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        css_selector = "h1, h2, h3"
+        result = await crawler.arun(
+            url=url, bypass_cache=True, css_selector=css_selector
+        )
+        assert result.success
+        assert result.markdown
+        assert all(heading in result.markdown for heading in ["#", "##", "###"])
+
+@pytest.mark.asyncio
+async def test_base_tag_link_extraction():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://sohamkukreti.github.io/portfolio"
+        result = await crawler.arun(url=url)
+        assert result.success
+        assert result.links
+        assert isinstance(result.links, dict)
+        assert "internal" in result.links
+        assert "external" in result.links
+        assert any("github.com" in x["href"] for x in result.links["external"])
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_content_filter_bm25.py

@@ -0,0 +1,183 @@
+import os, sys
+import pytest
+from bs4 import BeautifulSoup
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+
+@pytest.fixture
+def basic_html():
+    return """
+    <html>
+        <head>
+            <title>Test Article</title>
+            <meta name="description" content="Test description">
+            <meta name="keywords" content="test, keywords">
+        </head>
+        <body>
+            <h1>Main Heading</h1>
+            <article>
+                <p>This is a long paragraph with more than fifty words. It continues with more text to ensure we meet the minimum word count threshold. We need to make sure this paragraph is substantial enough to be considered for extraction according to our filtering rules. This should be enough words now.</p>
+                <div class="navigation">Skip this nav content</div>
+            </article>
+        </body>
+    </html>
+    """
+
+
+@pytest.fixture
+def wiki_html():
+    return """
+    <html>
+        <head>
+            <title>Wikipedia Article</title>
+        </head>
+        <body>
+            <h1>Article Title</h1>
+            <h2>Section 1</h2>
+            <p>Short but important section header description.</p>
+            <div class="content">
+                <p>Long paragraph with sufficient words to meet the minimum threshold. This paragraph continues with more text to ensure we have enough content for proper testing. We need to make sure this has enough words to pass our filters and be considered valid content for extraction purposes.</p>
+            </div>
+        </body>
+    </html>
+    """
+
+
+@pytest.fixture
+def no_meta_html():
+    return """
+    <html>
+        <body>
+            <h1>Simple Page</h1>
+            <p>First paragraph that should be used as fallback for query when no meta tags exist. This text needs to be long enough to serve as a meaningful fallback for our content extraction process.</p>
+        </body>
+    </html>
+    """
+
+
+class TestBM25ContentFilter:
+    def test_basic_extraction(self, basic_html):
+        """Test basic content extraction functionality"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(basic_html)
+
+        assert contents, "Should extract content"
+        assert len(contents) >= 1, "Should extract at least one content block"
+        assert "long paragraph" in " ".join(contents).lower()
+        assert "navigation" not in " ".join(contents).lower()
+
+    def test_user_query_override(self, basic_html):
+        """Test that user query overrides metadata extraction"""
+        user_query = "specific test query"
+        filter = BM25ContentFilter(user_query=user_query)
+
+        # Access internal state to verify query usage
+        soup = BeautifulSoup(basic_html, "lxml")
+        extracted_query = filter.extract_page_query(soup.find("head"))
+
+        assert extracted_query == user_query
+        assert "Test description" not in extracted_query
+
+    def test_header_extraction(self, wiki_html):
+        """Test that headers are properly extracted despite length"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(wiki_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "section 1" in combined_content, "Should include section header"
+        assert "article title" in combined_content, "Should include main title"
+
+    def test_no_metadata_fallback(self, no_meta_html):
+        """Test fallback behavior when no metadata is present"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(no_meta_html)
+
+        assert contents, "Should extract content even without metadata"
+        assert "First paragraph" in " ".join(
+            contents
+        ), "Should use first paragraph content"
+
+    def test_empty_input(self):
+        """Test handling of empty input"""
+        filter = BM25ContentFilter()
+        assert filter.filter_content("") == []
+        assert filter.filter_content(None) == []
+
+    def test_malformed_html(self):
+        """Test handling of malformed HTML"""
+        malformed_html = "<p>Unclosed paragraph<div>Nested content</p></div>"
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(malformed_html)
+
+        assert isinstance(contents, list), "Should return list even with malformed HTML"
+
+    def test_threshold_behavior(self, basic_html):
+        """Test different BM25 threshold values"""
+        strict_filter = BM25ContentFilter(bm25_threshold=2.0)
+        lenient_filter = BM25ContentFilter(bm25_threshold=0.5)
+
+        strict_contents = strict_filter.filter_content(basic_html)
+        lenient_contents = lenient_filter.filter_content(basic_html)
+
+        assert len(strict_contents) <= len(
+            lenient_contents
+        ), "Strict threshold should extract fewer elements"
+
+    def test_html_cleaning(self, basic_html):
+        """Test HTML cleaning functionality"""
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(basic_html)
+
+        cleaned_content = " ".join(contents)
+        assert "class=" not in cleaned_content, "Should remove class attributes"
+        assert "style=" not in cleaned_content, "Should remove style attributes"
+        assert "<script" not in cleaned_content, "Should remove script tags"
+
+    def test_large_content(self):
+        """Test handling of large content blocks"""
+        large_html = f"""
+        <html><body>
+            <article>{'<p>Test content. ' * 1000}</article>
+        </body></html>
+        """
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(large_html)
+        assert contents, "Should handle large content blocks"
+
+    @pytest.mark.parametrize(
+        "unwanted_tag", ["script", "style", "nav", "footer", "header"]
+    )
+    def test_excluded_tags(self, unwanted_tag):
+        """Test that specific tags are properly excluded"""
+        html = f"""
+        <html><body>
+            <{unwanted_tag}>Should not appear</{unwanted_tag}>
+            <p>Should appear</p>
+        </body></html>
+        """
+        filter = BM25ContentFilter()
+        contents = filter.filter_content(html)
+
+        combined_content = " ".join(contents).lower()
+        assert "should not appear" not in combined_content
+
+    def test_performance(self, basic_html):
+        """Test performance with timer"""
+        filter = BM25ContentFilter()
+
+        import time
+
+        start = time.perf_counter()
+        filter.filter_content(basic_html)
+        duration = time.perf_counter() - start
+
+        assert duration < 1.0, f"Processing took too long: {duration:.2f} seconds"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

tests/async/test_content_filter_prune.py

@@ -0,0 +1,170 @@
+import os, sys
+import pytest
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+
+@pytest.fixture
+def basic_html():
+    return """
+    <html>
+        <body>
+            <article>
+                <h1>Main Article</h1>
+                <p>This is a high-quality paragraph with substantial text content. It contains enough words to pass the threshold and has good text density without too many links. This kind of content should survive the pruning process.</p>
+                <div class="sidebar">Low quality sidebar content</div>
+                <div class="social-share">Share buttons</div>
+            </article>
+        </body>
+    </html>
+    """
+
+
+@pytest.fixture
+def link_heavy_html():
+    return """
+    <html>
+        <body>
+            <div class="content">
+                <p>Good content paragraph that should remain.</p>
+                <div class="links">
+                    <a href="#">Link 1</a>
+                    <a href="#">Link 2</a>
+                    <a href="#">Link 3</a>
+                    <a href="#">Link 4</a>
+                </div>
+            </div>
+        </body>
+    </html>
+    """
+
+
+@pytest.fixture
+def mixed_content_html():
+    return """
+    <html>
+        <body>
+            <article>
+                <h1>Article Title</h1>
+                <p class="summary">Short summary.</p>
+                <div class="content">
+                    <p>Long high-quality paragraph with substantial content that should definitely survive the pruning process. This content has good text density and proper formatting which makes it valuable for retention.</p>
+                </div>
+                <div class="comments">
+                    <p>Short comment 1</p>
+                    <p>Short comment 2</p>
+                </div>
+            </article>
+        </body>
+    </html>
+    """
+
+
+class TestPruningContentFilter:
+    def test_basic_pruning(self, basic_html):
+        """Test basic content pruning functionality"""
+        filter = PruningContentFilter(min_word_threshold=5)
+        contents = filter.filter_content(basic_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "high-quality paragraph" in combined_content
+        assert "sidebar content" not in combined_content
+        assert "share buttons" not in combined_content
+
+    def test_min_word_threshold(self, mixed_content_html):
+        """Test minimum word threshold filtering"""
+        filter = PruningContentFilter(min_word_threshold=10)
+        contents = filter.filter_content(mixed_content_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "short summary" not in combined_content
+        assert "long high-quality paragraph" in combined_content
+        assert "short comment" not in combined_content
+
+    def test_threshold_types(self, basic_html):
+        """Test fixed vs dynamic thresholds"""
+        fixed_filter = PruningContentFilter(threshold_type="fixed", threshold=0.48)
+        dynamic_filter = PruningContentFilter(threshold_type="dynamic", threshold=0.45)
+
+        fixed_contents = fixed_filter.filter_content(basic_html)
+        dynamic_contents = dynamic_filter.filter_content(basic_html)
+
+        assert len(fixed_contents) != len(
+            dynamic_contents
+        ), "Fixed and dynamic thresholds should yield different results"
+
+    def test_link_density_impact(self, link_heavy_html):
+        """Test handling of link-heavy content"""
+        filter = PruningContentFilter(threshold_type="dynamic")
+        contents = filter.filter_content(link_heavy_html)
+
+        combined_content = " ".join(contents).lower()
+        assert "good content paragraph" in combined_content
+        assert (
+            len([c for c in contents if "href" in c]) < 2
+        ), "Should prune link-heavy sections"
+
+    def test_tag_importance(self, mixed_content_html):
+        """Test tag importance in scoring"""
+        filter = PruningContentFilter(threshold_type="dynamic")
+        contents = filter.filter_content(mixed_content_html)
+
+        has_article = any("article" in c.lower() for c in contents)
+        has_h1 = any("h1" in c.lower() for c in contents)
+        assert has_article or has_h1, "Should retain important tags"
+
+    def test_empty_input(self):
+        """Test handling of empty input"""
+        filter = PruningContentFilter()
+        assert filter.filter_content("") == []
+        assert filter.filter_content(None) == []
+
+    def test_malformed_html(self):
+        """Test handling of malformed HTML"""
+        malformed_html = "<div>Unclosed div<p>Nested<span>content</div>"
+        filter = PruningContentFilter()
+        contents = filter.filter_content(malformed_html)
+        assert isinstance(contents, list)
+
+    def test_performance(self, basic_html):
+        """Test performance with timer"""
+        filter = PruningContentFilter()
+
+        import time
+
+        start = time.perf_counter()
+        filter.filter_content(basic_html)
+        duration = time.perf_counter() - start
+
+        # Extra strict on performance since you mentioned milliseconds matter
+        assert duration < 0.1, f"Processing took too long: {duration:.3f} seconds"
+
+    @pytest.mark.parametrize(
+        "threshold,expected_count",
+        [
+            (0.3, 4),  # Very lenient
+            (0.48, 2),  # Default
+            (0.7, 1),  # Very strict
+        ],
+    )
+    def test_threshold_levels(self, mixed_content_html, threshold, expected_count):
+        """Test different threshold levels"""
+        filter = PruningContentFilter(threshold_type="fixed", threshold=threshold)
+        contents = filter.filter_content(mixed_content_html)
+        assert (
+            len(contents) <= expected_count
+        ), f"Expected {expected_count} or fewer elements with threshold {threshold}"
+
+    def test_consistent_output(self, basic_html):
+        """Test output consistency across multiple runs"""
+        filter = PruningContentFilter()
+        first_run = filter.filter_content(basic_html)
+        second_run = filter.filter_content(basic_html)
+        assert first_run == second_run, "Output should be consistent"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

tests/async/test_content_scraper_strategy.py

@@ -0,0 +1,216 @@
+import os
+import sys
+import time
+import csv
+from tabulate import tabulate
+from dataclasses import dataclass
+from typing import List
+
+parent_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+# This test compares the same strategy with itself now since WebScrapingStrategy is deprecated
+
+
+@dataclass
+class TestResult:
+    name: str
+    success: bool
+    images: int
+    internal_links: int
+    external_links: int
+    markdown_length: int
+    execution_time: float
+
+
+class StrategyTester:
+    def __init__(self):
+        self.new_scraper = LXMLWebScrapingStrategy()
+        self.current_scraper = LXMLWebScrapingStrategy()  # Same strategy now
+        with open(__location__ + "/sample_wikipedia.html", "r", encoding="utf-8") as f:
+            self.WIKI_HTML = f.read()
+        self.results = {"new": [], "current": []}
+
+    def run_test(self, name: str, **kwargs) -> tuple[TestResult, TestResult]:
+        results = []
+        for scraper in [self.new_scraper, self.current_scraper]:
+            start_time = time.time()
+            result = scraper._get_content_of_website_optimized(
+                url="https://en.wikipedia.org/wiki/Test", html=self.WIKI_HTML, **kwargs
+            )
+            execution_time = time.time() - start_time
+
+            test_result = TestResult(
+                name=name,
+                success=result["success"],
+                images=len(result["media"]["images"]),
+                internal_links=len(result["links"]["internal"]),
+                external_links=len(result["links"]["external"]),
+                markdown_length=len(result["markdown"]),
+                execution_time=execution_time,
+            )
+            results.append(test_result)
+
+        return results[0], results[1]  # new, current
+
+    def run_all_tests(self):
+        test_cases = [
+            ("Basic Extraction", {}),
+            ("Exclude Tags", {"excluded_tags": ["table", "div.infobox", "div.navbox"]}),
+            ("Word Threshold", {"word_count_threshold": 50}),
+            ("CSS Selector", {"css_selector": "div.mw-parser-output > p"}),
+            (
+                "Link Exclusions",
+                {
+                    "exclude_external_links": True,
+                    "exclude_social_media_links": True,
+                    "exclude_domains": ["facebook.com", "twitter.com"],
+                },
+            ),
+            (
+                "Media Handling",
+                {
+                    "exclude_external_images": True,
+                    "image_description_min_word_threshold": 20,
+                },
+            ),
+            ("Text Only", {"only_text": True, "remove_forms": True}),
+            ("HTML Cleaning", {"clean_html": True, "keep_data_attributes": True}),
+            (
+                "HTML2Text Options",
+                {
+                    "html2text": {
+                        "skip_internal_links": True,
+                        "single_line_break": True,
+                        "mark_code": True,
+                        "preserve_tags": ["pre", "code"],
+                    }
+                },
+            ),
+        ]
+
+        all_results = []
+        for name, kwargs in test_cases:
+            try:
+                new_result, current_result = self.run_test(name, **kwargs)
+                all_results.append((name, new_result, current_result))
+            except Exception as e:
+                print(f"Error in {name}: {str(e)}")
+
+        self.save_results_to_csv(all_results)
+        self.print_comparison_table(all_results)
+
+    def save_results_to_csv(self, all_results: List[tuple]):
+        csv_file = os.path.join(__location__, "strategy_comparison_results.csv")
+        with open(csv_file, "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(
+                [
+                    "Test Name",
+                    "Strategy",
+                    "Success",
+                    "Images",
+                    "Internal Links",
+                    "External Links",
+                    "Markdown Length",
+                    "Execution Time",
+                ]
+            )
+
+            for name, new_result, current_result in all_results:
+                writer.writerow(
+                    [
+                        name,
+                        "New",
+                        new_result.success,
+                        new_result.images,
+                        new_result.internal_links,
+                        new_result.external_links,
+                        new_result.markdown_length,
+                        f"{new_result.execution_time:.3f}",
+                    ]
+                )
+                writer.writerow(
+                    [
+                        name,
+                        "Current",
+                        current_result.success,
+                        current_result.images,
+                        current_result.internal_links,
+                        current_result.external_links,
+                        current_result.markdown_length,
+                        f"{current_result.execution_time:.3f}",
+                    ]
+                )
+
+    def print_comparison_table(self, all_results: List[tuple]):
+        table_data = []
+        headers = [
+            "Test Name",
+            "Strategy",
+            "Success",
+            "Images",
+            "Internal Links",
+            "External Links",
+            "Markdown Length",
+            "Time (s)",
+        ]
+
+        for name, new_result, current_result in all_results:
+            # Check for differences
+            differences = []
+            if new_result.images != current_result.images:
+                differences.append("images")
+            if new_result.internal_links != current_result.internal_links:
+                differences.append("internal_links")
+            if new_result.external_links != current_result.external_links:
+                differences.append("external_links")
+            if new_result.markdown_length != current_result.markdown_length:
+                differences.append("markdown")
+
+            # Add row for new strategy
+            new_row = [
+                name,
+                "New",
+                new_result.success,
+                new_result.images,
+                new_result.internal_links,
+                new_result.external_links,
+                new_result.markdown_length,
+                f"{new_result.execution_time:.3f}",
+            ]
+            table_data.append(new_row)
+
+            # Add row for current strategy
+            current_row = [
+                "",
+                "Current",
+                current_result.success,
+                current_result.images,
+                current_result.internal_links,
+                current_result.external_links,
+                current_result.markdown_length,
+                f"{current_result.execution_time:.3f}",
+            ]
+            table_data.append(current_row)
+
+            # Add difference summary if any
+            if differences:
+                table_data.append(
+                    ["", "⚠️ Differences", ", ".join(differences), "", "", "", "", ""]
+                )
+
+            # Add empty row for better readability
+            table_data.append([""] * len(headers))
+
+        print("\nStrategy Comparison Results:")
+        print(tabulate(table_data, headers=headers, tablefmt="grid"))
+
+
+if __name__ == "__main__":
+    tester = StrategyTester()
+    tester.run_all_tests()

tests/async/test_crawler_strategy.py

@@ -0,0 +1,73 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_custom_user_agent():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        custom_user_agent = "MyCustomUserAgent/1.0"
+        crawler.crawler_strategy.update_user_agent(custom_user_agent)
+        url = "https://httpbin.org/user-agent"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert custom_user_agent in result.html
+
+
+@pytest.mark.asyncio
+async def test_custom_headers():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        custom_headers = {"X-Test-Header": "TestValue"}
+        crawler.crawler_strategy.set_custom_headers(custom_headers)
+        url = "https://httpbin.org/headers"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert "X-Test-Header" in result.html
+        assert "TestValue" in result.html
+
+
+@pytest.mark.asyncio
+async def test_javascript_execution():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        js_code = "document.body.innerHTML = '<h1>Modified by JS</h1>';"
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True, js_code=js_code)
+        assert result.success
+        assert "<h1>Modified by JS</h1>" in result.html
+
+
+@pytest.mark.asyncio
+async def test_hook_execution():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+
+        async def test_hook(page):
+            await page.evaluate("document.body.style.backgroundColor = 'red';")
+            return page
+
+        crawler.crawler_strategy.set_hook("after_goto", test_hook)
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True)
+        assert result.success
+        assert "background-color: red" in result.html
+
+
+@pytest.mark.asyncio
+async def test_screenshot():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.com"
+        result = await crawler.arun(url=url, bypass_cache=True, screenshot=True)
+        assert result.success
+        assert result.screenshot
+        assert isinstance(result.screenshot, str)
+        assert len(result.screenshot) > 0
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_database_operations.py

@@ -0,0 +1,90 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_cache_url():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.com"
+        # First run to cache the URL
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        assert result1.success
+
+        # Second run to retrieve from cache
+        result2 = await crawler.arun(url=url, bypass_cache=False)
+        assert result2.success
+        assert result2.html == result1.html
+
+
+@pytest.mark.asyncio
+async def test_bypass_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.python.org"
+        # First run to cache the URL
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        assert result1.success
+
+        # Second run bypassing cache
+        result2 = await crawler.arun(url=url, bypass_cache=True)
+        assert result2.success
+        assert (
+            result2.html != result1.html
+        )  # Content might be different due to dynamic nature of websites
+
+
+@pytest.mark.asyncio
+async def test_cache_size():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        initial_size = await crawler.aget_cache_size()
+
+        url = "https://www.nbcnews.com/business"
+        await crawler.arun(url=url, bypass_cache=True)
+
+        new_size = await crawler.aget_cache_size()
+        assert new_size == initial_size + 1
+
+
+@pytest.mark.asyncio
+async def test_clear_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.org"
+        await crawler.arun(url=url, bypass_cache=True)
+
+        initial_size = await crawler.aget_cache_size()
+        assert initial_size > 0
+
+        await crawler.aclear_cache()
+        new_size = await crawler.aget_cache_size()
+        assert new_size == 0
+
+
+@pytest.mark.asyncio
+async def test_flush_cache():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.example.net"
+        await crawler.arun(url=url, bypass_cache=True)
+
+        initial_size = await crawler.aget_cache_size()
+        assert initial_size > 0
+
+        await crawler.aflush_cache()
+        new_size = await crawler.aget_cache_size()
+        assert new_size == 0
+
+        # Try to retrieve the previously cached URL
+        result = await crawler.arun(url=url, bypass_cache=False)
+        assert (
+            result.success
+        )  # The crawler should still succeed, but it will fetch the content anew
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_dispatchers.py

@@ -0,0 +1,170 @@
+import pytest
+import time
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CrawlerRunConfig,
+    MemoryAdaptiveDispatcher,
+    SemaphoreDispatcher,
+    RateLimiter,
+    CrawlerMonitor,
+    DisplayMode,
+    CacheMode,
+)
+
+
+@pytest.fixture
+def browser_config():
+    return BrowserConfig(headless=True, verbose=False)
+
+
+@pytest.fixture
+def run_config():
+    return CrawlerRunConfig(cache_mode=CacheMode.BYPASS, verbose=False)
+
+
+@pytest.fixture
+def test_urls():
+    return [
+        "http://example.com",
+        "http://example.com/page1",
+        "http://example.com/page2",
+    ]
+
+
+@pytest.mark.asyncio
+class TestDispatchStrategies:
+    async def test_memory_adaptive_basic(self, browser_config, run_config, test_urls):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=70.0, max_session_permit=2, check_interval=0.1
+            )
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_memory_adaptive_with_rate_limit(
+        self, browser_config, run_config, test_urls
+    ):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=70.0,
+                max_session_permit=2,
+                check_interval=0.1,
+                rate_limiter=RateLimiter(
+                    base_delay=(0.1, 0.2), max_delay=1.0, max_retries=2
+                ),
+            )
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_semaphore_basic(self, browser_config, run_config, test_urls):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = SemaphoreDispatcher(semaphore_count=2)
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_semaphore_with_rate_limit(
+        self, browser_config, run_config, test_urls
+    ):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = SemaphoreDispatcher(
+                semaphore_count=2,
+                rate_limiter=RateLimiter(
+                    base_delay=(0.1, 0.2), max_delay=1.0, max_retries=2
+                ),
+            )
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            assert all(r.success for r in results)
+
+    async def test_memory_adaptive_memory_error(
+        self, browser_config, run_config, test_urls
+    ):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                memory_threshold_percent=1.0,  # Set unrealistically low threshold
+                max_session_permit=2,
+                check_interval=0.1,
+                memory_wait_timeout=1.0,  # Short timeout for testing
+            )
+            with pytest.raises(MemoryError):
+                await crawler.arun_many(
+                    test_urls, config=run_config, dispatcher=dispatcher
+                )
+
+    async def test_empty_urls(self, browser_config, run_config):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2)
+            results = await crawler.arun_many(
+                [], config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == 0
+
+    async def test_single_url(self, browser_config, run_config):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2)
+            results = await crawler.arun_many(
+                ["http://example.com"], config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == 1
+            assert results[0].success
+
+    async def test_invalid_urls(self, browser_config, run_config):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2)
+            results = await crawler.arun_many(
+                ["http://invalid.url.that.doesnt.exist"],
+                config=run_config,
+                dispatcher=dispatcher,
+            )
+            assert len(results) == 1
+            assert not results[0].success
+
+    async def test_rate_limit_backoff(self, browser_config, run_config):
+        urls = ["http://example.com"] * 5  # Multiple requests to same domain
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            dispatcher = MemoryAdaptiveDispatcher(
+                max_session_permit=2,
+                rate_limiter=RateLimiter(
+                    base_delay=(0.1, 0.2),
+                    max_delay=1.0,
+                    max_retries=2,
+                    rate_limit_codes=[200],  # Force rate limiting for testing
+                ),
+            )
+            start_time = time.time()
+            results = await crawler.arun_many(
+                urls, config=run_config, dispatcher=dispatcher
+            )
+            duration = time.time() - start_time
+            assert len(results) == len(urls)
+            assert duration > 1.0  # Ensure rate limiting caused delays
+
+    async def test_monitor_integration(self, browser_config, run_config, test_urls):
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            monitor = CrawlerMonitor(
+                max_visible_rows=5, display_mode=DisplayMode.DETAILED
+            )
+            dispatcher = MemoryAdaptiveDispatcher(max_session_permit=2, monitor=monitor)
+            results = await crawler.arun_many(
+                test_urls, config=run_config, dispatcher=dispatcher
+            )
+            assert len(results) == len(test_urls)
+            # Check monitor stats
+            assert len(monitor.stats) == len(test_urls)
+            assert all(stat.end_time is not None for stat in monitor.stats.values())
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "--asyncio-mode=auto"])

tests/async/test_edge_cases.py

@@ -0,0 +1,133 @@
+import os
+import re
+import sys
+import pytest
+from bs4 import BeautifulSoup
+import asyncio
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+# @pytest.mark.asyncio
+# async def test_large_content_page():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://en.wikipedia.org/wiki/List_of_largest_known_stars"  # A page with a large table
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert len(result.html) > 1000000  # Expecting more than 1MB of content
+
+# @pytest.mark.asyncio
+# async def test_minimal_content_page():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://example.com"  # A very simple page
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert len(result.html) < 10000  # Expecting less than 10KB of content
+
+# @pytest.mark.asyncio
+# async def test_single_page_application():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://reactjs.org/"  # React's website is a SPA
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "react" in result.html.lower()
+
+# @pytest.mark.asyncio
+# async def test_page_with_infinite_scroll():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://news.ycombinator.com/"  # Hacker News has infinite scroll
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "hacker news" in result.html.lower()
+
+# @pytest.mark.asyncio
+# async def test_page_with_heavy_javascript():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://www.airbnb.com/"  # Airbnb uses a lot of JavaScript
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "airbnb" in result.html.lower()
+
+# @pytest.mark.asyncio
+# async def test_page_with_mixed_content():
+#     async with AsyncWebCrawler(verbose=True) as crawler:
+#         url = "https://github.com/"  # GitHub has a mix of static and dynamic content
+#         result = await crawler.arun(url=url, bypass_cache=True)
+#         assert result.success
+#         assert "github" in result.html.lower()
+
+
+# Add this test to your existing test file
+@pytest.mark.asyncio
+async def test_typescript_commits_multi_page():
+    first_commit = ""
+
+    async def on_execution_started(page):
+        nonlocal first_commit
+        try:
+            # Check if the page firct commit h4 text is different from the first commit (use document.querySelector('li.Box-sc-g0xbh4-0 h4'))
+            while True:
+                await page.wait_for_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await page.query_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await commit.evaluate("(element) => element.textContent")
+                commit = re.sub(r"\s+", "", commit)
+                if commit and commit != first_commit:
+                    first_commit = commit
+                    break
+                await asyncio.sleep(0.5)
+        except Exception as e:
+            print(f"Warning: New content didn't appear after JavaScript execution: {e}")
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+        """
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,  # Only use URL for the first page
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                js=js_next_page
+                if page > 0
+                else None,  # Don't click 'next' on the first page
+                bypass_cache=True,
+                js_only=page > 0,  # Use js_only for subsequent pages
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            # Parse the HTML and extract commits
+            soup = BeautifulSoup(result.cleaned_html, "html.parser")
+            commits = soup.select("li")
+            # Take first commit find h4 extract text
+            first_commit = commits[0].find("h4").text
+            first_commit = re.sub(r"\s+", "", first_commit)
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        # Clean up the session
+        await crawler.crawler_strategy.kill_session(session_id)
+
+        # Assertions
+        assert (
+            len(all_commits) >= 90
+        ), f"Expected at least 90 commits, but got {len(all_commits)}"
+
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_error_handling.py

@@ -0,0 +1,78 @@
+# import os
+# import sys
+# import pytest
+# import asyncio
+
+# # Add the parent directory to the Python path
+# parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# sys.path.append(parent_dir)
+
+# from crawl4ai.async_webcrawler import AsyncWebCrawler
+# from crawl4ai.utils import InvalidCSSSelectorError
+
+# class AsyncCrawlerWrapper:
+#     def __init__(self):
+#         self.crawler = None
+
+#     async def setup(self):
+#         self.crawler = AsyncWebCrawler(verbose=True)
+#         await self.crawler.awarmup()
+
+#     async def cleanup(self):
+#         if self.crawler:
+#             await self.crawler.aclear_cache()
+
+# @pytest.fixture(scope="module")
+# def crawler_wrapper():
+#     wrapper = AsyncCrawlerWrapper()
+#     asyncio.get_event_loop().run_until_complete(wrapper.setup())
+#     yield wrapper
+#     asyncio.get_event_loop().run_until_complete(wrapper.cleanup())
+
+# @pytest.mark.asyncio
+# async def test_network_error(crawler_wrapper):
+#     url = "https://www.nonexistentwebsite123456789.com"
+#     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True)
+#     assert not result.success
+#     assert "Failed to crawl" in result.error_message
+
+# # @pytest.mark.asyncio
+# # async def test_timeout_error(crawler_wrapper):
+# #     # Simulating a timeout by using a very short timeout value
+# #     url = "https://www.nbcnews.com/business"
+# #     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True, timeout=0.001)
+# #     assert not result.success
+# #     assert "timeout" in result.error_message.lower()
+
+# # @pytest.mark.asyncio
+# # async def test_invalid_css_selector(crawler_wrapper):
+# #     url = "https://www.nbcnews.com/business"
+# #     with pytest.raises(InvalidCSSSelectorError):
+# #         await crawler_wrapper.crawler.arun(url=url, bypass_cache=True, css_selector="invalid>>selector")
+
+# # @pytest.mark.asyncio
+# # async def test_js_execution_error(crawler_wrapper):
+# #     url = "https://www.nbcnews.com/business"
+# #     invalid_js = "This is not valid JavaScript code;"
+# #     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True, js=invalid_js)
+# #     assert not result.success
+# #     assert "JavaScript" in result.error_message
+
+# # @pytest.mark.asyncio
+# # async def test_empty_page(crawler_wrapper):
+# #     # Use a URL that typically returns an empty page
+# #     url = "http://example.com/empty"
+# #     result = await crawler_wrapper.crawler.arun(url=url, bypass_cache=True)
+# #     assert result.success  # The crawl itself should succeed
+# #     assert not result.markdown.strip()  # The markdown content should be empty or just whitespace
+
+# # @pytest.mark.asyncio
+# # async def test_rate_limiting(crawler_wrapper):
+# #     # Simulate rate limiting by making multiple rapid requests
+# #     url = "https://www.nbcnews.com/business"
+# #     results = await asyncio.gather(*[crawler_wrapper.crawler.arun(url=url, bypass_cache=True) for _ in range(10)])
+# #     assert any(not result.success and "rate limit" in result.error_message.lower() for result in results)
+
+# # Entry point for debugging
+# if __name__ == "__main__":
+#     pytest.main([__file__, "-v"])

tests/async/test_evaluation_scraping_methods_performance.configs.py

@@ -0,0 +1,705 @@
+import json
+import time
+from bs4 import BeautifulSoup
+from crawl4ai.content_scraping_strategy import (
+    WebScrapingStrategy,
+    LXMLWebScrapingStrategy,
+)
+from typing import Dict, List, Tuple
+import difflib
+from lxml import html as lhtml, etree
+
+
+def normalize_dom(element):
+    """
+    Recursively normalizes an lxml HTML element:
+      - Removes comment nodes
+      - Sorts attributes on each node
+      - Removes <head> if you want (optional)
+    Returns the same element (mutated).
+    """
+    # Remove comment nodes
+    comments = element.xpath("//comment()")
+    for c in comments:
+        p = c.getparent()
+        if p is not None:
+            p.remove(c)
+
+    # If you'd like to remove <head>, or unify <html>/<body>, you could do so here.
+    # For example, remove <head> entirely:
+    # heads = element.xpath('//head')
+    # for h in heads:
+    #     parent = h.getparent()
+    #     if parent is not None:
+    #         parent.remove(h)
+
+    # Sort attributes (to avoid false positives due to attr order)
+    for el in element.iter():
+        if el.attrib:
+            # Convert to a sorted list of (k, v), then reassign
+            sorted_attribs = sorted(el.attrib.items())
+            el.attrib.clear()
+            for k, v in sorted_attribs:
+                el.set(k, v)
+
+    return element
+
+
+def strip_html_body(root):
+    """
+    If 'root' is <html>, find its <body> child and move all of <body>'s children
+    into a new <div>. Return that <div>.
+
+    If 'root' is <body>, similarly move all of its children into a new <div> and return it.
+
+    Otherwise, return 'root' as-is.
+    """
+    tag_name = (root.tag or "").lower()
+
+    # Case 1: The root is <html>
+    if tag_name == "html":
+        bodies = root.xpath("./body")
+        if bodies:
+            body = bodies[0]
+            new_div = lhtml.Element("div")
+            for child in body:
+                new_div.append(child)
+            return new_div
+        else:
+            # No <body> found; just return the <html> root
+            return root
+
+    # Case 2: The root is <body>
+    elif tag_name == "body":
+        new_div = lhtml.Element("div")
+        for child in root:
+            new_div.append(child)
+        return new_div
+
+    # Case 3: Neither <html> nor <body>
+    else:
+        return root
+
+
+def compare_nodes(node1, node2, differences, path="/"):
+    """
+    Recursively compare two lxml nodes, appending textual differences to `differences`.
+    `path` is used to indicate the location in the tree (like an XPath).
+    """
+    # 1) Compare tag names
+    if node1.tag != node2.tag:
+        differences.append(f"Tag mismatch at {path}: '{node1.tag}' vs. '{node2.tag}'")
+        return
+
+    # 2) Compare attributes
+    # By now, they are sorted in normalize_dom()
+    attrs1 = list(node1.attrib.items())
+    attrs2 = list(node2.attrib.items())
+    if attrs1 != attrs2:
+        differences.append(
+            f"Attribute mismatch at {path}/{node1.tag}: {attrs1} vs. {attrs2}"
+        )
+
+    # 3) Compare text (trim or unify whitespace as needed)
+    text1 = (node1.text or "").strip()
+    text2 = (node2.text or "").strip()
+    # Normalize whitespace
+    text1 = " ".join(text1.split())
+    text2 = " ".join(text2.split())
+    if text1 != text2:
+        # If you prefer ignoring newlines or multiple whitespace, do a more robust cleanup
+        differences.append(
+            f"Text mismatch at {path}/{node1.tag}: '{text1}' vs. '{text2}'"
+        )
+
+    # 4) Compare number of children
+    children1 = list(node1)
+    children2 = list(node2)
+    if len(children1) != len(children2):
+        differences.append(
+            f"Child count mismatch at {path}/{node1.tag}: {len(children1)} vs. {len(children2)}"
+        )
+        return  # If counts differ, no point comparing child by child
+
+    # 5) Recursively compare each child
+    for i, (c1, c2) in enumerate(zip(children1, children2)):
+        # Build a path for child
+        child_path = f"{path}/{node1.tag}[{i}]"
+        compare_nodes(c1, c2, differences, child_path)
+
+    # 6) Compare tail text
+    tail1 = (node1.tail or "").strip()
+    tail2 = (node2.tail or "").strip()
+    if tail1 != tail2:
+        differences.append(
+            f"Tail mismatch after {path}/{node1.tag}: '{tail1}' vs. '{tail2}'"
+        )
+
+
+def compare_html_structurally(html1, html2):
+    """
+    Compare two HTML strings using a structural approach with lxml.
+    Returns a list of differences (if any). If empty, they're effectively the same.
+    """
+    # 1) Parse both
+    try:
+        tree1 = lhtml.fromstring(html1)
+    except etree.ParserError:
+        return ["Error parsing HTML1"]
+
+    try:
+        tree2 = lhtml.fromstring(html2)
+    except etree.ParserError:
+        return ["Error parsing HTML2"]
+
+    # 2) Normalize both DOMs (remove comments, sort attributes, etc.)
+    tree1 = normalize_dom(tree1)
+    tree2 = normalize_dom(tree2)
+
+    # 3) Possibly strip <html>/<body> wrappers for better apples-to-apples comparison
+    tree1 = strip_html_body(tree1)
+    tree2 = strip_html_body(tree2)
+
+    # 4) Compare recursively
+    differences = []
+    compare_nodes(tree1, tree2, differences, path="")
+    return differences
+
+
+def generate_large_html(n_elements=1000):
+    html = ["<!DOCTYPE html><html><head></head><body>"]
+    for i in range(n_elements):
+        html.append(
+            f"""
+            <div class="article">
+                <h2>Heading {i}</h2>
+                <p>This is paragraph {i} with some content and a <a href="http://example.com/{i}">link</a></p>
+                <img src="image{i}.jpg" alt="Image {i}">
+                <ul>
+                    <li>List item {i}.1</li>
+                    <li>List item {i}.2</li>
+                </ul>
+            </div>
+        """
+        )
+    html.append("</body></html>")
+    return "".join(html)
+
+
+def generate_complicated_html():
+    """
+    HTML with multiple domains, forms, data attributes,
+    various images, comments, style, and noscript to test all parameter toggles.
+    """
+    return """
+    <!DOCTYPE html>
+    <html>
+      <head>
+        <title>Complicated Test Page</title>
+        <meta name="description" content="A very complicated page for testing.">
+        
+        <style>
+          .hidden { display: none; }
+          .highlight { color: red; }
+        </style>
+      </head>
+      <body>
+        <!-- This is a comment that we may remove if remove_comments=True -->
+        
+        <header>
+          <h1>Main Title of the Page</h1>
+          <nav>
+            <a href="http://example.com/home">Home</a>
+            <a href="http://social.com/profile">Social Profile</a>
+            <a href="javascript:void(0)">JS Void Link</a>
+          </nav>
+        </header>
+        
+        <noscript>
+          <p>JavaScript is disabled or not supported.</p>
+        </noscript>
+        
+        <form action="submit.php" method="post">
+          <input type="text" name="username" />
+          <button type="submit">Submit</button>
+        </form>
+        
+        <section>
+          <article>
+            <h2>Article Title</h2>
+            <p>
+              This paragraph has a good amount of text to exceed word_count_threshold if it's 
+              set to something small. But it might not exceed a very high threshold.
+            </p>
+            
+            <img src="http://images.example.com/photo.jpg" alt="Descriptive alt text"
+                 style="width:200px;height:150px;" data-lazy="true">
+            
+            <img src="icon.png" alt="Icon" style="display:none;">
+            
+            <p>Another short text. <a href="/local-link">Local Link</a></p>
+          </article>
+        </section>
+        
+        <section id="promo-section">
+          <p>Promo text <a href="http://ads.example.com/ad">Ad Link</a></p>
+        </section>
+        
+        <aside class="sidebar">
+          <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA..." alt="Base64 Image">
+          <div data-info="secret" class="social-widget">
+            <p>Follow us on <a href="http://facebook.com/brand">Facebook</a></p>
+          </div>
+        </aside>
+        
+        <!-- Another comment below this line -->
+        <script>console.log("script that might be removed");</script>
+        
+        <div style="display:none;">
+          <p>This is hidden</p>
+        </div>
+        
+        <footer>
+          <small>Footer Info &copy; 2025</small>
+        </footer>
+      </body>
+    </html>
+    """
+
+
+def get_test_scenarios():
+    """
+    Returns a dictionary of parameter sets (test scenarios) for the scraper.
+    Each scenario name maps to a dictionary of keyword arguments
+    that will be passed into scrap() for testing various features.
+    """
+    TEST_SCENARIOS = {
+        # "default": {},
+        # "exclude_domains": {
+        #     "exclude_domains": {"images.example.com", "ads.example.com"}
+        # },
+        # "exclude_social_media_links": {
+        #     "exclude_social_media_links": True
+        # },
+        # "high_word_threshold": {
+        #     "word_count_threshold": 100
+        # },
+        # "keep_data_attrs": {
+        #     "keep_data_attributes": True
+        # },
+        # "remove_forms_and_comments": {
+        #     "remove_forms": True,
+        #     "remove_comments": True
+        # },
+        # "exclude_tags_and_selector": {
+        #     "excluded_tags": ["aside", "script"],
+        #     "excluded_selector": ".social-widget"
+        # },
+        # "only_text_mode": {
+        #     "only_text": True
+        # },
+        # "combo_mode": {
+        #     "exclude_domains": {"images.example.com", "ads.example.com"},
+        #     "exclude_social_media_links": True,
+        #     "remove_forms": True,
+        #     "remove_comments": True,
+        #     "excluded_tags": ["aside"],
+        #     "excluded_selector": "#promo-section",
+        #     "only_text": False,
+        #     "keep_data_attributes": True,
+        #     "word_count_threshold": 20
+        # },
+        # "exclude_external_images": {
+        #     "exclude_external_images": True,
+        #     "exclude_social_media_links": True
+        # },
+        # "strict_image_scoring": {
+        #     "image_score_threshold": 3,
+        #     "image_description_min_word_threshold": 10
+        # },
+        # "custom_css_selector": {
+        #     "css_selector": "section#promo-section"
+        # },
+        # "remove_noscript": {
+        #     "excluded_tags": ["noscript"]
+        # },
+        # "exclude_external_links": {
+        #     "exclude_external_links": True
+        # },
+        # "large_word_count": {
+        #     "word_count_threshold": 500
+        # },
+        # "super_strict_images": {
+        #     "image_score_threshold": 5,
+        #     "image_description_min_word_threshold": 15
+        # },
+        # "exclude_style_and_script": {
+        #     "excluded_tags": ["style", "script"]
+        # },
+        # "keep_data_and_remove_forms": {
+        #     "keep_data_attributes": True,
+        #     "remove_forms": True
+        # },
+        # "only_text_high_word_count": {
+        #     "only_text": True,
+        #     "word_count_threshold": 40
+        # },
+        # "reduce_to_selector": {
+        #     "css_selector": "section > article"
+        # },
+        # "exclude_all_links": {
+        #     # Removes all external links and also excludes example.com & social.com
+        #     "exclude_domains": {"example.com", "social.com", "facebook.com"},
+        #     "exclude_external_links": True
+        # },
+        # "comprehensive_removal": {
+        #     # Exclude multiple tags, remove forms & comments,
+        #     # and also remove targeted selectors
+        #     "excluded_tags": ["aside", "noscript", "script"],
+        #     "excluded_selector": "#promo-section, .social-widget",
+        #     "remove_comments": True,
+        #     "remove_forms": True
+        # }
+    }
+    return TEST_SCENARIOS
+
+
+class ScraperEquivalenceTester:
+    def __init__(self):
+        self.test_cases = {
+            "basic": self.generate_basic_html(),
+            "complex": self.generate_complex_html(),
+            "malformed": self.generate_malformed_html(),
+            # 'real_world': self.load_real_samples()
+        }
+
+    def generate_basic_html(self):
+        return generate_large_html(1000)  # Your existing function
+
+    def generate_complex_html(self):
+        return """
+        <html><body>
+            <div class="nested-content">
+                <article>
+                    <h1>Main Title</h1>
+                    <img src="test.jpg" srcset="test-1x.jpg 1x, test-2x.jpg 2x" data-src="lazy.jpg">
+                    <p>Text with <a href="http://test.com">mixed <b>formatting</b></a></p>
+                    <iframe src="embedded.html"></iframe>
+                </article>
+                <nav>
+                    <ul>
+                        <li><a href="/page1">Link 1</a></li>
+                        <li><a href="javascript:void(0)">JS Link</a></li>
+                    </ul>
+                </nav>
+            </div>
+        </body></html>
+        """
+
+    def generate_malformed_html(self):
+        return """
+        <div>Unclosed div
+        <p>Unclosed paragraph
+        <a href="test.com">Link</a>
+        <img src=no-quotes>
+        <script>document.write("<div>Dynamic</div>");</script>
+        <!-- Malformed comment -- > -->
+        <![CDATA[Test CDATA]]>
+        """
+
+    def load_real_samples(self):
+        # Load some real-world HTML samples you've collected
+        samples = {
+            "article": open("tests/samples/article.html").read(),
+            "product": open("tests/samples/product.html").read(),
+            "blog": open("tests/samples/blog.html").read(),
+        }
+        return samples
+
+    def deep_compare_links(self, old_links: Dict, new_links: Dict) -> List[str]:
+        """Detailed comparison of link structures"""
+        differences = []
+
+        for category in ["internal", "external"]:
+            old_urls = {link["href"] for link in old_links[category]}
+            new_urls = {link["href"] for link in new_links[category]}
+
+            missing = old_urls - new_urls
+            extra = new_urls - old_urls
+
+            if missing:
+                differences.append(f"Missing {category} links: {missing}")
+            if extra:
+                differences.append(f"Extra {category} links: {extra}")
+
+            # Compare link attributes for common URLs
+            common = old_urls & new_urls
+            for url in common:
+                old_link = next(l for l in old_links[category] if l["href"] == url)
+                new_link = next(l for l in new_links[category] if l["href"] == url)
+
+                for attr in ["text", "title"]:
+                    if old_link[attr] != new_link[attr]:
+                        differences.append(
+                            f"Link attribute mismatch for {url} - {attr}:"
+                            f" old='{old_link[attr]}' vs new='{new_link[attr]}'"
+                        )
+
+        return differences
+
+    def deep_compare_media(self, old_media: Dict, new_media: Dict) -> List[str]:
+        """Detailed comparison of media elements"""
+        differences = []
+
+        for media_type in ["images", "videos", "audios"]:
+            old_srcs = {item["src"] for item in old_media[media_type]}
+            new_srcs = {item["src"] for item in new_media[media_type]}
+
+            missing = old_srcs - new_srcs
+            extra = new_srcs - old_srcs
+
+            if missing:
+                differences.append(f"Missing {media_type}: {missing}")
+            if extra:
+                differences.append(f"Extra {media_type}: {extra}")
+
+            # Compare media attributes for common sources
+            common = old_srcs & new_srcs
+            for src in common:
+                old_item = next(m for m in old_media[media_type] if m["src"] == src)
+                new_item = next(m for m in new_media[media_type] if m["src"] == src)
+
+                for attr in ["alt", "description"]:
+                    if old_item.get(attr) != new_item.get(attr):
+                        differences.append(
+                            f"{media_type} attribute mismatch for {src} - {attr}:"
+                            f" old='{old_item.get(attr)}' vs new='{new_item.get(attr)}'"
+                        )
+
+        return differences
+
+    def compare_html_content(self, old_html: str, new_html: str) -> List[str]:
+        """Compare HTML content structure and text"""
+        # return compare_html_structurally(old_html, new_html)
+        differences = []
+
+        def normalize_html(html: str) -> Tuple[str, str]:
+            soup = BeautifulSoup(html, "lxml")
+            # Get both structure and text
+            structure = " ".join(tag.name for tag in soup.find_all())
+            text = " ".join(soup.get_text().split())
+            return structure, text
+
+        old_structure, old_text = normalize_html(old_html)
+        new_structure, new_text = normalize_html(new_html)
+
+        # Compare structure
+        if abs(len(old_structure) - len(new_structure)) > 100:
+            # if old_structure != new_structure:
+            diff = difflib.unified_diff(
+                old_structure.split(), new_structure.split(), lineterm=""
+            )
+            differences.append("HTML structure differences:\n" + "\n".join(diff))
+
+        # Compare text content
+        if abs(len(old_text) - len(new_text)) > 100:
+            # if old_text != new_text:
+            # Show detailed text differences
+            text_diff = difflib.unified_diff(
+                old_text.split(), new_text.split(), lineterm=""
+            )
+            differences.append("Text content differences:\n" + "\n".join(text_diff))
+
+        return differences
+
+    def compare_results(
+        self, old_result: Dict, new_result: Dict
+    ) -> Dict[str, List[str]]:
+        """Comprehensive comparison of scraper outputs"""
+        differences = {}
+
+        # Compare links
+        link_differences = self.deep_compare_links(
+            old_result["links"], new_result["links"]
+        )
+        if link_differences:
+            differences["links"] = link_differences
+
+        # Compare media
+        media_differences = self.deep_compare_media(
+            old_result["media"], new_result["media"]
+        )
+        if media_differences:
+            differences["media"] = media_differences
+
+        # Compare HTML
+        html_differences = self.compare_html_content(
+            old_result["cleaned_html"], new_result["cleaned_html"]
+        )
+        if html_differences:
+            differences["html"] = html_differences
+
+        return differences
+
+    def run_tests(self) -> Dict:
+        """Run comparison tests using the complicated HTML with multiple parameter scenarios."""
+        # We'll still keep some "test_cases" logic from above (basic, complex, malformed).
+        # But we add a new section for the complicated HTML scenarios.
+
+        results = {"tests": [], "summary": {"passed": 0, "failed": 0}}
+
+        # 1) First, run the existing 3 built-in test cases (basic, complex, malformed).
+        # for case_name, html in self.test_cases.items():
+        #     print(f"\nTesting built-in case: {case_name}...")
+
+        #     original = WebScrapingStrategy()
+        #     lxml = LXMLWebScrapingStrategy()
+
+        #     start = time.time()
+        #     orig_result = original.scrap("http://test.com", html)
+        #     orig_time = time.time() - start
+
+        #     print("\nOriginal Mode:")
+        #     print(f"Cleaned HTML size: {len(orig_result['cleaned_html'])/1024:.2f} KB")
+        #     print(f"Images: {len(orig_result['media']['images'])}")
+        #     print(f"External links: {len(orig_result['links']['external'])}")
+        #     print(f"Times - Original: {orig_time:.3f}s")
+
+        #     start = time.time()
+        #     lxml_result = lxml.scrap("http://test.com", html)
+        #     lxml_time = time.time() - start
+
+        #     print("\nLXML Mode:")
+        #     print(f"Cleaned HTML size: {len(lxml_result['cleaned_html'])/1024:.2f} KB")
+        #     print(f"Images: {len(lxml_result['media']['images'])}")
+        #     print(f"External links: {len(lxml_result['links']['external'])}")
+        #     print(f"Times - LXML: {lxml_time:.3f}s")
+
+        #     # Compare
+        #     diffs = {}
+        #     link_diff = self.deep_compare_links(orig_result['links'], lxml_result['links'])
+        #     if link_diff:
+        #         diffs['links'] = link_diff
+
+        #     media_diff = self.deep_compare_media(orig_result['media'], lxml_result['media'])
+        #     if media_diff:
+        #         diffs['media'] = media_diff
+
+        #     html_diff = self.compare_html_content(orig_result['cleaned_html'], lxml_result['cleaned_html'])
+        #     if html_diff:
+        #         diffs['html'] = html_diff
+
+        #     test_result = {
+        #         'case': case_name,
+        #         'lxml_mode': {
+        #             'differences': diffs,
+        #             'execution_time': lxml_time
+        #         },
+        #         'original_time': orig_time
+        #     }
+        #     results['tests'].append(test_result)
+
+        #     if not diffs:
+        #         results['summary']['passed'] += 1
+        #     else:
+        #         results['summary']['failed'] += 1
+
+        # 2) Now, run the complicated HTML with multiple parameter scenarios.
+        complicated_html = generate_complicated_html()
+        print("\n=== Testing complicated HTML with multiple parameter scenarios ===")
+
+        # Create the scrapers once (or you can re-create if needed)
+        original = WebScrapingStrategy()
+        lxml = LXMLWebScrapingStrategy()
+
+        for scenario_name, params in get_test_scenarios().items():
+            print(f"\nScenario: {scenario_name}")
+
+            start = time.time()
+            orig_result = original.scrap("http://test.com", complicated_html, **params)
+            orig_time = time.time() - start
+
+            start = time.time()
+            lxml_result = lxml.scrap("http://test.com", complicated_html, **params)
+            lxml_time = time.time() - start
+
+            diffs = {}
+            link_diff = self.deep_compare_links(
+                orig_result["links"], lxml_result["links"]
+            )
+            if link_diff:
+                diffs["links"] = link_diff
+
+            media_diff = self.deep_compare_media(
+                orig_result["media"], lxml_result["media"]
+            )
+            if media_diff:
+                diffs["media"] = media_diff
+
+            html_diff = self.compare_html_content(
+                orig_result["cleaned_html"], lxml_result["cleaned_html"]
+            )
+            if html_diff:
+                diffs["html"] = html_diff
+
+            test_result = {
+                "case": f"complicated_{scenario_name}",
+                "lxml_mode": {"differences": diffs, "execution_time": lxml_time},
+                "original_time": orig_time,
+            }
+            results["tests"].append(test_result)
+
+            if not diffs:
+                results["summary"]["passed"] += 1
+                print(
+                    f"✅ [OK] No differences found. Time(Orig: {orig_time:.3f}s, LXML: {lxml_time:.3f}s)"
+                )
+            else:
+                results["summary"]["failed"] += 1
+                print("❌ Differences found:")
+                for category, dlist in diffs.items():
+                    print(f"  {category}:")
+                    for d in dlist:
+                        print(f"    - {d}")
+
+        return results
+
+    def print_report(self, results: Dict):
+        """Generate detailed equivalence report"""
+        print("\n=== Scraper Equivalence Test Report ===\n")
+        print(f"Total Cases: {len(results['tests'])}")
+        print(f"Passed: {results['summary']['passed']}")
+        print(f"Failed: {results['summary']['failed']}")
+
+        for test in results["tests"]:
+            print(f"\nTest Case: {test['case']}")
+
+            if not test["lxml_mode"]["differences"]:
+                print("✅ All implementations produced identical results")
+                print(
+                    f"Times - Original: {test['original_time']:.3f}s, "
+                    f"LXML: {test['lxml_mode']['execution_time']:.3f}s"
+                )
+            else:
+                print("❌ Differences found:")
+
+                if test["lxml_mode"]["differences"]:
+                    print("\nLXML Mode Differences:")
+                    for category, diffs in test["lxml_mode"]["differences"].items():
+                        print(f"\n{category}:")
+                        for diff in diffs:
+                            print(f"  - {diff}")
+
+
+def main():
+    tester = ScraperEquivalenceTester()
+    results = tester.run_tests()
+    tester.print_report(results)
+
+    # Save detailed results for debugging
+    with open("scraper_equivalence_results.json", "w") as f:
+        json.dump(results, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

tests/async/test_http_file_download.py

@@ -0,0 +1,423 @@
+"""
+Tests for HTTP strategy file download detection and handling.
+
+Tests the Content-Type/Content-Disposition detection logic in AsyncHTTPCrawlerStrategy
+that saves non-HTML responses to disk and populates downloaded_files.
+"""
+
+import os
+import sys
+import asyncio
+import tempfile
+import shutil
+import json
+import socket
+
+import pytest
+from aiohttp import web
+
+# Add parent to path so crawl4ai is importable
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, parent_dir)
+
+from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
+from crawl4ai.async_configs import HTTPCrawlerConfig, CrawlerRunConfig
+
+
+# ---------------------------------------------------------------------------
+# Test HTTP server
+# ---------------------------------------------------------------------------
+
+async def handle_html(request):
+    return web.Response(text="<html><body>Hello</body></html>", content_type="text/html")
+
+async def handle_csv(request):
+    csv_data = "id,name,value\n1,alpha,100\n2,beta,200\n3,gamma,300\n"
+    return web.Response(
+        text=csv_data,
+        content_type="text/csv",
+        headers={"Content-Disposition": 'attachment; filename="data.csv"'},
+    )
+
+async def handle_csv_no_disposition(request):
+    return web.Response(text="col1,col2\na,b\nc,d\n", content_type="text/csv")
+
+async def handle_json(request):
+    return web.Response(
+        text=json.dumps({"key": "value", "items": [1, 2, 3]}),
+        content_type="application/json",
+    )
+
+async def handle_pdf(request):
+    pdf_bytes = b"%PDF-1.4 fake pdf content " + (b"\x00\xff" * 500)
+    return web.Response(
+        body=pdf_bytes,
+        content_type="application/pdf",
+        headers={"Content-Disposition": 'attachment; filename="report.pdf"'},
+    )
+
+async def handle_binary_no_name(request):
+    return web.Response(body=b"\x89PNG\r\n" + b"\x00" * 100, content_type="image/png")
+
+async def handle_plain_text(request):
+    return web.Response(text="Just plain text content.", content_type="text/plain")
+
+async def handle_xml(request):
+    return web.Response(
+        text='<?xml version="1.0"?><root><item>test</item></root>',
+        content_type="application/xml",
+    )
+
+async def handle_attachment_html(request):
+    return web.Response(
+        text="<html><body>download me</body></html>",
+        content_type="text/html",
+        headers={"Content-Disposition": 'attachment; filename="page.html"'},
+    )
+
+async def handle_csv_url_filename(request):
+    return web.Response(text="x,y\n1,2\n", content_type="text/csv")
+
+
+def _find_free_port():
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+class _TestServer:
+    """Minimal test server lifecycle manager."""
+
+    def __init__(self):
+        self.port = _find_free_port()
+        self.runner = None
+
+    @property
+    def base_url(self):
+        return f"http://127.0.0.1:{self.port}"
+
+    def url(self, path):
+        return f"{self.base_url}{path}"
+
+    async def start(self):
+        app = web.Application()
+        app.router.add_get("/page.html", handle_html)
+        app.router.add_get("/data.csv", handle_csv)
+        app.router.add_get("/inline.csv", handle_csv_no_disposition)
+        app.router.add_get("/api/data.json", handle_json)
+        app.router.add_get("/report.pdf", handle_pdf)
+        app.router.add_get("/image", handle_binary_no_name)
+        app.router.add_get("/readme.txt", handle_plain_text)
+        app.router.add_get("/feed.xml", handle_xml)
+        app.router.add_get("/attachment.html", handle_attachment_html)
+        app.router.add_get("/files/export.csv", handle_csv_url_filename)
+
+        self.runner = web.AppRunner(app)
+        await self.runner.setup()
+        site = web.TCPSite(self.runner, "127.0.0.1", self.port)
+        await site.start()
+
+    async def stop(self):
+        if self.runner:
+            await self.runner.cleanup()
+
+
+# ---------------------------------------------------------------------------
+# Helper to run an async crawl test
+# ---------------------------------------------------------------------------
+
+async def _crawl(server, path, downloads_dir=None):
+    config = HTTPCrawlerConfig(downloads_path=downloads_dir)
+    strategy = AsyncHTTPCrawlerStrategy(browser_config=config)
+    return await strategy.crawl(server.url(path))
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+class TestHTMLPassthrough:
+    """Normal HTML responses should behave exactly as before."""
+
+    def test_html_response_unchanged(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/page.html", dl_dir)
+
+                assert "<html>" in result.html
+                assert result.downloaded_files is None
+                assert result.status_code == 200
+                assert len(os.listdir(dl_dir)) == 0
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestTextFileDownloads:
+    """Text-based file downloads (CSV, JSON, XML, plain text)."""
+
+    def test_csv_with_disposition(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/data.csv", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert len(result.downloaded_files) == 1
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith("data.csv")
+                assert os.path.isfile(filepath)
+                assert "alpha" in result.html
+                assert "id,name,value" in result.html
+                with open(filepath) as f:
+                    assert "alpha" in f.read()
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_csv_without_disposition(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/inline.csv", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert len(result.downloaded_files) == 1
+                assert result.downloaded_files[0].endswith("inline.csv")
+                assert "col1,col2" in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_json_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/api/data.json", dl_dir)
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith("data.json")
+                assert '"key"' in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_plain_text(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/readme.txt", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert "Just plain text content." in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_xml_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/feed.xml", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert "<root>" in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_csv_filename_from_url(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/files/export.csv", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert result.downloaded_files[0].endswith("export.csv")
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestBinaryFileDownloads:
+    """Binary file downloads (PDF, images) — html should be empty."""
+
+    def test_pdf_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/report.pdf", dl_dir)
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith("report.pdf")
+                assert os.path.isfile(filepath)
+                assert result.html == ""
+                with open(filepath, "rb") as f:
+                    data = f.read()
+                    assert data.startswith(b"%PDF")
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_binary_no_filename(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/image", dl_dir)
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert filepath.endswith(".png")
+                assert os.path.isfile(filepath)
+                assert result.html == ""
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestEdgeCases:
+    """Edge cases and backward compatibility."""
+
+    def test_attachment_html_treated_as_download(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/attachment.html", dl_dir)
+
+                assert result.downloaded_files is not None
+                assert result.downloaded_files[0].endswith("page.html")
+                assert "download me" in result.html
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_default_downloads_path(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                config = HTTPCrawlerConfig()  # no downloads_path
+                strategy = AsyncHTTPCrawlerStrategy(browser_config=config)
+                result = await strategy.crawl(srv.url("/data.csv"))
+
+                assert result.downloaded_files is not None
+                filepath = result.downloaded_files[0]
+                assert ".crawl4ai/downloads" in filepath
+                if os.path.isfile(filepath):
+                    os.unlink(filepath)
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_response_headers_contain_content_type(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/data.csv", dl_dir)
+                assert "text/csv" in result.response_headers.get("Content-Type", "")
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+    def test_status_code_preserved(self, tmp_path):
+        async def _test():
+            srv = _TestServer()
+            await srv.start()
+            try:
+                dl_dir = str(tmp_path / "dl")
+                os.makedirs(dl_dir)
+                result = await _crawl(srv, "/report.pdf", dl_dir)
+                assert result.status_code == 200
+            finally:
+                await srv.stop()
+
+        asyncio.get_event_loop().run_until_complete(_test())
+
+
+class TestDetectionHelpers:
+    """Unit tests for the detection helper methods."""
+
+    def test_is_file_download(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._is_file_download("text/csv", "") is True
+        assert s._is_file_download("application/pdf", "") is True
+        assert s._is_file_download("image/png", "") is True
+        assert s._is_file_download("text/html", "") is False
+        assert s._is_file_download("text/html", "attachment; filename=x") is True
+        assert s._is_file_download("", "") is False
+
+    def test_is_text_content(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._is_text_content("text/csv") is True
+        assert s._is_text_content("text/plain") is True
+        assert s._is_text_content("application/json") is True
+        assert s._is_text_content("application/pdf") is False
+        assert s._is_text_content("image/png") is False
+        assert s._is_text_content("text/tab-separated-values") is True
+
+    def test_extract_filename_from_disposition(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._extract_filename('attachment; filename="data.csv"', "http://x/y", "text/csv") == "data.csv"
+        assert s._extract_filename("attachment; filename=report.pdf", "http://x/y", "application/pdf") == "report.pdf"
+
+    def test_extract_filename_from_url(self):
+        s = AsyncHTTPCrawlerStrategy()
+        assert s._extract_filename("", "http://example.com/files/export.csv", "text/csv") == "export.csv"
+
+    def test_extract_filename_fallback(self):
+        s = AsyncHTTPCrawlerStrategy()
+        name = s._extract_filename("", "http://example.com/download", "application/pdf")
+        assert name.startswith("download_")
+        assert name.endswith(".pdf")
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_markdown_genertor.py

@@ -0,0 +1,181 @@
+# ## Issue #236
+# - **Last Updated:** 2024-11-11 01:42:14
+# - **Title:** [user data crawling opens two windows, unable to control correct user browser](https://github.com/unclecode/crawl4ai/issues/236)
+# - **State:** open
+
+import os, sys, time
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+import os
+import time
+from typing import Dict, Any
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+# Get current directory
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+
+def print_test_result(name: str, result: Dict[str, Any], execution_time: float):
+    """Helper function to print test results."""
+    print(f"\n{'='*20} {name} {'='*20}")
+    print(f"Execution time: {execution_time:.4f} seconds")
+
+    # Save markdown to files
+    for key, content in result.items():
+        if isinstance(content, str):
+            with open(__location__ + f"/output/{name.lower()}_{key}.md", "w") as f:
+                f.write(content)
+
+    # # Print first few lines of each markdown version
+    # for key, content in result.items():
+    #     if isinstance(content, str):
+    #         preview = '\n'.join(content.split('\n')[:3])
+    #         print(f"\n{key} (first 3 lines):")
+    #         print(preview)
+    #         print(f"Total length: {len(content)} characters")
+
+
+def test_basic_markdown_conversion():
+    """Test basic markdown conversion with links."""
+    with open(__location__ + "/data/wikipedia.html", "r") as f:
+        cleaned_html = f.read()
+
+    generator = DefaultMarkdownGenerator()
+
+    start_time = time.perf_counter()
+    result = generator.generate_markdown(
+        cleaned_html=cleaned_html, base_url="https://en.wikipedia.org"
+    )
+    execution_time = time.perf_counter() - start_time
+
+    print_test_result(
+        "Basic Markdown Conversion",
+        {
+            "raw": result.raw_markdown,
+            "with_citations": result.markdown_with_citations,
+            "references": result.references_markdown,
+        },
+        execution_time,
+    )
+
+    # Basic assertions
+    assert result.raw_markdown, "Raw markdown should not be empty"
+    assert result.markdown_with_citations, "Markdown with citations should not be empty"
+    assert result.references_markdown, "References should not be empty"
+    assert "⟨" in result.markdown_with_citations, "Citations should use ⟨⟩ brackets"
+    assert (
+        "## References" in result.references_markdown
+    ), "Should contain references section"
+
+
+def test_relative_links():
+    """Test handling of relative links with base URL."""
+    markdown = """
+    Here's a [relative link](/wiki/Apple) and an [absolute link](https://example.com).
+    Also an [image](/images/test.png) and another [page](/wiki/Banana).
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://en.wikipedia.org"
+    )
+
+    assert "https://en.wikipedia.org/wiki/Apple" in result.references_markdown
+    assert "https://example.com" in result.references_markdown
+    assert "https://en.wikipedia.org/images/test.png" in result.references_markdown
+
+
+def test_duplicate_links():
+    """Test handling of duplicate links."""
+    markdown = """
+    Here's a [link](/test) and another [link](/test) and a [different link](/other).
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://example.com"
+    )
+
+    # Count citations in markdown
+    citations = result.markdown_with_citations.count("⟨1⟩")
+    assert citations == 2, "Same link should use same citation number"
+
+
+def test_link_descriptions():
+    """Test handling of link titles and descriptions."""
+    markdown = """
+    Here's a [link with title](/test "Test Title") and a [link with description](/other) to test.
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://example.com"
+    )
+
+    assert (
+        "Test Title" in result.references_markdown
+    ), "Link title should be in references"
+    assert (
+        "link with description" in result.references_markdown
+    ), "Link text should be in references"
+
+
+def test_performance_large_document():
+    """Test performance with large document."""
+    with open(__location__ + "/data/wikipedia.md", "r") as f:
+        markdown = f.read()
+
+    # Test with multiple iterations
+    iterations = 5
+    times = []
+
+    generator = DefaultMarkdownGenerator()
+
+    for i in range(iterations):
+        start_time = time.perf_counter()
+        result = generator.generate_markdown(
+            cleaned_html=markdown, base_url="https://en.wikipedia.org"
+        )
+        end_time = time.perf_counter()
+        times.append(end_time - start_time)
+
+    avg_time = sum(times) / len(times)
+    print(f"\n{'='*20} Performance Test {'='*20}")
+    print(
+        f"Average execution time over {iterations} iterations: {avg_time:.4f} seconds"
+    )
+    print(f"Min time: {min(times):.4f} seconds")
+    print(f"Max time: {max(times):.4f} seconds")
+
+
+def test_image_links():
+    """Test handling of image links."""
+    markdown = """
+    Here's an ![image](/image.png "Image Title") and another ![image](/other.jpg).
+    And a regular [link](/page).
+    """
+
+    generator = DefaultMarkdownGenerator()
+    result = generator.generate_markdown(
+        cleaned_html=markdown, base_url="https://example.com"
+    )
+
+    assert (
+        "![" in result.markdown_with_citations
+    ), "Image markdown syntax should be preserved"
+    assert (
+        "Image Title" in result.references_markdown
+    ), "Image title should be in references"
+
+
+if __name__ == "__main__":
+    print("Running markdown generation strategy tests...")
+
+    test_basic_markdown_conversion()
+    test_relative_links()
+    test_duplicate_links()
+    test_link_descriptions()
+    test_performance_large_document()
+    test_image_links()

tests/async/test_parameters_and_options.py

@@ -0,0 +1,116 @@
+import os
+import sys
+import pytest
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_word_count_threshold():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result_no_threshold = await crawler.arun(
+            url=url, word_count_threshold=0, bypass_cache=True
+        )
+        result_with_threshold = await crawler.arun(
+            url=url, word_count_threshold=50, bypass_cache=True
+        )
+
+        assert len(result_no_threshold.markdown) > len(result_with_threshold.markdown)
+
+
+@pytest.mark.asyncio
+async def test_css_selector():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        css_selector = "h1, h2, h3"
+        result = await crawler.arun(
+            url=url, css_selector=css_selector, bypass_cache=True
+        )
+
+        assert result.success
+        assert (
+            "<h1" in result.cleaned_html
+            or "<h2" in result.cleaned_html
+            or "<h3" in result.cleaned_html
+        )
+
+
+@pytest.mark.asyncio
+async def test_javascript_execution():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        # Crawl without JS
+        result_without_more = await crawler.arun(url=url, bypass_cache=True)
+
+        js_code = [
+            "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+        ]
+        result_with_more = await crawler.arun(url=url, js=js_code, bypass_cache=True)
+
+        assert result_with_more.success
+        assert len(result_with_more.markdown) > len(result_without_more.markdown)
+
+
+@pytest.mark.asyncio
+async def test_screenshot():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, screenshot=True, bypass_cache=True)
+
+        assert result.success
+        assert result.screenshot
+        assert isinstance(result.screenshot, str)  # Should be a base64 encoded string
+
+
+@pytest.mark.asyncio
+async def test_custom_user_agent():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        custom_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Crawl4AI/1.0"
+        result = await crawler.arun(
+            url=url, user_agent=custom_user_agent, bypass_cache=True
+        )
+
+        assert result.success
+        # Note: We can't directly verify the user agent in the result, but we can check if the crawl was successful
+
+
+@pytest.mark.asyncio
+async def test_extract_media_and_links():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+
+        assert result.success
+        assert result.media
+        assert isinstance(result.media, dict)
+        assert "images" in result.media
+        assert result.links
+        assert isinstance(result.links, dict)
+        assert "internal" in result.links and "external" in result.links
+
+
+@pytest.mark.asyncio
+async def test_metadata_extraction():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        result = await crawler.arun(url=url, bypass_cache=True)
+
+        assert result.success
+        assert result.metadata
+        assert isinstance(result.metadata, dict)
+        # Check for common metadata fields
+        assert any(
+            key in result.metadata for key in ["title", "description", "keywords"]
+        )
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_performance.py

@@ -0,0 +1,79 @@
+import os
+import sys
+import pytest
+import time
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_crawl_speed():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+        start_time = time.time()
+        result = await crawler.arun(url=url, bypass_cache=True)
+        end_time = time.time()
+
+        assert result.success
+        crawl_time = end_time - start_time
+        print(f"Crawl time: {crawl_time:.2f} seconds")
+
+        assert crawl_time < 10, f"Crawl took too long: {crawl_time:.2f} seconds"
+
+
+@pytest.mark.asyncio
+async def test_concurrent_crawling_performance():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        urls = [
+            "https://www.nbcnews.com/business",
+            "https://www.example.com",
+            "https://www.python.org",
+            "https://www.github.com",
+            "https://www.stackoverflow.com",
+        ]
+
+        start_time = time.time()
+        results = await crawler.arun_many(urls=urls, bypass_cache=True)
+        end_time = time.time()
+
+        total_time = end_time - start_time
+        print(f"Total time for concurrent crawling: {total_time:.2f} seconds")
+
+        assert all(result.success for result in results)
+        assert len(results) == len(urls)
+
+        assert (
+            total_time < len(urls) * 5
+        ), f"Concurrent crawling not significantly faster: {total_time:.2f} seconds"
+
+
+@pytest.mark.asyncio
+async def test_crawl_speed_with_caching():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nbcnews.com/business"
+
+        start_time = time.time()
+        result1 = await crawler.arun(url=url, bypass_cache=True)
+        end_time = time.time()
+        first_crawl_time = end_time - start_time
+
+        start_time = time.time()
+        result2 = await crawler.arun(url=url, bypass_cache=False)
+        end_time = time.time()
+        second_crawl_time = end_time - start_time
+
+        assert result1.success and result2.success
+        print(f"First crawl time: {first_crawl_time:.2f} seconds")
+        print(f"Second crawl time (cached): {second_crawl_time:.2f} seconds")
+
+        assert (
+            second_crawl_time < first_crawl_time / 2
+        ), "Cached crawl not significantly faster"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/async/test_redirect_url_resolution.py

@@ -0,0 +1,118 @@
+"""Test delayed redirect WITH wait_for - does link resolution use correct URL?"""
+import asyncio
+import threading
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+class RedirectTestHandler(SimpleHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass
+
+    def do_GET(self):
+        if self.path == "/page-a":
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            content = """
+            <!DOCTYPE html>
+            <html>
+            <head><title>Page A</title></head>
+            <body>
+                <h1>Page A - Will redirect after 200ms</h1>
+                <script>
+                    setTimeout(function() {
+                        window.location.href = '/redirect-target/';
+                    }, 200);
+                </script>
+            </body>
+            </html>
+            """
+            self.wfile.write(content.encode())
+        elif self.path.startswith("/redirect-target"):
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+            content = """
+            <!DOCTYPE html>
+            <html>
+            <head><title>Redirect Target</title></head>
+            <body>
+                <h1>Redirect Target</h1>
+                <nav id="target-nav">
+                    <a href="subpage-1">Subpage 1</a>
+                    <a href="subpage-2">Subpage 2</a>
+                </nav>
+            </body>
+            </html>
+            """
+            self.wfile.write(content.encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+async def main():
+    import socket
+    class ReuseAddrHTTPServer(HTTPServer):
+        allow_reuse_address = True
+    
+    server = ReuseAddrHTTPServer(("localhost", 8769), RedirectTestHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.daemon = True
+    thread.start()
+    
+    try:
+        import sys
+        sys.path.insert(0, '/Users/nasrin/vscode/c4ai-uc/develop')
+        from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+        
+        print("=" * 60)
+        print("TEST: Delayed JS redirect WITH wait_for='css:#target-nav'")
+        print("This waits for the redirect to complete")
+        print("=" * 60)
+        
+        browser_config = BrowserConfig(headless=True, verbose=False)
+        crawl_config = CrawlerRunConfig(
+            cache_mode="bypass",
+            wait_for="css:#target-nav"  # Wait for element on redirect target
+        )
+        
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            result = await crawler.arun(
+                url="http://localhost:8769/page-a",
+                config=crawl_config
+            )
+            
+            print(f"Original URL: http://localhost:8769/page-a")
+            print(f"Redirected URL returned: {result.redirected_url}")
+            print(f"HTML contains 'Redirect Target': {'Redirect Target' in result.html}")
+            print()
+            
+            if "/redirect-target" in (result.redirected_url or ""):
+                print("✓ redirected_url is CORRECT")
+            else:
+                print("✗ BUG #1: redirected_url is WRONG - still shows original URL!")
+                
+            # Check links
+            all_links = []
+            if isinstance(result.links, dict):
+                all_links = result.links.get("internal", []) + result.links.get("external", [])
+            
+            print(f"\nLinks found ({len(all_links)} total):")
+            bug_found = False
+            for link in all_links:
+                href = link.get("href", "") if isinstance(link, dict) else getattr(link, 'href', "")
+                if "subpage" in href:
+                    print(f"  {href}")
+                    if "/page-a/" in href:
+                        print("    ^^^ BUG #2: Link resolved with WRONG base URL!")
+                        bug_found = True
+                    elif "/redirect-target/" in href:
+                        print("    ^^^ CORRECT")
+            
+            if not bug_found and all_links:
+                print("\n✓ Link resolution is CORRECT")
+                        
+    finally:
+        server.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/async/test_screenshot.py

@@ -0,0 +1,122 @@
+import os
+import sys
+import pytest
+import base64
+from PIL import Image
+import io
+
+# Add the parent directory to the Python path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+
+
+@pytest.mark.asyncio
+async def test_basic_screenshot():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://example.com"  # A static website
+        result = await crawler.arun(url=url, bypass_cache=True, screenshot=True)
+
+        assert result.success
+        assert result.screenshot is not None
+
+        # Verify the screenshot is a valid image
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_with_wait_for():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        # Using a website with dynamic content
+        url = "https://www.youtube.com"
+        wait_for = "css:#content"  # Wait for the main content to load
+
+        result = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True, wait_for=wait_for
+        )
+
+        assert result.success
+        assert result.screenshot is not None
+
+        # Verify the screenshot is a valid image
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+        # You might want to add more specific checks here, like image dimensions
+        # or even use image recognition to verify certain elements are present
+
+
+@pytest.mark.asyncio
+async def test_screenshot_with_js_wait_for():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.amazon.com"
+        wait_for = "js:() => document.querySelector('#nav-logo-sprites') !== null"
+
+        result = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True, wait_for=wait_for
+        )
+
+        assert result.success
+        assert result.screenshot is not None
+
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_without_wait_for():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.nytimes.com"  # A website with lots of dynamic content
+
+        result = await crawler.arun(url=url, bypass_cache=True, screenshot=True)
+
+        assert result.success
+        assert result.screenshot is not None
+
+        image_data = base64.b64decode(result.screenshot)
+        image = Image.open(io.BytesIO(image_data))
+        assert image.format == "PNG"
+
+
+@pytest.mark.asyncio
+async def test_screenshot_comparison():
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://www.reddit.com"
+        wait_for = "css:#SHORTCUT_FOCUSABLE_DIV"
+
+        # Take screenshot without wait_for
+        result_without_wait = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True
+        )
+
+        # Take screenshot with wait_for
+        result_with_wait = await crawler.arun(
+            url=url, bypass_cache=True, screenshot=True, wait_for=wait_for
+        )
+
+        assert result_without_wait.success and result_with_wait.success
+        assert result_without_wait.screenshot is not None
+        assert result_with_wait.screenshot is not None
+
+        # Compare the two screenshots
+        image_without_wait = Image.open(
+            io.BytesIO(base64.b64decode(result_without_wait.screenshot))
+        )
+        image_with_wait = Image.open(
+            io.BytesIO(base64.b64decode(result_with_wait.screenshot))
+        )
+
+        # This is a simple size comparison. In a real-world scenario, you might want to use
+        # more sophisticated image comparison techniques.
+        assert image_with_wait.size[0] >= image_without_wait.size[0]
+        assert image_with_wait.size[1] >= image_without_wait.size[1]
+
+
+# Entry point for debugging
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])