chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,4 @@
|
||||
"""Docker browser strategy tests.
|
||||
|
||||
This package contains tests for the Docker browser strategy implementation.
|
||||
"""
|
||||
@@ -0,0 +1,651 @@
|
||||
"""Test examples for Docker Browser Strategy.
|
||||
|
||||
These examples demonstrate the functionality of Docker Browser Strategy
|
||||
and serve as functional tests.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import uuid
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../..')))
|
||||
|
||||
from crawl4ai.browser import BrowserManager
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
from crawl4ai.browser import DockerConfig
|
||||
from crawl4ai.browser import DockerRegistry
|
||||
from crawl4ai.browser import DockerUtils
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
# Global Docker utils instance
|
||||
docker_utils = DockerUtils(logger)
|
||||
|
||||
async def test_docker_components():
|
||||
"""Test Docker utilities, registry, and image building.
|
||||
|
||||
This function tests the core Docker components before running the browser tests.
|
||||
It validates DockerRegistry, DockerUtils, and builds test images to ensure
|
||||
everything is functioning correctly.
|
||||
"""
|
||||
logger.info("Testing Docker components", tag="SETUP")
|
||||
|
||||
# Create a test registry directory
|
||||
registry_dir = os.path.join(os.path.dirname(__file__), "test_registry")
|
||||
registry_file = os.path.join(registry_dir, "test_registry.json")
|
||||
os.makedirs(registry_dir, exist_ok=True)
|
||||
|
||||
try:
|
||||
# 1. Test DockerRegistry
|
||||
logger.info("Testing DockerRegistry...", tag="SETUP")
|
||||
registry = DockerRegistry(registry_file)
|
||||
|
||||
# Test saving and loading registry
|
||||
test_container_id = "test-container-123"
|
||||
registry.register_container(test_container_id, 9876, "test-hash-123")
|
||||
registry.save()
|
||||
|
||||
# Create a new registry instance that loads from the file
|
||||
registry2 = DockerRegistry(registry_file)
|
||||
port = registry2.get_container_host_port(test_container_id)
|
||||
hash_value = registry2.get_container_config_hash(test_container_id)
|
||||
|
||||
if port != 9876 or hash_value != "test-hash-123":
|
||||
logger.error("DockerRegistry persistence failed", tag="SETUP")
|
||||
return False
|
||||
|
||||
# Clean up test container from registry
|
||||
registry2.unregister_container(test_container_id)
|
||||
logger.success("DockerRegistry works correctly", tag="SETUP")
|
||||
|
||||
# 2. Test DockerUtils
|
||||
logger.info("Testing DockerUtils...", tag="SETUP")
|
||||
|
||||
# Test port detection
|
||||
in_use = docker_utils.is_port_in_use(22) # SSH port is usually in use
|
||||
logger.info(f"Port 22 in use: {in_use}", tag="SETUP")
|
||||
|
||||
# Get next available port
|
||||
available_port = docker_utils.get_next_available_port(9000)
|
||||
logger.info(f"Next available port: {available_port}", tag="SETUP")
|
||||
|
||||
# Test config hash generation
|
||||
config_dict = {"mode": "connect", "headless": True}
|
||||
config_hash = docker_utils.generate_config_hash(config_dict)
|
||||
logger.info(f"Generated config hash: {config_hash[:8]}...", tag="SETUP")
|
||||
|
||||
# 3. Test Docker is available
|
||||
logger.info("Checking Docker availability...", tag="SETUP")
|
||||
if not await check_docker_available():
|
||||
logger.error("Docker is not available - cannot continue tests", tag="SETUP")
|
||||
return False
|
||||
|
||||
# 4. Test building connect image
|
||||
logger.info("Building connect mode Docker image...", tag="SETUP")
|
||||
connect_image = await docker_utils.ensure_docker_image_exists(None, "connect")
|
||||
if not connect_image:
|
||||
logger.error("Failed to build connect mode image", tag="SETUP")
|
||||
return False
|
||||
logger.success(f"Successfully built connect image: {connect_image}", tag="SETUP")
|
||||
|
||||
# 5. Test building launch image
|
||||
logger.info("Building launch mode Docker image...", tag="SETUP")
|
||||
launch_image = await docker_utils.ensure_docker_image_exists(None, "launch")
|
||||
if not launch_image:
|
||||
logger.error("Failed to build launch mode image", tag="SETUP")
|
||||
return False
|
||||
logger.success(f"Successfully built launch image: {launch_image}", tag="SETUP")
|
||||
|
||||
# 6. Test creating and removing container
|
||||
logger.info("Testing container creation and removal...", tag="SETUP")
|
||||
container_id = await docker_utils.create_container(
|
||||
image_name=launch_image,
|
||||
host_port=available_port,
|
||||
container_name="crawl4ai-test-container"
|
||||
)
|
||||
|
||||
if not container_id:
|
||||
logger.error("Failed to create test container", tag="SETUP")
|
||||
return False
|
||||
|
||||
logger.info(f"Created test container: {container_id[:12]}", tag="SETUP")
|
||||
|
||||
# Verify container is running
|
||||
running = await docker_utils.is_container_running(container_id)
|
||||
if not running:
|
||||
logger.error("Test container is not running", tag="SETUP")
|
||||
await docker_utils.remove_container(container_id)
|
||||
return False
|
||||
|
||||
# Test commands in container
|
||||
logger.info("Testing command execution in container...", tag="SETUP")
|
||||
returncode, stdout, stderr = await docker_utils.exec_in_container(
|
||||
container_id, ["ls", "-la", "/"]
|
||||
)
|
||||
|
||||
if returncode != 0:
|
||||
logger.error(f"Command execution failed: {stderr}", tag="SETUP")
|
||||
await docker_utils.remove_container(container_id)
|
||||
return False
|
||||
|
||||
# Verify Chrome is installed in the container
|
||||
returncode, stdout, stderr = await docker_utils.exec_in_container(
|
||||
container_id, ["which", "chromium"]
|
||||
)
|
||||
|
||||
if returncode != 0:
|
||||
logger.error("Chrome not found in container", tag="SETUP")
|
||||
await docker_utils.remove_container(container_id)
|
||||
return False
|
||||
|
||||
chrome_path = stdout.strip()
|
||||
logger.info(f"Chrome found at: {chrome_path}", tag="SETUP")
|
||||
|
||||
# Test Chrome version
|
||||
returncode, stdout, stderr = await docker_utils.exec_in_container(
|
||||
container_id, ["chromium", "--version"]
|
||||
)
|
||||
|
||||
if returncode != 0:
|
||||
logger.error(f"Failed to get Chrome version: {stderr}", tag="SETUP")
|
||||
await docker_utils.remove_container(container_id)
|
||||
return False
|
||||
|
||||
logger.info(f"Chrome version: {stdout.strip()}", tag="SETUP")
|
||||
|
||||
# Remove test container
|
||||
removed = await docker_utils.remove_container(container_id)
|
||||
if not removed:
|
||||
logger.error("Failed to remove test container", tag="SETUP")
|
||||
return False
|
||||
|
||||
logger.success("Test container removed successfully", tag="SETUP")
|
||||
|
||||
# All components tested successfully
|
||||
logger.success("All Docker components tested successfully", tag="SETUP")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Docker component tests failed: {str(e)}", tag="SETUP")
|
||||
return False
|
||||
finally:
|
||||
# Clean up registry test directory
|
||||
if os.path.exists(registry_dir):
|
||||
shutil.rmtree(registry_dir)
|
||||
|
||||
async def test_docker_connect_mode():
|
||||
"""Test Docker browser in connect mode.
|
||||
|
||||
This tests the basic functionality of creating a browser in Docker
|
||||
connect mode and using it for navigation.
|
||||
"""
|
||||
logger.info("Testing Docker browser in connect mode", tag="TEST")
|
||||
|
||||
# Create temp directory for user data
|
||||
temp_dir = os.path.join(os.path.dirname(__file__), "tmp_user_data")
|
||||
os.makedirs(temp_dir, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Create Docker configuration
|
||||
docker_config = DockerConfig(
|
||||
mode="connect",
|
||||
persistent=False,
|
||||
remove_on_exit=True,
|
||||
user_data_dir=temp_dir
|
||||
)
|
||||
|
||||
# Create browser configuration
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="docker",
|
||||
headless=True,
|
||||
docker_config=docker_config
|
||||
)
|
||||
|
||||
# Create browser manager
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Create crawler config
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
|
||||
# Get a page
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
logger.info("Got page successfully", tag="TEST")
|
||||
|
||||
# Navigate to a website
|
||||
await page.goto("https://example.com")
|
||||
logger.info("Navigated to example.com", tag="TEST")
|
||||
|
||||
# Get page title
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
finally:
|
||||
# Clean up the temp directory
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
async def test_docker_launch_mode():
|
||||
"""Test Docker browser in launch mode.
|
||||
|
||||
This tests launching a Chrome browser within a Docker container
|
||||
on demand with custom settings.
|
||||
"""
|
||||
logger.info("Testing Docker browser in launch mode", tag="TEST")
|
||||
|
||||
# Create temp directory for user data
|
||||
temp_dir = os.path.join(os.path.dirname(__file__), "tmp_user_data_launch")
|
||||
os.makedirs(temp_dir, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Create Docker configuration
|
||||
docker_config = DockerConfig(
|
||||
mode="launch",
|
||||
persistent=False,
|
||||
remove_on_exit=True,
|
||||
user_data_dir=temp_dir
|
||||
)
|
||||
|
||||
# Create browser configuration
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="docker",
|
||||
headless=True,
|
||||
text_mode=True, # Enable text mode for faster operation
|
||||
docker_config=docker_config
|
||||
)
|
||||
|
||||
# Create browser manager
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Create crawler config
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
|
||||
# Get a page
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
logger.info("Got page successfully", tag="TEST")
|
||||
|
||||
# Navigate to a website
|
||||
await page.goto("https://example.com")
|
||||
logger.info("Navigated to example.com", tag="TEST")
|
||||
|
||||
# Get page title
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
finally:
|
||||
# Clean up the temp directory
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
async def test_docker_persistent_storage():
|
||||
"""Test Docker browser with persistent storage.
|
||||
|
||||
This tests creating localStorage data in one session and verifying
|
||||
it persists to another session when using persistent storage.
|
||||
"""
|
||||
logger.info("Testing Docker browser with persistent storage", tag="TEST")
|
||||
|
||||
# Create a unique temp directory
|
||||
test_id = uuid.uuid4().hex[:8]
|
||||
temp_dir = os.path.join(os.path.dirname(__file__), f"tmp_user_data_persist_{test_id}")
|
||||
os.makedirs(temp_dir, exist_ok=True)
|
||||
|
||||
manager1 = None
|
||||
manager2 = None
|
||||
|
||||
try:
|
||||
# Create Docker configuration with persistence
|
||||
docker_config = DockerConfig(
|
||||
mode="connect",
|
||||
persistent=True, # Keep container running between sessions
|
||||
user_data_dir=temp_dir,
|
||||
container_user_data_dir="/data"
|
||||
)
|
||||
|
||||
# Create browser configuration
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="docker",
|
||||
headless=True,
|
||||
docker_config=docker_config
|
||||
)
|
||||
|
||||
# Create first browser manager
|
||||
manager1 = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager1.start()
|
||||
logger.info("First browser started successfully", tag="TEST")
|
||||
|
||||
# Create crawler config
|
||||
crawler_config = CrawlerRunConfig()
|
||||
|
||||
# Get a page
|
||||
page1, context1 = await manager1.get_page(crawler_config)
|
||||
|
||||
# Navigate to example.com
|
||||
await page1.goto("https://example.com")
|
||||
|
||||
# Set localStorage item
|
||||
test_value = f"test_value_{test_id}"
|
||||
await page1.evaluate(f"localStorage.setItem('test_key', '{test_value}')")
|
||||
logger.info(f"Set localStorage test_key = {test_value}", tag="TEST")
|
||||
|
||||
# Close the first browser manager
|
||||
await manager1.close()
|
||||
logger.info("First browser closed", tag="TEST")
|
||||
|
||||
# Create second browser manager with same config
|
||||
manager2 = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager2.start()
|
||||
logger.info("Second browser started successfully", tag="TEST")
|
||||
|
||||
# Get a page
|
||||
page2, context2 = await manager2.get_page(crawler_config)
|
||||
|
||||
# Navigate to same site
|
||||
await page2.goto("https://example.com")
|
||||
|
||||
# Get localStorage item
|
||||
value = await page2.evaluate("localStorage.getItem('test_key')")
|
||||
logger.info(f"Retrieved localStorage test_key = {value}", tag="TEST")
|
||||
|
||||
# Check if persistence worked
|
||||
if value == test_value:
|
||||
logger.success("Storage persistence verified!", tag="TEST")
|
||||
else:
|
||||
logger.error(f"Storage persistence failed! Expected {test_value}, got {value}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager2.close()
|
||||
logger.info("Second browser closed successfully", tag="TEST")
|
||||
|
||||
return value == test_value
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
if manager1:
|
||||
await manager1.close()
|
||||
if manager2:
|
||||
await manager2.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
finally:
|
||||
# Clean up the temp directory
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
async def test_docker_parallel_pages():
|
||||
"""Test Docker browser with parallel page creation.
|
||||
|
||||
This tests the ability to create and use multiple pages in parallel
|
||||
from a single Docker browser instance.
|
||||
"""
|
||||
logger.info("Testing Docker browser with parallel pages", tag="TEST")
|
||||
|
||||
try:
|
||||
# Create Docker configuration
|
||||
docker_config = DockerConfig(
|
||||
mode="connect",
|
||||
persistent=False,
|
||||
remove_on_exit=True
|
||||
)
|
||||
|
||||
# Create browser configuration
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="docker",
|
||||
headless=True,
|
||||
docker_config=docker_config
|
||||
)
|
||||
|
||||
# Create browser manager
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Create crawler config
|
||||
crawler_config = CrawlerRunConfig()
|
||||
|
||||
# Get multiple pages
|
||||
page_count = 3
|
||||
pages = await manager.get_pages(crawler_config, count=page_count)
|
||||
logger.info(f"Got {len(pages)} pages successfully", tag="TEST")
|
||||
|
||||
if len(pages) != page_count:
|
||||
logger.error(f"Expected {page_count} pages, got {len(pages)}", tag="TEST")
|
||||
await manager.close()
|
||||
return False
|
||||
|
||||
# Navigate to different sites with each page
|
||||
tasks = []
|
||||
for i, (page, _) in enumerate(pages):
|
||||
tasks.append(page.goto(f"https://example.com?page={i}"))
|
||||
|
||||
# Wait for all navigations to complete
|
||||
await asyncio.gather(*tasks)
|
||||
logger.info("All pages navigated successfully", tag="TEST")
|
||||
|
||||
# Get titles from all pages
|
||||
titles = []
|
||||
for i, (page, _) in enumerate(pages):
|
||||
title = await page.title()
|
||||
titles.append(title)
|
||||
logger.info(f"Page {i+1} title: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_docker_registry_reuse():
|
||||
"""Test Docker container reuse via registry.
|
||||
|
||||
This tests that containers with matching configurations
|
||||
are reused rather than creating new ones.
|
||||
"""
|
||||
logger.info("Testing Docker container reuse via registry", tag="TEST")
|
||||
|
||||
# Create registry for this test
|
||||
registry_dir = os.path.join(os.path.dirname(__file__), "registry_reuse_test")
|
||||
registry_file = os.path.join(registry_dir, "registry.json")
|
||||
os.makedirs(registry_dir, exist_ok=True)
|
||||
|
||||
manager1 = None
|
||||
manager2 = None
|
||||
container_id1 = None
|
||||
|
||||
try:
|
||||
# Create identical Docker configurations with custom registry
|
||||
docker_config1 = DockerConfig(
|
||||
mode="connect",
|
||||
persistent=True, # Keep container running after closing
|
||||
registry_file=registry_file
|
||||
)
|
||||
|
||||
# Create first browser configuration
|
||||
browser_config1 = BrowserConfig(
|
||||
browser_mode="docker",
|
||||
headless=True,
|
||||
docker_config=docker_config1
|
||||
)
|
||||
|
||||
# Create first browser manager
|
||||
manager1 = BrowserManager(browser_config=browser_config1, logger=logger)
|
||||
|
||||
# Start the first browser
|
||||
await manager1.start()
|
||||
logger.info("First browser started successfully", tag="TEST")
|
||||
|
||||
# Get container ID from the strategy
|
||||
docker_strategy1 = manager1.strategy
|
||||
container_id1 = docker_strategy1.container_id
|
||||
logger.info(f"First browser container ID: {container_id1[:12]}", tag="TEST")
|
||||
|
||||
# Close the first manager but keep container running
|
||||
await manager1.close()
|
||||
logger.info("First browser closed", tag="TEST")
|
||||
|
||||
# Create second Docker configuration identical to first
|
||||
docker_config2 = DockerConfig(
|
||||
mode="connect",
|
||||
persistent=True,
|
||||
registry_file=registry_file
|
||||
)
|
||||
|
||||
# Create second browser configuration
|
||||
browser_config2 = BrowserConfig(
|
||||
browser_mode="docker",
|
||||
headless=True,
|
||||
docker_config=docker_config2
|
||||
)
|
||||
|
||||
# Create second browser manager
|
||||
manager2 = BrowserManager(browser_config=browser_config2, logger=logger)
|
||||
|
||||
# Start the second browser - should reuse existing container
|
||||
await manager2.start()
|
||||
logger.info("Second browser started successfully", tag="TEST")
|
||||
|
||||
# Get container ID from the second strategy
|
||||
docker_strategy2 = manager2.strategy
|
||||
container_id2 = docker_strategy2.container_id
|
||||
logger.info(f"Second browser container ID: {container_id2[:12]}", tag="TEST")
|
||||
|
||||
# Verify container reuse
|
||||
if container_id1 == container_id2:
|
||||
logger.success("Container reuse successful - using same container!", tag="TEST")
|
||||
else:
|
||||
logger.error("Container reuse failed - new container created!", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
docker_strategy2.docker_config.persistent = False
|
||||
docker_strategy2.docker_config.remove_on_exit = True
|
||||
await manager2.close()
|
||||
logger.info("Second browser closed and container removed", tag="TEST")
|
||||
|
||||
return container_id1 == container_id2
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
if manager1:
|
||||
await manager1.close()
|
||||
if manager2:
|
||||
await manager2.close()
|
||||
# Make sure container is removed
|
||||
if container_id1:
|
||||
await docker_utils.remove_container(container_id1, force=True)
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
finally:
|
||||
# Clean up registry directory
|
||||
if os.path.exists(registry_dir):
|
||||
shutil.rmtree(registry_dir)
|
||||
|
||||
async def run_tests():
|
||||
"""Run all tests sequentially."""
|
||||
results = []
|
||||
|
||||
logger.info("Starting Docker Browser Strategy tests", tag="TEST")
|
||||
|
||||
# Check if Docker is available
|
||||
if not await check_docker_available():
|
||||
logger.error("Docker is not available - skipping tests", tag="TEST")
|
||||
return
|
||||
|
||||
# First test Docker components
|
||||
# setup_result = await test_docker_components()
|
||||
# if not setup_result:
|
||||
# logger.error("Docker component tests failed - skipping browser tests", tag="TEST")
|
||||
# return
|
||||
|
||||
# Run browser tests
|
||||
results.append(await test_docker_connect_mode())
|
||||
results.append(await test_docker_launch_mode())
|
||||
results.append(await test_docker_persistent_storage())
|
||||
results.append(await test_docker_parallel_pages())
|
||||
results.append(await test_docker_registry_reuse())
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r)
|
||||
logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
|
||||
|
||||
if passed == total:
|
||||
logger.success("All tests passed!", tag="SUMMARY")
|
||||
else:
|
||||
logger.error(f"{total - passed} tests failed", tag="SUMMARY")
|
||||
|
||||
async def check_docker_available() -> bool:
|
||||
"""Check if Docker is available on the system.
|
||||
|
||||
Returns:
|
||||
bool: True if Docker is available, False otherwise
|
||||
"""
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"docker", "--version",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, _ = await proc.communicate()
|
||||
return proc.returncode == 0 and stdout
|
||||
except:
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
@@ -0,0 +1,525 @@
|
||||
"""Demo script for testing the enhanced BrowserManager.
|
||||
|
||||
This script demonstrates the browser pooling capabilities of the enhanced
|
||||
BrowserManager with various configurations and usage patterns.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import random
|
||||
|
||||
from crawl4ai.browser.manager import BrowserManager, UnavailableBehavior
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
import playwright
|
||||
|
||||
SAFE_URLS = [
|
||||
"https://example.com",
|
||||
"https://example.com/page1",
|
||||
"https://httpbin.org/get",
|
||||
"https://httpbin.org/html",
|
||||
"https://httpbin.org/ip",
|
||||
"https://httpbin.org/user-agent",
|
||||
"https://httpbin.org/headers",
|
||||
"https://httpbin.org/cookies",
|
||||
"https://httpstat.us/200",
|
||||
"https://httpstat.us/301",
|
||||
"https://httpstat.us/404",
|
||||
"https://httpstat.us/500",
|
||||
"https://jsonplaceholder.typicode.com/posts/1",
|
||||
"https://jsonplaceholder.typicode.com/posts/2",
|
||||
"https://jsonplaceholder.typicode.com/posts/3",
|
||||
"https://jsonplaceholder.typicode.com/posts/4",
|
||||
"https://jsonplaceholder.typicode.com/posts/5",
|
||||
"https://jsonplaceholder.typicode.com/comments/1",
|
||||
"https://jsonplaceholder.typicode.com/comments/2",
|
||||
"https://jsonplaceholder.typicode.com/users/1",
|
||||
"https://jsonplaceholder.typicode.com/users/2",
|
||||
"https://jsonplaceholder.typicode.com/albums/1",
|
||||
"https://jsonplaceholder.typicode.com/albums/2",
|
||||
"https://jsonplaceholder.typicode.com/photos/1",
|
||||
"https://jsonplaceholder.typicode.com/photos/2",
|
||||
"https://jsonplaceholder.typicode.com/todos/1",
|
||||
"https://jsonplaceholder.typicode.com/todos/2",
|
||||
"https://www.iana.org",
|
||||
"https://www.iana.org/domains",
|
||||
"https://www.iana.org/numbers",
|
||||
"https://www.iana.org/protocols",
|
||||
"https://www.iana.org/about",
|
||||
"https://www.iana.org/time-zones",
|
||||
"https://www.data.gov",
|
||||
"https://catalog.data.gov/dataset",
|
||||
"https://www.archives.gov",
|
||||
"https://www.usa.gov",
|
||||
"https://www.loc.gov",
|
||||
"https://www.irs.gov",
|
||||
"https://www.census.gov",
|
||||
"https://www.bls.gov",
|
||||
"https://www.gpo.gov",
|
||||
"https://www.w3.org",
|
||||
"https://www.w3.org/standards",
|
||||
"https://www.w3.org/WAI",
|
||||
"https://www.rfc-editor.org",
|
||||
"https://www.ietf.org",
|
||||
"https://www.icann.org",
|
||||
"https://www.internetsociety.org",
|
||||
"https://www.python.org"
|
||||
]
|
||||
|
||||
async def basic_pooling_demo():
|
||||
"""Demonstrate basic browser pooling functionality."""
|
||||
print("\n=== Basic Browser Pooling Demo ===")
|
||||
|
||||
# Create logger
|
||||
logger = AsyncLogger(verbose=True)
|
||||
|
||||
# Create browser configurations
|
||||
config1 = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
browser_mode="playwright"
|
||||
)
|
||||
|
||||
config2 = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
browser_mode="cdp"
|
||||
)
|
||||
|
||||
# Create browser manager with on-demand behavior
|
||||
manager = BrowserManager(
|
||||
browser_config=config1,
|
||||
logger=logger,
|
||||
unavailable_behavior=UnavailableBehavior.ON_DEMAND,
|
||||
max_browsers_per_config=3
|
||||
)
|
||||
|
||||
try:
|
||||
# Initialize pool with both configurations
|
||||
print("Initializing browser pool...")
|
||||
await manager.initialize_pool(
|
||||
browser_configs=[config1, config2],
|
||||
browsers_per_config=2
|
||||
)
|
||||
|
||||
# Display initial pool status
|
||||
status = await manager.get_pool_status()
|
||||
print(f"Initial pool status: {status}")
|
||||
|
||||
# Create crawler run configurations
|
||||
run_config1 = CrawlerRunConfig()
|
||||
run_config2 = CrawlerRunConfig()
|
||||
|
||||
# Simulate concurrent page requests
|
||||
print("\nGetting pages for parallel crawling...")
|
||||
|
||||
# Function to simulate crawling
|
||||
async def simulate_crawl(index: int, config: BrowserConfig, run_config: CrawlerRunConfig):
|
||||
print(f"Crawler {index}: Requesting page...")
|
||||
page, context, strategy = await manager.get_page(run_config, config)
|
||||
print(f"Crawler {index}: Got page, navigating to example.com...")
|
||||
|
||||
try:
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
print(f"Crawler {index}: Page title: {title}")
|
||||
|
||||
# Simulate work
|
||||
await asyncio.sleep(random.uniform(1, 3))
|
||||
print(f"Crawler {index}: Work completed, releasing page...")
|
||||
|
||||
# Check dynamic page content
|
||||
content = await page.content()
|
||||
content_length = len(content)
|
||||
print(f"Crawler {index}: Page content length: {content_length}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Crawler {index}: Error: {str(e)}")
|
||||
finally:
|
||||
# Release the page
|
||||
await manager.release_page(page, strategy, config)
|
||||
print(f"Crawler {index}: Page released")
|
||||
|
||||
# Create 5 parallel crawls
|
||||
crawl_tasks = []
|
||||
for i in range(5):
|
||||
# Alternate between configurations
|
||||
config = config1 if i % 2 == 0 else config2
|
||||
run_config = run_config1 if i % 2 == 0 else run_config2
|
||||
|
||||
task = asyncio.create_task(simulate_crawl(i+1, config, run_config))
|
||||
crawl_tasks.append(task)
|
||||
|
||||
# Wait for all crawls to complete
|
||||
await asyncio.gather(*crawl_tasks)
|
||||
|
||||
# Display final pool status
|
||||
status = await manager.get_pool_status()
|
||||
print(f"\nFinal pool status: {status}")
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
print("\nClosing browser manager...")
|
||||
await manager.close()
|
||||
print("Browser manager closed")
|
||||
|
||||
|
||||
async def prewarm_pages_demo():
|
||||
"""Demonstrate page pre-warming functionality."""
|
||||
print("\n=== Page Pre-warming Demo ===")
|
||||
|
||||
# Create logger
|
||||
logger = AsyncLogger(verbose=True)
|
||||
|
||||
# Create browser configuration
|
||||
config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
browser_mode="playwright"
|
||||
)
|
||||
|
||||
# Create crawler run configurations for pre-warming
|
||||
run_config1 = CrawlerRunConfig(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
)
|
||||
|
||||
run_config2 = CrawlerRunConfig(
|
||||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15"
|
||||
)
|
||||
|
||||
# Create page pre-warm configurations
|
||||
page_configs = [
|
||||
(config, run_config1, 2), # 2 pages with run_config1
|
||||
(config, run_config2, 3) # 3 pages with run_config2
|
||||
]
|
||||
|
||||
# Create browser manager
|
||||
manager = BrowserManager(
|
||||
browser_config=config,
|
||||
logger=logger,
|
||||
unavailable_behavior=UnavailableBehavior.EXCEPTION
|
||||
)
|
||||
|
||||
try:
|
||||
# Initialize pool with pre-warmed pages
|
||||
print("Initializing browser pool with pre-warmed pages...")
|
||||
await manager.initialize_pool(
|
||||
browser_configs=[config],
|
||||
browsers_per_config=2,
|
||||
page_configs=page_configs
|
||||
)
|
||||
|
||||
# Display pool status
|
||||
status = await manager.get_pool_status()
|
||||
print(f"Pool status after pre-warming: {status}")
|
||||
|
||||
# Simulate using pre-warmed pages
|
||||
print("\nUsing pre-warmed pages...")
|
||||
|
||||
async def use_prewarm_page(index: int, run_config: CrawlerRunConfig):
|
||||
print(f"Task {index}: Requesting pre-warmed page...")
|
||||
page, context, strategy = await manager.get_page(run_config, config)
|
||||
|
||||
try:
|
||||
print(f"Task {index}: Got page, navigating to example.com...")
|
||||
await page.goto("https://example.com")
|
||||
|
||||
# Verify user agent was applied correctly
|
||||
user_agent = await page.evaluate("() => navigator.userAgent")
|
||||
print(f"Task {index}: User agent: {user_agent}")
|
||||
|
||||
# Get page title
|
||||
title = await page.title()
|
||||
print(f"Task {index}: Page title: {title}")
|
||||
|
||||
# Simulate work
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
# Release the page
|
||||
print(f"Task {index}: Releasing page...")
|
||||
await manager.release_page(page, strategy, config)
|
||||
|
||||
# Create tasks to use pre-warmed pages
|
||||
tasks = []
|
||||
# Use run_config1 pages
|
||||
for i in range(2):
|
||||
tasks.append(asyncio.create_task(use_prewarm_page(i+1, run_config1)))
|
||||
|
||||
# Use run_config2 pages
|
||||
for i in range(3):
|
||||
tasks.append(asyncio.create_task(use_prewarm_page(i+3, run_config2)))
|
||||
|
||||
# Wait for all tasks to complete
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
# Try to use more pages than we pre-warmed (should raise exception)
|
||||
print("\nTrying to use more pages than pre-warmed...")
|
||||
try:
|
||||
page, context, strategy = await manager.get_page(run_config1, config)
|
||||
try:
|
||||
print("Got extra page (unexpected)")
|
||||
await page.goto("https://example.com")
|
||||
finally:
|
||||
await manager.release_page(page, strategy, config)
|
||||
except Exception as e:
|
||||
print(f"Expected exception when requesting more pages: {str(e)}")
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
print("\nClosing browser manager...")
|
||||
await manager.close()
|
||||
print("Browser manager closed")
|
||||
|
||||
|
||||
async def prewarm_on_demand_demo():
|
||||
"""Demonstrate pre-warming with on-demand browser creation."""
|
||||
print("\n=== Pre-warming with On-Demand Browser Creation Demo ===")
|
||||
|
||||
# Create logger
|
||||
logger = AsyncLogger(verbose=True)
|
||||
|
||||
# Create browser configuration
|
||||
config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
browser_mode="playwright"
|
||||
)
|
||||
|
||||
# Create crawler run configurations
|
||||
run_config = CrawlerRunConfig(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
)
|
||||
|
||||
# Create page pre-warm configurations - just pre-warm 2 pages
|
||||
page_configs = [
|
||||
(config, run_config, 2)
|
||||
]
|
||||
|
||||
# Create browser manager with ON_DEMAND behavior
|
||||
manager = BrowserManager(
|
||||
browser_config=config,
|
||||
logger=logger,
|
||||
unavailable_behavior=UnavailableBehavior.ON_DEMAND,
|
||||
max_browsers_per_config=5 # Allow up to 5 browsers
|
||||
)
|
||||
|
||||
try:
|
||||
# Initialize pool with pre-warmed pages
|
||||
print("Initializing browser pool with pre-warmed pages...")
|
||||
await manager.initialize_pool(
|
||||
browser_configs=[config],
|
||||
browsers_per_config=1, # Start with just 1 browser
|
||||
page_configs=page_configs
|
||||
)
|
||||
|
||||
# Display initial pool status
|
||||
status = await manager.get_pool_status()
|
||||
print(f"Initial pool status: {status}")
|
||||
|
||||
# Simulate using more pages than pre-warmed - should create browsers on demand
|
||||
print("\nUsing more pages than pre-warmed (should create on demand)...")
|
||||
|
||||
async def use_page(index: int):
|
||||
print(f"Task {index}: Requesting page...")
|
||||
page, context, strategy = await manager.get_page(run_config, config)
|
||||
|
||||
try:
|
||||
print(f"Task {index}: Got page, navigating to example.com...")
|
||||
await page.goto("https://example.com")
|
||||
|
||||
# Get page title
|
||||
title = await page.title()
|
||||
print(f"Task {index}: Page title: {title}")
|
||||
|
||||
# Simulate work for a varying amount of time
|
||||
work_time = 1 + (index * 0.5) # Stagger completion times
|
||||
print(f"Task {index}: Working for {work_time} seconds...")
|
||||
await asyncio.sleep(work_time)
|
||||
print(f"Task {index}: Work completed")
|
||||
finally:
|
||||
# Release the page
|
||||
print(f"Task {index}: Releasing page...")
|
||||
await manager.release_page(page, strategy, config)
|
||||
|
||||
# Create more tasks than pre-warmed pages
|
||||
tasks = []
|
||||
for i in range(5): # Try to use 5 pages when only 2 are pre-warmed
|
||||
tasks.append(asyncio.create_task(use_page(i+1)))
|
||||
|
||||
# Wait for all tasks to complete
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
# Display final pool status - should show on-demand created browsers
|
||||
status = await manager.get_pool_status()
|
||||
print(f"\nFinal pool status: {status}")
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
print("\nClosing browser manager...")
|
||||
await manager.close()
|
||||
print("Browser manager closed")
|
||||
|
||||
|
||||
async def high_volume_demo():
|
||||
"""Demonstrate high-volume access to pre-warmed pages."""
|
||||
print("\n=== High Volume Pre-warmed Pages Demo ===")
|
||||
|
||||
# Create logger
|
||||
logger = AsyncLogger(verbose=True)
|
||||
|
||||
# Create browser configuration
|
||||
config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
browser_mode="playwright"
|
||||
)
|
||||
|
||||
# Create crawler run configuration
|
||||
run_config = CrawlerRunConfig(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
)
|
||||
|
||||
# Set up dimensions
|
||||
browser_count = 10
|
||||
pages_per_browser = 5
|
||||
total_pages = browser_count * pages_per_browser
|
||||
|
||||
# Create page pre-warm configuration
|
||||
page_configs = [
|
||||
(config, run_config, total_pages)
|
||||
]
|
||||
|
||||
print(f"Preparing {browser_count} browsers with {pages_per_browser} pages each ({total_pages} total pages)")
|
||||
|
||||
# Create browser manager with ON_DEMAND behavior as fallback
|
||||
# No need to specify max_browsers_per_config as it will be calculated automatically
|
||||
manager = BrowserManager(
|
||||
browser_config=config,
|
||||
logger=logger,
|
||||
unavailable_behavior=UnavailableBehavior.ON_DEMAND
|
||||
)
|
||||
|
||||
try:
|
||||
# Initialize pool with browsers and pre-warmed pages
|
||||
print(f"Pre-warming {total_pages} pages...")
|
||||
start_time = time.time()
|
||||
await manager.initialize_pool(
|
||||
browser_configs=[config],
|
||||
browsers_per_config=browser_count,
|
||||
page_configs=page_configs
|
||||
)
|
||||
warmup_time = time.time() - start_time
|
||||
print(f"Pre-warming completed in {warmup_time:.2f} seconds")
|
||||
|
||||
# Display pool status
|
||||
status = await manager.get_pool_status()
|
||||
print(f"Pool status after pre-warming: {status}")
|
||||
|
||||
# Simulate using all pre-warmed pages simultaneously
|
||||
print(f"\nSending {total_pages} crawl requests simultaneously...")
|
||||
|
||||
async def crawl_page(index: int):
|
||||
# url = f"https://example.com/page{index}"
|
||||
url = SAFE_URLS[index % len(SAFE_URLS)]
|
||||
print(f"Page {index}: Requesting page...")
|
||||
# Measure time to acquire page
|
||||
page_start = time.time()
|
||||
page, context, strategy = await manager.get_page(run_config, config)
|
||||
page_acquisition_time = time.time() - page_start
|
||||
|
||||
try:
|
||||
# Navigate to the URL
|
||||
nav_start = time.time()
|
||||
await page.goto(url, timeout=5000)
|
||||
navigation_time = time.time() - nav_start
|
||||
|
||||
# Get the page title
|
||||
title = await page.title()
|
||||
|
||||
return {
|
||||
"index": index,
|
||||
"url": url,
|
||||
"title": title,
|
||||
"page_acquisition_time": page_acquisition_time,
|
||||
"navigation_time": navigation_time
|
||||
}
|
||||
except playwright._impl._errors.TimeoutError as e:
|
||||
# print(f"Page {index}: Navigation timed out - {e}")
|
||||
return {
|
||||
"index": index,
|
||||
"url": url,
|
||||
"title": "Navigation timed out",
|
||||
"page_acquisition_time": page_acquisition_time,
|
||||
"navigation_time": 0
|
||||
}
|
||||
finally:
|
||||
# Release the page
|
||||
await manager.release_page(page, strategy, config)
|
||||
|
||||
# Create and execute all tasks simultaneously
|
||||
start_time = time.time()
|
||||
|
||||
# Non-parallel way
|
||||
# for i in range(total_pages):
|
||||
# await crawl_page(i+1)
|
||||
|
||||
tasks = [crawl_page(i+1) for i in range(total_pages)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
total_time = time.time() - start_time
|
||||
|
||||
# # Print all titles
|
||||
# for result in results:
|
||||
# print(f"Page {result['index']} ({result['url']}): Title: {result['title']}")
|
||||
# print(f" Page acquisition time: {result['page_acquisition_time']:.4f}s")
|
||||
# print(f" Navigation time: {result['navigation_time']:.4f}s")
|
||||
# print(f" Total time: {result['page_acquisition_time'] + result['navigation_time']:.4f}s")
|
||||
# print("-" * 40)
|
||||
|
||||
# Report results
|
||||
print(f"\nAll {total_pages} crawls completed in {total_time:.2f} seconds")
|
||||
|
||||
# Calculate statistics
|
||||
acquisition_times = [r["page_acquisition_time"] for r in results]
|
||||
navigation_times = [r["navigation_time"] for r in results]
|
||||
|
||||
avg_acquisition = sum(acquisition_times) / len(acquisition_times)
|
||||
max_acquisition = max(acquisition_times)
|
||||
min_acquisition = min(acquisition_times)
|
||||
|
||||
avg_navigation = sum(navigation_times) / len(navigation_times)
|
||||
max_navigation = max(navigation_times)
|
||||
min_navigation = min(navigation_times)
|
||||
|
||||
print("\nPage acquisition times:")
|
||||
print(f" Average: {avg_acquisition:.4f}s")
|
||||
print(f" Min: {min_acquisition:.4f}s")
|
||||
print(f" Max: {max_acquisition:.4f}s")
|
||||
|
||||
print("\nPage navigation times:")
|
||||
print(f" Average: {avg_navigation:.4f}s")
|
||||
print(f" Min: {min_navigation:.4f}s")
|
||||
print(f" Max: {max_navigation:.4f}s")
|
||||
|
||||
# Display final pool status
|
||||
status = await manager.get_pool_status()
|
||||
print(f"\nFinal pool status: {status}")
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
print("\nClosing browser manager...")
|
||||
await manager.close()
|
||||
print("Browser manager closed")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all demos."""
|
||||
# await basic_pooling_demo()
|
||||
# await prewarm_pages_demo()
|
||||
# await prewarm_on_demand_demo()
|
||||
await high_volume_demo()
|
||||
# Additional demo functions can be added here
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,489 @@
|
||||
"""Test for browser_context_id and target_id parameters.
|
||||
|
||||
These tests verify that Crawl4AI can connect to and use pre-created
|
||||
browser contexts, which is essential for cloud browser services that
|
||||
pre-create isolated contexts for each user.
|
||||
|
||||
The flow being tested:
|
||||
1. Start a browser with CDP
|
||||
2. Create a context via raw CDP commands (simulating cloud service)
|
||||
3. Create a page/target in that context
|
||||
4. Have Crawl4AI connect using browser_context_id and target_id
|
||||
5. Verify Crawl4AI uses the existing context/page instead of creating new ones
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import websockets
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.browser_manager import BrowserManager, ManagedBrowser
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
|
||||
class CDPContextCreator:
|
||||
"""
|
||||
Helper class to create browser contexts via raw CDP commands.
|
||||
This simulates what a cloud browser service would do.
|
||||
"""
|
||||
|
||||
def __init__(self, cdp_url: str):
|
||||
self.cdp_url = cdp_url
|
||||
self._message_id = 0
|
||||
self._ws = None
|
||||
self._pending_responses = {}
|
||||
self._receiver_task = None
|
||||
|
||||
async def connect(self):
|
||||
"""Establish WebSocket connection to browser."""
|
||||
# Convert HTTP URL to WebSocket URL if needed
|
||||
ws_url = self.cdp_url.replace("http://", "ws://").replace("https://", "wss://")
|
||||
if not ws_url.endswith("/devtools/browser"):
|
||||
# Get the browser websocket URL from /json/version
|
||||
import aiohttp
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f"{self.cdp_url}/json/version") as response:
|
||||
data = await response.json()
|
||||
ws_url = data.get("webSocketDebuggerUrl", ws_url)
|
||||
|
||||
self._ws = await websockets.connect(ws_url, max_size=None, ping_interval=None)
|
||||
self._receiver_task = asyncio.create_task(self._receive_messages())
|
||||
logger.info(f"Connected to CDP at {ws_url}", tag="CDP")
|
||||
|
||||
async def disconnect(self):
|
||||
"""Close WebSocket connection."""
|
||||
if self._receiver_task:
|
||||
self._receiver_task.cancel()
|
||||
try:
|
||||
await self._receiver_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if self._ws:
|
||||
await self._ws.close()
|
||||
self._ws = None
|
||||
|
||||
async def _receive_messages(self):
|
||||
"""Background task to receive CDP messages."""
|
||||
try:
|
||||
async for message in self._ws:
|
||||
data = json.loads(message)
|
||||
msg_id = data.get('id')
|
||||
if msg_id is not None and msg_id in self._pending_responses:
|
||||
self._pending_responses[msg_id].set_result(data)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"CDP receiver error: {e}", tag="CDP")
|
||||
|
||||
async def _send_command(self, method: str, params: dict = None) -> dict:
|
||||
"""Send CDP command and wait for response."""
|
||||
self._message_id += 1
|
||||
msg_id = self._message_id
|
||||
|
||||
message = {
|
||||
"id": msg_id,
|
||||
"method": method,
|
||||
"params": params or {}
|
||||
}
|
||||
|
||||
future = asyncio.get_event_loop().create_future()
|
||||
self._pending_responses[msg_id] = future
|
||||
|
||||
try:
|
||||
await self._ws.send(json.dumps(message))
|
||||
response = await asyncio.wait_for(future, timeout=30.0)
|
||||
|
||||
if 'error' in response:
|
||||
raise Exception(f"CDP error: {response['error']}")
|
||||
|
||||
return response.get('result', {})
|
||||
finally:
|
||||
self._pending_responses.pop(msg_id, None)
|
||||
|
||||
async def create_context(self) -> dict:
|
||||
"""
|
||||
Create an isolated browser context with a blank page.
|
||||
|
||||
Returns:
|
||||
dict with browser_context_id, target_id, and cdp_session_id
|
||||
"""
|
||||
await self.connect()
|
||||
|
||||
# 1. Create isolated browser context
|
||||
result = await self._send_command("Target.createBrowserContext", {
|
||||
"disposeOnDetach": False # Keep context alive
|
||||
})
|
||||
browser_context_id = result["browserContextId"]
|
||||
logger.info(f"Created browser context: {browser_context_id}", tag="CDP")
|
||||
|
||||
# 2. Create a new page (target) in the context
|
||||
result = await self._send_command("Target.createTarget", {
|
||||
"url": "about:blank",
|
||||
"browserContextId": browser_context_id
|
||||
})
|
||||
target_id = result["targetId"]
|
||||
logger.info(f"Created target: {target_id}", tag="CDP")
|
||||
|
||||
# 3. Attach to the target to get a session ID
|
||||
result = await self._send_command("Target.attachToTarget", {
|
||||
"targetId": target_id,
|
||||
"flatten": True
|
||||
})
|
||||
cdp_session_id = result["sessionId"]
|
||||
logger.info(f"Attached to target, sessionId: {cdp_session_id}", tag="CDP")
|
||||
|
||||
return {
|
||||
"browser_context_id": browser_context_id,
|
||||
"target_id": target_id,
|
||||
"cdp_session_id": cdp_session_id
|
||||
}
|
||||
|
||||
async def get_targets(self) -> list:
|
||||
"""Get list of all targets in the browser."""
|
||||
result = await self._send_command("Target.getTargets")
|
||||
return result.get("targetInfos", [])
|
||||
|
||||
async def dispose_context(self, browser_context_id: str):
|
||||
"""Dispose of a browser context."""
|
||||
try:
|
||||
await self._send_command("Target.disposeBrowserContext", {
|
||||
"browserContextId": browser_context_id
|
||||
})
|
||||
logger.info(f"Disposed browser context: {browser_context_id}", tag="CDP")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error disposing context: {e}", tag="CDP")
|
||||
|
||||
|
||||
async def test_browser_context_id_basic():
|
||||
"""
|
||||
Test that BrowserConfig accepts browser_context_id and target_id parameters.
|
||||
"""
|
||||
logger.info("Testing BrowserConfig browser_context_id parameter", tag="TEST")
|
||||
|
||||
try:
|
||||
# Test that BrowserConfig accepts the new parameters
|
||||
config = BrowserConfig(
|
||||
cdp_url="http://localhost:9222",
|
||||
browser_context_id="test-context-id",
|
||||
target_id="test-target-id",
|
||||
headless=True
|
||||
)
|
||||
|
||||
# Verify parameters are set correctly
|
||||
assert config.browser_context_id == "test-context-id", "browser_context_id not set"
|
||||
assert config.target_id == "test-target-id", "target_id not set"
|
||||
|
||||
# Test from_kwargs
|
||||
config2 = BrowserConfig.from_kwargs({
|
||||
"cdp_url": "http://localhost:9222",
|
||||
"browser_context_id": "test-context-id-2",
|
||||
"target_id": "test-target-id-2"
|
||||
})
|
||||
|
||||
assert config2.browser_context_id == "test-context-id-2", "browser_context_id not set via from_kwargs"
|
||||
assert config2.target_id == "test-target-id-2", "target_id not set via from_kwargs"
|
||||
|
||||
# Test to_dict
|
||||
config_dict = config.to_dict()
|
||||
assert config_dict.get("browser_context_id") == "test-context-id", "browser_context_id not in to_dict"
|
||||
assert config_dict.get("target_id") == "test-target-id", "target_id not in to_dict"
|
||||
|
||||
logger.success("BrowserConfig browser_context_id test passed", tag="TEST")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
return False
|
||||
|
||||
|
||||
async def test_pre_created_context_usage():
|
||||
"""
|
||||
Test that Crawl4AI uses a pre-created browser context instead of creating a new one.
|
||||
|
||||
This simulates the cloud browser service flow:
|
||||
1. Start browser with CDP
|
||||
2. Create context via raw CDP (simulating cloud service)
|
||||
3. Have Crawl4AI connect with browser_context_id
|
||||
4. Verify it uses existing context
|
||||
"""
|
||||
logger.info("Testing pre-created context usage", tag="TEST")
|
||||
|
||||
# Start a managed browser first
|
||||
browser_config_initial = BrowserConfig(
|
||||
use_managed_browser=True,
|
||||
headless=True,
|
||||
debugging_port=9226, # Use unique port
|
||||
verbose=True
|
||||
)
|
||||
|
||||
managed_browser = ManagedBrowser(browser_config=browser_config_initial, logger=logger)
|
||||
cdp_creator = None
|
||||
manager = None
|
||||
context_info = None
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
cdp_url = await managed_browser.start()
|
||||
logger.info(f"Browser started at {cdp_url}", tag="TEST")
|
||||
|
||||
# Create a context via raw CDP (simulating cloud service)
|
||||
cdp_creator = CDPContextCreator(cdp_url)
|
||||
context_info = await cdp_creator.create_context()
|
||||
|
||||
logger.info(f"Pre-created context: {context_info['browser_context_id']}", tag="TEST")
|
||||
logger.info(f"Pre-created target: {context_info['target_id']}", tag="TEST")
|
||||
|
||||
# Get initial target count
|
||||
targets_before = await cdp_creator.get_targets()
|
||||
initial_target_count = len(targets_before)
|
||||
logger.info(f"Initial target count: {initial_target_count}", tag="TEST")
|
||||
|
||||
# Now create BrowserManager with browser_context_id and target_id
|
||||
browser_config = BrowserConfig(
|
||||
cdp_url=cdp_url,
|
||||
browser_context_id=context_info['browser_context_id'],
|
||||
target_id=context_info['target_id'],
|
||||
headless=True,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
await manager.start()
|
||||
|
||||
logger.info("BrowserManager started with pre-created context", tag="TEST")
|
||||
|
||||
# Get a page
|
||||
crawler_config = CrawlerRunConfig()
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
|
||||
# Navigate to a test page
|
||||
await page.goto("https://example.com", wait_until="domcontentloaded")
|
||||
title = await page.title()
|
||||
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Get target count after
|
||||
targets_after = await cdp_creator.get_targets()
|
||||
final_target_count = len(targets_after)
|
||||
logger.info(f"Final target count: {final_target_count}", tag="TEST")
|
||||
|
||||
# Verify: target count should not have increased significantly
|
||||
# (allow for 1 extra target for internal use, but not many more)
|
||||
target_diff = final_target_count - initial_target_count
|
||||
logger.info(f"Target count difference: {target_diff}", tag="TEST")
|
||||
|
||||
# Success criteria:
|
||||
# 1. Page navigation worked
|
||||
# 2. Target count didn't explode (reused existing context)
|
||||
success = title == "Example Domain" and target_diff <= 1
|
||||
|
||||
if success:
|
||||
logger.success("Pre-created context usage test passed", tag="TEST")
|
||||
else:
|
||||
logger.error(f"Test failed - Title: {title}, Target diff: {target_diff}", tag="TEST")
|
||||
|
||||
return success
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
if manager:
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
if cdp_creator and context_info:
|
||||
try:
|
||||
await cdp_creator.dispose_context(context_info['browser_context_id'])
|
||||
await cdp_creator.disconnect()
|
||||
except:
|
||||
pass
|
||||
|
||||
if managed_browser:
|
||||
try:
|
||||
await managed_browser.cleanup()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
async def test_context_isolation():
|
||||
"""
|
||||
Test that using browser_context_id actually provides isolation.
|
||||
Create two contexts and verify they don't share state.
|
||||
"""
|
||||
logger.info("Testing context isolation with browser_context_id", tag="TEST")
|
||||
|
||||
browser_config_initial = BrowserConfig(
|
||||
use_managed_browser=True,
|
||||
headless=True,
|
||||
debugging_port=9227,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
managed_browser = ManagedBrowser(browser_config=browser_config_initial, logger=logger)
|
||||
cdp_creator = None
|
||||
manager1 = None
|
||||
manager2 = None
|
||||
context_info_1 = None
|
||||
context_info_2 = None
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
cdp_url = await managed_browser.start()
|
||||
logger.info(f"Browser started at {cdp_url}", tag="TEST")
|
||||
|
||||
# Create two separate contexts
|
||||
cdp_creator = CDPContextCreator(cdp_url)
|
||||
context_info_1 = await cdp_creator.create_context()
|
||||
logger.info(f"Context 1: {context_info_1['browser_context_id']}", tag="TEST")
|
||||
|
||||
# Need to reconnect for second context (or use same connection)
|
||||
await cdp_creator.disconnect()
|
||||
cdp_creator2 = CDPContextCreator(cdp_url)
|
||||
context_info_2 = await cdp_creator2.create_context()
|
||||
logger.info(f"Context 2: {context_info_2['browser_context_id']}", tag="TEST")
|
||||
|
||||
# Verify contexts are different
|
||||
assert context_info_1['browser_context_id'] != context_info_2['browser_context_id'], \
|
||||
"Contexts should have different IDs"
|
||||
|
||||
# Connect with first context
|
||||
browser_config_1 = BrowserConfig(
|
||||
cdp_url=cdp_url,
|
||||
browser_context_id=context_info_1['browser_context_id'],
|
||||
target_id=context_info_1['target_id'],
|
||||
headless=True
|
||||
)
|
||||
|
||||
manager1 = BrowserManager(browser_config=browser_config_1, logger=logger)
|
||||
await manager1.start()
|
||||
|
||||
# Set a cookie in context 1
|
||||
page1, ctx1 = await manager1.get_page(CrawlerRunConfig())
|
||||
await page1.goto("https://example.com", wait_until="domcontentloaded")
|
||||
await ctx1.add_cookies([{
|
||||
"name": "test_isolation",
|
||||
"value": "context_1_value",
|
||||
"domain": "example.com",
|
||||
"path": "/"
|
||||
}])
|
||||
|
||||
cookies1 = await ctx1.cookies(["https://example.com"])
|
||||
cookie1_value = next((c["value"] for c in cookies1 if c["name"] == "test_isolation"), None)
|
||||
logger.info(f"Cookie in context 1: {cookie1_value}", tag="TEST")
|
||||
|
||||
# Connect with second context
|
||||
browser_config_2 = BrowserConfig(
|
||||
cdp_url=cdp_url,
|
||||
browser_context_id=context_info_2['browser_context_id'],
|
||||
target_id=context_info_2['target_id'],
|
||||
headless=True
|
||||
)
|
||||
|
||||
manager2 = BrowserManager(browser_config=browser_config_2, logger=logger)
|
||||
await manager2.start()
|
||||
|
||||
# Check cookies in context 2 - should not have the cookie from context 1
|
||||
page2, ctx2 = await manager2.get_page(CrawlerRunConfig())
|
||||
await page2.goto("https://example.com", wait_until="domcontentloaded")
|
||||
|
||||
cookies2 = await ctx2.cookies(["https://example.com"])
|
||||
cookie2_value = next((c["value"] for c in cookies2 if c["name"] == "test_isolation"), None)
|
||||
logger.info(f"Cookie in context 2: {cookie2_value}", tag="TEST")
|
||||
|
||||
# Verify isolation
|
||||
isolation_works = cookie1_value == "context_1_value" and cookie2_value is None
|
||||
|
||||
if isolation_works:
|
||||
logger.success("Context isolation test passed", tag="TEST")
|
||||
else:
|
||||
logger.error(f"Isolation failed - Cookie1: {cookie1_value}, Cookie2: {cookie2_value}", tag="TEST")
|
||||
|
||||
return isolation_works
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
for mgr in [manager1, manager2]:
|
||||
if mgr:
|
||||
try:
|
||||
await mgr.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
for ctx_info, creator in [(context_info_1, cdp_creator), (context_info_2, cdp_creator2 if 'cdp_creator2' in dir() else None)]:
|
||||
if ctx_info and creator:
|
||||
try:
|
||||
await creator.dispose_context(ctx_info['browser_context_id'])
|
||||
await creator.disconnect()
|
||||
except:
|
||||
pass
|
||||
|
||||
if managed_browser:
|
||||
try:
|
||||
await managed_browser.cleanup()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
async def run_tests():
|
||||
"""Run all browser_context_id tests."""
|
||||
results = []
|
||||
|
||||
logger.info("Running browser_context_id tests", tag="SUITE")
|
||||
|
||||
# Basic parameter test
|
||||
results.append(("browser_context_id_basic", await test_browser_context_id_basic()))
|
||||
|
||||
# Pre-created context usage test
|
||||
results.append(("pre_created_context_usage", await test_pre_created_context_usage()))
|
||||
|
||||
# Note: Context isolation test is commented out because isolation is enforced
|
||||
# at the CDP level by the cloud browser service, not at the Playwright level.
|
||||
# When multiple BrowserManagers connect to the same browser, Playwright sees
|
||||
# all contexts. In production, each worker gets exactly one pre-created context.
|
||||
# results.append(("context_isolation", await test_context_isolation()))
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(1 for _, r in results if r)
|
||||
|
||||
logger.info("=" * 50, tag="SUMMARY")
|
||||
logger.info(f"Test Results: {passed}/{total} passed", tag="SUMMARY")
|
||||
logger.info("=" * 50, tag="SUMMARY")
|
||||
|
||||
for name, result in results:
|
||||
status = "PASSED" if result else "FAILED"
|
||||
logger.info(f" {name}: {status}", tag="SUMMARY")
|
||||
|
||||
if passed == total:
|
||||
logger.success("All tests passed!", tag="SUMMARY")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"{total - passed} tests failed", tag="SUMMARY")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = asyncio.run(run_tests())
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Test examples for BrowserManager.
|
||||
|
||||
These examples demonstrate the functionality of BrowserManager
|
||||
and serve as functional tests.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.browser import BrowserManager
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
async def test_basic_browser_manager():
|
||||
"""Test basic BrowserManager functionality with default configuration."""
|
||||
logger.info("Starting test_basic_browser_manager", tag="TEST")
|
||||
|
||||
try:
|
||||
# Create a browser manager with default config
|
||||
manager = BrowserManager(logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Get a page
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
logger.info("Page created successfully", tag="TEST")
|
||||
|
||||
# Navigate to a website
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.success("test_basic_browser_manager completed successfully", tag="TEST")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"test_basic_browser_manager failed: {str(e)}", tag="TEST")
|
||||
return False
|
||||
|
||||
async def test_custom_browser_config():
|
||||
"""Test BrowserManager with custom browser configuration."""
|
||||
logger.info("Starting test_custom_browser_config", tag="TEST")
|
||||
|
||||
try:
|
||||
# Create a custom browser config
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
viewport_width=1280,
|
||||
viewport_height=800,
|
||||
light_mode=True
|
||||
)
|
||||
|
||||
# Create browser manager with the config
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully with custom config", tag="TEST")
|
||||
|
||||
# Get a page
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
|
||||
# Navigate to a website
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Verify viewport size
|
||||
viewport_size = await page.evaluate("() => ({ width: window.innerWidth, height: window.innerHeight })")
|
||||
logger.info(f"Viewport size: {viewport_size}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.success("test_custom_browser_config completed successfully", tag="TEST")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"test_custom_browser_config failed: {str(e)}", tag="TEST")
|
||||
return False
|
||||
|
||||
async def test_multiple_pages():
|
||||
"""Test BrowserManager with multiple pages."""
|
||||
logger.info("Starting test_multiple_pages", tag="TEST")
|
||||
|
||||
try:
|
||||
# Create browser manager
|
||||
manager = BrowserManager(logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Create multiple pages
|
||||
pages = []
|
||||
urls = ["https://example.com", "https://example.org", "https://mozilla.org"]
|
||||
|
||||
for i, url in enumerate(urls):
|
||||
crawler_config = CrawlerRunConfig(url=url)
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
await page.goto(url)
|
||||
pages.append((page, url))
|
||||
logger.info(f"Created page {i+1} for {url}", tag="TEST")
|
||||
|
||||
# Verify all pages are loaded correctly
|
||||
for i, (page, url) in enumerate(pages):
|
||||
title = await page.title()
|
||||
logger.info(f"Page {i+1} title: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.success("test_multiple_pages completed successfully", tag="TEST")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"test_multiple_pages failed: {str(e)}", tag="TEST")
|
||||
return False
|
||||
|
||||
async def test_session_management():
|
||||
"""Test session management in BrowserManager."""
|
||||
logger.info("Starting test_session_management", tag="TEST")
|
||||
|
||||
try:
|
||||
# Create browser manager
|
||||
manager = BrowserManager(logger=logger)
|
||||
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Create a session
|
||||
session_id = "test_session_1"
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com", session_id=session_id)
|
||||
page1, context1 = await manager.get_page(crawler_config)
|
||||
await page1.goto("https://example.com")
|
||||
logger.info(f"Created session with ID: {session_id}", tag="TEST")
|
||||
|
||||
# Get the same session again
|
||||
page2, context2 = await manager.get_page(crawler_config)
|
||||
|
||||
# Verify it's the same page/context
|
||||
is_same_page = page1 == page2
|
||||
is_same_context = context1 == context2
|
||||
logger.info(f"Same page: {is_same_page}, Same context: {is_same_context}", tag="TEST")
|
||||
|
||||
# Kill the session
|
||||
await manager.kill_session(session_id)
|
||||
logger.info(f"Killed session with ID: {session_id}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.success("test_session_management completed successfully", tag="TEST")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"test_session_management failed: {str(e)}", tag="TEST")
|
||||
return False
|
||||
|
||||
async def run_tests():
|
||||
"""Run all tests sequentially."""
|
||||
results = []
|
||||
|
||||
results.append(await test_basic_browser_manager())
|
||||
results.append(await test_custom_browser_config())
|
||||
results.append(await test_multiple_pages())
|
||||
results.append(await test_session_management())
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(results)
|
||||
logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
|
||||
|
||||
if passed == total:
|
||||
logger.success("All tests passed!", tag="SUMMARY")
|
||||
else:
|
||||
logger.error(f"{total - passed} tests failed", tag="SUMMARY")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
@@ -0,0 +1,35 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from crawl4ai.browser_manager import BrowserManager
|
||||
|
||||
|
||||
class _Context:
|
||||
def __init__(self, manager=None):
|
||||
self.manager = manager
|
||||
self.closed = False
|
||||
|
||||
async def close(self):
|
||||
self.closed = True
|
||||
if self.manager:
|
||||
self.manager.contexts_by_config.pop("second", None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_iterates_over_context_snapshot():
|
||||
manager = BrowserManager.__new__(BrowserManager)
|
||||
manager.config = SimpleNamespace(cdp_url=None, sleep_on_close=False)
|
||||
manager.sessions = {}
|
||||
manager.browser = manager.managed_browser = manager.playwright = None
|
||||
manager.logger = SimpleNamespace(error=lambda **_: None)
|
||||
manager._using_cached_cdp = manager._launched_persistent = False
|
||||
manager._context_refcounts = manager._context_last_used = manager._page_to_sig = {}
|
||||
first, second = _Context(manager), _Context()
|
||||
manager.contexts_by_config = {"first": first, "second": second}
|
||||
|
||||
await manager.close()
|
||||
|
||||
assert first.closed
|
||||
assert second.closed
|
||||
assert manager.contexts_by_config == {}
|
||||
@@ -0,0 +1,809 @@
|
||||
"""
|
||||
Test script for builtin browser functionality in the browser module.
|
||||
|
||||
This script tests:
|
||||
1. Creating a builtin browser
|
||||
2. Getting browser information
|
||||
3. Killing the browser
|
||||
4. Restarting the browser
|
||||
5. Testing operations with different browser strategies
|
||||
6. Testing edge cases
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import List, Dict, Any
|
||||
from colorama import Fore, Style, init
|
||||
|
||||
# Add the project root to the path for imports
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
|
||||
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
from rich.box import Box, SIMPLE
|
||||
|
||||
from crawl4ai.browser import BrowserManager
|
||||
from crawl4ai.browser.strategies import BuiltinBrowserStrategy
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Initialize colorama for cross-platform colored terminal output
|
||||
init()
|
||||
|
||||
# Define colors for pretty output
|
||||
SUCCESS = Fore.GREEN
|
||||
WARNING = Fore.YELLOW
|
||||
ERROR = Fore.RED
|
||||
INFO = Fore.CYAN
|
||||
RESET = Fore.RESET
|
||||
|
||||
# Create logger
|
||||
logger = AsyncLogger(verbose=True)
|
||||
|
||||
|
||||
async def test_builtin_browser_creation():
|
||||
"""Test creating a builtin browser using the BrowserManager with BuiltinBrowserStrategy"""
|
||||
print(f"\n{INFO}========== Testing Builtin Browser Creation =========={RESET}")
|
||||
|
||||
# Step 1: Create a BrowserManager with builtin mode
|
||||
print(f"\n{INFO}1. Creating BrowserManager with builtin mode{RESET}")
|
||||
browser_config = BrowserConfig(browser_mode="builtin", headless=True, verbose=True)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Step 2: Check if we have a BuiltinBrowserStrategy
|
||||
print(f"\n{INFO}2. Checking if we have a BuiltinBrowserStrategy{RESET}")
|
||||
if isinstance(manager.strategy, BuiltinBrowserStrategy):
|
||||
print(
|
||||
f"{SUCCESS}Correct strategy type: {manager.strategy.__class__.__name__}{RESET}"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"{ERROR}Wrong strategy type: {manager.strategy.__class__.__name__}{RESET}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Step 3: Start the manager to launch or connect to builtin browser
|
||||
print(f"\n{INFO}3. Starting the browser manager{RESET}")
|
||||
try:
|
||||
await manager.start()
|
||||
print(f"{SUCCESS}Browser manager started successfully{RESET}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to start browser manager: {str(e)}{RESET}")
|
||||
return None
|
||||
|
||||
# Step 4: Get browser info from the strategy
|
||||
print(f"\n{INFO}4. Getting browser information{RESET}")
|
||||
browser_info = manager.strategy.get_browser_info()
|
||||
if browser_info:
|
||||
print(f"{SUCCESS}Browser info retrieved:{RESET}")
|
||||
for key, value in browser_info.items():
|
||||
if key != "config": # Skip the verbose config section
|
||||
print(f" {key}: {value}")
|
||||
|
||||
cdp_url = browser_info.get("cdp_url")
|
||||
print(f"{SUCCESS}CDP URL: {cdp_url}{RESET}")
|
||||
else:
|
||||
print(f"{ERROR}Failed to get browser information{RESET}")
|
||||
cdp_url = None
|
||||
|
||||
# Save manager for later tests
|
||||
return manager, cdp_url
|
||||
|
||||
|
||||
async def test_page_operations(manager: BrowserManager):
|
||||
"""Test page operations with the builtin browser"""
|
||||
print(
|
||||
f"\n{INFO}========== Testing Page Operations with Builtin Browser =========={RESET}"
|
||||
)
|
||||
|
||||
# Step 1: Get a single page
|
||||
print(f"\n{INFO}1. Getting a single page{RESET}")
|
||||
try:
|
||||
crawler_config = CrawlerRunConfig()
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
print(f"{SUCCESS}Got page successfully{RESET}")
|
||||
|
||||
# Navigate to a test URL
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
print(f"{SUCCESS}Page title: {title}{RESET}")
|
||||
|
||||
# Close the page
|
||||
await page.close()
|
||||
print(f"{SUCCESS}Page closed successfully{RESET}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Page operation failed: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
# Step 2: Get multiple pages
|
||||
print(f"\n{INFO}2. Getting multiple pages with get_pages(){RESET}")
|
||||
try:
|
||||
# Request 3 pages
|
||||
crawler_config = CrawlerRunConfig()
|
||||
pages = await manager.get_pages(crawler_config, count=3)
|
||||
print(f"{SUCCESS}Got {len(pages)} pages{RESET}")
|
||||
|
||||
# Test each page
|
||||
for i, (page, context) in enumerate(pages):
|
||||
await page.goto(f"https://example.com?test={i}")
|
||||
title = await page.title()
|
||||
print(f"{SUCCESS}Page {i + 1} title: {title}{RESET}")
|
||||
await page.close()
|
||||
|
||||
print(f"{SUCCESS}All pages tested and closed successfully{RESET}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Multiple page operation failed: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
async def test_browser_status_management(manager: BrowserManager):
|
||||
"""Test browser status and management operations"""
|
||||
print(f"\n{INFO}========== Testing Browser Status and Management =========={RESET}")
|
||||
|
||||
# Step 1: Get browser status
|
||||
print(f"\n{INFO}1. Getting browser status{RESET}")
|
||||
try:
|
||||
status = await manager.strategy.get_builtin_browser_status()
|
||||
print(f"{SUCCESS}Browser status:{RESET}")
|
||||
print(f" Running: {status['running']}")
|
||||
print(f" CDP URL: {status['cdp_url']}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to get browser status: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
# Step 2: Test killing the browser
|
||||
print(f"\n{INFO}2. Testing killing the browser{RESET}")
|
||||
try:
|
||||
result = await manager.strategy.kill_builtin_browser()
|
||||
if result:
|
||||
print(f"{SUCCESS}Browser killed successfully{RESET}")
|
||||
else:
|
||||
print(f"{ERROR}Failed to kill browser{RESET}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Browser kill operation failed: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
# Step 3: Check status after kill
|
||||
print(f"\n{INFO}3. Checking status after kill{RESET}")
|
||||
try:
|
||||
status = await manager.strategy.get_builtin_browser_status()
|
||||
if not status["running"]:
|
||||
print(f"{SUCCESS}Browser is correctly reported as not running{RESET}")
|
||||
else:
|
||||
print(f"{ERROR}Browser is incorrectly reported as still running{RESET}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to get browser status: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
# Step 4: Launch a new browser
|
||||
print(f"\n{INFO}4. Launching a new browser{RESET}")
|
||||
try:
|
||||
cdp_url = await manager.strategy.launch_builtin_browser(
|
||||
browser_type="chromium", headless=True
|
||||
)
|
||||
if cdp_url:
|
||||
print(f"{SUCCESS}New browser launched at: {cdp_url}{RESET}")
|
||||
else:
|
||||
print(f"{ERROR}Failed to launch new browser{RESET}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Browser launch failed: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
async def test_multiple_managers():
|
||||
"""Test creating multiple BrowserManagers that use the same builtin browser"""
|
||||
print(f"\n{INFO}========== Testing Multiple Browser Managers =========={RESET}")
|
||||
|
||||
# Step 1: Create first manager
|
||||
print(f"\n{INFO}1. Creating first browser manager{RESET}")
|
||||
browser_config1 = BrowserConfig(browser_mode="builtin", headless=True)
|
||||
manager1 = BrowserManager(browser_config=browser_config1, logger=logger)
|
||||
|
||||
# Step 2: Create second manager
|
||||
print(f"\n{INFO}2. Creating second browser manager{RESET}")
|
||||
browser_config2 = BrowserConfig(browser_mode="builtin", headless=True)
|
||||
manager2 = BrowserManager(browser_config=browser_config2, logger=logger)
|
||||
|
||||
# Step 3: Start both managers (should connect to the same builtin browser)
|
||||
print(f"\n{INFO}3. Starting both managers{RESET}")
|
||||
try:
|
||||
await manager1.start()
|
||||
print(f"{SUCCESS}First manager started{RESET}")
|
||||
|
||||
await manager2.start()
|
||||
print(f"{SUCCESS}Second manager started{RESET}")
|
||||
|
||||
# Check if they got the same CDP URL
|
||||
cdp_url1 = manager1.strategy.config.cdp_url
|
||||
cdp_url2 = manager2.strategy.config.cdp_url
|
||||
|
||||
if cdp_url1 == cdp_url2:
|
||||
print(
|
||||
f"{SUCCESS}Both managers connected to the same browser: {cdp_url1}{RESET}"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"{WARNING}Managers connected to different browsers: {cdp_url1} and {cdp_url2}{RESET}"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to start managers: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
# Step 4: Test using both managers
|
||||
print(f"\n{INFO}4. Testing operations with both managers{RESET}")
|
||||
try:
|
||||
# First manager creates a page
|
||||
page1, ctx1 = await manager1.get_page(CrawlerRunConfig())
|
||||
await page1.goto("https://example.com")
|
||||
title1 = await page1.title()
|
||||
print(f"{SUCCESS}Manager 1 page title: {title1}{RESET}")
|
||||
|
||||
# Second manager creates a page
|
||||
page2, ctx2 = await manager2.get_page(CrawlerRunConfig())
|
||||
await page2.goto("https://example.org")
|
||||
title2 = await page2.title()
|
||||
print(f"{SUCCESS}Manager 2 page title: {title2}{RESET}")
|
||||
|
||||
# Clean up
|
||||
await page1.close()
|
||||
await page2.close()
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to use both managers: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
# Step 5: Close both managers
|
||||
print(f"\n{INFO}5. Closing both managers{RESET}")
|
||||
try:
|
||||
await manager1.close()
|
||||
print(f"{SUCCESS}First manager closed{RESET}")
|
||||
|
||||
await manager2.close()
|
||||
print(f"{SUCCESS}Second manager closed{RESET}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to close managers: {str(e)}{RESET}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
async def test_edge_cases():
|
||||
"""Test edge cases like multiple starts, killing browser during operations, etc."""
|
||||
print(f"\n{INFO}========== Testing Edge Cases =========={RESET}")
|
||||
|
||||
# Step 1: Test multiple starts with the same manager
|
||||
print(f"\n{INFO}1. Testing multiple starts with the same manager{RESET}")
|
||||
browser_config = BrowserConfig(browser_mode="builtin", headless=True)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
print(f"{SUCCESS}First start successful{RESET}")
|
||||
|
||||
# Try to start again
|
||||
await manager.start()
|
||||
print(f"{SUCCESS}Second start completed without errors{RESET}")
|
||||
|
||||
# Test if it's still functional
|
||||
page, context = await manager.get_page(CrawlerRunConfig())
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
print(
|
||||
f"{SUCCESS}Page operations work after multiple starts. Title: {title}{RESET}"
|
||||
)
|
||||
await page.close()
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Multiple starts test failed: {str(e)}{RESET}")
|
||||
return False
|
||||
finally:
|
||||
await manager.close()
|
||||
|
||||
# Step 2: Test killing the browser while manager is active
|
||||
print(f"\n{INFO}2. Testing killing the browser while manager is active{RESET}")
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
print(f"{SUCCESS}Manager started{RESET}")
|
||||
|
||||
# Kill the browser directly
|
||||
print(f"{INFO}Killing the browser...{RESET}")
|
||||
await manager.strategy.kill_builtin_browser()
|
||||
print(f"{SUCCESS}Browser killed{RESET}")
|
||||
|
||||
# Try to get a page (should fail or launch a new browser)
|
||||
try:
|
||||
page, context = await manager.get_page(CrawlerRunConfig())
|
||||
print(
|
||||
f"{WARNING}Page request succeeded despite killed browser (might have auto-restarted){RESET}"
|
||||
)
|
||||
title = await page.title()
|
||||
print(f"{SUCCESS}Got page title: {title}{RESET}")
|
||||
await page.close()
|
||||
except Exception as e:
|
||||
print(
|
||||
f"{SUCCESS}Page request failed as expected after browser was killed: {str(e)}{RESET}"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Kill during operation test failed: {str(e)}{RESET}")
|
||||
return False
|
||||
finally:
|
||||
await manager.close()
|
||||
|
||||
return True
|
||||
|
||||
|
||||
async def cleanup_browsers():
|
||||
"""Clean up any remaining builtin browsers"""
|
||||
print(f"\n{INFO}========== Cleaning Up Builtin Browsers =========={RESET}")
|
||||
|
||||
browser_config = BrowserConfig(browser_mode="builtin", headless=True)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
# No need to start, just access the strategy directly
|
||||
strategy = manager.strategy
|
||||
if isinstance(strategy, BuiltinBrowserStrategy):
|
||||
result = await strategy.kill_builtin_browser()
|
||||
if result:
|
||||
print(f"{SUCCESS}Successfully killed all builtin browsers{RESET}")
|
||||
else:
|
||||
print(f"{WARNING}No builtin browsers found to kill{RESET}")
|
||||
else:
|
||||
print(f"{ERROR}Wrong strategy type: {strategy.__class__.__name__}{RESET}")
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Cleanup failed: {str(e)}{RESET}")
|
||||
finally:
|
||||
# Just to be safe
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
async def test_performance_scaling():
|
||||
"""Test performance with multiple browsers and pages.
|
||||
|
||||
This test creates multiple browsers on different ports,
|
||||
spawns multiple pages per browser, and measures performance metrics.
|
||||
"""
|
||||
print(f"\n{INFO}========== Testing Performance Scaling =========={RESET}")
|
||||
|
||||
# Configuration parameters
|
||||
num_browsers = 10
|
||||
pages_per_browser = 10
|
||||
total_pages = num_browsers * pages_per_browser
|
||||
base_port = 9222
|
||||
|
||||
# Set up a measuring mechanism for memory
|
||||
import psutil
|
||||
import gc
|
||||
|
||||
# Force garbage collection before starting
|
||||
gc.collect()
|
||||
process = psutil.Process()
|
||||
initial_memory = process.memory_info().rss / 1024 / 1024 # in MB
|
||||
peak_memory = initial_memory
|
||||
|
||||
# Report initial configuration
|
||||
print(
|
||||
f"{INFO}Test configuration: {num_browsers} browsers × {pages_per_browser} pages = {total_pages} total crawls{RESET}"
|
||||
)
|
||||
|
||||
# List to track managers
|
||||
managers: List[BrowserManager] = []
|
||||
all_pages = []
|
||||
|
||||
|
||||
|
||||
# Get crawl4ai home directory
|
||||
crawl4ai_home = os.path.expanduser("~/.crawl4ai")
|
||||
temp_dir = os.path.join(crawl4ai_home, "temp")
|
||||
os.makedirs(temp_dir, exist_ok=True)
|
||||
|
||||
# Create all managers but don't start them yet
|
||||
manager_configs = []
|
||||
for i in range(num_browsers):
|
||||
port = base_port + i
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="builtin",
|
||||
headless=True,
|
||||
debugging_port=port,
|
||||
user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
|
||||
)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
manager.strategy.shutting_down = True
|
||||
manager_configs.append((manager, i, port))
|
||||
|
||||
# Define async function to start a single manager
|
||||
async def start_manager(manager, index, port):
|
||||
try:
|
||||
await manager.start()
|
||||
return manager
|
||||
except Exception as e:
|
||||
print(
|
||||
f"{ERROR}Failed to start browser {index + 1} on port {port}: {str(e)}{RESET}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Start all managers in parallel
|
||||
start_tasks = [
|
||||
start_manager(manager, i, port) for manager, i, port in manager_configs
|
||||
]
|
||||
started_managers = await asyncio.gather(*start_tasks)
|
||||
|
||||
# Filter out None values (failed starts) and add to managers list
|
||||
managers = [m for m in started_managers if m is not None]
|
||||
|
||||
if len(managers) == 0:
|
||||
print(f"{ERROR}All browser managers failed to start. Aborting test.{RESET}")
|
||||
return False
|
||||
|
||||
if len(managers) < num_browsers:
|
||||
print(
|
||||
f"{WARNING}Only {len(managers)} out of {num_browsers} browser managers started successfully{RESET}"
|
||||
)
|
||||
|
||||
# Create pages for each browser
|
||||
for i, manager in enumerate(managers):
|
||||
try:
|
||||
pages = await manager.get_pages(CrawlerRunConfig(), count=pages_per_browser)
|
||||
all_pages.extend(pages)
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to create pages for browser {i + 1}: {str(e)}{RESET}")
|
||||
|
||||
# Check memory after page creation
|
||||
gc.collect()
|
||||
current_memory = process.memory_info().rss / 1024 / 1024
|
||||
peak_memory = max(peak_memory, current_memory)
|
||||
|
||||
# Ask for confirmation before loading
|
||||
confirmation = input(
|
||||
f"{WARNING}Do you want to proceed with loading pages? (y/n): {RESET}"
|
||||
)
|
||||
# Step 1: Create and start multiple browser managers in parallel
|
||||
start_time = time.time()
|
||||
|
||||
if confirmation.lower() == "y":
|
||||
load_start_time = time.time()
|
||||
|
||||
# Function to load a single page
|
||||
async def load_page(page_ctx, index):
|
||||
page, _ = page_ctx
|
||||
try:
|
||||
await page.goto(f"https://example.com/page{index}", timeout=30000)
|
||||
title = await page.title()
|
||||
return title
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
# Load all pages concurrently
|
||||
load_tasks = [load_page(page_ctx, i) for i, page_ctx in enumerate(all_pages)]
|
||||
load_results = await asyncio.gather(*load_tasks, return_exceptions=True)
|
||||
|
||||
# Count successes and failures
|
||||
successes = sum(
|
||||
1 for r in load_results if isinstance(r, str) and not r.startswith("Error")
|
||||
)
|
||||
failures = len(load_results) - successes
|
||||
|
||||
load_time = time.time() - load_start_time
|
||||
total_test_time = time.time() - start_time
|
||||
|
||||
# Check memory after loading (peak memory)
|
||||
gc.collect()
|
||||
current_memory = process.memory_info().rss / 1024 / 1024
|
||||
peak_memory = max(peak_memory, current_memory)
|
||||
|
||||
# Calculate key metrics
|
||||
memory_per_page = peak_memory / successes if successes > 0 else 0
|
||||
time_per_crawl = total_test_time / successes if successes > 0 else 0
|
||||
crawls_per_second = successes / total_test_time if total_test_time > 0 else 0
|
||||
crawls_per_minute = crawls_per_second * 60
|
||||
crawls_per_hour = crawls_per_minute * 60
|
||||
|
||||
# Print simplified performance summary
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
# Create a simple summary table
|
||||
table = Table(title="CRAWL4AI PERFORMANCE SUMMARY")
|
||||
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Total Crawls Completed", f"{successes}")
|
||||
table.add_row("Total Time", f"{total_test_time:.2f} seconds")
|
||||
table.add_row("Time Per Crawl", f"{time_per_crawl:.2f} seconds")
|
||||
table.add_row("Crawling Speed", f"{crawls_per_second:.2f} crawls/second")
|
||||
table.add_row("Projected Rate (1 minute)", f"{crawls_per_minute:.0f} crawls")
|
||||
table.add_row("Projected Rate (1 hour)", f"{crawls_per_hour:.0f} crawls")
|
||||
table.add_row("Peak Memory Usage", f"{peak_memory:.2f} MB")
|
||||
table.add_row("Memory Per Crawl", f"{memory_per_page:.2f} MB")
|
||||
|
||||
# Display the table
|
||||
console.print(table)
|
||||
|
||||
# Ask confirmation before cleanup
|
||||
confirmation = input(
|
||||
f"{WARNING}Do you want to proceed with cleanup? (y/n): {RESET}"
|
||||
)
|
||||
if confirmation.lower() != "y":
|
||||
print(f"{WARNING}Cleanup aborted by user{RESET}")
|
||||
return False
|
||||
|
||||
# Close all pages
|
||||
for page, _ in all_pages:
|
||||
try:
|
||||
await page.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Close all managers
|
||||
for manager in managers:
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Remove the temp directory
|
||||
import shutil
|
||||
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
async def test_performance_scaling_lab( num_browsers: int = 10, pages_per_browser: int = 10):
|
||||
"""Test performance with multiple browsers and pages.
|
||||
|
||||
This test creates multiple browsers on different ports,
|
||||
spawns multiple pages per browser, and measures performance metrics.
|
||||
"""
|
||||
print(f"\n{INFO}========== Testing Performance Scaling =========={RESET}")
|
||||
|
||||
# Configuration parameters
|
||||
num_browsers = num_browsers
|
||||
pages_per_browser = pages_per_browser
|
||||
total_pages = num_browsers * pages_per_browser
|
||||
base_port = 9222
|
||||
|
||||
# Set up a measuring mechanism for memory
|
||||
import psutil
|
||||
import gc
|
||||
|
||||
# Force garbage collection before starting
|
||||
gc.collect()
|
||||
process = psutil.Process()
|
||||
initial_memory = process.memory_info().rss / 1024 / 1024 # in MB
|
||||
peak_memory = initial_memory
|
||||
|
||||
# Report initial configuration
|
||||
print(
|
||||
f"{INFO}Test configuration: {num_browsers} browsers × {pages_per_browser} pages = {total_pages} total crawls{RESET}"
|
||||
)
|
||||
|
||||
# List to track managers
|
||||
managers: List[BrowserManager] = []
|
||||
all_pages = []
|
||||
|
||||
# Get crawl4ai home directory
|
||||
crawl4ai_home = os.path.expanduser("~/.crawl4ai")
|
||||
temp_dir = os.path.join(crawl4ai_home, "temp")
|
||||
os.makedirs(temp_dir, exist_ok=True)
|
||||
|
||||
# Create all managers but don't start them yet
|
||||
manager_configs = []
|
||||
for i in range(num_browsers):
|
||||
port = base_port + i
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="builtin",
|
||||
headless=True,
|
||||
debugging_port=port,
|
||||
user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
|
||||
)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
manager.strategy.shutting_down = True
|
||||
manager_configs.append((manager, i, port))
|
||||
|
||||
# Define async function to start a single manager
|
||||
async def start_manager(manager, index, port):
|
||||
try:
|
||||
await manager.start()
|
||||
return manager
|
||||
except Exception as e:
|
||||
print(
|
||||
f"{ERROR}Failed to start browser {index + 1} on port {port}: {str(e)}{RESET}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Start all managers in parallel
|
||||
start_tasks = [
|
||||
start_manager(manager, i, port) for manager, i, port in manager_configs
|
||||
]
|
||||
started_managers = await asyncio.gather(*start_tasks)
|
||||
|
||||
# Filter out None values (failed starts) and add to managers list
|
||||
managers = [m for m in started_managers if m is not None]
|
||||
|
||||
if len(managers) == 0:
|
||||
print(f"{ERROR}All browser managers failed to start. Aborting test.{RESET}")
|
||||
return False
|
||||
|
||||
if len(managers) < num_browsers:
|
||||
print(
|
||||
f"{WARNING}Only {len(managers)} out of {num_browsers} browser managers started successfully{RESET}"
|
||||
)
|
||||
|
||||
# Create pages for each browser
|
||||
for i, manager in enumerate(managers):
|
||||
try:
|
||||
pages = await manager.get_pages(CrawlerRunConfig(), count=pages_per_browser)
|
||||
all_pages.extend(pages)
|
||||
except Exception as e:
|
||||
print(f"{ERROR}Failed to create pages for browser {i + 1}: {str(e)}{RESET}")
|
||||
|
||||
# Check memory after page creation
|
||||
gc.collect()
|
||||
current_memory = process.memory_info().rss / 1024 / 1024
|
||||
peak_memory = max(peak_memory, current_memory)
|
||||
|
||||
# Ask for confirmation before loading
|
||||
confirmation = input(
|
||||
f"{WARNING}Do you want to proceed with loading pages? (y/n): {RESET}"
|
||||
)
|
||||
# Step 1: Create and start multiple browser managers in parallel
|
||||
start_time = time.time()
|
||||
|
||||
if confirmation.lower() == "y":
|
||||
load_start_time = time.time()
|
||||
|
||||
# Function to load a single page
|
||||
async def load_page(page_ctx, index):
|
||||
page, _ = page_ctx
|
||||
try:
|
||||
await page.goto(f"https://example.com/page{index}", timeout=30000)
|
||||
title = await page.title()
|
||||
return title
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
# Load all pages concurrently
|
||||
load_tasks = [load_page(page_ctx, i) for i, page_ctx in enumerate(all_pages)]
|
||||
load_results = await asyncio.gather(*load_tasks, return_exceptions=True)
|
||||
|
||||
# Count successes and failures
|
||||
successes = sum(
|
||||
1 for r in load_results if isinstance(r, str) and not r.startswith("Error")
|
||||
)
|
||||
failures = len(load_results) - successes
|
||||
|
||||
load_time = time.time() - load_start_time
|
||||
total_test_time = time.time() - start_time
|
||||
|
||||
# Check memory after loading (peak memory)
|
||||
gc.collect()
|
||||
current_memory = process.memory_info().rss / 1024 / 1024
|
||||
peak_memory = max(peak_memory, current_memory)
|
||||
|
||||
# Calculate key metrics
|
||||
memory_per_page = peak_memory / successes if successes > 0 else 0
|
||||
time_per_crawl = total_test_time / successes if successes > 0 else 0
|
||||
crawls_per_second = successes / total_test_time if total_test_time > 0 else 0
|
||||
crawls_per_minute = crawls_per_second * 60
|
||||
crawls_per_hour = crawls_per_minute * 60
|
||||
|
||||
# Print simplified performance summary
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
# Create a simple summary table
|
||||
table = Table(title="CRAWL4AI PERFORMANCE SUMMARY")
|
||||
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Total Crawls Completed", f"{successes}")
|
||||
table.add_row("Total Time", f"{total_test_time:.2f} seconds")
|
||||
table.add_row("Time Per Crawl", f"{time_per_crawl:.2f} seconds")
|
||||
table.add_row("Crawling Speed", f"{crawls_per_second:.2f} crawls/second")
|
||||
table.add_row("Projected Rate (1 minute)", f"{crawls_per_minute:.0f} crawls")
|
||||
table.add_row("Projected Rate (1 hour)", f"{crawls_per_hour:.0f} crawls")
|
||||
table.add_row("Peak Memory Usage", f"{peak_memory:.2f} MB")
|
||||
table.add_row("Memory Per Crawl", f"{memory_per_page:.2f} MB")
|
||||
|
||||
# Display the table
|
||||
console.print(table)
|
||||
|
||||
# Ask confirmation before cleanup
|
||||
confirmation = input(
|
||||
f"{WARNING}Do you want to proceed with cleanup? (y/n): {RESET}"
|
||||
)
|
||||
if confirmation.lower() != "y":
|
||||
print(f"{WARNING}Cleanup aborted by user{RESET}")
|
||||
return False
|
||||
|
||||
# Close all pages
|
||||
for page, _ in all_pages:
|
||||
try:
|
||||
await page.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Close all managers
|
||||
for manager in managers:
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Remove the temp directory
|
||||
import shutil
|
||||
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all tests"""
|
||||
try:
|
||||
print(f"{INFO}Starting builtin browser tests with browser module{RESET}")
|
||||
|
||||
# # Run browser creation test
|
||||
# manager, cdp_url = await test_builtin_browser_creation()
|
||||
# if not manager:
|
||||
# print(f"{ERROR}Browser creation failed, cannot continue tests{RESET}")
|
||||
# return
|
||||
|
||||
# # Run page operations test
|
||||
# await test_page_operations(manager)
|
||||
|
||||
# # Run browser status and management test
|
||||
# await test_browser_status_management(manager)
|
||||
|
||||
# # Close manager before multiple manager test
|
||||
# await manager.close()
|
||||
|
||||
# Run multiple managers test
|
||||
await test_multiple_managers()
|
||||
|
||||
# Run performance scaling test
|
||||
await test_performance_scaling()
|
||||
|
||||
# Run cleanup test
|
||||
await cleanup_browsers()
|
||||
|
||||
# Run edge cases test
|
||||
await test_edge_cases()
|
||||
|
||||
print(f"\n{SUCCESS}All tests completed!{RESET}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n{ERROR}Test failed with error: {str(e)}{RESET}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
# Clean up: kill any remaining builtin browsers
|
||||
await cleanup_browsers()
|
||||
print(f"{SUCCESS}Test cleanup complete{RESET}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Test examples for BuiltinBrowserStrategy.
|
||||
|
||||
These examples demonstrate the functionality of BuiltinBrowserStrategy
|
||||
and serve as functional tests.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.browser import BrowserManager
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
async def test_builtin_browser():
|
||||
"""Test using a builtin browser that persists between sessions."""
|
||||
logger.info("Testing builtin browser", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="builtin",
|
||||
headless=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
# Start should connect to existing builtin browser or create one
|
||||
await manager.start()
|
||||
logger.info("Connected to builtin browser", tag="TEST")
|
||||
|
||||
# Test page creation
|
||||
crawler_config = CrawlerRunConfig()
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
|
||||
# Test navigation
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Close manager (should not close the builtin browser)
|
||||
await manager.close()
|
||||
logger.info("First session closed", tag="TEST")
|
||||
|
||||
# Create a second manager to verify browser persistence
|
||||
logger.info("Creating second session to verify persistence", tag="TEST")
|
||||
manager2 = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
await manager2.start()
|
||||
logger.info("Connected to existing builtin browser", tag="TEST")
|
||||
|
||||
page2, context2 = await manager2.get_page(crawler_config)
|
||||
await page2.goto("https://example.org")
|
||||
title2 = await page2.title()
|
||||
logger.info(f"Second session page title: {title2}", tag="TEST")
|
||||
|
||||
await manager2.close()
|
||||
logger.info("Second session closed successfully", tag="TEST")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_builtin_browser_status():
|
||||
"""Test getting status of the builtin browser."""
|
||||
logger.info("Testing builtin browser status", tag="TEST")
|
||||
|
||||
from crawl4ai.browser.strategies import BuiltinBrowserStrategy
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="builtin",
|
||||
headless=True
|
||||
)
|
||||
|
||||
# Create strategy directly to access its status methods
|
||||
strategy = BuiltinBrowserStrategy(browser_config, logger)
|
||||
|
||||
try:
|
||||
# Get status before starting (should be not running)
|
||||
status_before = await strategy.get_builtin_browser_status()
|
||||
logger.info(f"Initial status: {status_before}", tag="TEST")
|
||||
|
||||
# Start the browser
|
||||
await strategy.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Get status after starting
|
||||
status_after = await strategy.get_builtin_browser_status()
|
||||
logger.info(f"Status after start: {status_after}", tag="TEST")
|
||||
|
||||
# Create a page to verify functionality
|
||||
crawler_config = CrawlerRunConfig()
|
||||
page, context = await strategy.get_page(crawler_config)
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Close strategy (should not kill the builtin browser)
|
||||
await strategy.close()
|
||||
logger.info("Strategy closed successfully", tag="TEST")
|
||||
|
||||
# Create a new strategy object
|
||||
strategy2 = BuiltinBrowserStrategy(browser_config, logger)
|
||||
|
||||
# Get status again (should still be running)
|
||||
status_final = await strategy2.get_builtin_browser_status()
|
||||
logger.info(f"Final status: {status_final}", tag="TEST")
|
||||
|
||||
# Verify that the status shows the browser is running
|
||||
is_running = status_final.get('running', False)
|
||||
logger.info(f"Builtin browser persistence confirmed: {is_running}", tag="TEST")
|
||||
|
||||
# Kill the builtin browser to clean up
|
||||
logger.info("Killing builtin browser", tag="TEST")
|
||||
success = await strategy2.kill_builtin_browser()
|
||||
logger.info(f"Killed builtin browser successfully: {success}", tag="TEST")
|
||||
|
||||
return is_running and success
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await strategy.close()
|
||||
|
||||
# Try to kill the builtin browser to clean up
|
||||
strategy2 = BuiltinBrowserStrategy(browser_config, logger)
|
||||
await strategy2.kill_builtin_browser()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def run_tests():
|
||||
"""Run all tests sequentially."""
|
||||
results = []
|
||||
|
||||
results.append(await test_builtin_browser())
|
||||
results.append(await test_builtin_browser_status())
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(results)
|
||||
logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
|
||||
|
||||
if passed == total:
|
||||
logger.success("All tests passed!", tag="SUMMARY")
|
||||
else:
|
||||
logger.error(f"{total - passed} tests failed", tag="SUMMARY")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for CDP connection cleanup and browser reuse.
|
||||
|
||||
These tests verify that:
|
||||
1. WebSocket URLs are properly handled (skip HTTP verification)
|
||||
2. cdp_cleanup_on_close properly disconnects without terminating the browser
|
||||
3. The same browser can be reused by multiple sequential connections
|
||||
|
||||
Requirements:
|
||||
- A CDP-compatible browser pool service running (e.g., chromepoold)
|
||||
- Service should be accessible at CDP_SERVICE_URL (default: http://localhost:11235)
|
||||
|
||||
Usage:
|
||||
pytest tests/browser/test_cdp_cleanup_reuse.py -v
|
||||
|
||||
Or run directly:
|
||||
python tests/browser/test_cdp_cleanup_reuse.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import pytest
|
||||
import requests
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
|
||||
# Configuration
|
||||
CDP_SERVICE_URL = os.getenv("CDP_SERVICE_URL", "http://localhost:11235")
|
||||
|
||||
|
||||
def is_cdp_service_available():
|
||||
"""Check if CDP service is running."""
|
||||
try:
|
||||
resp = requests.get(f"{CDP_SERVICE_URL}/health", timeout=2)
|
||||
return resp.status_code == 200
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def create_browser():
|
||||
"""Create a browser via CDP service API."""
|
||||
resp = requests.post(
|
||||
f"{CDP_SERVICE_URL}/v1/browsers",
|
||||
json={"headless": True},
|
||||
timeout=10
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def get_browser_info(browser_id):
|
||||
"""Get browser info from CDP service."""
|
||||
resp = requests.get(f"{CDP_SERVICE_URL}/v1/browsers", timeout=5)
|
||||
for browser in resp.json():
|
||||
if browser["id"] == browser_id:
|
||||
return browser
|
||||
return None
|
||||
|
||||
|
||||
def delete_browser(browser_id):
|
||||
"""Delete a browser via CDP service API."""
|
||||
try:
|
||||
requests.delete(f"{CDP_SERVICE_URL}/v1/browsers/{browser_id}", timeout=5)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
# Skip all tests if CDP service is not available
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not is_cdp_service_available(),
|
||||
reason=f"CDP service not available at {CDP_SERVICE_URL}"
|
||||
)
|
||||
|
||||
|
||||
class TestCDPWebSocketURL:
|
||||
"""Tests for WebSocket URL handling."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_url_skips_http_verification(self):
|
||||
"""WebSocket URLs should skip HTTP /json/version verification."""
|
||||
browser = create_browser()
|
||||
try:
|
||||
ws_url = browser["ws_url"]
|
||||
assert ws_url.startswith("ws://") or ws_url.startswith("wss://")
|
||||
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
browser_mode="cdp",
|
||||
cdp_url=ws_url,
|
||||
headless=True,
|
||||
cdp_cleanup_on_close=True,
|
||||
)
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=CrawlerRunConfig(verbose=False),
|
||||
)
|
||||
assert result.success
|
||||
assert "Example Domain" in result.metadata.get("title", "")
|
||||
finally:
|
||||
delete_browser(browser["browser_id"])
|
||||
|
||||
|
||||
class TestCDPCleanupOnClose:
|
||||
"""Tests for cdp_cleanup_on_close behavior."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_survives_after_cleanup_close(self):
|
||||
"""Browser should remain alive after close with cdp_cleanup_on_close=True."""
|
||||
browser = create_browser()
|
||||
browser_id = browser["browser_id"]
|
||||
ws_url = browser["ws_url"]
|
||||
|
||||
try:
|
||||
# Verify browser exists
|
||||
info_before = get_browser_info(browser_id)
|
||||
assert info_before is not None
|
||||
pid_before = info_before["pid"]
|
||||
|
||||
# Connect, crawl, and close with cleanup
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
browser_mode="cdp",
|
||||
cdp_url=ws_url,
|
||||
headless=True,
|
||||
cdp_cleanup_on_close=True,
|
||||
)
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=CrawlerRunConfig(verbose=False),
|
||||
)
|
||||
assert result.success
|
||||
|
||||
# Browser should still exist with same PID
|
||||
info_after = get_browser_info(browser_id)
|
||||
assert info_after is not None, "Browser was terminated but should only disconnect"
|
||||
assert info_after["pid"] == pid_before, "Browser PID changed unexpectedly"
|
||||
finally:
|
||||
delete_browser(browser_id)
|
||||
|
||||
|
||||
class TestCDPBrowserReuse:
|
||||
"""Tests for reusing the same browser with multiple connections."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sequential_connections_same_browser(self):
|
||||
"""Multiple sequential connections to the same browser should work."""
|
||||
browser = create_browser()
|
||||
browser_id = browser["browser_id"]
|
||||
ws_url = browser["ws_url"]
|
||||
|
||||
try:
|
||||
urls = [
|
||||
"https://example.com",
|
||||
"https://httpbin.org/ip",
|
||||
"https://httpbin.org/headers",
|
||||
]
|
||||
|
||||
for i, url in enumerate(urls, 1):
|
||||
# Each connection uses cdp_cleanup_on_close=True
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
browser_mode="cdp",
|
||||
cdp_url=ws_url,
|
||||
headless=True,
|
||||
cdp_cleanup_on_close=True,
|
||||
)
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
url=url,
|
||||
config=CrawlerRunConfig(verbose=False),
|
||||
)
|
||||
assert result.success, f"Connection {i} failed for {url}"
|
||||
|
||||
# Verify browser is still healthy
|
||||
info = get_browser_info(browser_id)
|
||||
assert info is not None, f"Browser died after connection {i}"
|
||||
|
||||
finally:
|
||||
delete_browser(browser_id)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_user_wait_needed_between_connections(self):
|
||||
"""With cdp_cleanup_on_close=True, no user wait should be needed."""
|
||||
browser = create_browser()
|
||||
browser_id = browser["browser_id"]
|
||||
ws_url = browser["ws_url"]
|
||||
|
||||
try:
|
||||
# Rapid-fire connections with NO sleep between them
|
||||
for i in range(3):
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
browser_mode="cdp",
|
||||
cdp_url=ws_url,
|
||||
headless=True,
|
||||
cdp_cleanup_on_close=True,
|
||||
)
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=CrawlerRunConfig(verbose=False),
|
||||
)
|
||||
assert result.success, f"Rapid connection {i+1} failed"
|
||||
# NO asyncio.sleep() here - internal delay should be sufficient
|
||||
finally:
|
||||
delete_browser(browser_id)
|
||||
|
||||
|
||||
class TestCDPBackwardCompatibility:
|
||||
"""Tests for backward compatibility with existing CDP usage."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_url_with_browser_id_works(self):
|
||||
"""HTTP URL with browser_id query param should work (backward compatibility)."""
|
||||
browser = create_browser()
|
||||
browser_id = browser["browser_id"]
|
||||
try:
|
||||
# Use HTTP URL with browser_id query parameter
|
||||
http_url = f"{CDP_SERVICE_URL}?browser_id={browser_id}"
|
||||
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
browser_mode="cdp",
|
||||
cdp_url=http_url,
|
||||
headless=True,
|
||||
cdp_cleanup_on_close=True,
|
||||
)
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=CrawlerRunConfig(verbose=False),
|
||||
)
|
||||
assert result.success
|
||||
finally:
|
||||
delete_browser(browser_id)
|
||||
|
||||
|
||||
# Allow running directly
|
||||
if __name__ == "__main__":
|
||||
if not is_cdp_service_available():
|
||||
print(f"CDP service not available at {CDP_SERVICE_URL}")
|
||||
print("Please start a CDP-compatible browser pool service first.")
|
||||
exit(1)
|
||||
|
||||
async def run_tests():
|
||||
print("=" * 60)
|
||||
print("CDP Cleanup and Browser Reuse Tests")
|
||||
print("=" * 60)
|
||||
|
||||
tests = [
|
||||
("WebSocket URL handling", TestCDPWebSocketURL().test_websocket_url_skips_http_verification),
|
||||
("Browser survives after cleanup", TestCDPCleanupOnClose().test_browser_survives_after_cleanup_close),
|
||||
("Sequential connections", TestCDPBrowserReuse().test_sequential_connections_same_browser),
|
||||
("No user wait needed", TestCDPBrowserReuse().test_no_user_wait_needed_between_connections),
|
||||
("HTTP URL with browser_id", TestCDPBackwardCompatibility().test_http_url_with_browser_id_works),
|
||||
]
|
||||
|
||||
results = []
|
||||
for name, test_func in tests:
|
||||
print(f"\n--- {name} ---")
|
||||
try:
|
||||
await test_func()
|
||||
print(f"PASS")
|
||||
results.append((name, True))
|
||||
except Exception as e:
|
||||
print(f"FAIL: {e}")
|
||||
results.append((name, False))
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("SUMMARY")
|
||||
print("=" * 60)
|
||||
for name, passed in results:
|
||||
print(f" {name}: {'PASS' if passed else 'FAIL'}")
|
||||
|
||||
all_passed = all(r[1] for r in results)
|
||||
print(f"\nOverall: {'ALL TESTS PASSED' if all_passed else 'SOME TESTS FAILED'}")
|
||||
return 0 if all_passed else 1
|
||||
|
||||
exit(asyncio.run(run_tests()))
|
||||
@@ -0,0 +1,470 @@
|
||||
"""Test examples for CDPBrowserStrategy.
|
||||
|
||||
These examples demonstrate the functionality of CDPBrowserStrategy
|
||||
and serve as functional tests.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.browser_manager import BrowserManager
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
async def test_cdp_launch_connect():
|
||||
"""Test launching a browser and connecting via CDP."""
|
||||
logger.info("Testing launch and connect via CDP", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
browser_mode="cdp",
|
||||
use_managed_browser=True,
|
||||
headless=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
logger.info("Browser launched and connected via CDP", tag="TEST")
|
||||
|
||||
# Test with multiple pages
|
||||
pages = []
|
||||
for i in range(3):
|
||||
crawler_config = CrawlerRunConfig()
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
await page.goto(f"https://example.com?test={i}")
|
||||
pages.append(page)
|
||||
logger.info(f"Created page {i+1}", tag="TEST")
|
||||
|
||||
# Verify all pages are working
|
||||
for i, page in enumerate(pages):
|
||||
title = await page.title()
|
||||
logger.info(f"Page {i+1} title: {title}", tag="TEST")
|
||||
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_cdp_with_user_data_dir():
|
||||
"""Test CDP browser with a user data directory and storage state."""
|
||||
logger.info("Testing CDP browser with user data directory", tag="TEST")
|
||||
|
||||
# Create a temporary user data directory
|
||||
import tempfile
|
||||
user_data_dir = tempfile.mkdtemp(prefix="crawl4ai-test-")
|
||||
storage_state_file = os.path.join(user_data_dir, "storage_state.json")
|
||||
logger.info(f"Created temporary user data directory: {user_data_dir}", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
user_data_dir=user_data_dir
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
logger.info("Browser launched with user data directory", tag="TEST")
|
||||
|
||||
# Navigate to a page and store some data
|
||||
crawler_config = CrawlerRunConfig()
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
|
||||
# Visit the site first
|
||||
await page.goto("https://example.com", wait_until="domcontentloaded")
|
||||
|
||||
# Set a cookie via JavaScript (more reliable for persistence)
|
||||
await page.evaluate("""
|
||||
document.cookie = 'test_cookie=test_value; path=/; max-age=86400';
|
||||
""")
|
||||
|
||||
# Also set via context API for double coverage
|
||||
await context.add_cookies([{
|
||||
"name": "test_cookie_api",
|
||||
"value": "test_value_api",
|
||||
"domain": "example.com",
|
||||
"path": "/"
|
||||
}])
|
||||
|
||||
# Verify cookies were set
|
||||
cookies = await context.cookies(["https://example.com"])
|
||||
has_test_cookie = any(cookie["name"] in ["test_cookie", "test_cookie_api"] for cookie in cookies)
|
||||
logger.info(f"Cookie set successfully: {has_test_cookie}", tag="TEST")
|
||||
|
||||
# Save storage state before closing
|
||||
await context.storage_state(path=storage_state_file)
|
||||
logger.info(f"Storage state saved to: {storage_state_file}", tag="TEST")
|
||||
|
||||
# Close the browser
|
||||
await manager.close()
|
||||
logger.info("First browser session closed", tag="TEST")
|
||||
|
||||
# Wait a moment for clean shutdown
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Start a new browser with the same user data directory and storage state
|
||||
logger.info("Starting second browser session with same user data directory", tag="TEST")
|
||||
browser_config2 = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
user_data_dir=user_data_dir,
|
||||
storage_state=storage_state_file
|
||||
)
|
||||
|
||||
manager2 = BrowserManager(browser_config=browser_config2, logger=logger)
|
||||
await manager2.start()
|
||||
|
||||
# Get a new page and check if the cookie persists
|
||||
page2, context2 = await manager2.get_page(crawler_config)
|
||||
await page2.goto("https://example.com", wait_until="domcontentloaded")
|
||||
|
||||
# Verify cookie persisted
|
||||
cookies2 = await context2.cookies(["https://example.com"])
|
||||
has_test_cookie2 = any(cookie["name"] in ["test_cookie", "test_cookie_api"] for cookie in cookies2)
|
||||
logger.info(f"Cookie persisted across sessions: {has_test_cookie2}", tag="TEST")
|
||||
logger.info(f"Cookies found: {[c['name'] for c in cookies2]}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager2.close()
|
||||
|
||||
# Remove temporary directory
|
||||
import shutil
|
||||
shutil.rmtree(user_data_dir, ignore_errors=True)
|
||||
logger.info(f"Removed temporary user data directory", tag="TEST")
|
||||
|
||||
return has_test_cookie and has_test_cookie2
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
await manager2.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Clean up temporary directory
|
||||
try:
|
||||
import shutil
|
||||
shutil.rmtree(user_data_dir, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
async def test_cdp_session_management():
|
||||
"""Test session management with CDP browser - focused on session tracking."""
|
||||
logger.info("Testing session management with CDP browser", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
use_managed_browser=True,
|
||||
headless=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
logger.info("Browser launched successfully", tag="TEST")
|
||||
|
||||
# Test session tracking and lifecycle management
|
||||
session1_id = "test_session_1"
|
||||
session2_id = "test_session_2"
|
||||
|
||||
# Set up first session
|
||||
crawler_config1 = CrawlerRunConfig(session_id=session1_id)
|
||||
page1, context1 = await manager.get_page(crawler_config1)
|
||||
await page1.goto("https://example.com", wait_until="domcontentloaded")
|
||||
|
||||
# Get page URL and title for verification
|
||||
page1_url = page1.url
|
||||
page1_title = await page1.title()
|
||||
logger.info(f"Session 1 setup - URL: {page1_url}, Title: {page1_title}", tag="TEST")
|
||||
|
||||
# Set up second session
|
||||
crawler_config2 = CrawlerRunConfig(session_id=session2_id)
|
||||
page2, context2 = await manager.get_page(crawler_config2)
|
||||
await page2.goto("https://httpbin.org/html", wait_until="domcontentloaded")
|
||||
|
||||
page2_url = page2.url
|
||||
page2_title = await page2.title()
|
||||
logger.info(f"Session 2 setup - URL: {page2_url}, Title: {page2_title}", tag="TEST")
|
||||
|
||||
# Verify sessions exist in manager
|
||||
session1_exists = session1_id in manager.sessions
|
||||
session2_exists = session2_id in manager.sessions
|
||||
logger.info(f"Sessions in manager - S1: {session1_exists}, S2: {session2_exists}", tag="TEST")
|
||||
|
||||
# Test session reuse
|
||||
page1_again, context1_again = await manager.get_page(crawler_config1)
|
||||
is_same_page = page1 == page1_again
|
||||
is_same_context = context1 == context1_again
|
||||
|
||||
logger.info(f"Session 1 reuse - Same page: {is_same_page}, Same context: {is_same_context}", tag="TEST")
|
||||
|
||||
# Test that sessions are properly tracked with timestamps
|
||||
session1_info = manager.sessions.get(session1_id)
|
||||
session2_info = manager.sessions.get(session2_id)
|
||||
|
||||
session1_has_timestamp = session1_info and len(session1_info) == 3
|
||||
session2_has_timestamp = session2_info and len(session2_info) == 3
|
||||
|
||||
logger.info(f"Session tracking - S1 complete: {session1_has_timestamp}, S2 complete: {session2_has_timestamp}", tag="TEST")
|
||||
|
||||
# In managed browser mode, pages might be shared. Let's test what actually happens
|
||||
pages_same_or_different = page1 == page2
|
||||
logger.info(f"Pages same object: {pages_same_or_different}", tag="TEST")
|
||||
|
||||
# Test that we can distinguish sessions by their stored info
|
||||
session1_context, session1_page, session1_time = session1_info
|
||||
session2_context, session2_page, session2_time = session2_info
|
||||
|
||||
sessions_have_different_timestamps = session1_time != session2_time
|
||||
logger.info(f"Sessions have different timestamps: {sessions_have_different_timestamps}", tag="TEST")
|
||||
|
||||
# Test session killing
|
||||
await manager.kill_session(session1_id)
|
||||
logger.info(f"Killed session 1", tag="TEST")
|
||||
|
||||
# Verify session was removed
|
||||
session1_removed = session1_id not in manager.sessions
|
||||
session2_still_exists = session2_id in manager.sessions
|
||||
logger.info(f"After kill - S1 removed: {session1_removed}, S2 exists: {session2_still_exists}", tag="TEST")
|
||||
|
||||
# Test page state after killing session
|
||||
page1_closed = page1.is_closed()
|
||||
logger.info(f"Page1 closed after kill: {page1_closed}", tag="TEST")
|
||||
|
||||
# Clean up remaining session
|
||||
try:
|
||||
await manager.kill_session(session2_id)
|
||||
logger.info("Killed session 2", tag="TEST")
|
||||
session2_removed = session2_id not in manager.sessions
|
||||
except Exception as e:
|
||||
logger.info(f"Session 2 cleanup: {e}", tag="TEST")
|
||||
session2_removed = False
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
# Success criteria for managed browser sessions:
|
||||
# 1. Sessions can be created and tracked with proper info
|
||||
# 2. Same page/context returned for same session ID
|
||||
# 3. Sessions have proper timestamp tracking
|
||||
# 4. Sessions can be killed and removed from tracking
|
||||
# 5. Session cleanup works properly
|
||||
success = (session1_exists and
|
||||
session2_exists and
|
||||
is_same_page and
|
||||
session1_has_timestamp and
|
||||
session2_has_timestamp and
|
||||
sessions_have_different_timestamps and
|
||||
session1_removed and
|
||||
session2_removed)
|
||||
|
||||
logger.info(f"Test success: {success}", tag="TEST")
|
||||
return success
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_cdp_timing_fix_fast_startup():
|
||||
"""
|
||||
Test that the CDP timing fix handles fast browser startup correctly.
|
||||
This should work without any delays or retries.
|
||||
"""
|
||||
logger.info("Testing CDP timing fix with fast startup", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
use_managed_browser=True,
|
||||
browser_mode="cdp",
|
||||
headless=True,
|
||||
debugging_port=9223, # Use different port to avoid conflicts
|
||||
verbose=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
await manager.start()
|
||||
startup_time = time.time() - start_time
|
||||
|
||||
logger.info(f"Browser started successfully in {startup_time:.2f}s", tag="TEST")
|
||||
|
||||
# Test basic functionality
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
|
||||
await page.goto("https://example.com", wait_until="domcontentloaded")
|
||||
title = await page.title()
|
||||
|
||||
logger.info(f"Successfully navigated to page: {title}", tag="TEST")
|
||||
|
||||
await manager.close()
|
||||
logger.success("test_cdp_timing_fix_fast_startup completed successfully", tag="TEST")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"test_cdp_timing_fix_fast_startup failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
async def test_cdp_timing_fix_delayed_browser_start():
|
||||
"""
|
||||
Test CDP timing fix by actually delaying the browser startup process.
|
||||
This simulates a real scenario where the browser takes time to expose CDP.
|
||||
"""
|
||||
logger.info("Testing CDP timing fix with delayed browser startup", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
use_managed_browser=True,
|
||||
browser_mode="cdp",
|
||||
headless=True,
|
||||
debugging_port=9224,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Start the managed browser separately to control timing
|
||||
from crawl4ai.browser_manager import ManagedBrowser
|
||||
managed_browser = ManagedBrowser(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
# Start browser process but it will take time for CDP to be ready
|
||||
cdp_url = await managed_browser.start()
|
||||
logger.info(f"Managed browser started at {cdp_url}", tag="TEST")
|
||||
|
||||
# Small delay to simulate the browser needing time to fully initialize CDP
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Now create BrowserManager and connect - this should use the CDP verification fix
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
manager.config.cdp_url = cdp_url # Use the CDP URL from managed browser
|
||||
|
||||
start_time = time.time()
|
||||
await manager.start()
|
||||
startup_time = time.time() - start_time
|
||||
|
||||
logger.info(f"BrowserManager connected successfully in {startup_time:.2f}s", tag="TEST")
|
||||
|
||||
# Test basic functionality
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
await page.goto("https://example.com", wait_until="domcontentloaded")
|
||||
title = await page.title()
|
||||
|
||||
logger.info(f"Successfully navigated to page: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
await managed_browser.cleanup()
|
||||
|
||||
logger.success("test_cdp_timing_fix_delayed_browser_start completed successfully", tag="TEST")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"test_cdp_timing_fix_delayed_browser_start failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
await managed_browser.cleanup()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
async def test_cdp_verification_backoff_behavior():
|
||||
"""
|
||||
Test the exponential backoff behavior of CDP verification in isolation.
|
||||
"""
|
||||
logger.info("Testing CDP verification exponential backoff behavior", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
use_managed_browser=True,
|
||||
debugging_port=9225, # Use different port
|
||||
verbose=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
# Test with a non-existent CDP URL to trigger retries
|
||||
fake_cdp_url = "http://localhost:19999" # This should not exist
|
||||
|
||||
start_time = time.time()
|
||||
result = await manager._verify_cdp_ready(fake_cdp_url)
|
||||
elapsed_time = time.time() - start_time
|
||||
|
||||
# Should return False after all retries
|
||||
assert result is False, "Expected CDP verification to fail with non-existent endpoint"
|
||||
|
||||
# Should take some time due to retries and backoff
|
||||
assert elapsed_time > 2.0, f"Expected backoff delays, but took only {elapsed_time:.2f}s"
|
||||
|
||||
logger.info(f"CDP verification correctly failed after {elapsed_time:.2f}s with exponential backoff", tag="TEST")
|
||||
logger.success("test_cdp_verification_backoff_behavior completed successfully", tag="TEST")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"test_cdp_verification_backoff_behavior failed: {str(e)}", tag="TEST")
|
||||
return False
|
||||
|
||||
|
||||
|
||||
async def run_tests():
|
||||
"""Run all tests sequentially."""
|
||||
import time
|
||||
|
||||
results = []
|
||||
|
||||
# Original CDP strategy tests
|
||||
logger.info("Running original CDP strategy tests", tag="SUITE")
|
||||
# results.append(await test_cdp_launch_connect())
|
||||
results.append(await test_cdp_with_user_data_dir())
|
||||
results.append(await test_cdp_session_management())
|
||||
|
||||
# CDP timing fix tests
|
||||
logger.info("Running CDP timing fix tests", tag="SUITE")
|
||||
results.append(await test_cdp_timing_fix_fast_startup())
|
||||
results.append(await test_cdp_timing_fix_delayed_browser_start())
|
||||
results.append(await test_cdp_verification_backoff_behavior())
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(results)
|
||||
logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
|
||||
|
||||
if passed == total:
|
||||
logger.success("All tests passed!", tag="SUMMARY")
|
||||
else:
|
||||
logger.error(f"{total - passed} tests failed", tag="SUMMARY")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Combined test runner for all browser module tests.
|
||||
|
||||
This script runs all the browser module tests in sequence and
|
||||
provides a comprehensive summary.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
async def run_test_module(module_name, header):
|
||||
"""Run all tests in a module and return results."""
|
||||
logger.info(f"\n{'-'*30}", tag="TEST")
|
||||
logger.info(f"RUNNING: {header}", tag="TEST")
|
||||
logger.info(f"{'-'*30}", tag="TEST")
|
||||
|
||||
# Import the module dynamically
|
||||
module = __import__(f"tests.browser.{module_name}", fromlist=["run_tests"])
|
||||
|
||||
# Track time for performance measurement
|
||||
start_time = time.time()
|
||||
|
||||
# Run the tests
|
||||
await module.run_tests()
|
||||
|
||||
# Calculate time taken
|
||||
time_taken = time.time() - start_time
|
||||
logger.info(f"Time taken: {time_taken:.2f} seconds", tag="TIMING")
|
||||
|
||||
return time_taken
|
||||
|
||||
async def main():
|
||||
"""Run all test modules."""
|
||||
logger.info("STARTING COMPREHENSIVE BROWSER MODULE TESTS", tag="MAIN")
|
||||
|
||||
# List of test modules to run
|
||||
test_modules = [
|
||||
("test_browser_manager", "Browser Manager Tests"),
|
||||
("test_playwright_strategy", "Playwright Strategy Tests"),
|
||||
("test_cdp_strategy", "CDP Strategy Tests"),
|
||||
("test_builtin_strategy", "Builtin Browser Strategy Tests"),
|
||||
("test_profiles", "Profile Management Tests")
|
||||
]
|
||||
|
||||
# Run each test module
|
||||
timings = {}
|
||||
for module_name, header in test_modules:
|
||||
try:
|
||||
time_taken = await run_test_module(module_name, header)
|
||||
timings[module_name] = time_taken
|
||||
except Exception as e:
|
||||
logger.error(f"Error running {module_name}: {str(e)}", tag="ERROR")
|
||||
|
||||
# Print summary
|
||||
logger.info("\n\nTEST SUMMARY:", tag="SUMMARY")
|
||||
logger.info(f"{'-'*50}", tag="SUMMARY")
|
||||
for module_name, header in test_modules:
|
||||
if module_name in timings:
|
||||
logger.info(f"{header}: {timings[module_name]:.2f} seconds", tag="SUMMARY")
|
||||
else:
|
||||
logger.error(f"{header}: FAILED TO RUN", tag="SUMMARY")
|
||||
logger.info(f"{'-'*50}", tag="SUMMARY")
|
||||
total_time = sum(timings.values())
|
||||
logger.info(f"Total time: {total_time:.2f} seconds", tag="SUMMARY")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,358 @@
|
||||
"""
|
||||
Integration tests for the browser context memory leak fix.
|
||||
|
||||
Tests:
|
||||
1. Signature shrink: non-context fields produce same hash
|
||||
2. Signature correctness: context-affecting fields produce different hashes
|
||||
3. Refcount lifecycle: increment on get_page, decrement on release
|
||||
4. LRU eviction: oldest idle context is evicted when over limit
|
||||
5. Eviction respects active refcounts
|
||||
6. Real browser: contexts don't leak under varying configs
|
||||
7. Real browser: batch crawl reuses same context
|
||||
8. Storage state path: temporary context is closed
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_configs import ProxyConfig, GeolocationConfig
|
||||
from crawl4ai.browser_manager import BrowserManager
|
||||
|
||||
|
||||
# ── Unit tests (no browser needed) ──────────────────────────────────────
|
||||
|
||||
class TestSignatureShrink:
|
||||
"""Verify the whitelist signature only considers context-affecting fields."""
|
||||
|
||||
def _bm(self):
|
||||
return BrowserManager(BrowserConfig(), logger=None)
|
||||
|
||||
def test_non_context_fields_same_signature(self):
|
||||
"""Fields that don't affect browser context must produce identical sigs."""
|
||||
bm = self._bm()
|
||||
configs = [
|
||||
CrawlerRunConfig(word_count_threshold=200),
|
||||
CrawlerRunConfig(word_count_threshold=50),
|
||||
CrawlerRunConfig(css_selector=".main"),
|
||||
CrawlerRunConfig(screenshot=True),
|
||||
CrawlerRunConfig(pdf=True, verbose=False),
|
||||
CrawlerRunConfig(scan_full_page=True, scroll_delay=0.5),
|
||||
CrawlerRunConfig(only_text=True),
|
||||
CrawlerRunConfig(wait_until="networkidle", page_timeout=30000),
|
||||
CrawlerRunConfig(capture_network_requests=True),
|
||||
CrawlerRunConfig(exclude_external_links=True),
|
||||
]
|
||||
sigs = [bm._make_config_signature(c) for c in configs]
|
||||
assert len(set(sigs)) == 1, (
|
||||
f"Expected all same sig, got {len(set(sigs))} unique: {sigs[:3]}"
|
||||
)
|
||||
|
||||
def test_proxy_changes_signature(self):
|
||||
bm = self._bm()
|
||||
c1 = CrawlerRunConfig()
|
||||
c2 = CrawlerRunConfig(proxy_config=ProxyConfig(server="http://p1:8080"))
|
||||
c3 = CrawlerRunConfig(proxy_config=ProxyConfig(server="http://p2:8080"))
|
||||
s1 = bm._make_config_signature(c1)
|
||||
s2 = bm._make_config_signature(c2)
|
||||
s3 = bm._make_config_signature(c3)
|
||||
assert s1 != s2, "proxy vs no-proxy should differ"
|
||||
assert s2 != s3, "different proxies should differ"
|
||||
|
||||
def test_locale_changes_signature(self):
|
||||
bm = self._bm()
|
||||
s1 = bm._make_config_signature(CrawlerRunConfig())
|
||||
s2 = bm._make_config_signature(CrawlerRunConfig(locale="en-US"))
|
||||
s3 = bm._make_config_signature(CrawlerRunConfig(locale="fr-FR"))
|
||||
assert s1 != s2
|
||||
assert s2 != s3
|
||||
|
||||
def test_timezone_changes_signature(self):
|
||||
bm = self._bm()
|
||||
s1 = bm._make_config_signature(CrawlerRunConfig())
|
||||
s2 = bm._make_config_signature(CrawlerRunConfig(timezone_id="America/New_York"))
|
||||
assert s1 != s2
|
||||
|
||||
def test_geolocation_changes_signature(self):
|
||||
bm = self._bm()
|
||||
s1 = bm._make_config_signature(CrawlerRunConfig())
|
||||
s2 = bm._make_config_signature(CrawlerRunConfig(
|
||||
geolocation=GeolocationConfig(latitude=40.7, longitude=-74.0)
|
||||
))
|
||||
assert s1 != s2
|
||||
|
||||
def test_navigator_overrides_change_signature(self):
|
||||
bm = self._bm()
|
||||
base = bm._make_config_signature(CrawlerRunConfig())
|
||||
s_nav = bm._make_config_signature(CrawlerRunConfig(override_navigator=True))
|
||||
s_sim = bm._make_config_signature(CrawlerRunConfig(simulate_user=True))
|
||||
s_mag = bm._make_config_signature(CrawlerRunConfig(magic=True))
|
||||
assert base != s_nav
|
||||
assert base != s_sim
|
||||
assert base != s_mag
|
||||
|
||||
def test_signature_stability(self):
|
||||
"""Same config always produces the same hash."""
|
||||
bm = self._bm()
|
||||
c = CrawlerRunConfig(locale="ja-JP", override_navigator=True)
|
||||
assert bm._make_config_signature(c) == bm._make_config_signature(c)
|
||||
|
||||
def test_proxy_config_with_credentials(self):
|
||||
"""ProxyConfig with username/password produces distinct stable sigs."""
|
||||
bm = self._bm()
|
||||
c1 = CrawlerRunConfig(proxy_config=ProxyConfig(
|
||||
server="http://proxy:8080", username="user1", password="pass1"
|
||||
))
|
||||
c2 = CrawlerRunConfig(proxy_config=ProxyConfig(
|
||||
server="http://proxy:8080", username="user2", password="pass2"
|
||||
))
|
||||
s1 = bm._make_config_signature(c1)
|
||||
s2 = bm._make_config_signature(c2)
|
||||
assert s1 != s2, "different credentials should differ"
|
||||
assert s1 == bm._make_config_signature(c1), "should be stable"
|
||||
|
||||
|
||||
class TestLRUEviction:
|
||||
"""Verify eviction logic (no browser needed)."""
|
||||
|
||||
def _bm(self, max_ctx=3):
|
||||
bm = BrowserManager(BrowserConfig(), logger=None)
|
||||
bm._max_contexts = max_ctx
|
||||
return bm
|
||||
|
||||
def test_no_eviction_under_limit(self):
|
||||
bm = self._bm(max_ctx=5)
|
||||
for i in range(5):
|
||||
sig = f"sig_{i}"
|
||||
bm.contexts_by_config[sig] = f"ctx_{i}"
|
||||
bm._context_refcounts[sig] = 0
|
||||
bm._context_last_used[sig] = time.monotonic()
|
||||
assert bm._evict_lru_context_locked() is None
|
||||
|
||||
def test_evicts_oldest_idle(self):
|
||||
bm = self._bm(max_ctx=3)
|
||||
for i in range(5):
|
||||
sig = f"sig_{i}"
|
||||
bm.contexts_by_config[sig] = f"ctx_{i}"
|
||||
bm._context_refcounts[sig] = 0
|
||||
bm._context_last_used[sig] = time.monotonic()
|
||||
time.sleep(0.002)
|
||||
|
||||
evicted = bm._evict_lru_context_locked()
|
||||
assert evicted == "ctx_0", f"expected oldest ctx_0, got {evicted}"
|
||||
assert "sig_0" not in bm.contexts_by_config
|
||||
assert "sig_0" not in bm._context_refcounts
|
||||
assert "sig_0" not in bm._context_last_used
|
||||
|
||||
def test_skips_active_contexts(self):
|
||||
bm = self._bm(max_ctx=2)
|
||||
# sig_0: old but active
|
||||
bm.contexts_by_config["sig_0"] = "ctx_0"
|
||||
bm._context_refcounts["sig_0"] = 3
|
||||
bm._context_last_used["sig_0"] = 0 # very old
|
||||
|
||||
# sig_1: newer, idle
|
||||
bm.contexts_by_config["sig_1"] = "ctx_1"
|
||||
bm._context_refcounts["sig_1"] = 0
|
||||
bm._context_last_used["sig_1"] = time.monotonic()
|
||||
|
||||
# sig_2: newest, idle
|
||||
bm.contexts_by_config["sig_2"] = "ctx_2"
|
||||
bm._context_refcounts["sig_2"] = 0
|
||||
bm._context_last_used["sig_2"] = time.monotonic()
|
||||
|
||||
evicted = bm._evict_lru_context_locked()
|
||||
# sig_0 is oldest but active (refcount=3) — must skip it
|
||||
assert evicted == "ctx_1", f"expected ctx_1 (oldest idle), got {evicted}"
|
||||
assert "sig_0" in bm.contexts_by_config, "active context must NOT be evicted"
|
||||
|
||||
def test_all_active_no_eviction(self):
|
||||
bm = self._bm(max_ctx=1)
|
||||
for i in range(3):
|
||||
sig = f"sig_{i}"
|
||||
bm.contexts_by_config[sig] = f"ctx_{i}"
|
||||
bm._context_refcounts[sig] = 1 # all active
|
||||
bm._context_last_used[sig] = time.monotonic()
|
||||
|
||||
evicted = bm._evict_lru_context_locked()
|
||||
assert evicted is None, "cannot evict when all are active"
|
||||
assert len(bm.contexts_by_config) == 3, "all contexts should remain"
|
||||
|
||||
def test_eviction_cleans_page_to_sig(self):
|
||||
bm = self._bm(max_ctx=1)
|
||||
bm.contexts_by_config["sig_old"] = "ctx_old"
|
||||
bm._context_refcounts["sig_old"] = 0
|
||||
bm._context_last_used["sig_old"] = 0
|
||||
|
||||
bm.contexts_by_config["sig_new"] = "ctx_new"
|
||||
bm._context_refcounts["sig_new"] = 0
|
||||
bm._context_last_used["sig_new"] = time.monotonic()
|
||||
|
||||
# Simulate a stale page mapping for the old context
|
||||
mock_page = object()
|
||||
bm._page_to_sig[mock_page] = "sig_old"
|
||||
|
||||
evicted = bm._evict_lru_context_locked()
|
||||
assert evicted == "ctx_old"
|
||||
assert mock_page not in bm._page_to_sig, "stale page mapping should be cleaned"
|
||||
|
||||
|
||||
# ── Integration tests (real browser) ────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def event_loop():
|
||||
loop = asyncio.new_event_loop()
|
||||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
def run(coro):
|
||||
"""Run an async function synchronously."""
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
class TestRealBrowserContextLifecycle:
|
||||
"""Real browser tests — verify contexts aren't leaked."""
|
||||
|
||||
def test_varying_configs_same_context(self):
|
||||
"""Different non-context fields should reuse the same context."""
|
||||
async def _test():
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
|
||||
# Crawl with different non-context configs
|
||||
html = "<html><body><p>Hello World with enough words to pass threshold</p></body></html>"
|
||||
for wct in [10, 50, 200]:
|
||||
config = CrawlerRunConfig(word_count_threshold=wct)
|
||||
result = await crawler.arun(f"raw:{html}", config=config)
|
||||
assert result.success
|
||||
|
||||
# Should have at most 1 context (all configs hash the same)
|
||||
ctx_count = len(bm.contexts_by_config)
|
||||
assert ctx_count <= 1, (
|
||||
f"Expected 1 context for identical browser config, got {ctx_count}"
|
||||
)
|
||||
run(_test())
|
||||
|
||||
def test_batch_crawl_reuses_context(self):
|
||||
"""Multiple URLs with same config should reuse a single context."""
|
||||
async def _test():
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
|
||||
html1 = "<html><body><p>Page one content here</p></body></html>"
|
||||
html2 = "<html><body><p>Page two content here</p></body></html>"
|
||||
html3 = "<html><body><p>Page three content here</p></body></html>"
|
||||
|
||||
config = CrawlerRunConfig()
|
||||
for h in [html1, html2, html3]:
|
||||
result = await crawler.arun(f"raw:{h}", config=config)
|
||||
assert result.success
|
||||
|
||||
ctx_count = len(bm.contexts_by_config)
|
||||
assert ctx_count <= 1, f"Batch should reuse context, got {ctx_count}"
|
||||
run(_test())
|
||||
|
||||
def test_refcount_drops_to_zero_after_crawl(self):
|
||||
"""After a crawl completes, the context refcount should be 0."""
|
||||
async def _test():
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
html = "<html><body><p>Test content</p></body></html>"
|
||||
config = CrawlerRunConfig()
|
||||
result = await crawler.arun(f"raw:{html}", config=config)
|
||||
assert result.success
|
||||
|
||||
# All refcounts should be 0 after crawl completes
|
||||
for sig, count in bm._context_refcounts.items():
|
||||
assert count == 0, (
|
||||
f"Refcount for {sig[:8]} should be 0 after crawl, got {count}"
|
||||
)
|
||||
run(_test())
|
||||
|
||||
def test_page_to_sig_cleaned_after_crawl(self):
|
||||
"""After crawl, the page->sig mapping should be empty (pages released)."""
|
||||
async def _test():
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
html = "<html><body><p>Test</p></body></html>"
|
||||
result = await crawler.arun(f"raw:{html}", config=CrawlerRunConfig())
|
||||
assert result.success
|
||||
|
||||
assert len(bm._page_to_sig) == 0, (
|
||||
f"Expected empty _page_to_sig after crawl, got {len(bm._page_to_sig)} entries"
|
||||
)
|
||||
run(_test())
|
||||
|
||||
def test_concurrent_crawls_refcount_tracking(self):
|
||||
"""Concurrent crawls should all properly increment/decrement refcounts."""
|
||||
async def _test():
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
config = CrawlerRunConfig()
|
||||
|
||||
htmls = [
|
||||
f"raw:<html><body><p>Concurrent page {i}</p></body></html>"
|
||||
for i in range(5)
|
||||
]
|
||||
tasks = [crawler.arun(h, config=config) for h in htmls]
|
||||
results = await asyncio.gather(*tasks)
|
||||
for r in results:
|
||||
assert r.success
|
||||
|
||||
# All done — refcounts should be 0
|
||||
for sig, count in bm._context_refcounts.items():
|
||||
assert count == 0, (
|
||||
f"After concurrent crawls, refcount for {sig[:8]} = {count}"
|
||||
)
|
||||
assert len(bm._page_to_sig) == 0
|
||||
run(_test())
|
||||
|
||||
def test_lru_eviction_real_browser(self):
|
||||
"""Verify LRU eviction actually closes contexts when limit exceeded."""
|
||||
async def _test():
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
bm._max_contexts = 2 # Low limit to trigger eviction
|
||||
|
||||
html = "<html><body><p>Test</p></body></html>"
|
||||
|
||||
# Crawl with 4 different locales → 4 different context signatures
|
||||
for locale in ["en-US", "fr-FR", "de-DE", "ja-JP"]:
|
||||
config = CrawlerRunConfig(locale=locale)
|
||||
result = await crawler.arun(f"raw:{html}", config=config)
|
||||
assert result.success
|
||||
|
||||
# Should have at most 2 contexts (limit)
|
||||
ctx_count = len(bm.contexts_by_config)
|
||||
assert ctx_count <= 2, (
|
||||
f"Expected <= 2 contexts (limit), got {ctx_count}"
|
||||
)
|
||||
|
||||
# Refcounts should all be 0
|
||||
for sig, count in bm._context_refcounts.items():
|
||||
assert count == 0, f"refcount {sig[:8]} = {count}"
|
||||
run(_test())
|
||||
|
||||
def test_close_clears_everything(self):
|
||||
"""close() should clear all tracking dicts."""
|
||||
async def _test():
|
||||
crawler = AsyncWebCrawler(config=BrowserConfig(headless=True))
|
||||
await crawler.start()
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
|
||||
html = "<html><body><p>Test</p></body></html>"
|
||||
result = await crawler.arun(f"raw:{html}", config=CrawlerRunConfig())
|
||||
assert result.success
|
||||
|
||||
await crawler.close()
|
||||
|
||||
assert len(bm.contexts_by_config) == 0
|
||||
assert len(bm._context_refcounts) == 0
|
||||
assert len(bm._context_last_used) == 0
|
||||
assert len(bm._page_to_sig) == 0
|
||||
run(_test())
|
||||
@@ -0,0 +1,399 @@
|
||||
"""
|
||||
Regression tests for init-script deduplication fix (PR #1768).
|
||||
|
||||
Problem: context.add_init_script() was called in BOTH setup_context() and
|
||||
_crawl_web(), causing unbounded script accumulation on shared contexts
|
||||
under concurrent load — ultimately crashing the context.
|
||||
|
||||
Fix: Flag-based guard (_crawl4ai_nav_overrider_injected,
|
||||
_crawl4ai_shadow_dom_injected) ensures each script type is injected
|
||||
at most once per context.
|
||||
|
||||
Tests:
|
||||
1. setup_context sets flags when crawlerRunConfig has anti-bot options
|
||||
2. setup_context without crawlerRunConfig does NOT set flags
|
||||
3. _crawl_web skips injection when flags already set (no duplication)
|
||||
4. _crawl_web injects and sets flags when they're missing (fallback path)
|
||||
5. Concurrent crawls on shared context don't accumulate scripts
|
||||
6. Navigator overrides actually work after dedup (functional check)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.browser_manager import BrowserManager
|
||||
|
||||
PASS = 0
|
||||
FAIL = 0
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
global PASS, FAIL
|
||||
if condition:
|
||||
PASS += 1
|
||||
print(f" PASS: {name}")
|
||||
else:
|
||||
FAIL += 1
|
||||
print(f" FAIL: {name}")
|
||||
|
||||
|
||||
async def test_setup_context_sets_flags():
|
||||
"""setup_context() with crawlerRunConfig should set injection flags."""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: setup_context sets flags when crawlerRunConfig has anti-bot opts")
|
||||
print("=" * 70)
|
||||
|
||||
bm = BrowserManager(BrowserConfig(headless=True, extra_args=['--no-sandbox']))
|
||||
await bm.start()
|
||||
|
||||
try:
|
||||
# Create context with navigator overrides
|
||||
config_nav = CrawlerRunConfig(override_navigator=True)
|
||||
ctx_nav = await bm.create_browser_context(config_nav)
|
||||
await bm.setup_context(ctx_nav, config_nav)
|
||||
|
||||
check("nav_overrider flag set after setup_context(override_navigator=True)",
|
||||
getattr(ctx_nav, '_crawl4ai_nav_overrider_injected', False) is True)
|
||||
check("shadow_dom flag NOT set (not requested)",
|
||||
getattr(ctx_nav, '_crawl4ai_shadow_dom_injected', False) is False)
|
||||
|
||||
await ctx_nav.close()
|
||||
|
||||
# Create context with magic mode
|
||||
config_magic = CrawlerRunConfig(magic=True)
|
||||
ctx_magic = await bm.create_browser_context(config_magic)
|
||||
await bm.setup_context(ctx_magic, config_magic)
|
||||
|
||||
check("nav_overrider flag set after setup_context(magic=True)",
|
||||
getattr(ctx_magic, '_crawl4ai_nav_overrider_injected', False) is True)
|
||||
|
||||
await ctx_magic.close()
|
||||
|
||||
# Create context with simulate_user
|
||||
config_sim = CrawlerRunConfig(simulate_user=True)
|
||||
ctx_sim = await bm.create_browser_context(config_sim)
|
||||
await bm.setup_context(ctx_sim, config_sim)
|
||||
|
||||
check("nav_overrider flag set after setup_context(simulate_user=True)",
|
||||
getattr(ctx_sim, '_crawl4ai_nav_overrider_injected', False) is True)
|
||||
|
||||
await ctx_sim.close()
|
||||
|
||||
# Create context with flatten_shadow_dom
|
||||
config_shadow = CrawlerRunConfig(flatten_shadow_dom=True)
|
||||
ctx_shadow = await bm.create_browser_context(config_shadow)
|
||||
await bm.setup_context(ctx_shadow, config_shadow)
|
||||
|
||||
check("shadow_dom flag set after setup_context(flatten_shadow_dom=True)",
|
||||
getattr(ctx_shadow, '_crawl4ai_shadow_dom_injected', False) is True)
|
||||
check("nav_overrider flag NOT set (not requested)",
|
||||
getattr(ctx_shadow, '_crawl4ai_nav_overrider_injected', False) is False)
|
||||
|
||||
await ctx_shadow.close()
|
||||
|
||||
# Create context with both
|
||||
config_both = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
|
||||
ctx_both = await bm.create_browser_context(config_both)
|
||||
await bm.setup_context(ctx_both, config_both)
|
||||
|
||||
check("both flags set when both features requested",
|
||||
getattr(ctx_both, '_crawl4ai_nav_overrider_injected', False) is True
|
||||
and getattr(ctx_both, '_crawl4ai_shadow_dom_injected', False) is True)
|
||||
|
||||
await ctx_both.close()
|
||||
|
||||
finally:
|
||||
await bm.close()
|
||||
|
||||
|
||||
async def test_setup_context_no_config_no_flags():
|
||||
"""setup_context() without crawlerRunConfig should NOT set flags."""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: setup_context without crawlerRunConfig does NOT set flags")
|
||||
print("=" * 70)
|
||||
|
||||
bm = BrowserManager(BrowserConfig(headless=True, extra_args=['--no-sandbox']))
|
||||
await bm.start()
|
||||
|
||||
try:
|
||||
ctx = await bm.create_browser_context()
|
||||
await bm.setup_context(ctx) # No crawlerRunConfig
|
||||
|
||||
check("nav_overrider flag NOT set (no crawlerRunConfig)",
|
||||
getattr(ctx, '_crawl4ai_nav_overrider_injected', False) is False)
|
||||
check("shadow_dom flag NOT set (no crawlerRunConfig)",
|
||||
getattr(ctx, '_crawl4ai_shadow_dom_injected', False) is False)
|
||||
|
||||
await ctx.close()
|
||||
|
||||
finally:
|
||||
await bm.close()
|
||||
|
||||
|
||||
async def test_no_duplication_standard_path():
|
||||
"""
|
||||
Standard path: setup_context() injects scripts + sets flags,
|
||||
then _crawl_web() should skip re-injection.
|
||||
|
||||
We verify by counting add_init_script calls on the context.
|
||||
"""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: No duplication on standard path (setup_context + _crawl_web)")
|
||||
print("=" * 70)
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
|
||||
config = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
|
||||
html = "<html><body><p>Test content for dedup check</p></body></html>"
|
||||
|
||||
result = await crawler.arun(f"raw:{html}", config=config)
|
||||
check("crawl succeeded", result.success)
|
||||
|
||||
# Get the context through the browser manager
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
for sig, ctx in bm.contexts_by_config.items():
|
||||
check("nav_overrider flag is set on context",
|
||||
getattr(ctx, '_crawl4ai_nav_overrider_injected', False) is True)
|
||||
check("shadow_dom flag is set on context",
|
||||
getattr(ctx, '_crawl4ai_shadow_dom_injected', False) is True)
|
||||
|
||||
|
||||
async def test_fallback_path_injects_once():
|
||||
"""
|
||||
Fallback path: manually create a context without crawlerRunConfig
|
||||
(simulating managed/persistent/CDP path), then verify _crawl_web()
|
||||
injects scripts exactly once and sets the flags.
|
||||
"""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: Fallback path injects once and sets flags")
|
||||
print("=" * 70)
|
||||
|
||||
bm = BrowserManager(BrowserConfig(headless=True, extra_args=['--no-sandbox']))
|
||||
await bm.start()
|
||||
|
||||
try:
|
||||
# Create context WITHOUT crawlerRunConfig (simulates persistent/CDP path)
|
||||
ctx = await bm.create_browser_context()
|
||||
await bm.setup_context(ctx) # No crawlerRunConfig — no flags set
|
||||
|
||||
check("flags NOT set before _crawl_web",
|
||||
not getattr(ctx, '_crawl4ai_nav_overrider_injected', False)
|
||||
and not getattr(ctx, '_crawl4ai_shadow_dom_injected', False))
|
||||
|
||||
# Track add_init_script calls
|
||||
original_add_init_script = ctx.add_init_script
|
||||
call_count = 0
|
||||
|
||||
async def counting_add_init_script(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return await original_add_init_script(*args, **kwargs)
|
||||
|
||||
ctx.add_init_script = counting_add_init_script
|
||||
|
||||
# Create a page and simulate what _crawl_web does
|
||||
page = await ctx.new_page()
|
||||
|
||||
config = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
|
||||
|
||||
# First "crawl" — should inject both scripts
|
||||
from crawl4ai.js_snippet import load_js_script
|
||||
|
||||
if config.override_navigator or config.simulate_user or config.magic:
|
||||
if not getattr(ctx, '_crawl4ai_nav_overrider_injected', False):
|
||||
await ctx.add_init_script(load_js_script("navigator_overrider"))
|
||||
ctx._crawl4ai_nav_overrider_injected = True
|
||||
|
||||
if config.flatten_shadow_dom:
|
||||
if not getattr(ctx, '_crawl4ai_shadow_dom_injected', False):
|
||||
await ctx.add_init_script("""
|
||||
const _origAttachShadow = Element.prototype.attachShadow;
|
||||
Element.prototype.attachShadow = function(init) {
|
||||
return _origAttachShadow.call(this, {...init, mode: 'open'});
|
||||
};
|
||||
""")
|
||||
ctx._crawl4ai_shadow_dom_injected = True
|
||||
|
||||
check("first pass: 2 add_init_script calls (nav + shadow)", call_count == 2)
|
||||
|
||||
# Second "crawl" — should skip both
|
||||
call_count = 0
|
||||
|
||||
if config.override_navigator or config.simulate_user or config.magic:
|
||||
if not getattr(ctx, '_crawl4ai_nav_overrider_injected', False):
|
||||
await ctx.add_init_script(load_js_script("navigator_overrider"))
|
||||
ctx._crawl4ai_nav_overrider_injected = True
|
||||
|
||||
if config.flatten_shadow_dom:
|
||||
if not getattr(ctx, '_crawl4ai_shadow_dom_injected', False):
|
||||
await ctx.add_init_script("""...""")
|
||||
ctx._crawl4ai_shadow_dom_injected = True
|
||||
|
||||
check("second pass: 0 add_init_script calls (flags set)", call_count == 0)
|
||||
|
||||
await page.close()
|
||||
await ctx.close()
|
||||
|
||||
finally:
|
||||
await bm.close()
|
||||
|
||||
|
||||
async def test_concurrent_crawls_no_accumulation():
|
||||
"""
|
||||
The core bug: N concurrent crawls on a shared context should NOT
|
||||
cause N * add_init_script() calls. With the fix, only 1 call
|
||||
should happen (the first crawl), and the rest should skip.
|
||||
"""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: Concurrent crawls don't accumulate init scripts")
|
||||
print("=" * 70)
|
||||
|
||||
CONCURRENCY = 10
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
|
||||
config = CrawlerRunConfig(magic=True, flatten_shadow_dom=True)
|
||||
|
||||
# Run N concurrent crawls with identical config (same context)
|
||||
htmls = [
|
||||
f"raw:<html><body><p>Concurrent page {i} with enough words</p></body></html>"
|
||||
for i in range(CONCURRENCY)
|
||||
]
|
||||
|
||||
tasks = [crawler.arun(h, config=config) for h in htmls]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
success_count = sum(1 for r in results if r.success)
|
||||
check(f"all {CONCURRENCY} concurrent crawls succeeded", success_count == CONCURRENCY)
|
||||
|
||||
# Check that the shared context has the flags set (proving injection happened)
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
for sig, ctx in bm.contexts_by_config.items():
|
||||
check("shared context has nav_overrider flag",
|
||||
getattr(ctx, '_crawl4ai_nav_overrider_injected', False) is True)
|
||||
check("shared context has shadow_dom flag",
|
||||
getattr(ctx, '_crawl4ai_shadow_dom_injected', False) is True)
|
||||
|
||||
# Verify no context crash — all refcounts should be 0
|
||||
for sig, count in bm._context_refcounts.items():
|
||||
check(f"refcount for {sig[:12]}... is 0 after all crawls", count == 0)
|
||||
|
||||
|
||||
async def test_navigator_overrides_functional():
|
||||
"""
|
||||
Functional check: after dedup fix, navigator overrides still work.
|
||||
The webdriver property should be undefined (not true).
|
||||
"""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: Navigator overrides still functional after dedup")
|
||||
print("=" * 70)
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
|
||||
config = CrawlerRunConfig(override_navigator=True)
|
||||
html = "<html><body><p>Navigator test</p></body></html>"
|
||||
|
||||
result = await crawler.arun(f"raw:{html}", config=config)
|
||||
check("crawl succeeded", result.success)
|
||||
|
||||
# Run a second crawl (same context) to verify scripts still work
|
||||
result2 = await crawler.arun(f"raw:{html}", config=config)
|
||||
check("second crawl on same context succeeded", result2.success)
|
||||
|
||||
|
||||
async def test_concurrent_different_configs():
|
||||
"""
|
||||
Concurrent crawls with DIFFERENT configs: one with magic, one without.
|
||||
Each config gets its own context. Verify no cross-contamination and
|
||||
no crashes.
|
||||
"""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: Concurrent crawls with different configs")
|
||||
print("=" * 70)
|
||||
|
||||
CRAWLS_PER_CONFIG = 5
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
|
||||
configs = [
|
||||
CrawlerRunConfig(magic=True),
|
||||
CrawlerRunConfig(magic=False),
|
||||
CrawlerRunConfig(magic=True, flatten_shadow_dom=True),
|
||||
]
|
||||
|
||||
tasks = []
|
||||
for i in range(CRAWLS_PER_CONFIG):
|
||||
for j, config in enumerate(configs):
|
||||
html = f"raw:<html><body><p>Config {j} crawl {i}</p></body></html>"
|
||||
tasks.append(crawler.arun(html, config=config))
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
total = CRAWLS_PER_CONFIG * len(configs)
|
||||
success_count = sum(1 for r in results if r.success)
|
||||
check(f"all {total} mixed-config crawls succeeded", success_count == total)
|
||||
|
||||
bm = crawler.crawler_strategy.browser_manager
|
||||
|
||||
# All refcounts should be 0
|
||||
for sig, count in bm._context_refcounts.items():
|
||||
check(f"refcount 0 for sig {sig[:12]}...", count == 0)
|
||||
|
||||
|
||||
async def test_shadow_dom_flattening_functional():
|
||||
"""
|
||||
Functional check: shadow DOM flattening works after dedup.
|
||||
The attachShadow override should force shadow roots to open mode.
|
||||
"""
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST: Shadow DOM flattening still functional after dedup")
|
||||
print("=" * 70)
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True, extra_args=['--no-sandbox'])) as crawler:
|
||||
config = CrawlerRunConfig(flatten_shadow_dom=True)
|
||||
|
||||
# HTML with a shadow DOM component
|
||||
html = """<html><body>
|
||||
<div id="host"></div>
|
||||
<script>
|
||||
const host = document.getElementById('host');
|
||||
const shadow = host.attachShadow({mode: 'closed'});
|
||||
shadow.innerHTML = '<p>Shadow content visible</p>';
|
||||
</script>
|
||||
</body></html>"""
|
||||
|
||||
result = await crawler.arun(f"raw:{html}", config=config)
|
||||
check("shadow DOM crawl succeeded", result.success)
|
||||
|
||||
# Second crawl on same context
|
||||
result2 = await crawler.arun(f"raw:{html}", config=config)
|
||||
check("second shadow DOM crawl succeeded", result2.success)
|
||||
|
||||
|
||||
async def main():
|
||||
print("=" * 70)
|
||||
print("Init Script Deduplication Tests (PR #1768 fix)")
|
||||
print("=" * 70)
|
||||
|
||||
await test_setup_context_sets_flags()
|
||||
await test_setup_context_no_config_no_flags()
|
||||
await test_no_duplication_standard_path()
|
||||
await test_fallback_path_injects_once()
|
||||
await test_concurrent_crawls_no_accumulation()
|
||||
await test_navigator_overrides_functional()
|
||||
await test_concurrent_different_configs()
|
||||
await test_shadow_dom_flattening_functional()
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
if FAIL == 0:
|
||||
print(f"ALL {PASS} CHECKS PASSED")
|
||||
else:
|
||||
print(f"FAILED: {FAIL} checks failed, {PASS} passed")
|
||||
print("=" * 70)
|
||||
|
||||
sys.exit(1 if FAIL > 0 else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,17 @@
|
||||
from crawl4ai.browser_profiler import BrowserProfiler
|
||||
import asyncio
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test launching a standalone browser
|
||||
async def test_standalone_browser():
|
||||
profiler = BrowserProfiler()
|
||||
cdp_url = await profiler.launch_standalone_browser(
|
||||
browser_type="chromium",
|
||||
user_data_dir="~/.crawl4ai/browser_profile/test-browser-data",
|
||||
debugging_port=9222,
|
||||
headless=False
|
||||
)
|
||||
print(f"CDP URL: {cdp_url}")
|
||||
|
||||
asyncio.run(test_standalone_browser())
|
||||
@@ -0,0 +1,606 @@
|
||||
"""
|
||||
Real integration tests for page reuse race condition fix.
|
||||
|
||||
Tests that when create_isolated_context=False:
|
||||
1. Single crawls still work correctly
|
||||
2. Concurrent crawls don't cause race conditions
|
||||
3. Pages are properly tracked and released
|
||||
4. Page reuse works when pages become available
|
||||
|
||||
These are REAL tests - no mocking, actual browser operations.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
|
||||
|
||||
async def test_single_crawl_still_works():
|
||||
"""
|
||||
Test 1: Basic single crawl functionality still works with create_isolated_context=False.
|
||||
This ensures we haven't broken existing functionality.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 1: Single crawl with create_isolated_context=False")
|
||||
print("="*70)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun("https://example.com")
|
||||
|
||||
assert result.success, f"Crawl failed: {result.error_message}"
|
||||
assert result.html, "No HTML content returned"
|
||||
assert "Example Domain" in result.html, "Expected content not found"
|
||||
|
||||
print(f" Status: {result.status_code}")
|
||||
print(f" HTML length: {len(result.html)} chars")
|
||||
print(" PASSED: Single crawl works correctly")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
async def test_sequential_crawls_work():
|
||||
"""
|
||||
Test 2: Sequential crawls reuse the same page (when released).
|
||||
This tests that page tracking and release works correctly.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 2: Sequential crawls with page reuse")
|
||||
print("="*70)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
urls = [
|
||||
"https://example.com",
|
||||
"https://httpbin.org/html",
|
||||
"https://example.org",
|
||||
]
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
results = []
|
||||
for url in urls:
|
||||
result = await crawler.arun(url)
|
||||
results.append(result)
|
||||
print(f" Crawled {url}: success={result.success}, status={result.status_code}")
|
||||
|
||||
# All should succeed
|
||||
for i, result in enumerate(results):
|
||||
assert result.success, f"Crawl {i+1} failed: {result.error_message}"
|
||||
|
||||
print(" PASSED: Sequential crawls work correctly")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
async def test_concurrent_crawls_no_race_condition():
|
||||
"""
|
||||
Test 3: Multiple concurrent crawls don't cause race conditions.
|
||||
This is the main bug we're fixing - concurrent crawls should each get their own page.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 3: Concurrent crawls with create_isolated_context=False")
|
||||
print("="*70)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
# Use different URLs to ensure they can't accidentally succeed by being on the same page
|
||||
urls = [
|
||||
"https://example.com",
|
||||
"https://httpbin.org/html",
|
||||
"https://example.org",
|
||||
"https://httpbin.org/get",
|
||||
"https://www.iana.org/domains/reserved",
|
||||
]
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
print(f" Launching {len(urls)} concurrent crawls...")
|
||||
start_time = time.time()
|
||||
|
||||
# Launch all crawls concurrently
|
||||
tasks = [crawler.arun(url) for url in urls]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f" Completed in {elapsed:.2f}s")
|
||||
|
||||
# Check results
|
||||
success_count = 0
|
||||
for i, (url, result) in enumerate(zip(urls, results)):
|
||||
if isinstance(result, Exception):
|
||||
print(f" [{i+1}] {url}: EXCEPTION - {result}")
|
||||
elif result.success:
|
||||
success_count += 1
|
||||
print(f" [{i+1}] {url}: OK (status={result.status_code})")
|
||||
else:
|
||||
print(f" [{i+1}] {url}: FAILED - {result.error_message}")
|
||||
|
||||
# All should succeed
|
||||
assert success_count == len(urls), f"Only {success_count}/{len(urls)} succeeded"
|
||||
|
||||
print(f" PASSED: All {len(urls)} concurrent crawls succeeded without race conditions")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
async def test_high_concurrency_stress():
|
||||
"""
|
||||
Test 4: High concurrency stress test - many concurrent crawls.
|
||||
This stresses the page tracking system to ensure it handles many concurrent operations.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 4: High concurrency stress test (10 concurrent crawls)")
|
||||
print("="*70)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
# Generate multiple unique URLs
|
||||
base_urls = [
|
||||
"https://example.com",
|
||||
"https://httpbin.org/html",
|
||||
"https://example.org",
|
||||
"https://httpbin.org/get",
|
||||
"https://www.iana.org/domains/reserved",
|
||||
]
|
||||
|
||||
# Create 10 URLs by adding query params
|
||||
urls = []
|
||||
for i in range(10):
|
||||
url = f"{base_urls[i % len(base_urls)]}?test={i}&t={int(time.time())}"
|
||||
urls.append(url)
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
print(f" Launching {len(urls)} concurrent crawls...")
|
||||
start_time = time.time()
|
||||
|
||||
# Launch all crawls concurrently
|
||||
tasks = [crawler.arun(url) for url in urls]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f" Completed in {elapsed:.2f}s")
|
||||
|
||||
# Count results
|
||||
success_count = 0
|
||||
error_count = 0
|
||||
exception_count = 0
|
||||
|
||||
for url, result in zip(urls, results):
|
||||
if isinstance(result, Exception):
|
||||
exception_count += 1
|
||||
elif result.success:
|
||||
success_count += 1
|
||||
else:
|
||||
error_count += 1
|
||||
|
||||
print(f" Results: {success_count} success, {error_count} errors, {exception_count} exceptions")
|
||||
|
||||
# At least 80% should succeed (allowing for some network issues)
|
||||
min_success = int(len(urls) * 0.8)
|
||||
assert success_count >= min_success, f"Only {success_count}/{len(urls)} succeeded (min: {min_success})"
|
||||
|
||||
print(f" PASSED: High concurrency test ({success_count}/{len(urls)} succeeded)")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
async def test_page_tracking_internal_state():
|
||||
"""
|
||||
Test 5: Verify internal page tracking state is correct.
|
||||
This directly tests the global page tracking mechanism.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 5: Internal page tracking state verification")
|
||||
print("="*70)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
browser_manager = crawler.crawler_strategy.browser_manager
|
||||
|
||||
# Check endpoint key is set
|
||||
endpoint_key = browser_manager._browser_endpoint_key
|
||||
print(f" Browser endpoint key: {endpoint_key}")
|
||||
assert endpoint_key, "Endpoint key should be set"
|
||||
|
||||
# Initially, no pages should be in use
|
||||
initial_in_use = len(browser_manager._get_pages_in_use())
|
||||
print(f" Initial pages in use: {initial_in_use}")
|
||||
|
||||
# Do a crawl
|
||||
result = await crawler.arun("https://example.com")
|
||||
assert result.success, f"Crawl failed: {result.error_message}"
|
||||
|
||||
# After crawl completes, page should be released
|
||||
after_crawl_in_use = len(browser_manager._get_pages_in_use())
|
||||
print(f" Pages in use after crawl: {after_crawl_in_use}")
|
||||
|
||||
# The page should have been released (or kept as the last page)
|
||||
# Either way, tracking should be consistent
|
||||
|
||||
# Do another crawl - should work fine
|
||||
result2 = await crawler.arun("https://example.org")
|
||||
assert result2.success, f"Second crawl failed: {result2.error_message}"
|
||||
|
||||
final_in_use = len(browser_manager._get_pages_in_use())
|
||||
print(f" Pages in use after second crawl: {final_in_use}")
|
||||
|
||||
print(" PASSED: Page tracking state is consistent")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
async def test_mixed_sequential_and_concurrent():
|
||||
"""
|
||||
Test 6: Mixed sequential and concurrent crawls.
|
||||
Tests realistic usage pattern where some crawls are sequential and some concurrent.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 6: Mixed sequential and concurrent crawls")
|
||||
print("="*70)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# Sequential crawl 1
|
||||
print(" Phase 1: Sequential crawl")
|
||||
result1 = await crawler.arun("https://example.com")
|
||||
assert result1.success, f"Sequential crawl 1 failed"
|
||||
print(f" Crawl 1: OK")
|
||||
|
||||
# Concurrent crawls
|
||||
print(" Phase 2: Concurrent crawls (3 URLs)")
|
||||
concurrent_urls = [
|
||||
"https://httpbin.org/html",
|
||||
"https://example.org",
|
||||
"https://httpbin.org/get",
|
||||
]
|
||||
tasks = [crawler.arun(url) for url in concurrent_urls]
|
||||
concurrent_results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for i, result in enumerate(concurrent_results):
|
||||
if isinstance(result, Exception):
|
||||
print(f" Concurrent {i+1}: EXCEPTION - {result}")
|
||||
else:
|
||||
assert result.success, f"Concurrent crawl {i+1} failed"
|
||||
print(f" Concurrent {i+1}: OK")
|
||||
|
||||
# Sequential crawl 2
|
||||
print(" Phase 3: Sequential crawl")
|
||||
result2 = await crawler.arun("https://www.iana.org/domains/reserved")
|
||||
assert result2.success, f"Sequential crawl 2 failed"
|
||||
print(f" Crawl 2: OK")
|
||||
|
||||
# Another batch of concurrent
|
||||
print(" Phase 4: More concurrent crawls (2 URLs)")
|
||||
tasks2 = [
|
||||
crawler.arun("https://example.com?test=1"),
|
||||
crawler.arun("https://example.org?test=2"),
|
||||
]
|
||||
results2 = await asyncio.gather(*tasks2, return_exceptions=True)
|
||||
for i, result in enumerate(results2):
|
||||
if isinstance(result, Exception):
|
||||
print(f" Concurrent {i+1}: EXCEPTION - {result}")
|
||||
else:
|
||||
assert result.success, f"Batch 2 crawl {i+1} failed"
|
||||
print(f" Concurrent {i+1}: OK")
|
||||
|
||||
print(" PASSED: Mixed sequential and concurrent crawls work correctly")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
async def test_compare_isolated_vs_shared_context():
|
||||
"""
|
||||
Test 7: Compare behavior between isolated and shared context modes.
|
||||
Both should work for concurrent crawls now.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 7: Compare isolated vs shared context modes")
|
||||
print("="*70)
|
||||
|
||||
urls = [
|
||||
"https://example.com",
|
||||
"https://httpbin.org/html",
|
||||
"https://example.org",
|
||||
]
|
||||
|
||||
# Test with create_isolated_context=True
|
||||
print(" Testing with create_isolated_context=True:")
|
||||
browser_config_isolated = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=True,
|
||||
)
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config_isolated) as crawler:
|
||||
tasks = [crawler.arun(url) for url in urls]
|
||||
results_isolated = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
isolated_success = sum(1 for r in results_isolated if not isinstance(r, Exception) and r.success)
|
||||
print(f" Isolated context: {isolated_success}/{len(urls)} succeeded")
|
||||
except Exception as e:
|
||||
print(f" Isolated context: FAILED - {e}")
|
||||
isolated_success = 0
|
||||
|
||||
# Test with create_isolated_context=False
|
||||
print(" Testing with create_isolated_context=False:")
|
||||
browser_config_shared = BrowserConfig(
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config_shared) as crawler:
|
||||
tasks = [crawler.arun(url) for url in urls]
|
||||
results_shared = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
shared_success = sum(1 for r in results_shared if not isinstance(r, Exception) and r.success)
|
||||
print(f" Shared context: {shared_success}/{len(urls)} succeeded")
|
||||
except Exception as e:
|
||||
print(f" Shared context: FAILED - {e}")
|
||||
shared_success = 0
|
||||
|
||||
# Both modes should work
|
||||
assert isolated_success == len(urls), f"Isolated context: only {isolated_success}/{len(urls)} succeeded"
|
||||
assert shared_success == len(urls), f"Shared context: only {shared_success}/{len(urls)} succeeded"
|
||||
|
||||
print(" PASSED: Both context modes work correctly for concurrent crawls")
|
||||
return True
|
||||
|
||||
|
||||
async def test_multiple_crawlers_same_cdp():
|
||||
"""
|
||||
Test 8: Multiple AsyncWebCrawler instances connecting to the same CDP endpoint.
|
||||
|
||||
This tests the realistic scenario where:
|
||||
1. A browser is started externally (or by a managed browser)
|
||||
2. Multiple crawler instances connect to it via CDP URL
|
||||
3. All use create_isolated_context=False to share cookies/session
|
||||
4. Each should get its own page to avoid race conditions
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST 8: Multiple crawlers connecting to same CDP endpoint")
|
||||
print("="*70)
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
# Start a browser manually using subprocess
|
||||
port = 9444
|
||||
temp_dir = tempfile.mkdtemp(prefix="browser-test-")
|
||||
|
||||
browser_process = None
|
||||
try:
|
||||
# Start chromium with remote debugging - use Playwright's bundled chromium
|
||||
import os
|
||||
playwright_path = os.path.expanduser("~/.cache/ms-playwright/chromium-1200/chrome-linux64/chrome")
|
||||
if not os.path.exists(playwright_path):
|
||||
# Fallback - try to find it
|
||||
for path in [
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
"/usr/bin/google-chrome",
|
||||
]:
|
||||
if os.path.exists(path):
|
||||
playwright_path = path
|
||||
break
|
||||
chrome_path = playwright_path
|
||||
|
||||
cmd = [
|
||||
chrome_path,
|
||||
f"--remote-debugging-port={port}",
|
||||
f"--user-data-dir={temp_dir}",
|
||||
"--headless=new",
|
||||
"--no-sandbox",
|
||||
"--disable-gpu",
|
||||
"--disable-dev-shm-usage",
|
||||
]
|
||||
|
||||
browser_process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
await asyncio.sleep(2) # Wait for browser to start
|
||||
|
||||
cdp_url = f"http://localhost:{port}"
|
||||
print(f" Started browser at {cdp_url}")
|
||||
|
||||
# Both crawlers connect via CDP URL
|
||||
browser_config1 = BrowserConfig(
|
||||
headless=True,
|
||||
cdp_url=cdp_url,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
browser_config2 = BrowserConfig(
|
||||
headless=True,
|
||||
cdp_url=cdp_url,
|
||||
create_isolated_context=False,
|
||||
)
|
||||
|
||||
urls_crawler1 = [
|
||||
"https://example.com?crawler=1",
|
||||
"https://example.org?crawler=1",
|
||||
]
|
||||
urls_crawler2 = [
|
||||
"https://httpbin.org/html?crawler=2",
|
||||
"https://httpbin.org/get?crawler=2",
|
||||
]
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config1) as crawler1:
|
||||
async with AsyncWebCrawler(config=browser_config2) as crawler2:
|
||||
bm1 = crawler1.crawler_strategy.browser_manager
|
||||
bm2 = crawler2.crawler_strategy.browser_manager
|
||||
|
||||
print(f" Crawler 1 endpoint key: {bm1._browser_endpoint_key}")
|
||||
print(f" Crawler 2 endpoint key: {bm2._browser_endpoint_key}")
|
||||
print(f" Keys match: {bm1._browser_endpoint_key == bm2._browser_endpoint_key}")
|
||||
|
||||
# Launch concurrent crawls from BOTH crawlers simultaneously
|
||||
print(f" Launching {len(urls_crawler1) + len(urls_crawler2)} concurrent crawls...")
|
||||
|
||||
tasks1 = [crawler1.arun(url) for url in urls_crawler1]
|
||||
tasks2 = [crawler2.arun(url) for url in urls_crawler2]
|
||||
|
||||
all_results = await asyncio.gather(
|
||||
*tasks1, *tasks2,
|
||||
return_exceptions=True
|
||||
)
|
||||
|
||||
# Check results
|
||||
success_count = 0
|
||||
for i, result in enumerate(all_results):
|
||||
crawler_id = 1 if i < len(urls_crawler1) else 2
|
||||
url_idx = i if i < len(urls_crawler1) else i - len(urls_crawler1)
|
||||
|
||||
if isinstance(result, Exception):
|
||||
print(f" Crawler {crawler_id}, URL {url_idx+1}: EXCEPTION - {result}")
|
||||
elif result.success:
|
||||
success_count += 1
|
||||
print(f" Crawler {crawler_id}, URL {url_idx+1}: OK")
|
||||
else:
|
||||
print(f" Crawler {crawler_id}, URL {url_idx+1}: FAILED - {result.error_message}")
|
||||
|
||||
total = len(urls_crawler1) + len(urls_crawler2)
|
||||
assert success_count == total, f"Only {success_count}/{total} succeeded"
|
||||
|
||||
print(f" PASSED: All {total} concurrent crawls from 2 crawlers succeeded")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
finally:
|
||||
# Clean up browser process
|
||||
if browser_process:
|
||||
browser_process.terminate()
|
||||
try:
|
||||
browser_process.wait(timeout=5)
|
||||
except:
|
||||
browser_process.kill()
|
||||
# Clean up temp dir
|
||||
import shutil
|
||||
try:
|
||||
shutil.rmtree(temp_dir)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
async def run_all_tests():
|
||||
"""Run all tests and report results."""
|
||||
print("\n" + "#"*70)
|
||||
print("# PAGE REUSE RACE CONDITION FIX - INTEGRATION TESTS")
|
||||
print("#"*70)
|
||||
|
||||
tests = [
|
||||
("Single crawl works", test_single_crawl_still_works),
|
||||
("Sequential crawls work", test_sequential_crawls_work),
|
||||
("Concurrent crawls no race", test_concurrent_crawls_no_race_condition),
|
||||
("High concurrency stress", test_high_concurrency_stress),
|
||||
("Page tracking state", test_page_tracking_internal_state),
|
||||
("Mixed sequential/concurrent", test_mixed_sequential_and_concurrent),
|
||||
("Isolated vs shared context", test_compare_isolated_vs_shared_context),
|
||||
]
|
||||
|
||||
results = []
|
||||
for name, test_func in tests:
|
||||
try:
|
||||
passed = await test_func()
|
||||
results.append((name, passed))
|
||||
except Exception as e:
|
||||
print(f" EXCEPTION in {name}: {e}")
|
||||
results.append((name, False))
|
||||
|
||||
# Summary
|
||||
print("\n" + "="*70)
|
||||
print("TEST SUMMARY")
|
||||
print("="*70)
|
||||
|
||||
passed = sum(1 for _, p in results if p)
|
||||
total = len(results)
|
||||
|
||||
for name, p in results:
|
||||
status = "PASS" if p else "FAIL"
|
||||
print(f" [{status}] {name}")
|
||||
|
||||
print("-"*70)
|
||||
print(f" Total: {passed}/{total} tests passed")
|
||||
|
||||
if passed == total:
|
||||
print("\n ALL TESTS PASSED!")
|
||||
return 0
|
||||
else:
|
||||
print(f"\n {total - passed} TESTS FAILED!")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(run_all_tests())
|
||||
sys.exit(exit_code)
|
||||
@@ -0,0 +1,902 @@
|
||||
"""
|
||||
Test examples for parallel crawling with the browser module.
|
||||
|
||||
These examples demonstrate the functionality of parallel page creation
|
||||
and serve as functional tests for multi-page crawling performance.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import List
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.browser import BrowserManager
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
async def test_get_pages_basic():
|
||||
"""Test basic functionality of get_pages method."""
|
||||
logger.info("Testing basic get_pages functionality", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
|
||||
# Request 3 pages
|
||||
crawler_config = CrawlerRunConfig()
|
||||
pages = await manager.get_pages(crawler_config, count=3)
|
||||
|
||||
# Verify we got the correct number of pages
|
||||
assert len(pages) == 3, f"Expected 3 pages, got {len(pages)}"
|
||||
|
||||
# Verify each page is valid
|
||||
for i, (page, context) in enumerate(pages):
|
||||
await page.goto("https://example.com")
|
||||
title = await page.title()
|
||||
logger.info(f"Page {i+1} title: {title}", tag="TEST")
|
||||
assert title, f"Page {i+1} has no title"
|
||||
|
||||
await manager.close()
|
||||
logger.success("Basic get_pages test completed successfully", tag="TEST")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_parallel_approaches_comparison():
|
||||
"""Compare two parallel crawling approaches:
|
||||
1. Create a page for each URL on-demand (get_page + gather)
|
||||
2. Get all pages upfront with get_pages, then use them (get_pages + gather)
|
||||
"""
|
||||
logger.info("Comparing different parallel crawling approaches", tag="TEST")
|
||||
|
||||
urls = [
|
||||
"https://example.com/page1",
|
||||
"https://crawl4ai.com",
|
||||
"https://kidocode.com",
|
||||
"https://bbc.com",
|
||||
# "https://example.com/page1",
|
||||
# "https://example.com/page2",
|
||||
# "https://example.com/page3",
|
||||
# "https://example.com/page4",
|
||||
]
|
||||
|
||||
browser_config = BrowserConfig(headless=False)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
|
||||
# Approach 1: Create a page for each URL on-demand and run in parallel
|
||||
logger.info("Testing approach 1: get_page for each URL + gather", tag="TEST")
|
||||
start_time = time.time()
|
||||
|
||||
async def fetch_title_approach1(url):
|
||||
"""Create a new page for each URL, go to the URL, and get title"""
|
||||
crawler_config = CrawlerRunConfig(url=url)
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
try:
|
||||
await page.goto(url)
|
||||
title = await page.title()
|
||||
return title
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
# Run fetch_title_approach1 for each URL in parallel
|
||||
tasks = [fetch_title_approach1(url) for url in urls]
|
||||
approach1_results = await asyncio.gather(*tasks)
|
||||
|
||||
approach1_time = time.time() - start_time
|
||||
logger.info(f"Approach 1 time (get_page + gather): {approach1_time:.2f}s", tag="TEST")
|
||||
|
||||
# Approach 2: Get all pages upfront with get_pages, then use them in parallel
|
||||
logger.info("Testing approach 2: get_pages upfront + gather", tag="TEST")
|
||||
start_time = time.time()
|
||||
|
||||
# Get all pages upfront
|
||||
crawler_config = CrawlerRunConfig()
|
||||
pages = await manager.get_pages(crawler_config, count=len(urls))
|
||||
|
||||
async def fetch_title_approach2(page_ctx, url):
|
||||
"""Use a pre-created page to go to URL and get title"""
|
||||
page, _ = page_ctx
|
||||
try:
|
||||
await page.goto(url)
|
||||
title = await page.title()
|
||||
return title
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
# Use the pre-created pages to fetch titles in parallel
|
||||
tasks = [fetch_title_approach2(page_ctx, url) for page_ctx, url in zip(pages, urls)]
|
||||
approach2_results = await asyncio.gather(*tasks)
|
||||
|
||||
approach2_time = time.time() - start_time
|
||||
logger.info(f"Approach 2 time (get_pages + gather): {approach2_time:.2f}s", tag="TEST")
|
||||
|
||||
# Compare results and performance
|
||||
speedup = approach1_time / approach2_time if approach2_time > 0 else 0
|
||||
if speedup > 1:
|
||||
logger.success(f"Approach 2 (get_pages upfront) was {speedup:.2f}x faster", tag="TEST")
|
||||
else:
|
||||
logger.info(f"Approach 1 (get_page + gather) was {1/speedup:.2f}x faster", tag="TEST")
|
||||
|
||||
# Verify same content was retrieved in both approaches
|
||||
assert len(approach1_results) == len(approach2_results), "Result count mismatch"
|
||||
|
||||
# Sort results for comparison since parallel execution might complete in different order
|
||||
assert sorted(approach1_results) == sorted(approach2_results), "Results content mismatch"
|
||||
|
||||
await manager.close()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_multi_browser_scaling(num_browsers=3, pages_per_browser=5):
|
||||
"""Test performance with multiple browsers and pages per browser.
|
||||
Compares two approaches:
|
||||
1. On-demand page creation (get_page + gather)
|
||||
2. Pre-created pages (get_pages + gather)
|
||||
"""
|
||||
logger.info(f"Testing multi-browser scaling with {num_browsers} browsers × {pages_per_browser} pages", tag="TEST")
|
||||
|
||||
# Generate test URLs
|
||||
total_pages = num_browsers * pages_per_browser
|
||||
urls = [f"https://example.com/page_{i}" for i in range(total_pages)]
|
||||
|
||||
# Create browser managers
|
||||
managers = []
|
||||
base_port = 9222
|
||||
|
||||
try:
|
||||
# Start all browsers in parallel
|
||||
start_tasks = []
|
||||
for i in range(num_browsers):
|
||||
browser_config = BrowserConfig(
|
||||
headless=True # Using default browser mode like in test_parallel_approaches_comparison
|
||||
)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
start_tasks.append(manager.start())
|
||||
managers.append(manager)
|
||||
|
||||
await asyncio.gather(*start_tasks)
|
||||
|
||||
# Distribute URLs among managers
|
||||
urls_per_manager = {}
|
||||
for i, manager in enumerate(managers):
|
||||
start_idx = i * pages_per_browser
|
||||
end_idx = min(start_idx + pages_per_browser, len(urls))
|
||||
urls_per_manager[manager] = urls[start_idx:end_idx]
|
||||
|
||||
# Approach 1: Create a page for each URL on-demand and run in parallel
|
||||
logger.info("Testing approach 1: get_page for each URL + gather", tag="TEST")
|
||||
start_time = time.time()
|
||||
|
||||
async def fetch_title_approach1(manager, url):
|
||||
"""Create a new page for the URL, go to the URL, and get title"""
|
||||
crawler_config = CrawlerRunConfig(url=url)
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
try:
|
||||
await page.goto(url)
|
||||
title = await page.title()
|
||||
return title
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
# Run fetch_title_approach1 for each URL in parallel
|
||||
tasks = []
|
||||
for manager, manager_urls in urls_per_manager.items():
|
||||
for url in manager_urls:
|
||||
tasks.append(fetch_title_approach1(manager, url))
|
||||
|
||||
approach1_results = await asyncio.gather(*tasks)
|
||||
|
||||
approach1_time = time.time() - start_time
|
||||
logger.info(f"Approach 1 time (get_page + gather): {approach1_time:.2f}s", tag="TEST")
|
||||
|
||||
# Approach 2: Get all pages upfront with get_pages, then use them in parallel
|
||||
logger.info("Testing approach 2: get_pages upfront + gather", tag="TEST")
|
||||
start_time = time.time()
|
||||
|
||||
# Get all pages upfront for each manager
|
||||
all_pages = []
|
||||
for manager, manager_urls in urls_per_manager.items():
|
||||
crawler_config = CrawlerRunConfig()
|
||||
pages = await manager.get_pages(crawler_config, count=len(manager_urls))
|
||||
all_pages.extend(zip(pages, manager_urls))
|
||||
|
||||
async def fetch_title_approach2(page_ctx, url):
|
||||
"""Use a pre-created page to go to URL and get title"""
|
||||
page, _ = page_ctx
|
||||
try:
|
||||
await page.goto(url)
|
||||
title = await page.title()
|
||||
return title
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
# Use the pre-created pages to fetch titles in parallel
|
||||
tasks = [fetch_title_approach2(page_ctx, url) for page_ctx, url in all_pages]
|
||||
approach2_results = await asyncio.gather(*tasks)
|
||||
|
||||
approach2_time = time.time() - start_time
|
||||
logger.info(f"Approach 2 time (get_pages + gather): {approach2_time:.2f}s", tag="TEST")
|
||||
|
||||
# Compare results and performance
|
||||
speedup = approach1_time / approach2_time if approach2_time > 0 else 0
|
||||
pages_per_second = total_pages / approach2_time
|
||||
|
||||
# Show a simple summary
|
||||
logger.info(f"📊 Summary: {num_browsers} browsers × {pages_per_browser} pages = {total_pages} total crawls", tag="TEST")
|
||||
logger.info(f"⚡ Performance: {pages_per_second:.1f} pages/second ({pages_per_second*60:.0f} pages/minute)", tag="TEST")
|
||||
logger.info(f"🚀 Total crawl time: {approach2_time:.2f} seconds", tag="TEST")
|
||||
|
||||
if speedup > 1:
|
||||
logger.success(f"✅ Approach 2 (get_pages upfront) was {speedup:.2f}x faster", tag="TEST")
|
||||
else:
|
||||
logger.info(f"✅ Approach 1 (get_page + gather) was {1/speedup:.2f}x faster", tag="TEST")
|
||||
|
||||
# Close all managers
|
||||
for manager in managers:
|
||||
await manager.close()
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Clean up
|
||||
for manager in managers:
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def grid_search_optimal_configuration(total_urls=50):
|
||||
"""Perform a grid search to find the optimal balance between number of browsers and pages per browser.
|
||||
|
||||
This function tests different combinations of browser count and pages per browser,
|
||||
while keeping the total number of URLs constant. It measures performance metrics
|
||||
for each configuration to find the "sweet spot" that provides the best speed
|
||||
with reasonable memory usage.
|
||||
|
||||
Args:
|
||||
total_urls: Total number of URLs to crawl (default: 50)
|
||||
"""
|
||||
logger.info(f"=== GRID SEARCH FOR OPTIMAL CRAWLING CONFIGURATION ({total_urls} URLs) ===", tag="TEST")
|
||||
|
||||
# Generate test URLs once
|
||||
urls = [f"https://example.com/page_{i}" for i in range(total_urls)]
|
||||
|
||||
# Define grid search configurations
|
||||
# We'll use more flexible approach: test all browser counts from 1 to min(20, total_urls)
|
||||
# and distribute pages evenly (some browsers may have 1 more page than others)
|
||||
configurations = []
|
||||
|
||||
# Maximum number of browsers to test
|
||||
max_browsers_to_test = min(20, total_urls)
|
||||
|
||||
# Try configurations with 1 to max_browsers_to_test browsers
|
||||
for num_browsers in range(1, max_browsers_to_test + 1):
|
||||
base_pages_per_browser = total_urls // num_browsers
|
||||
remainder = total_urls % num_browsers
|
||||
|
||||
# Generate exact page distribution array
|
||||
if remainder > 0:
|
||||
# First 'remainder' browsers get one more page
|
||||
page_distribution = [base_pages_per_browser + 1] * remainder + [base_pages_per_browser] * (num_browsers - remainder)
|
||||
pages_distribution = f"{base_pages_per_browser+1} pages × {remainder} browsers, {base_pages_per_browser} pages × {num_browsers - remainder} browsers"
|
||||
else:
|
||||
# All browsers get the same number of pages
|
||||
page_distribution = [base_pages_per_browser] * num_browsers
|
||||
pages_distribution = f"{base_pages_per_browser} pages × {num_browsers} browsers"
|
||||
|
||||
# Format the distribution as a tuple string like (4, 4, 3, 3)
|
||||
distribution_str = str(tuple(page_distribution))
|
||||
|
||||
configurations.append((num_browsers, base_pages_per_browser, pages_distribution, page_distribution, distribution_str))
|
||||
|
||||
# Track results
|
||||
results = []
|
||||
|
||||
# Test each configuration
|
||||
for num_browsers, pages_per_browser, pages_distribution, page_distribution, distribution_str in configurations:
|
||||
logger.info("-" * 80, tag="TEST")
|
||||
logger.info(f"Testing configuration: {num_browsers} browsers with distribution: {distribution_str}", tag="TEST")
|
||||
logger.info(f"Details: {pages_distribution}", tag="TEST")
|
||||
# Sleep a bit for randomness
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
try:
|
||||
# Import psutil for memory tracking
|
||||
try:
|
||||
import psutil
|
||||
process = psutil.Process()
|
||||
initial_memory = process.memory_info().rss / (1024 * 1024) # MB
|
||||
except ImportError:
|
||||
logger.warning("psutil not available, memory metrics will not be tracked", tag="TEST")
|
||||
initial_memory = 0
|
||||
|
||||
# Create and start browser managers
|
||||
managers = []
|
||||
start_time = time.time()
|
||||
|
||||
# Start all browsers in parallel
|
||||
start_tasks = []
|
||||
for i in range(num_browsers):
|
||||
browser_config = BrowserConfig(
|
||||
headless=True
|
||||
)
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
start_tasks.append(manager.start())
|
||||
managers.append(manager)
|
||||
|
||||
await asyncio.gather(*start_tasks)
|
||||
browser_startup_time = time.time() - start_time
|
||||
|
||||
# Measure memory after browser startup
|
||||
if initial_memory > 0:
|
||||
browser_memory = process.memory_info().rss / (1024 * 1024) - initial_memory
|
||||
else:
|
||||
browser_memory = 0
|
||||
|
||||
# Distribute URLs among managers using the exact page distribution
|
||||
urls_per_manager = {}
|
||||
total_assigned = 0
|
||||
|
||||
for i, manager in enumerate(managers):
|
||||
if i < len(page_distribution):
|
||||
# Get the exact number of pages for this browser from our distribution
|
||||
manager_pages = page_distribution[i]
|
||||
|
||||
# Get the URL slice for this manager
|
||||
start_idx = total_assigned
|
||||
end_idx = start_idx + manager_pages
|
||||
urls_per_manager[manager] = urls[start_idx:end_idx]
|
||||
total_assigned += manager_pages
|
||||
else:
|
||||
# If we have more managers than our distribution (should never happen)
|
||||
urls_per_manager[manager] = []
|
||||
|
||||
# Use the more efficient approach (pre-created pages)
|
||||
logger.info("Running page crawling test...", tag="TEST")
|
||||
crawl_start_time = time.time()
|
||||
|
||||
# Get all pages upfront for each manager
|
||||
all_pages = []
|
||||
for manager, manager_urls in urls_per_manager.items():
|
||||
if not manager_urls: # Skip managers with no URLs
|
||||
continue
|
||||
crawler_config = CrawlerRunConfig()
|
||||
pages = await manager.get_pages(crawler_config, count=len(manager_urls))
|
||||
all_pages.extend(zip(pages, manager_urls))
|
||||
|
||||
# Measure memory after page creation
|
||||
if initial_memory > 0:
|
||||
pages_memory = process.memory_info().rss / (1024 * 1024) - browser_memory - initial_memory
|
||||
else:
|
||||
pages_memory = 0
|
||||
|
||||
# Function to crawl a URL with a pre-created page
|
||||
async def fetch_title(page_ctx, url):
|
||||
page, _ = page_ctx
|
||||
try:
|
||||
await page.goto(url)
|
||||
title = await page.title()
|
||||
return title
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
# Use the pre-created pages to fetch titles in parallel
|
||||
tasks = [fetch_title(page_ctx, url) for page_ctx, url in all_pages]
|
||||
crawl_results = await asyncio.gather(*tasks)
|
||||
|
||||
crawl_time = time.time() - crawl_start_time
|
||||
total_time = time.time() - start_time
|
||||
|
||||
# Final memory measurement
|
||||
if initial_memory > 0:
|
||||
peak_memory = max(browser_memory + pages_memory, process.memory_info().rss / (1024 * 1024) - initial_memory)
|
||||
else:
|
||||
peak_memory = 0
|
||||
|
||||
# Close all managers
|
||||
for manager in managers:
|
||||
await manager.close()
|
||||
|
||||
# Calculate metrics
|
||||
pages_per_second = total_urls / crawl_time
|
||||
|
||||
# Store result metrics
|
||||
result = {
|
||||
"num_browsers": num_browsers,
|
||||
"pages_per_browser": pages_per_browser,
|
||||
"page_distribution": page_distribution,
|
||||
"distribution_str": distribution_str,
|
||||
"total_urls": total_urls,
|
||||
"browser_startup_time": browser_startup_time,
|
||||
"crawl_time": crawl_time,
|
||||
"total_time": total_time,
|
||||
"browser_memory": browser_memory,
|
||||
"pages_memory": pages_memory,
|
||||
"peak_memory": peak_memory,
|
||||
"pages_per_second": pages_per_second,
|
||||
# Calculate efficiency score (higher is better)
|
||||
# This balances speed vs memory usage
|
||||
"efficiency_score": pages_per_second / (peak_memory + 1) if peak_memory > 0 else pages_per_second,
|
||||
}
|
||||
|
||||
results.append(result)
|
||||
|
||||
# Log the results
|
||||
logger.info(f"Browser startup: {browser_startup_time:.2f}s", tag="TEST")
|
||||
logger.info(f"Crawl time: {crawl_time:.2f}s", tag="TEST")
|
||||
logger.info(f"Total time: {total_time:.2f}s", tag="TEST")
|
||||
logger.info(f"Performance: {pages_per_second:.1f} pages/second", tag="TEST")
|
||||
|
||||
if peak_memory > 0:
|
||||
logger.info(f"Browser memory: {browser_memory:.1f}MB", tag="TEST")
|
||||
logger.info(f"Pages memory: {pages_memory:.1f}MB", tag="TEST")
|
||||
logger.info(f"Peak memory: {peak_memory:.1f}MB", tag="TEST")
|
||||
logger.info(f"Efficiency score: {result['efficiency_score']:.6f}", tag="TEST")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error testing configuration: {str(e)}", tag="TEST")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Clean up
|
||||
for manager in managers:
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Print summary of all configurations
|
||||
logger.info("=" * 100, tag="TEST")
|
||||
logger.info("GRID SEARCH RESULTS SUMMARY", tag="TEST")
|
||||
logger.info("=" * 100, tag="TEST")
|
||||
|
||||
# Rank configurations by efficiency score
|
||||
ranked_results = sorted(results, key=lambda x: x["efficiency_score"], reverse=True)
|
||||
|
||||
# Also determine rankings by different metrics
|
||||
fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
|
||||
lowest_memory = sorted(results, key=lambda x: x["peak_memory"] if x["peak_memory"] > 0 else float('inf'))[0]
|
||||
most_efficient = ranked_results[0]
|
||||
|
||||
# Print top performers by category
|
||||
logger.info("🏆 TOP PERFORMERS BY CATEGORY:", tag="TEST")
|
||||
logger.info(f"⚡ Fastest: {fastest['num_browsers']} browsers × ~{fastest['pages_per_browser']} pages " +
|
||||
f"({fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/s)", tag="TEST")
|
||||
|
||||
if lowest_memory["peak_memory"] > 0:
|
||||
logger.info(f"💾 Lowest memory: {lowest_memory['num_browsers']} browsers × ~{lowest_memory['pages_per_browser']} pages " +
|
||||
f"({lowest_memory['peak_memory']:.1f}MB)", tag="TEST")
|
||||
|
||||
logger.info(f"🌟 Most efficient: {most_efficient['num_browsers']} browsers × ~{most_efficient['pages_per_browser']} pages " +
|
||||
f"(score: {most_efficient['efficiency_score']:.6f})", tag="TEST")
|
||||
|
||||
# Print result table header
|
||||
logger.info("\n📊 COMPLETE RANKING TABLE (SORTED BY EFFICIENCY SCORE):", tag="TEST")
|
||||
logger.info("-" * 120, tag="TEST")
|
||||
|
||||
# Define table header
|
||||
header = f"{'Rank':<5} | {'Browsers':<8} | {'Distribution':<55} | {'Total Time(s)':<12} | {'Speed(p/s)':<12} | {'Memory(MB)':<12} | {'Efficiency':<10} | {'Notes'}"
|
||||
logger.info(header, tag="TEST")
|
||||
logger.info("-" * 120, tag="TEST")
|
||||
|
||||
# Print each configuration in ranked order
|
||||
for rank, result in enumerate(ranked_results, 1):
|
||||
# Add special notes for top performers
|
||||
notes = []
|
||||
if result == fastest:
|
||||
notes.append("⚡ Fastest")
|
||||
if result == lowest_memory:
|
||||
notes.append("💾 Lowest Memory")
|
||||
if result == most_efficient:
|
||||
notes.append("🌟 Most Efficient")
|
||||
|
||||
notes_str = " | ".join(notes) if notes else ""
|
||||
|
||||
# Format memory if available
|
||||
memory_str = f"{result['peak_memory']:.1f}" if result['peak_memory'] > 0 else "N/A"
|
||||
|
||||
# Get the distribution string
|
||||
dist_str = result.get('distribution_str', str(tuple([result['pages_per_browser']] * result['num_browsers'])))
|
||||
|
||||
# Build the row
|
||||
row = f"{rank:<5} | {result['num_browsers']:<8} | {dist_str:<55} | {result['total_time']:.2f}s{' ':<7} | "
|
||||
row += f"{result['pages_per_second']:.2f}{' ':<6} | {memory_str}{' ':<6} | {result['efficiency_score']:.4f}{' ':<4} | {notes_str}"
|
||||
|
||||
logger.info(row, tag="TEST")
|
||||
|
||||
logger.info("-" * 120, tag="TEST")
|
||||
|
||||
# Generate visualization if matplotlib is available
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
# Extract data for plotting from ranked results
|
||||
browser_counts = [r["num_browsers"] for r in ranked_results]
|
||||
efficiency_scores = [r["efficiency_score"] for r in ranked_results]
|
||||
crawl_times = [r["crawl_time"] for r in ranked_results]
|
||||
total_times = [r["total_time"] for r in ranked_results]
|
||||
|
||||
# Filter results with memory data
|
||||
memory_results = [r for r in ranked_results if r["peak_memory"] > 0]
|
||||
memory_browser_counts = [r["num_browsers"] for r in memory_results]
|
||||
peak_memories = [r["peak_memory"] for r in memory_results]
|
||||
|
||||
# Create figure with clean design
|
||||
plt.figure(figsize=(14, 12), facecolor='white')
|
||||
plt.style.use('ggplot')
|
||||
|
||||
# Create grid for subplots
|
||||
gs = plt.GridSpec(3, 1, height_ratios=[1, 1, 1], hspace=0.3)
|
||||
|
||||
# Plot 1: Efficiency Score (higher is better)
|
||||
ax1 = plt.subplot(gs[0])
|
||||
bar_colors = ['#3498db'] * len(browser_counts)
|
||||
|
||||
# Highlight the most efficient
|
||||
most_efficient_idx = browser_counts.index(most_efficient["num_browsers"])
|
||||
bar_colors[most_efficient_idx] = '#e74c3c' # Red for most efficient
|
||||
|
||||
bars = ax1.bar(range(len(browser_counts)), efficiency_scores, color=bar_colors)
|
||||
ax1.set_xticks(range(len(browser_counts)))
|
||||
ax1.set_xticklabels([f"{bc}" for bc in browser_counts], rotation=45)
|
||||
ax1.set_xlabel('Number of Browsers')
|
||||
ax1.set_ylabel('Efficiency Score (higher is better)')
|
||||
ax1.set_title('Browser Configuration Efficiency (higher is better)')
|
||||
|
||||
# Add value labels on top of bars
|
||||
for bar, score in zip(bars, efficiency_scores):
|
||||
height = bar.get_height()
|
||||
ax1.text(bar.get_x() + bar.get_width()/2., height + 0.02*max(efficiency_scores),
|
||||
f'{score:.3f}', ha='center', va='bottom', rotation=90, fontsize=8)
|
||||
|
||||
# Highlight best configuration
|
||||
ax1.text(0.02, 0.90, f"🌟 Most Efficient: {most_efficient['num_browsers']} browsers with ~{most_efficient['pages_per_browser']} pages",
|
||||
transform=ax1.transAxes, fontsize=12, verticalalignment='top',
|
||||
bbox=dict(boxstyle='round,pad=0.5', facecolor='yellow', alpha=0.3))
|
||||
|
||||
# Plot 2: Time Performance
|
||||
ax2 = plt.subplot(gs[1])
|
||||
|
||||
# Plot both total time and crawl time
|
||||
ax2.plot(browser_counts, crawl_times, 'bo-', label='Crawl Time (s)', linewidth=2)
|
||||
ax2.plot(browser_counts, total_times, 'go--', label='Total Time (s)', linewidth=2, alpha=0.6)
|
||||
|
||||
# Mark the fastest configuration
|
||||
fastest_idx = browser_counts.index(fastest["num_browsers"])
|
||||
ax2.plot(browser_counts[fastest_idx], crawl_times[fastest_idx], 'ro', ms=10,
|
||||
label=f'Fastest: {fastest["num_browsers"]} browsers')
|
||||
|
||||
ax2.set_xlabel('Number of Browsers')
|
||||
ax2.set_ylabel('Time (seconds)')
|
||||
ax2.set_title(f'Time Performance for {total_urls} URLs by Browser Count')
|
||||
ax2.grid(True, linestyle='--', alpha=0.7)
|
||||
ax2.legend(loc='upper right')
|
||||
|
||||
# Plot pages per second on second y-axis
|
||||
pages_per_second = [total_urls/t for t in crawl_times]
|
||||
ax2_twin = ax2.twinx()
|
||||
ax2_twin.plot(browser_counts, pages_per_second, 'r^--', label='Pages/second', alpha=0.5)
|
||||
ax2_twin.set_ylabel('Pages per second')
|
||||
|
||||
# Add note about the fastest configuration
|
||||
ax2.text(0.02, 0.90, f"⚡ Fastest: {fastest['num_browsers']} browsers with ~{fastest['pages_per_browser']} pages" +
|
||||
f"\n {fastest['crawl_time']:.2f}s ({fastest['pages_per_second']:.1f} pages/s)",
|
||||
transform=ax2.transAxes, fontsize=12, verticalalignment='top',
|
||||
bbox=dict(boxstyle='round,pad=0.5', facecolor='lightblue', alpha=0.3))
|
||||
|
||||
# Plot 3: Memory Usage (if available)
|
||||
if memory_results:
|
||||
ax3 = plt.subplot(gs[2])
|
||||
|
||||
# Prepare data for grouped bar chart
|
||||
memory_per_browser = [m/n for m, n in zip(peak_memories, memory_browser_counts)]
|
||||
memory_per_page = [m/(n*p) for m, n, p in zip(
|
||||
[r["peak_memory"] for r in memory_results],
|
||||
[r["num_browsers"] for r in memory_results],
|
||||
[r["pages_per_browser"] for r in memory_results])]
|
||||
|
||||
x = np.arange(len(memory_browser_counts))
|
||||
width = 0.35
|
||||
|
||||
# Create grouped bars
|
||||
ax3.bar(x - width/2, peak_memories, width, label='Total Memory (MB)', color='#9b59b6')
|
||||
ax3.bar(x + width/2, memory_per_browser, width, label='Memory per Browser (MB)', color='#3498db')
|
||||
|
||||
# Configure axis
|
||||
ax3.set_xticks(x)
|
||||
ax3.set_xticklabels([f"{bc}" for bc in memory_browser_counts], rotation=45)
|
||||
ax3.set_xlabel('Number of Browsers')
|
||||
ax3.set_ylabel('Memory (MB)')
|
||||
ax3.set_title('Memory Usage by Browser Configuration')
|
||||
ax3.legend(loc='upper left')
|
||||
ax3.grid(True, linestyle='--', alpha=0.7)
|
||||
|
||||
# Add second y-axis for memory per page
|
||||
ax3_twin = ax3.twinx()
|
||||
ax3_twin.plot(x, memory_per_page, 'ro-', label='Memory per Page (MB)')
|
||||
ax3_twin.set_ylabel('Memory per Page (MB)')
|
||||
|
||||
# Get lowest memory configuration
|
||||
lowest_memory_idx = memory_browser_counts.index(lowest_memory["num_browsers"])
|
||||
|
||||
# Add note about lowest memory configuration
|
||||
ax3.text(0.02, 0.90, f"💾 Lowest Memory: {lowest_memory['num_browsers']} browsers with ~{lowest_memory['pages_per_browser']} pages" +
|
||||
f"\n {lowest_memory['peak_memory']:.1f}MB ({lowest_memory['peak_memory']/total_urls:.2f}MB per page)",
|
||||
transform=ax3.transAxes, fontsize=12, verticalalignment='top',
|
||||
bbox=dict(boxstyle='round,pad=0.5', facecolor='lightgreen', alpha=0.3))
|
||||
|
||||
# Add overall title
|
||||
plt.suptitle(f'Browser Scaling Grid Search Results for {total_urls} URLs', fontsize=16, y=0.98)
|
||||
|
||||
# Add timestamp and info at the bottom
|
||||
plt.figtext(0.5, 0.01, f"Generated by Crawl4AI at {time.strftime('%Y-%m-%d %H:%M:%S')}",
|
||||
ha="center", fontsize=10, style='italic')
|
||||
|
||||
# Get current directory and save the figure there
|
||||
import os
|
||||
__current_file = os.path.abspath(__file__)
|
||||
current_dir = os.path.dirname(__current_file)
|
||||
output_file = os.path.join(current_dir, 'browser_scaling_grid_search.png')
|
||||
|
||||
# Adjust layout and save figure with high DPI
|
||||
plt.tight_layout(rect=[0, 0.03, 1, 0.97])
|
||||
plt.savefig(output_file, dpi=200, bbox_inches='tight')
|
||||
logger.success(f"Visualization saved to {output_file}", tag="TEST")
|
||||
|
||||
except ImportError:
|
||||
logger.warning("matplotlib not available, skipping visualization", tag="TEST")
|
||||
|
||||
return most_efficient["num_browsers"], most_efficient["pages_per_browser"]
|
||||
|
||||
async def find_optimal_browser_config(total_urls=50, verbose=True, rate_limit_delay=0.2):
|
||||
"""Find optimal browser configuration for crawling a specific number of URLs.
|
||||
|
||||
Args:
|
||||
total_urls: Number of URLs to crawl
|
||||
verbose: Whether to print progress
|
||||
rate_limit_delay: Delay between page loads to avoid rate limiting
|
||||
|
||||
Returns:
|
||||
dict: Contains fastest, lowest_memory, and optimal configurations
|
||||
"""
|
||||
if verbose:
|
||||
print(f"\n=== Finding optimal configuration for crawling {total_urls} URLs ===\n")
|
||||
|
||||
# Generate test URLs with timestamp to avoid caching
|
||||
timestamp = int(time.time())
|
||||
urls = [f"https://example.com/page_{i}?t={timestamp}" for i in range(total_urls)]
|
||||
|
||||
# Limit browser configurations to test (1 browser to max 10)
|
||||
max_browsers = min(10, total_urls)
|
||||
configs_to_test = []
|
||||
|
||||
# Generate configurations (browser count, pages distribution)
|
||||
for num_browsers in range(1, max_browsers + 1):
|
||||
base_pages = total_urls // num_browsers
|
||||
remainder = total_urls % num_browsers
|
||||
|
||||
# Create distribution array like [3, 3, 2, 2] (some browsers get one more page)
|
||||
if remainder > 0:
|
||||
distribution = [base_pages + 1] * remainder + [base_pages] * (num_browsers - remainder)
|
||||
else:
|
||||
distribution = [base_pages] * num_browsers
|
||||
|
||||
configs_to_test.append((num_browsers, distribution))
|
||||
|
||||
results = []
|
||||
|
||||
# Test each configuration
|
||||
for browser_count, page_distribution in configs_to_test:
|
||||
if verbose:
|
||||
print(f"Testing {browser_count} browsers with distribution {tuple(page_distribution)}")
|
||||
|
||||
try:
|
||||
# Track memory if possible
|
||||
try:
|
||||
import psutil
|
||||
process = psutil.Process()
|
||||
start_memory = process.memory_info().rss / (1024 * 1024) # MB
|
||||
except ImportError:
|
||||
if verbose:
|
||||
print("Memory tracking not available (psutil not installed)")
|
||||
start_memory = 0
|
||||
|
||||
# Start browsers in parallel
|
||||
managers = []
|
||||
start_tasks = []
|
||||
start_time = time.time()
|
||||
|
||||
for i in range(browser_count):
|
||||
config = BrowserConfig(headless=True)
|
||||
manager = BrowserManager(browser_config=config, logger=logger)
|
||||
start_tasks.append(manager.start())
|
||||
managers.append(manager)
|
||||
|
||||
await asyncio.gather(*start_tasks)
|
||||
|
||||
# Distribute URLs among browsers
|
||||
urls_per_manager = {}
|
||||
url_index = 0
|
||||
|
||||
for i, manager in enumerate(managers):
|
||||
pages_for_this_browser = page_distribution[i]
|
||||
end_index = url_index + pages_for_this_browser
|
||||
urls_per_manager[manager] = urls[url_index:end_index]
|
||||
url_index = end_index
|
||||
|
||||
# Create pages for each browser
|
||||
all_pages = []
|
||||
for manager, manager_urls in urls_per_manager.items():
|
||||
if not manager_urls:
|
||||
continue
|
||||
pages = await manager.get_pages(CrawlerRunConfig(), count=len(manager_urls))
|
||||
all_pages.extend(zip(pages, manager_urls))
|
||||
|
||||
# Crawl pages with delay to avoid rate limiting
|
||||
async def crawl_page(page_ctx, url):
|
||||
page, _ = page_ctx
|
||||
try:
|
||||
await page.goto(url)
|
||||
if rate_limit_delay > 0:
|
||||
await asyncio.sleep(rate_limit_delay)
|
||||
title = await page.title()
|
||||
return title
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
crawl_start = time.time()
|
||||
crawl_tasks = [crawl_page(page_ctx, url) for page_ctx, url in all_pages]
|
||||
await asyncio.gather(*crawl_tasks)
|
||||
crawl_time = time.time() - crawl_start
|
||||
total_time = time.time() - start_time
|
||||
|
||||
# Measure final memory usage
|
||||
if start_memory > 0:
|
||||
end_memory = process.memory_info().rss / (1024 * 1024)
|
||||
memory_used = end_memory - start_memory
|
||||
else:
|
||||
memory_used = 0
|
||||
|
||||
# Close all browsers
|
||||
for manager in managers:
|
||||
await manager.close()
|
||||
|
||||
# Calculate metrics
|
||||
pages_per_second = total_urls / crawl_time
|
||||
|
||||
# Calculate efficiency score (higher is better)
|
||||
# This balances speed vs memory
|
||||
if memory_used > 0:
|
||||
efficiency = pages_per_second / (memory_used + 1)
|
||||
else:
|
||||
efficiency = pages_per_second
|
||||
|
||||
# Store result
|
||||
result = {
|
||||
"browser_count": browser_count,
|
||||
"distribution": tuple(page_distribution),
|
||||
"crawl_time": crawl_time,
|
||||
"total_time": total_time,
|
||||
"memory_used": memory_used,
|
||||
"pages_per_second": pages_per_second,
|
||||
"efficiency": efficiency
|
||||
}
|
||||
|
||||
results.append(result)
|
||||
|
||||
if verbose:
|
||||
print(f" ✓ Crawled {total_urls} pages in {crawl_time:.2f}s ({pages_per_second:.1f} pages/sec)")
|
||||
if memory_used > 0:
|
||||
print(f" ✓ Memory used: {memory_used:.1f}MB ({memory_used/total_urls:.1f}MB per page)")
|
||||
print(f" ✓ Efficiency score: {efficiency:.4f}")
|
||||
|
||||
except Exception as e:
|
||||
if verbose:
|
||||
print(f" ✗ Error: {str(e)}")
|
||||
|
||||
# Clean up
|
||||
for manager in managers:
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# If no successful results, return None
|
||||
if not results:
|
||||
return None
|
||||
|
||||
# Find best configurations
|
||||
fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
|
||||
|
||||
# Only consider memory if available
|
||||
memory_results = [r for r in results if r["memory_used"] > 0]
|
||||
if memory_results:
|
||||
lowest_memory = sorted(memory_results, key=lambda x: x["memory_used"])[0]
|
||||
else:
|
||||
lowest_memory = fastest
|
||||
|
||||
# Find most efficient (balanced speed vs memory)
|
||||
optimal = sorted(results, key=lambda x: x["efficiency"], reverse=True)[0]
|
||||
|
||||
# Print summary
|
||||
if verbose:
|
||||
print("\n=== OPTIMAL CONFIGURATIONS ===")
|
||||
print(f"⚡ Fastest: {fastest['browser_count']} browsers {fastest['distribution']}")
|
||||
print(f" {fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/sec")
|
||||
|
||||
print(f"💾 Memory-efficient: {lowest_memory['browser_count']} browsers {lowest_memory['distribution']}")
|
||||
if lowest_memory["memory_used"] > 0:
|
||||
print(f" {lowest_memory['memory_used']:.1f}MB, {lowest_memory['memory_used']/total_urls:.2f}MB per page")
|
||||
|
||||
print(f"🌟 Balanced optimal: {optimal['browser_count']} browsers {optimal['distribution']}")
|
||||
print(f" {optimal['crawl_time']:.2f}s, {optimal['pages_per_second']:.1f} pages/sec, score: {optimal['efficiency']:.4f}")
|
||||
|
||||
return {
|
||||
"fastest": fastest,
|
||||
"lowest_memory": lowest_memory,
|
||||
"optimal": optimal,
|
||||
"all_configs": results
|
||||
}
|
||||
|
||||
async def run_tests():
|
||||
"""Run all tests sequentially."""
|
||||
results = []
|
||||
|
||||
# Find optimal configuration using our utility function
|
||||
configs = await find_optimal_browser_config(
|
||||
total_urls=20, # Use a small number for faster testing
|
||||
verbose=True,
|
||||
rate_limit_delay=0.2 # 200ms delay between page loads to avoid rate limiting
|
||||
)
|
||||
|
||||
if configs:
|
||||
# Show the optimal configuration
|
||||
optimal = configs["optimal"]
|
||||
print(f"\n🎯 Recommended configuration for production use:")
|
||||
print(f" {optimal['browser_count']} browsers with distribution {optimal['distribution']}")
|
||||
print(f" Estimated performance: {optimal['pages_per_second']:.1f} pages/second")
|
||||
results.append(True)
|
||||
else:
|
||||
print("\n❌ Failed to find optimal configuration")
|
||||
results.append(False)
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(results)
|
||||
print(f"\nTests complete: {passed}/{total} passed")
|
||||
|
||||
if passed == total:
|
||||
print("All tests passed!")
|
||||
else:
|
||||
print(f"{total - passed} tests failed")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
@@ -0,0 +1,316 @@
|
||||
"""Test examples for PlaywrightBrowserStrategy.
|
||||
|
||||
These examples demonstrate the functionality of PlaywrightBrowserStrategy
|
||||
and serve as functional tests.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.browser import BrowserManager
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
|
||||
|
||||
async def test_start_close():
|
||||
# Create browser config for standard Playwright
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
viewport_width=1280,
|
||||
viewport_height=800
|
||||
)
|
||||
|
||||
# Create browser manager with the config
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
for _ in range(4):
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Get a page
|
||||
page, context = await manager.get_page(CrawlerRunConfig())
|
||||
logger.info("Got page successfully", tag="TEST")
|
||||
|
||||
# Navigate to a website
|
||||
await page.goto("https://example.com")
|
||||
logger.info("Navigated to example.com", tag="TEST")
|
||||
|
||||
# Get page title
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
await asyncio.sleep(1) # Wait for a moment before restarting
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
return True
|
||||
|
||||
async def test_playwright_basic():
|
||||
"""Test basic Playwright browser functionality."""
|
||||
logger.info("Testing standard Playwright browser", tag="TEST")
|
||||
|
||||
# Create browser config for standard Playwright
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
viewport_width=1280,
|
||||
viewport_height=800
|
||||
)
|
||||
|
||||
# Create browser manager with the config
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Create crawler config
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
|
||||
# Get a page
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
logger.info("Got page successfully", tag="TEST")
|
||||
|
||||
# Navigate to a website
|
||||
await page.goto("https://example.com")
|
||||
logger.info("Navigated to example.com", tag="TEST")
|
||||
|
||||
# Get page title
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_playwright_text_mode():
|
||||
"""Test Playwright browser in text-only mode."""
|
||||
logger.info("Testing Playwright text mode", tag="TEST")
|
||||
|
||||
# Create browser config with text mode enabled
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
text_mode=True # Enable text-only mode
|
||||
)
|
||||
|
||||
# Create browser manager with the config
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully in text mode", tag="TEST")
|
||||
|
||||
# Get a page
|
||||
crawler_config = CrawlerRunConfig(url="https://example.com")
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
|
||||
# Navigate to a website
|
||||
await page.goto("https://example.com")
|
||||
logger.info("Navigated to example.com", tag="TEST")
|
||||
|
||||
# Get page title
|
||||
title = await page.title()
|
||||
logger.info(f"Page title: {title}", tag="TEST")
|
||||
|
||||
# Check if images are blocked in text mode
|
||||
# We'll check if any image requests were made
|
||||
has_images = False
|
||||
async with page.expect_request("**/*.{png,jpg,jpeg,gif,webp,svg}", timeout=1000) as request_info:
|
||||
try:
|
||||
# Try to load a page with images
|
||||
await page.goto("https://picsum.photos/", wait_until="domcontentloaded")
|
||||
request = await request_info.value
|
||||
has_images = True
|
||||
except:
|
||||
# Timeout without image requests means text mode is working
|
||||
has_images = False
|
||||
|
||||
logger.info(f"Text mode image blocking working: {not has_images}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_playwright_context_reuse():
|
||||
"""Test context caching and reuse with identical configurations."""
|
||||
logger.info("Testing context reuse with identical configurations", tag="TEST")
|
||||
|
||||
# Create browser config
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
# Create browser manager
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await manager.start()
|
||||
logger.info("Browser started successfully", tag="TEST")
|
||||
|
||||
# Create identical crawler configs
|
||||
crawler_config1 = CrawlerRunConfig(
|
||||
css_selector="body",
|
||||
)
|
||||
|
||||
crawler_config2 = CrawlerRunConfig(
|
||||
css_selector="body",
|
||||
)
|
||||
|
||||
# Get pages with these configs
|
||||
page1, context1 = await manager.get_page(crawler_config1)
|
||||
page2, context2 = await manager.get_page(crawler_config2)
|
||||
|
||||
# Check if contexts are reused
|
||||
is_same_context = context1 == context2
|
||||
logger.info(f"Contexts reused: {is_same_context}", tag="TEST")
|
||||
|
||||
# Now try with a different config
|
||||
crawler_config3 = CrawlerRunConfig()
|
||||
|
||||
page3, context3 = await manager.get_page(crawler_config3)
|
||||
|
||||
# This should be a different context
|
||||
is_different_context = context1 != context3
|
||||
logger.info(f"Different contexts for different configs: {is_different_context}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
# Both tests should pass for success
|
||||
return is_same_context and is_different_context
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Ensure cleanup
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_playwright_session_management():
|
||||
"""Test session management with Playwright browser."""
|
||||
logger.info("Testing session management with Playwright browser", tag="TEST")
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
try:
|
||||
await manager.start()
|
||||
logger.info("Browser launched successfully", tag="TEST")
|
||||
|
||||
# Create two sessions
|
||||
session1_id = "playwright_session_1"
|
||||
session2_id = "playwright_session_2"
|
||||
|
||||
# Set up first session
|
||||
crawler_config1 = CrawlerRunConfig(session_id=session1_id, url="https://example.com")
|
||||
page1, context1 = await manager.get_page(crawler_config1)
|
||||
await page1.goto("https://example.com")
|
||||
await page1.evaluate("localStorage.setItem('playwright_session1_data', 'test_value1')")
|
||||
logger.info(f"Set up session 1 with ID: {session1_id}", tag="TEST")
|
||||
|
||||
# Set up second session
|
||||
crawler_config2 = CrawlerRunConfig(session_id=session2_id, url="https://example.org")
|
||||
page2, context2 = await manager.get_page(crawler_config2)
|
||||
await page2.goto("https://example.org")
|
||||
await page2.evaluate("localStorage.setItem('playwright_session2_data', 'test_value2')")
|
||||
logger.info(f"Set up session 2 with ID: {session2_id}", tag="TEST")
|
||||
|
||||
# Get first session again
|
||||
page1_again, context1_again = await manager.get_page(crawler_config1)
|
||||
|
||||
# Verify it's the same page and data persists
|
||||
is_same_page = page1 == page1_again
|
||||
is_same_context = context1 == context1_again
|
||||
data1 = await page1_again.evaluate("localStorage.getItem('playwright_session1_data')")
|
||||
logger.info(f"Session 1 reuse successful: {is_same_page}, data: {data1}", tag="TEST")
|
||||
|
||||
# Kill first session
|
||||
await manager.kill_session(session1_id)
|
||||
logger.info(f"Killed session 1", tag="TEST")
|
||||
|
||||
# Verify second session still works
|
||||
data2 = await page2.evaluate("localStorage.getItem('playwright_session2_data')")
|
||||
logger.info(f"Session 2 still functional after killing session 1, data: {data2}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager.close()
|
||||
logger.info("Browser closed successfully", tag="TEST")
|
||||
|
||||
return is_same_page and is_same_context and data1 == "test_value1" and data2 == "test_value2"
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
try:
|
||||
await manager.close()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def run_tests():
|
||||
"""Run all tests sequentially."""
|
||||
results = []
|
||||
|
||||
# results.append(await test_start_close())
|
||||
# results.append(await test_playwright_basic())
|
||||
# results.append(await test_playwright_text_mode())
|
||||
# results.append(await test_playwright_context_reuse())
|
||||
results.append(await test_playwright_session_management())
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(results)
|
||||
logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
|
||||
|
||||
if passed == total:
|
||||
logger.success("All tests passed!", tag="SUMMARY")
|
||||
else:
|
||||
logger.error(f"{total - passed} tests failed", tag="SUMMARY")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,180 @@
|
||||
"""Test examples for BrowserProfileManager.
|
||||
|
||||
These examples demonstrate the functionality of BrowserProfileManager
|
||||
and serve as functional tests.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
import shutil
|
||||
|
||||
from crawl4ai import BrowserProfiler
|
||||
from crawl4ai.browser_manager import BrowserManager
|
||||
|
||||
# Add the project root to Python path if running directly
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
# Create a logger for clear terminal output
|
||||
logger = AsyncLogger(verbose=True, log_file=None)
|
||||
|
||||
async def test_profile_creation():
|
||||
"""Test creating and managing browser profiles."""
|
||||
logger.info("Testing profile creation and management", tag="TEST")
|
||||
|
||||
profile_manager = BrowserProfiler(logger=logger)
|
||||
|
||||
try:
|
||||
# List existing profiles
|
||||
profiles = profile_manager.list_profiles()
|
||||
logger.info(f"Found {len(profiles)} existing profiles", tag="TEST")
|
||||
|
||||
# Generate a unique profile name for testing
|
||||
test_profile_name = f"test-profile-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
# Create a test profile directory
|
||||
profile_path = os.path.join(profile_manager.profiles_dir, test_profile_name)
|
||||
os.makedirs(os.path.join(profile_path, "Default"), exist_ok=True)
|
||||
|
||||
# Create a dummy Preferences file to simulate a Chrome profile
|
||||
with open(os.path.join(profile_path, "Default", "Preferences"), "w") as f:
|
||||
f.write("{\"test\": true}")
|
||||
|
||||
logger.info(f"Created test profile at: {profile_path}", tag="TEST")
|
||||
|
||||
# Verify the profile is now in the list
|
||||
profiles = profile_manager.list_profiles()
|
||||
profile_found = any(p["name"] == test_profile_name for p in profiles)
|
||||
logger.info(f"Profile found in list: {profile_found}", tag="TEST")
|
||||
|
||||
# Try to get the profile path
|
||||
retrieved_path = profile_manager.get_profile_path(test_profile_name)
|
||||
path_match = retrieved_path == profile_path
|
||||
logger.info(f"Retrieved correct profile path: {path_match}", tag="TEST")
|
||||
|
||||
# Delete the profile
|
||||
success = profile_manager.delete_profile(test_profile_name)
|
||||
logger.info(f"Profile deletion successful: {success}", tag="TEST")
|
||||
|
||||
# Verify it's gone
|
||||
profiles_after = profile_manager.list_profiles()
|
||||
profile_removed = not any(p["name"] == test_profile_name for p in profiles_after)
|
||||
logger.info(f"Profile removed from list: {profile_removed}", tag="TEST")
|
||||
|
||||
# Clean up just in case
|
||||
if os.path.exists(profile_path):
|
||||
shutil.rmtree(profile_path, ignore_errors=True)
|
||||
|
||||
return profile_found and path_match and success and profile_removed
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Clean up test directory
|
||||
try:
|
||||
if os.path.exists(profile_path):
|
||||
shutil.rmtree(profile_path, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def test_profile_with_browser():
|
||||
"""Test using a profile with a browser."""
|
||||
logger.info("Testing using a profile with a browser", tag="TEST")
|
||||
|
||||
profile_manager = BrowserProfiler(logger=logger)
|
||||
test_profile_name = f"test-browser-profile-{uuid.uuid4().hex[:8]}"
|
||||
profile_path = None
|
||||
|
||||
try:
|
||||
# Create a test profile directory
|
||||
profile_path = os.path.join(profile_manager.profiles_dir, test_profile_name)
|
||||
os.makedirs(os.path.join(profile_path, "Default"), exist_ok=True)
|
||||
|
||||
# Create a dummy Preferences file to simulate a Chrome profile
|
||||
with open(os.path.join(profile_path, "Default", "Preferences"), "w") as f:
|
||||
f.write("{\"test\": true}")
|
||||
|
||||
logger.info(f"Created test profile at: {profile_path}", tag="TEST")
|
||||
|
||||
# Now use this profile with a browser
|
||||
browser_config = BrowserConfig(
|
||||
user_data_dir=profile_path,
|
||||
use_managed_browser=True,
|
||||
use_persistent_context=True,
|
||||
headless=True
|
||||
)
|
||||
|
||||
manager = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
|
||||
# Start the browser with the profile
|
||||
await manager.start()
|
||||
logger.info("Browser started with profile", tag="TEST")
|
||||
|
||||
# Create a page
|
||||
crawler_config = CrawlerRunConfig()
|
||||
page, context = await manager.get_page(crawler_config)
|
||||
|
||||
# Navigate and set some data to verify profile works
|
||||
await page.goto("https://example.com")
|
||||
await page.evaluate("localStorage.setItem('test_data', 'profile_value')")
|
||||
|
||||
# Close browser
|
||||
await manager.close()
|
||||
logger.info("First browser session closed", tag="TEST")
|
||||
|
||||
# Create a new browser with the same profile
|
||||
manager2 = BrowserManager(browser_config=browser_config, logger=logger)
|
||||
await manager2.start()
|
||||
logger.info("Second browser session started with same profile", tag="TEST")
|
||||
|
||||
# Get a page and check if the data persists
|
||||
page2, context2 = await manager2.get_page(crawler_config)
|
||||
await page2.goto("https://example.com")
|
||||
data = await page2.evaluate("localStorage.getItem('test_data')")
|
||||
|
||||
# Verify data persisted
|
||||
data_persisted = data == "profile_value"
|
||||
logger.info(f"Data persisted across sessions: {data_persisted}", tag="TEST")
|
||||
|
||||
# Clean up
|
||||
await manager2.close()
|
||||
logger.info("Second browser session closed", tag="TEST")
|
||||
|
||||
# Delete the test profile
|
||||
success = profile_manager.delete_profile(test_profile_name)
|
||||
logger.info(f"Test profile deleted: {success}", tag="TEST")
|
||||
|
||||
return data_persisted and success
|
||||
except Exception as e:
|
||||
logger.error(f"Test failed: {str(e)}", tag="TEST")
|
||||
# Clean up
|
||||
try:
|
||||
if profile_path and os.path.exists(profile_path):
|
||||
shutil.rmtree(profile_path, ignore_errors=True)
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def run_tests():
|
||||
"""Run all tests sequentially."""
|
||||
results = []
|
||||
|
||||
results.append(await test_profile_creation())
|
||||
results.append(await test_profile_with_browser())
|
||||
|
||||
# Print summary
|
||||
total = len(results)
|
||||
passed = sum(results)
|
||||
logger.info(f"Tests complete: {passed}/{total} passed", tag="SUMMARY")
|
||||
|
||||
if passed == total:
|
||||
logger.success("All tests passed!", tag="SUMMARY")
|
||||
else:
|
||||
logger.error(f"{total - passed} tests failed", tag="SUMMARY")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
@@ -0,0 +1,424 @@
|
||||
"""
|
||||
Regression tests for PR #1640 — memory leak / hang under high concurrency
|
||||
with max_pages_before_recycle enabled.
|
||||
|
||||
Tests three bugs that were fixed:
|
||||
|
||||
Bug 1: Race condition — release_page_with_context() runs BEFORE
|
||||
_maybe_bump_browser_version() adds the sig to _pending_cleanup.
|
||||
FIX: Don't add refcount-0 sigs to pending; clean them up immediately.
|
||||
|
||||
Bug 2: The finally block in _crawl_web can fail before calling
|
||||
release_page_with_context(), leaking the refcount permanently.
|
||||
FIX: Call release_page_with_context() FIRST in the finally block.
|
||||
|
||||
Bug 3: Accumulated pending_cleanup entries hit _max_pending_browsers cap,
|
||||
blocking ALL get_page() calls → system-wide deadlock.
|
||||
FIX: 30s timeout on safety cap wait + force-clean stuck entries.
|
||||
|
||||
Exit code 0 = all tests pass. Exit code 1 = regression found.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.browser_manager import BrowserManager
|
||||
|
||||
PASS = 0
|
||||
FAIL = 0
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
global PASS, FAIL
|
||||
if condition:
|
||||
PASS += 1
|
||||
print(f" PASS: {name}")
|
||||
else:
|
||||
FAIL += 1
|
||||
print(f" FAIL: {name}")
|
||||
|
||||
|
||||
async def test_bug1_multi_config_race():
|
||||
"""
|
||||
Bug 1 fix: idle sigs (refcount=0) must NOT be added to _pending_cleanup.
|
||||
They should be cleaned up immediately during the version bump.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST: Bug 1 — idle sig must not get stuck in _pending_cleanup")
|
||||
print("="*70)
|
||||
|
||||
config = BrowserConfig(
|
||||
headless=True,
|
||||
extra_args=['--no-sandbox', '--disable-gpu'],
|
||||
max_pages_before_recycle=3,
|
||||
)
|
||||
bm = BrowserManager(config)
|
||||
await bm.start()
|
||||
|
||||
try:
|
||||
config_a = CrawlerRunConfig(magic=True, cache_mode="bypass")
|
||||
config_b = CrawlerRunConfig(magic=False, cache_mode="bypass")
|
||||
|
||||
# Use config A, then release → refcount 0
|
||||
page_a, _ = await bm.get_page(config_a)
|
||||
sig_a = bm._page_to_sig.get(page_a)
|
||||
await bm.release_page_with_context(page_a)
|
||||
await page_a.close()
|
||||
|
||||
print(f" sig_a refcount after release: {bm._context_refcounts.get(sig_a)}")
|
||||
|
||||
# Use config B twice → pages_served hits threshold → version bump
|
||||
page_b1, _ = await bm.get_page(config_b)
|
||||
page_b2, _ = await bm.get_page(config_b)
|
||||
sig_b = bm._page_to_sig.get(page_b1)
|
||||
|
||||
# At this point the version should have bumped (3 pages served >= threshold 3)
|
||||
print(f" _browser_version: {bm._browser_version}")
|
||||
print(f" _pending_cleanup sigs: {list(bm._pending_cleanup.keys())}")
|
||||
|
||||
# sig_a (refcount=0) must NOT be in _pending_cleanup
|
||||
check("sig_a NOT in _pending_cleanup",
|
||||
sig_a not in bm._pending_cleanup)
|
||||
|
||||
# sig_a should have been cleaned up from _context_refcounts
|
||||
check("sig_a cleaned from _context_refcounts",
|
||||
sig_a not in bm._context_refcounts)
|
||||
|
||||
# sig_b (refcount>0) SHOULD be in _pending_cleanup (it will drain naturally)
|
||||
check("sig_b IS in _pending_cleanup (active, will drain)",
|
||||
sig_b in bm._pending_cleanup)
|
||||
|
||||
# Release B pages → sig_b drains → cleaned up
|
||||
await bm.release_page_with_context(page_b1)
|
||||
await page_b1.close()
|
||||
await bm.release_page_with_context(page_b2)
|
||||
await page_b2.close()
|
||||
|
||||
check("sig_b cleaned after release",
|
||||
sig_b not in bm._pending_cleanup)
|
||||
|
||||
check("_pending_cleanup is empty",
|
||||
len(bm._pending_cleanup) == 0)
|
||||
|
||||
finally:
|
||||
await bm.close()
|
||||
|
||||
|
||||
async def test_bug2_release_always_called():
|
||||
"""
|
||||
Bug 2 fix: release_page_with_context() must be called even when
|
||||
the browser is in a bad state.
|
||||
|
||||
The fix moves release_page_with_context() to the FIRST line of
|
||||
the finally block in _crawl_web, wrapped in try/except.
|
||||
Here we verify that release_page_with_context itself works even
|
||||
after browser crash, and that the fixed finally block pattern
|
||||
always decrements the refcount.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST: Bug 2 — release_page_with_context must work after browser crash")
|
||||
print("="*70)
|
||||
|
||||
config = BrowserConfig(
|
||||
headless=True,
|
||||
extra_args=['--no-sandbox', '--disable-gpu'],
|
||||
max_pages_before_recycle=5,
|
||||
)
|
||||
bm = BrowserManager(config)
|
||||
await bm.start()
|
||||
|
||||
try:
|
||||
crawl_config = CrawlerRunConfig(magic=True, cache_mode="bypass")
|
||||
|
||||
page, ctx = await bm.get_page(crawl_config)
|
||||
sig = bm._page_to_sig.get(page)
|
||||
print(f" sig refcount before crash: {bm._context_refcounts.get(sig)}")
|
||||
|
||||
check("refcount is 1 before crash",
|
||||
bm._context_refcounts.get(sig) == 1)
|
||||
|
||||
# Simulate browser crash
|
||||
if bm.browser:
|
||||
await bm.browser.close()
|
||||
bm.browser = None
|
||||
|
||||
# The FIX: call release_page_with_context even after crash
|
||||
# (simulating what the fixed finally block does)
|
||||
try:
|
||||
await bm.release_page_with_context(page)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
refcount_after = bm._context_refcounts.get(sig, 0)
|
||||
print(f" sig refcount after crash + release: {refcount_after}")
|
||||
|
||||
check("refcount decremented to 0 after crash + release",
|
||||
refcount_after == 0)
|
||||
|
||||
check("page removed from _page_to_sig",
|
||||
page not in bm._page_to_sig)
|
||||
|
||||
finally:
|
||||
bm.browser = None
|
||||
bm.contexts_by_config.clear()
|
||||
bm._context_refcounts.clear()
|
||||
bm._context_last_used.clear()
|
||||
bm._page_to_sig.clear()
|
||||
if bm.playwright:
|
||||
await bm.playwright.stop()
|
||||
|
||||
|
||||
async def test_bug3_safety_cap_timeout():
|
||||
"""
|
||||
Bug 3 fix: the safety cap wait must have a timeout.
|
||||
When stuck entries accumulate, the timeout fires and force-cleans
|
||||
entries with refcount 0, preventing permanent deadlock.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST: Bug 3 — safety cap wait must not block forever")
|
||||
print("="*70)
|
||||
|
||||
config = BrowserConfig(
|
||||
headless=True,
|
||||
extra_args=['--no-sandbox', '--disable-gpu'],
|
||||
max_pages_before_recycle=2,
|
||||
)
|
||||
bm = BrowserManager(config)
|
||||
await bm.start()
|
||||
|
||||
try:
|
||||
crawl_config = CrawlerRunConfig(magic=True, cache_mode="bypass")
|
||||
|
||||
# Inject stuck entries WITH refcount 0 (simulating leaked refcounts
|
||||
# that were later force-decremented or never properly tracked)
|
||||
print(f" Safety cap: {bm._max_pending_browsers}")
|
||||
for i in range(bm._max_pending_browsers):
|
||||
fake_sig = f"stuck_sig_{i}"
|
||||
bm._pending_cleanup[fake_sig] = {"version": i, "done": asyncio.Event()}
|
||||
# refcount 0 = stuck (no future release will clean these up)
|
||||
bm._context_refcounts[fake_sig] = 0
|
||||
|
||||
print(f" Injected {len(bm._pending_cleanup)} stuck entries (refcount=0)")
|
||||
|
||||
bm._pages_served = bm.config.max_pages_before_recycle
|
||||
|
||||
# The fix: get_page should NOT block forever.
|
||||
# The 30s timeout will fire, force-clean stuck entries, and proceed.
|
||||
# We use a 35s test timeout to allow the 30s internal timeout to fire.
|
||||
print(f" Calling get_page() — should unblock after ~30s timeout...")
|
||||
start = time.monotonic()
|
||||
try:
|
||||
page, ctx = await asyncio.wait_for(
|
||||
bm.get_page(crawl_config),
|
||||
timeout=35.0
|
||||
)
|
||||
elapsed = time.monotonic() - start
|
||||
print(f" get_page() returned after {elapsed:.1f}s")
|
||||
|
||||
check("get_page() did NOT deadlock (returned within 35s)", True)
|
||||
check("stuck entries were force-cleaned",
|
||||
len(bm._pending_cleanup) < bm._max_pending_browsers)
|
||||
|
||||
await bm.release_page_with_context(page)
|
||||
await page.close()
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
elapsed = time.monotonic() - start
|
||||
print(f" get_page() STILL blocked after {elapsed:.1f}s")
|
||||
check("get_page() did NOT deadlock", False)
|
||||
|
||||
finally:
|
||||
bm._pending_cleanup.clear()
|
||||
bm._context_refcounts.clear()
|
||||
await bm.close()
|
||||
|
||||
|
||||
async def test_real_concurrent_crawl():
|
||||
"""
|
||||
Integration test: run many concurrent crawls with recycling
|
||||
and verify no stuck entries or deadlocks.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST: Real concurrent crawls with recycling")
|
||||
print("="*70)
|
||||
|
||||
config = BrowserConfig(
|
||||
headless=True,
|
||||
extra_args=['--no-sandbox', '--disable-gpu'],
|
||||
max_pages_before_recycle=10,
|
||||
)
|
||||
bm = BrowserManager(config)
|
||||
await bm.start()
|
||||
|
||||
TOTAL = 80
|
||||
CONCURRENT = 8
|
||||
completed = 0
|
||||
errors = 0
|
||||
|
||||
sem = asyncio.Semaphore(CONCURRENT)
|
||||
|
||||
async def do_crawl(i):
|
||||
nonlocal completed, errors
|
||||
async with sem:
|
||||
try:
|
||||
crawl_config = CrawlerRunConfig(magic=True, cache_mode="bypass")
|
||||
page, ctx = await asyncio.wait_for(
|
||||
bm.get_page(crawl_config),
|
||||
timeout=30.0
|
||||
)
|
||||
|
||||
try:
|
||||
await page.goto("https://example.com", timeout=15000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Use the FIXED finally pattern: release first, then close
|
||||
try:
|
||||
await bm.release_page_with_context(page)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await page.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
completed += 1
|
||||
if completed % 20 == 0:
|
||||
print(f" [{completed}/{TOTAL}] version={bm._browser_version} "
|
||||
f"pending={len(bm._pending_cleanup)} "
|
||||
f"pages_served={bm._pages_served}")
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
errors += 1
|
||||
print(f" [{i}] TIMEOUT in get_page()!")
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
if errors <= 3:
|
||||
print(f" [{i}] Error: {e}")
|
||||
|
||||
start = time.monotonic()
|
||||
tasks = [asyncio.create_task(do_crawl(i)) for i in range(TOTAL)]
|
||||
await asyncio.gather(*tasks)
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
print(f"\n Results: {completed}/{TOTAL} completed, {errors} errors, {elapsed:.1f}s")
|
||||
|
||||
stuck = [s for s in bm._pending_cleanup if bm._context_refcounts.get(s, 0) == 0]
|
||||
|
||||
check(f"all {TOTAL} crawls completed", completed == TOTAL)
|
||||
check("no errors", errors == 0)
|
||||
check("no stuck entries in _pending_cleanup", len(stuck) == 0)
|
||||
check("no timeouts (no deadlock)", errors == 0)
|
||||
|
||||
await bm.close()
|
||||
|
||||
|
||||
async def test_multi_config_concurrent():
|
||||
"""
|
||||
Integration test: concurrent crawls with DIFFERENT configs to
|
||||
exercise the multi-sig path that triggered Bug 1.
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("TEST: Multi-config concurrent crawls")
|
||||
print("="*70)
|
||||
|
||||
config = BrowserConfig(
|
||||
headless=True,
|
||||
extra_args=['--no-sandbox', '--disable-gpu'],
|
||||
max_pages_before_recycle=5,
|
||||
)
|
||||
bm = BrowserManager(config)
|
||||
await bm.start()
|
||||
|
||||
TOTAL = 40
|
||||
CONCURRENT = 6
|
||||
completed = 0
|
||||
errors = 0
|
||||
|
||||
sem = asyncio.Semaphore(CONCURRENT)
|
||||
configs = [
|
||||
CrawlerRunConfig(magic=True, cache_mode="bypass"),
|
||||
CrawlerRunConfig(magic=False, cache_mode="bypass"),
|
||||
CrawlerRunConfig(magic=True, simulate_user=True, cache_mode="bypass"),
|
||||
]
|
||||
|
||||
async def do_crawl(i):
|
||||
nonlocal completed, errors
|
||||
async with sem:
|
||||
try:
|
||||
crawl_config = configs[i % len(configs)]
|
||||
page, ctx = await asyncio.wait_for(
|
||||
bm.get_page(crawl_config),
|
||||
timeout=30.0
|
||||
)
|
||||
|
||||
try:
|
||||
await page.goto("https://example.com", timeout=15000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
await bm.release_page_with_context(page)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await page.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
completed += 1
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
errors += 1
|
||||
print(f" [{i}] TIMEOUT!")
|
||||
print(f" pending={len(bm._pending_cleanup)}")
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
if errors <= 3:
|
||||
print(f" [{i}] Error: {e}")
|
||||
|
||||
start = time.monotonic()
|
||||
tasks = [asyncio.create_task(do_crawl(i)) for i in range(TOTAL)]
|
||||
await asyncio.gather(*tasks)
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
stuck = [s for s in bm._pending_cleanup if bm._context_refcounts.get(s, 0) == 0]
|
||||
|
||||
print(f"\n Results: {completed}/{TOTAL}, {errors} errors, {elapsed:.1f}s")
|
||||
print(f" Final: version={bm._browser_version} pending={len(bm._pending_cleanup)} stuck={len(stuck)}")
|
||||
|
||||
check(f"all {TOTAL} multi-config crawls completed", completed == TOTAL)
|
||||
check("no stuck entries", len(stuck) == 0)
|
||||
check("no timeouts", errors == 0)
|
||||
|
||||
await bm.close()
|
||||
|
||||
|
||||
async def main():
|
||||
print("="*70)
|
||||
print("PR #1640 Regression Tests")
|
||||
print("="*70)
|
||||
|
||||
await test_bug2_release_always_called()
|
||||
await test_bug1_multi_config_race()
|
||||
await test_bug3_safety_cap_timeout()
|
||||
await test_real_concurrent_crawl()
|
||||
await test_multi_config_concurrent()
|
||||
|
||||
print("\n" + "="*70)
|
||||
if FAIL == 0:
|
||||
print(f"ALL {PASS} CHECKS PASSED")
|
||||
else:
|
||||
print(f"FAILED: {FAIL} checks failed, {PASS} passed")
|
||||
print("="*70)
|
||||
|
||||
sys.exit(1 if FAIL > 0 else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,178 @@
|
||||
"""E2E tests for avoid_ads / avoid_css resource filtering.
|
||||
|
||||
These tests launch real browsers and crawl real websites to verify
|
||||
that route-based resource blocking actually works.
|
||||
|
||||
Domains used:
|
||||
- books.toscrape.com (CSS-heavy practice site, designed for scraping)
|
||||
- quotes.toscrape.com (simple practice site)
|
||||
- httpbin.org/html (static HTML, no trackers)
|
||||
- en.wikipedia.org (real site with analytics)
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Basic success tests — flags should not break crawling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_with_avoid_css_succeeds():
|
||||
"""Crawl books.toscrape.com with avoid_css=True — page should load fine."""
|
||||
browser_config = BrowserConfig(headless=True, avoid_css=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://books.toscrape.com",
|
||||
config=CrawlerRunConfig(cache_mode="bypass"),
|
||||
)
|
||||
assert result.success, f"Crawl failed: {result.error_message}"
|
||||
assert len(result.html) > 500, "Page HTML is suspiciously short"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_with_avoid_ads_succeeds():
|
||||
"""Crawl Wikipedia with avoid_ads=True — content should be intact."""
|
||||
browser_config = BrowserConfig(headless=True, avoid_ads=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://en.wikipedia.org/wiki/Web_scraping",
|
||||
config=CrawlerRunConfig(cache_mode="bypass"),
|
||||
)
|
||||
assert result.success, f"Crawl failed: {result.error_message}"
|
||||
# Wikipedia article content must be present
|
||||
html_lower = result.html.lower()
|
||||
assert "web scraping" in html_lower, "Wikipedia content missing"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_with_both_flags_succeeds():
|
||||
"""Both avoid_css and avoid_ads enabled simultaneously."""
|
||||
browser_config = BrowserConfig(headless=True, avoid_css=True, avoid_ads=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://quotes.toscrape.com",
|
||||
config=CrawlerRunConfig(cache_mode="bypass"),
|
||||
)
|
||||
assert result.success, f"Crawl failed: {result.error_message}"
|
||||
html_lower = result.html.lower()
|
||||
assert "quote" in html_lower or "toscrape" in html_lower
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_avoid_ads_does_not_block_page_content():
|
||||
"""avoid_ads must not interfere with first-party page content."""
|
||||
browser_config = BrowserConfig(headless=True, avoid_ads=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://httpbin.org/html",
|
||||
config=CrawlerRunConfig(cache_mode="bypass"),
|
||||
)
|
||||
assert result.success, f"Crawl failed: {result.error_message}"
|
||||
# httpbin.org/html serves a Moby Dick excerpt
|
||||
assert "Herman Melville" in result.html, "First-party content missing"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Network-level verification — prove routes actually block requests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_without_flags_css_loads_normally():
|
||||
"""Baseline: without avoid_css, CSS responses should appear in network log."""
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://books.toscrape.com",
|
||||
config=CrawlerRunConfig(
|
||||
cache_mode="bypass",
|
||||
capture_network_requests=True,
|
||||
),
|
||||
)
|
||||
assert result.success
|
||||
assert result.network_requests is not None, "Network requests not captured"
|
||||
|
||||
# There should be successful CSS responses
|
||||
css_responses = [
|
||||
r
|
||||
for r in result.network_requests
|
||||
if r.get("event_type") == "response" and ".css" in r.get("url", "")
|
||||
]
|
||||
assert (
|
||||
len(css_responses) > 0
|
||||
), "CSS should load normally without avoid_css flag"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_avoid_css_blocks_css_requests():
|
||||
"""With avoid_css=True, CSS requests must be aborted (no successful responses)."""
|
||||
browser_config = BrowserConfig(headless=True, avoid_css=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://books.toscrape.com",
|
||||
config=CrawlerRunConfig(
|
||||
cache_mode="bypass",
|
||||
capture_network_requests=True,
|
||||
),
|
||||
)
|
||||
assert result.success
|
||||
assert result.network_requests is not None, "Network requests not captured"
|
||||
|
||||
# No CSS should have gotten a successful response
|
||||
css_responses = [
|
||||
r
|
||||
for r in result.network_requests
|
||||
if r.get("event_type") == "response" and ".css" in r.get("url", "")
|
||||
]
|
||||
assert (
|
||||
len(css_responses) == 0
|
||||
), f"CSS responses should be blocked, but found: {[r['url'] for r in css_responses]}"
|
||||
|
||||
# There SHOULD be request_failed events for CSS (proves blocking happened)
|
||||
css_failures = [
|
||||
r
|
||||
for r in result.network_requests
|
||||
if r.get("event_type") == "request_failed"
|
||||
and ".css" in r.get("url", "")
|
||||
]
|
||||
assert (
|
||||
len(css_failures) > 0
|
||||
), "Expected request_failed events for blocked CSS files"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_avoid_css_with_text_mode_combines():
|
||||
"""Both avoid_css and text_mode should combine their blocking rules."""
|
||||
browser_config = BrowserConfig(
|
||||
headless=True, avoid_css=True, text_mode=True
|
||||
)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://books.toscrape.com",
|
||||
config=CrawlerRunConfig(
|
||||
cache_mode="bypass",
|
||||
capture_network_requests=True,
|
||||
),
|
||||
)
|
||||
assert result.success
|
||||
assert result.network_requests is not None
|
||||
|
||||
successful = [
|
||||
r for r in result.network_requests if r.get("event_type") == "response"
|
||||
]
|
||||
|
||||
# CSS should be blocked (via avoid_css)
|
||||
css_hits = [r for r in successful if ".css" in r.get("url", "")]
|
||||
assert len(css_hits) == 0, "CSS should be blocked by avoid_css"
|
||||
|
||||
# Images should be blocked (via text_mode)
|
||||
img_exts = (".jpg", ".jpeg", ".png", ".gif", ".webp")
|
||||
img_hits = [
|
||||
r
|
||||
for r in successful
|
||||
if any(r.get("url", "").lower().endswith(ext) for ext in img_exts)
|
||||
]
|
||||
assert len(img_hits) == 0, "Images should be blocked by text_mode"
|
||||
Reference in New Issue
Block a user