chore: import upstream snapshot with attribution

browser_use/README.md

@@ -0,0 +1,51 @@
+# Codebase Structure
+
+> The code structure inspired by https://github.com/Netflix/dispatch.
+
+Very good structure on how to make a scalable codebase is also in [this repo](https://github.com/zhanymkanov/fastapi-best-practices).
+
+Just a brief document about how we should structure our backend codebase.
+
+## Code Structure
+
+```markdown
+src/
+/<service name>/
+models.py
+services.py
+prompts.py
+views.py
+utils.py
+routers.py
+
+    	/_<subservice name>/
+```
+
+### Service.py
+
+Always a single file, except if it becomes too long - more than ~500 lines, split it into \_subservices
+
+### Views.py
+
+Always split the views into two parts
+
+```python
+# All
+...
+
+# Requests
+...
+
+# Responses
+...
+```
+
+If too long → split into multiple files
+
+### Prompts.py
+
+Single file; if too long → split into multiple files (one prompt per file or so)
+
+### Routers.py
+
+Never split into more than one file

browser_use/__init__.py

@@ -0,0 +1,157 @@
+import os
+from typing import TYPE_CHECKING
+
+from browser_use.logging_config import setup_logging
+
+# Only set up logging if not in MCP mode or if explicitly requested
+if os.environ.get('BROWSER_USE_SETUP_LOGGING', 'true').lower() != 'false':
+	from browser_use.config import CONFIG
+
+	# Get log file paths from config/environment
+	debug_log_file = getattr(CONFIG, 'BROWSER_USE_DEBUG_LOG_FILE', None)
+	info_log_file = getattr(CONFIG, 'BROWSER_USE_INFO_LOG_FILE', None)
+
+	# Set up logging with file handlers if specified
+	logger = setup_logging(debug_log_file=debug_log_file, info_log_file=info_log_file)
+else:
+	import logging
+
+	logger = logging.getLogger('browser_use')
+
+# Monkeypatch BaseSubprocessTransport.__del__ to handle closed event loops gracefully
+from asyncio import base_subprocess
+
+_original_del = base_subprocess.BaseSubprocessTransport.__del__
+
+
+def _patched_del(self):
+	"""Patched __del__ that handles closed event loops without throwing noisy red-herring errors like RuntimeError: Event loop is closed"""
+	try:
+		# Check if the event loop is closed before calling the original
+		if hasattr(self, '_loop') and self._loop and self._loop.is_closed():
+			# Event loop is closed, skip cleanup that requires the loop
+			return
+		_original_del(self)
+	except RuntimeError as e:
+		if 'Event loop is closed' in str(e):
+			# Silently ignore this specific error
+			pass
+		else:
+			raise
+
+
+base_subprocess.BaseSubprocessTransport.__del__ = _patched_del
+
+
+# Type stubs for lazy imports - fixes linter warnings
+if TYPE_CHECKING:
+	from browser_use.agent.prompts import SystemPrompt
+	from browser_use.agent.service import Agent
+	from browser_use.agent.views import ActionModel, ActionResult, AgentHistoryList
+	from browser_use.browser import BrowserProfile, BrowserSession
+	from browser_use.browser import BrowserSession as Browser
+	from browser_use.dom.service import DomService
+	from browser_use.llm import models
+	from browser_use.llm.anthropic.chat import ChatAnthropic
+	from browser_use.llm.azure.chat import ChatAzureOpenAI
+	from browser_use.llm.browser_use.chat import ChatBrowserUse
+	from browser_use.llm.google.chat import ChatGoogle
+	from browser_use.llm.groq.chat import ChatGroq
+	from browser_use.llm.litellm.chat import ChatLiteLLM
+	from browser_use.llm.mistral.chat import ChatMistral
+	from browser_use.llm.oci_raw.chat import ChatOCIRaw
+	from browser_use.llm.ollama.chat import ChatOllama
+	from browser_use.llm.openai.chat import ChatOpenAI
+	from browser_use.llm.vercel.chat import ChatVercel
+	from browser_use.sandbox import sandbox
+	from browser_use.tools.service import Controller, Tools
+
+	# Lazy imports mapping - only import when actually accessed
+_LAZY_IMPORTS = {
+	# Agent service (heavy due to dependencies)
+	'Agent': ('browser_use.agent.service', 'Agent'),
+	# System prompt (moderate weight due to agent.views imports)
+	'SystemPrompt': ('browser_use.agent.prompts', 'SystemPrompt'),
+	# Agent views (very heavy - over 1 second!)
+	'ActionModel': ('browser_use.agent.views', 'ActionModel'),
+	'ActionResult': ('browser_use.agent.views', 'ActionResult'),
+	'AgentHistoryList': ('browser_use.agent.views', 'AgentHistoryList'),
+	'BrowserSession': ('browser_use.browser', 'BrowserSession'),
+	'Browser': ('browser_use.browser', 'BrowserSession'),  # Alias for BrowserSession
+	'BrowserProfile': ('browser_use.browser', 'BrowserProfile'),
+	# Tools (moderate weight)
+	'Tools': ('browser_use.tools.service', 'Tools'),
+	'Controller': ('browser_use.tools.service', 'Controller'),  # alias
+	# DOM service (moderate weight)
+	'DomService': ('browser_use.dom.service', 'DomService'),
+	# Chat models (very heavy imports)
+	'ChatOpenAI': ('browser_use.llm.openai.chat', 'ChatOpenAI'),
+	'ChatGoogle': ('browser_use.llm.google.chat', 'ChatGoogle'),
+	'ChatAnthropic': ('browser_use.llm.anthropic.chat', 'ChatAnthropic'),
+	'ChatBrowserUse': ('browser_use.llm.browser_use.chat', 'ChatBrowserUse'),
+	'ChatGroq': ('browser_use.llm.groq.chat', 'ChatGroq'),
+	'ChatLiteLLM': ('browser_use.llm.litellm.chat', 'ChatLiteLLM'),
+	'ChatMistral': ('browser_use.llm.mistral.chat', 'ChatMistral'),
+	'ChatAzureOpenAI': ('browser_use.llm.azure.chat', 'ChatAzureOpenAI'),
+	'ChatOCIRaw': ('browser_use.llm.oci_raw.chat', 'ChatOCIRaw'),
+	'ChatOllama': ('browser_use.llm.ollama.chat', 'ChatOllama'),
+	'ChatVercel': ('browser_use.llm.vercel.chat', 'ChatVercel'),
+	# LLM models module
+	'models': ('browser_use.llm.models', None),
+	# Sandbox execution
+	'sandbox': ('browser_use.sandbox', 'sandbox'),
+}
+
+
+def __getattr__(name: str):
+	"""Lazy import mechanism - only import modules when they're actually accessed."""
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		try:
+			from importlib import import_module
+
+			module = import_module(module_path)
+			if attr_name is None:
+				# For modules like 'models', return the module itself
+				attr = module
+			else:
+				attr = getattr(module, attr_name)
+			# Cache the imported attribute in the module's globals
+			globals()[name] = attr
+			return attr
+		except ImportError as e:
+			raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
+
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'Agent',
+	'BrowserSession',
+	'Browser',  # Alias for BrowserSession
+	'BrowserProfile',
+	'Controller',
+	'DomService',
+	'SystemPrompt',
+	'ActionResult',
+	'ActionModel',
+	'AgentHistoryList',
+	# Chat models
+	'ChatOpenAI',
+	'ChatGoogle',
+	'ChatAnthropic',
+	'ChatBrowserUse',
+	'ChatGroq',
+	'ChatLiteLLM',
+	'ChatMistral',
+	'ChatAzureOpenAI',
+	'ChatOCIRaw',
+	'ChatOllama',
+	'ChatVercel',
+	'Tools',
+	'Controller',
+	# LLM models module
+	'models',
+	# Sandbox execution
+	'sandbox',
+]

browser_use/actor/README.md

@@ -0,0 +1,251 @@
+# Browser Actor
+
+Browser Actor is a web automation library built on CDP (Chrome DevTools Protocol) that provides low-level browser automation capabilities within the browser-use ecosystem.
+
+## Usage
+
+### Integrated with Browser (Recommended)
+```python
+from browser_use import Browser  # Alias for BrowserSession
+
+# Create and start browser session
+browser = Browser()
+await browser.start()
+
+# Create new tabs and navigate
+page = await browser.new_page("https://example.com")
+pages = await browser.get_pages()
+current_page = await browser.get_current_page()
+```
+
+### Direct Page Access (Advanced)
+```python
+from browser_use.actor import Page, Element, Mouse
+
+# Create page with existing browser session
+page = Page(browser_session, target_id, session_id)
+```
+
+## Basic Operations
+
+```python
+# Tab Management
+page = await browser.new_page()  # Create blank tab
+page = await browser.new_page("https://example.com")  # Create tab with URL
+pages = await browser.get_pages()  # Get all existing tabs
+await browser.close_page(page)  # Close specific tab
+
+# Navigation
+await page.goto("https://example.com")
+await page.go_back()
+await page.go_forward()
+await page.reload()
+```
+
+## Element Operations
+
+```python
+# Find elements by CSS selector
+elements = await page.get_elements_by_css_selector("input[type='text']")
+buttons = await page.get_elements_by_css_selector("button.submit")
+
+# Get element by backend node ID
+element = await page.get_element(backend_node_id=12345)
+
+# AI-powered element finding (requires LLM)
+element = await page.get_element_by_prompt("search button", llm=your_llm)
+element = await page.must_get_element_by_prompt("login form", llm=your_llm)
+```
+
+> **Note**: `get_elements_by_css_selector` returns immediately without waiting for visibility.
+
+## Element Interactions
+
+```python
+# Element actions
+await element.click(button='left', click_count=1, modifiers=['Control'])
+await element.fill("Hello World")  # Clears first, then types
+await element.hover()
+await element.focus()
+await element.check()  # Toggle checkbox/radio
+await element.select_option(["option1", "option2"])  # For dropdown/select
+await element.drag_to(target_element)  # Drag and drop
+
+# Element properties
+value = await element.get_attribute("value")
+box = await element.get_bounding_box()  # Returns BoundingBox or None
+info = await element.get_basic_info()  # Comprehensive element info
+screenshot_b64 = await element.screenshot(format='png')
+
+# Execute JavaScript on element (this context is the element)
+text = await element.evaluate("() => this.textContent")
+await element.evaluate("(color) => this.style.backgroundColor = color", "yellow")
+classes = await element.evaluate("() => Array.from(this.classList)")
+```
+
+## Mouse Operations
+
+```python
+# Mouse operations
+mouse = await page.mouse
+await mouse.click(x=100, y=200, button='left', click_count=1)
+await mouse.move(x=300, y=400, steps=1)
+await mouse.down(button='left')  # Press button
+await mouse.up(button='left')    # Release button
+await mouse.scroll(x=0, y=100, delta_x=0, delta_y=-500)  # Scroll at coordinates
+```
+
+## Page Operations
+
+```python
+# JavaScript evaluation
+result = await page.evaluate('() => document.title')  # Must use arrow function format
+result = await page.evaluate('(x, y) => x + y', 10, 20)  # With arguments
+
+# Keyboard input
+await page.press("Control+A")  # Key combinations supported
+await page.press("Escape")     # Single keys
+
+# Page controls
+await page.set_viewport_size(width=1920, height=1080)
+page_screenshot = await page.screenshot()  # PNG by default
+page_png = await page.screenshot(format="png", quality=90)
+
+# Page information
+url = await page.get_url()
+title = await page.get_title()
+```
+
+## AI-Powered Features
+
+```python
+# Content extraction using LLM
+from pydantic import BaseModel
+
+class ProductInfo(BaseModel):
+    name: str
+    price: float
+    description: str
+
+# Extract structured data from current page
+products = await page.extract_content(
+    "Find all products with their names, prices and descriptions",
+    ProductInfo,
+    llm=your_llm
+)
+```
+
+## Core Classes
+
+- **BrowserSession** (aliased as **Browser**): Main browser session manager with tab operations
+- **Page**: Represents a single browser tab or iframe for page-level operations
+- **Element**: Individual DOM element for interactions and property access
+- **Mouse**: Mouse operations within a page (click, move, scroll)
+
+## API Reference
+
+### BrowserSession Methods (Tab Management)
+- `start()` - Initialize and start the browser session
+- `stop()` - Stop the browser session (keeps browser alive)
+- `kill()` - Kill the browser process and reset all state
+- `new_page(url=None)` → `Page` - Create blank tab or navigate to URL
+- `get_pages()` → `list[Page]` - Get all available pages
+- `get_current_page()` → `Page | None` - Get the currently focused page
+- `close_page(page: Page | str)` - Close page by object or ID
+- Session management and CDP client operations
+
+### Page Methods (Page Operations)
+- `get_elements_by_css_selector(selector: str)` → `list[Element]` - Find elements by CSS selector
+- `get_element(backend_node_id: int)` → `Element` - Get element by backend node ID
+- `get_element_by_prompt(prompt: str, llm)` → `Element | None` - AI-powered element finding
+- `must_get_element_by_prompt(prompt: str, llm)` → `Element` - AI element finding (raises if not found)
+- `extract_content(prompt: str, structured_output: type[T], llm)` → `T` - Extract structured data using LLM
+- `goto(url: str)` - Navigate this page to URL
+- `go_back()`, `go_forward()` - Navigate history (with error handling)
+- `reload()` - Reload the current page
+- `evaluate(page_function: str, *args)` → `str` - Execute JavaScript (MUST use (...args) => format)
+- `press(key: str)` - Press key on page (supports "Control+A" format)
+- `set_viewport_size(width: int, height: int)` - Set viewport dimensions
+- `screenshot(format='png', quality=None)` → `str` - Take page screenshot, return base64
+- `get_url()` → `str`, `get_title()` → `str` - Get page information
+- `mouse` → `Mouse` - Get mouse interface for this page
+
+### Element Methods (DOM Interactions)
+- `click(button='left', click_count=1, modifiers=None)` - Click element with advanced fallbacks
+- `fill(text: str, clear=True)` - Fill input with text (clears first by default)
+- `hover()` - Hover over element
+- `focus()` - Focus the element
+- `check()` - Toggle checkbox/radio button (clicks to change state)
+- `select_option(values: str | list[str])` - Select dropdown options
+- `drag_to(target_element: Element | Position, source_position=None, target_position=None)` - Drag to target element
+- `evaluate(page_function: str, *args)` → `str` - Execute JavaScript on element (this = element)
+- `get_attribute(name: str)` → `str | None` - Get attribute value
+- `get_bounding_box()` → `BoundingBox | None` - Get element position/size
+- `screenshot(format='png', quality=None)` → `str` - Take element screenshot, return base64
+- `get_basic_info()` → `ElementInfo` - Get comprehensive element information
+
+
+### Mouse Methods (Coordinate-Based Operations)
+- `click(x: int, y: int, button='left', click_count=1)` - Click at coordinates
+- `move(x: int, y: int, steps=1)` - Move to coordinates
+- `down(button='left', click_count=1)`, `up(button='left', click_count=1)` - Press/release button
+- `scroll(x=0, y=0, delta_x=None, delta_y=None)` - Scroll page at coordinates
+
+## Type Definitions
+
+### Position
+```python
+class Position(TypedDict):
+    x: float
+    y: float
+```
+
+### BoundingBox
+```python
+class BoundingBox(TypedDict):
+    x: float
+    y: float
+    width: float
+    height: float
+```
+
+### ElementInfo
+```python
+class ElementInfo(TypedDict):
+    backendNodeId: int          # CDP backend node ID
+    nodeId: int | None          # CDP node ID
+    nodeName: str               # HTML tag name (e.g., "DIV", "INPUT")
+    nodeType: int               # DOM node type
+    nodeValue: str | None       # Text content for text nodes
+    attributes: dict[str, str]  # HTML attributes
+    boundingBox: BoundingBox | None  # Element position and size
+    error: str | None           # Error message if info retrieval failed
+```
+
+## Important Usage Notes
+
+**This is browser-use actor, NOT Playwright or Selenium.** Only use the methods documented above.
+
+### Critical JavaScript Rules
+- `page.evaluate()` and `element.evaluate()` MUST use `(...args) => {}` arrow function format
+- Always returns string (objects are JSON-stringified automatically)
+- Use single quotes around the function: `page.evaluate('() => document.title')`
+- For complex selectors in JS: `'() => document.querySelector("input[name=\\"email\\"]")'`
+- `element.evaluate()`: `this` context is bound to the element automatically
+
+### Method Restrictions
+- `get_elements_by_css_selector()` returns immediately (no automatic waiting)
+- For dropdowns: use `element.select_option()`, NOT `element.fill()`
+- Form submission: click submit button or use `page.press("Enter")`
+- No methods like: `element.submit()`, `element.dispatch_event()`, `element.get_property()`
+
+### Error Prevention
+- Always verify page state changes with `page.get_url()`, `page.get_title()`
+- Use `element.get_attribute()` to check element properties
+- Validate CSS selectors before use
+- Handle navigation timing with appropriate `asyncio.sleep()` calls
+
+### AI Features
+- `get_element_by_prompt()` and `extract_content()` require an LLM instance
+- These methods use DOM analysis and structured output parsing
+- Best for complex page understanding and data extraction tasks

browser_use/actor/__init__.py

@@ -0,0 +1,11 @@
+"""CDP-Use High-Level Library
+
+A Playwright-like library built on top of CDP (Chrome DevTools Protocol).
+"""
+
+from .element import Element
+from .mouse import Mouse
+from .page import Page
+from .utils import Utils
+
+__all__ = ['Page', 'Element', 'Mouse', 'Utils']

browser_use/actor/mouse.py

@@ -0,0 +1,134 @@
+"""Mouse class for mouse operations."""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+	from cdp_use.cdp.input.commands import DispatchMouseEventParameters, SynthesizeScrollGestureParameters
+	from cdp_use.cdp.input.types import MouseButton
+
+	from browser_use.browser.session import BrowserSession
+
+
+class Mouse:
+	"""Mouse operations for a target."""
+
+	def __init__(self, browser_session: 'BrowserSession', session_id: str | None = None, target_id: str | None = None):
+		self._browser_session = browser_session
+		self._client = browser_session.cdp_client
+		self._session_id = session_id
+		self._target_id = target_id
+
+	async def click(self, x: int, y: int, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
+		"""Click at the specified coordinates."""
+		# Mouse press
+		press_params: 'DispatchMouseEventParameters' = {
+			'type': 'mousePressed',
+			'x': x,
+			'y': y,
+			'button': button,
+			'clickCount': click_count,
+		}
+		await self._client.send.Input.dispatchMouseEvent(
+			press_params,
+			session_id=self._session_id,
+		)
+
+		# Mouse release
+		release_params: 'DispatchMouseEventParameters' = {
+			'type': 'mouseReleased',
+			'x': x,
+			'y': y,
+			'button': button,
+			'clickCount': click_count,
+		}
+		await self._client.send.Input.dispatchMouseEvent(
+			release_params,
+			session_id=self._session_id,
+		)
+
+	async def down(self, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
+		"""Press mouse button down."""
+		params: 'DispatchMouseEventParameters' = {
+			'type': 'mousePressed',
+			'x': 0,  # Will use last mouse position
+			'y': 0,
+			'button': button,
+			'clickCount': click_count,
+		}
+		await self._client.send.Input.dispatchMouseEvent(
+			params,
+			session_id=self._session_id,
+		)
+
+	async def up(self, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
+		"""Release mouse button."""
+		params: 'DispatchMouseEventParameters' = {
+			'type': 'mouseReleased',
+			'x': 0,  # Will use last mouse position
+			'y': 0,
+			'button': button,
+			'clickCount': click_count,
+		}
+		await self._client.send.Input.dispatchMouseEvent(
+			params,
+			session_id=self._session_id,
+		)
+
+	async def move(self, x: int, y: int, steps: int = 1) -> None:
+		"""Move mouse to the specified coordinates."""
+		# TODO: Implement smooth movement with multiple steps if needed
+		_ = steps  # Acknowledge parameter for future use
+
+		params: 'DispatchMouseEventParameters' = {'type': 'mouseMoved', 'x': x, 'y': y}
+		await self._client.send.Input.dispatchMouseEvent(params, session_id=self._session_id)
+
+	async def scroll(self, x: int = 0, y: int = 0, delta_x: int | None = None, delta_y: int | None = None) -> None:
+		"""Scroll the page using robust CDP methods."""
+		if not self._session_id:
+			raise RuntimeError('Session ID is required for scroll operations')
+
+		# Method 1: Try mouse wheel event (most reliable)
+		try:
+			# Get viewport dimensions
+			layout_metrics = await self._client.send.Page.getLayoutMetrics(session_id=self._session_id)
+			viewport_width = layout_metrics['layoutViewport']['clientWidth']
+			viewport_height = layout_metrics['layoutViewport']['clientHeight']
+
+			# Use provided coordinates or center of viewport
+			scroll_x = x if x > 0 else viewport_width / 2
+			scroll_y = y if y > 0 else viewport_height / 2
+
+			# Calculate scroll deltas (positive = down/right)
+			scroll_delta_x = delta_x or 0
+			scroll_delta_y = delta_y or 0
+
+			# Dispatch mouse wheel event
+			await self._client.send.Input.dispatchMouseEvent(
+				params={
+					'type': 'mouseWheel',
+					'x': scroll_x,
+					'y': scroll_y,
+					'deltaX': scroll_delta_x,
+					'deltaY': scroll_delta_y,
+				},
+				session_id=self._session_id,
+			)
+			return
+
+		except Exception:
+			pass
+
+		# Method 2: Fallback to synthesizeScrollGesture
+		try:
+			params: 'SynthesizeScrollGestureParameters' = {'x': x, 'y': y, 'xDistance': delta_x or 0, 'yDistance': delta_y or 0}
+			await self._client.send.Input.synthesizeScrollGesture(
+				params,
+				session_id=self._session_id,
+			)
+		except Exception:
+			# Method 3: JavaScript fallback
+			scroll_js = f'window.scrollBy({delta_x or 0}, {delta_y or 0})'
+			await self._client.send.Runtime.evaluate(
+				params={'expression': scroll_js, 'returnByValue': True},
+				session_id=self._session_id,
+			)

browser_use/actor/page.py

@@ -0,0 +1,564 @@
+"""Page class for page-level operations."""
+
+from typing import TYPE_CHECKING, TypeVar
+
+from pydantic import BaseModel
+
+from browser_use import logger
+from browser_use.actor.utils import get_key_info
+from browser_use.dom.serializer.serializer import DOMTreeSerializer
+from browser_use.dom.service import DomService
+from browser_use.llm.messages import SystemMessage, UserMessage
+
+T = TypeVar('T', bound=BaseModel)
+
+if TYPE_CHECKING:
+	from cdp_use.cdp.dom.commands import (
+		DescribeNodeParameters,
+		QuerySelectorAllParameters,
+	)
+	from cdp_use.cdp.emulation.commands import SetDeviceMetricsOverrideParameters
+	from cdp_use.cdp.input.commands import (
+		DispatchKeyEventParameters,
+	)
+	from cdp_use.cdp.page.commands import CaptureScreenshotParameters, NavigateParameters, NavigateToHistoryEntryParameters
+	from cdp_use.cdp.runtime.commands import EvaluateParameters
+	from cdp_use.cdp.target.commands import (
+		AttachToTargetParameters,
+		GetTargetInfoParameters,
+	)
+	from cdp_use.cdp.target.types import TargetInfo
+
+	from browser_use.browser.session import BrowserSession
+	from browser_use.llm.base import BaseChatModel
+
+	from .element import Element
+	from .mouse import Mouse
+
+
+class Page:
+	"""Page operations (tab or iframe)."""
+
+	def __init__(
+		self, browser_session: 'BrowserSession', target_id: str, session_id: str | None = None, llm: 'BaseChatModel | None' = None
+	):
+		self._browser_session = browser_session
+		self._client = browser_session.cdp_client
+		self._target_id = target_id
+		self._session_id: str | None = session_id
+		self._mouse: 'Mouse | None' = None
+
+		self._llm = llm
+
+	async def _ensure_session(self) -> str:
+		"""Ensure we have a session ID for this target."""
+		if not self._session_id:
+			params: 'AttachToTargetParameters' = {'targetId': self._target_id, 'flatten': True}
+			result = await self._client.send.Target.attachToTarget(params)
+			self._session_id = result['sessionId']
+
+			# Enable necessary domains
+			import asyncio
+
+			await asyncio.gather(
+				self._client.send.Page.enable(session_id=self._session_id),
+				self._client.send.DOM.enable(session_id=self._session_id),
+				self._client.send.Runtime.enable(session_id=self._session_id),
+				self._client.send.Network.enable(session_id=self._session_id),
+			)
+
+		return self._session_id
+
+	@property
+	async def session_id(self) -> str:
+		"""Get the session ID for this target.
+
+		@dev Pass this to an arbitrary CDP call
+		"""
+		return await self._ensure_session()
+
+	@property
+	async def mouse(self) -> 'Mouse':
+		"""Get the mouse interface for this target."""
+		if not self._mouse:
+			session_id = await self._ensure_session()
+			from .mouse import Mouse
+
+			self._mouse = Mouse(self._browser_session, session_id, self._target_id)
+		return self._mouse
+
+	async def reload(self) -> None:
+		"""Reload the target."""
+		session_id = await self._ensure_session()
+		await self._client.send.Page.reload(session_id=session_id)
+
+	async def get_element(self, backend_node_id: int) -> 'Element':
+		"""Get an element by its backend node ID."""
+		session_id = await self._ensure_session()
+
+		from .element import Element as Element_
+
+		return Element_(self._browser_session, backend_node_id, session_id)
+
+	async def evaluate(self, page_function: str, *args) -> str:
+		"""Execute JavaScript in the target.
+
+		Args:
+			page_function: JavaScript code that MUST start with (...args) => format
+			*args: Arguments to pass to the function
+
+		Returns:
+			String representation of the JavaScript execution result.
+			Objects and arrays are JSON-stringified.
+		"""
+		session_id = await self._ensure_session()
+
+		# Clean and fix common JavaScript string parsing issues
+		page_function = self._fix_javascript_string(page_function)
+
+		# Enforce arrow function format
+		if not (page_function.startswith('(') and '=>' in page_function):
+			raise ValueError(f'JavaScript code must start with (...args) => format. Got: {page_function[:50]}...')
+
+		# Build the expression - call the arrow function with provided args
+		if args:
+			# Convert args to JSON representation for safe passing
+			import json
+
+			arg_strs = [json.dumps(arg) for arg in args]
+			expression = f'({page_function})({", ".join(arg_strs)})'
+		else:
+			expression = f'({page_function})()'
+
+		# Debug: log the actual expression being evaluated
+		logger.debug(f'Evaluating JavaScript: {repr(expression)}')
+
+		params: 'EvaluateParameters' = {'expression': expression, 'returnByValue': True, 'awaitPromise': True}
+		result = await self._client.send.Runtime.evaluate(
+			params,
+			session_id=session_id,
+		)
+
+		if 'exceptionDetails' in result:
+			raise RuntimeError(f'JavaScript evaluation failed: {result["exceptionDetails"]}')
+
+		value = result.get('result', {}).get('value')
+
+		# Always return string representation
+		if value is None:
+			return ''
+		elif isinstance(value, str):
+			return value
+		else:
+			# Convert objects, numbers, booleans to string
+			import json
+
+			try:
+				return json.dumps(value) if isinstance(value, (dict, list)) else str(value)
+			except (TypeError, ValueError):
+				return str(value)
+
+	def _fix_javascript_string(self, js_code: str) -> str:
+		"""Fix common JavaScript string parsing issues when written as Python string."""
+
+		# Just do minimal, safe cleaning
+		js_code = js_code.strip()
+
+		# Only fix the most common and safe issues:
+
+		# 1. Remove obvious Python string wrapper quotes if they exist
+		if (js_code.startswith('"') and js_code.endswith('"')) or (js_code.startswith("'") and js_code.endswith("'")):
+			# Check if it's a wrapped string (not part of JS syntax)
+			inner = js_code[1:-1]
+			if inner.count('"') + inner.count("'") == 0 or '() =>' in inner:
+				js_code = inner
+
+		# 2. Only fix clearly escaped quotes that shouldn't be
+		# But be very conservative - only if we're sure it's a Python string artifact
+		if '\\"' in js_code and js_code.count('\\"') > js_code.count('"'):
+			js_code = js_code.replace('\\"', '"')
+		if "\\'" in js_code and js_code.count("\\'") > js_code.count("'"):
+			js_code = js_code.replace("\\'", "'")
+
+		# 3. Basic whitespace normalization only
+		js_code = js_code.strip()
+
+		# Final validation - ensure it's not empty
+		if not js_code:
+			raise ValueError('JavaScript code is empty after cleaning')
+
+		return js_code
+
+	async def screenshot(self, format: str = 'png', quality: int | None = None) -> str:
+		"""Take a screenshot and return base64 encoded image.
+
+		Args:
+		    format: Image format ('jpeg', 'png', 'webp')
+		    quality: Quality 0-100 for JPEG format
+
+		Returns:
+		    Base64-encoded image data
+		"""
+		session_id = await self._ensure_session()
+
+		params: 'CaptureScreenshotParameters' = {'format': format}
+
+		if quality is not None and format.lower() == 'jpeg':
+			params['quality'] = quality
+
+		result = await self._client.send.Page.captureScreenshot(params, session_id=session_id)
+
+		return result['data']
+
+	async def press(self, key: str) -> None:
+		"""Press a key on the page (sends keyboard input to the focused element or page)."""
+		session_id = await self._ensure_session()
+
+		# Handle key combinations like "Control+A"
+		if '+' in key:
+			parts = key.split('+')
+			modifiers = parts[:-1]
+			main_key = parts[-1]
+
+			# Calculate modifier bitmask
+			modifier_value = 0
+			modifier_map = {'Alt': 1, 'Control': 2, 'Meta': 4, 'Shift': 8}
+			for mod in modifiers:
+				modifier_value |= modifier_map.get(mod, 0)
+
+			# Press modifier keys
+			for mod in modifiers:
+				code, vk_code = get_key_info(mod)
+				params: 'DispatchKeyEventParameters' = {'type': 'keyDown', 'key': mod, 'code': code}
+				if vk_code is not None:
+					params['windowsVirtualKeyCode'] = vk_code
+				await self._client.send.Input.dispatchKeyEvent(params, session_id=session_id)
+
+			# Press main key with modifiers bitmask
+			main_code, main_vk_code = get_key_info(main_key)
+			main_down_params: 'DispatchKeyEventParameters' = {
+				'type': 'keyDown',
+				'key': main_key,
+				'code': main_code,
+				'modifiers': modifier_value,
+			}
+			if main_vk_code is not None:
+				main_down_params['windowsVirtualKeyCode'] = main_vk_code
+			await self._client.send.Input.dispatchKeyEvent(main_down_params, session_id=session_id)
+
+			main_up_params: 'DispatchKeyEventParameters' = {
+				'type': 'keyUp',
+				'key': main_key,
+				'code': main_code,
+				'modifiers': modifier_value,
+			}
+			if main_vk_code is not None:
+				main_up_params['windowsVirtualKeyCode'] = main_vk_code
+			await self._client.send.Input.dispatchKeyEvent(main_up_params, session_id=session_id)
+
+			# Release modifier keys
+			for mod in reversed(modifiers):
+				code, vk_code = get_key_info(mod)
+				release_params: 'DispatchKeyEventParameters' = {'type': 'keyUp', 'key': mod, 'code': code}
+				if vk_code is not None:
+					release_params['windowsVirtualKeyCode'] = vk_code
+				await self._client.send.Input.dispatchKeyEvent(release_params, session_id=session_id)
+		else:
+			# Simple key press
+			code, vk_code = get_key_info(key)
+			key_down_params: 'DispatchKeyEventParameters' = {'type': 'keyDown', 'key': key, 'code': code}
+			if vk_code is not None:
+				key_down_params['windowsVirtualKeyCode'] = vk_code
+			await self._client.send.Input.dispatchKeyEvent(key_down_params, session_id=session_id)
+
+			key_up_params: 'DispatchKeyEventParameters' = {'type': 'keyUp', 'key': key, 'code': code}
+			if vk_code is not None:
+				key_up_params['windowsVirtualKeyCode'] = vk_code
+			await self._client.send.Input.dispatchKeyEvent(key_up_params, session_id=session_id)
+
+	async def set_viewport_size(self, width: int, height: int) -> None:
+		"""Set the viewport size."""
+		session_id = await self._ensure_session()
+
+		params: 'SetDeviceMetricsOverrideParameters' = {
+			'width': width,
+			'height': height,
+			'deviceScaleFactor': 1.0,
+			'mobile': False,
+		}
+		await self._client.send.Emulation.setDeviceMetricsOverride(
+			params,
+			session_id=session_id,
+		)
+
+	# Target properties (from CDP getTargetInfo)
+	async def get_target_info(self) -> 'TargetInfo':
+		"""Get target information."""
+		params: 'GetTargetInfoParameters' = {'targetId': self._target_id}
+		result = await self._client.send.Target.getTargetInfo(params)
+		return result['targetInfo']
+
+	async def get_url(self) -> str:
+		"""Get the current URL."""
+		info = await self.get_target_info()
+		return info.get('url', '')
+
+	async def get_title(self) -> str:
+		"""Get the current title."""
+		info = await self.get_target_info()
+		return info.get('title', '')
+
+	async def goto(self, url: str) -> None:
+		"""Navigate this target to a URL."""
+		session_id = await self._ensure_session()
+
+		params: 'NavigateParameters' = {'url': url}
+		await self._client.send.Page.navigate(params, session_id=session_id)
+
+	async def navigate(self, url: str) -> None:
+		"""Alias for goto."""
+		await self.goto(url)
+
+	async def go_back(self) -> None:
+		"""Navigate back in history."""
+		session_id = await self._ensure_session()
+
+		try:
+			# Get navigation history
+			history = await self._client.send.Page.getNavigationHistory(session_id=session_id)
+			current_index = history['currentIndex']
+			entries = history['entries']
+
+			# Check if we can go back
+			if current_index <= 0:
+				raise RuntimeError('Cannot go back - no previous entry in history')
+
+			# Navigate to the previous entry
+			previous_entry_id = entries[current_index - 1]['id']
+			params: 'NavigateToHistoryEntryParameters' = {'entryId': previous_entry_id}
+			await self._client.send.Page.navigateToHistoryEntry(params, session_id=session_id)
+
+		except Exception as e:
+			raise RuntimeError(f'Failed to navigate back: {e}')
+
+	async def go_forward(self) -> None:
+		"""Navigate forward in history."""
+		session_id = await self._ensure_session()
+
+		try:
+			# Get navigation history
+			history = await self._client.send.Page.getNavigationHistory(session_id=session_id)
+			current_index = history['currentIndex']
+			entries = history['entries']
+
+			# Check if we can go forward
+			if current_index >= len(entries) - 1:
+				raise RuntimeError('Cannot go forward - no next entry in history')
+
+			# Navigate to the next entry
+			next_entry_id = entries[current_index + 1]['id']
+			params: 'NavigateToHistoryEntryParameters' = {'entryId': next_entry_id}
+			await self._client.send.Page.navigateToHistoryEntry(params, session_id=session_id)
+
+		except Exception as e:
+			raise RuntimeError(f'Failed to navigate forward: {e}')
+
+	# Element finding methods (these would need to be implemented based on DOM queries)
+	async def get_elements_by_css_selector(self, selector: str) -> list['Element']:
+		"""Get elements by CSS selector."""
+		session_id = await self._ensure_session()
+
+		# Get document first
+		doc_result = await self._client.send.DOM.getDocument(session_id=session_id)
+		document_node_id = doc_result['root']['nodeId']
+
+		# Query selector all
+		query_params: 'QuerySelectorAllParameters' = {'nodeId': document_node_id, 'selector': selector}
+		result = await self._client.send.DOM.querySelectorAll(query_params, session_id=session_id)
+
+		elements = []
+		from .element import Element as Element_
+
+		# Convert node IDs to backend node IDs
+		for node_id in result['nodeIds']:
+			# Get backend node ID
+			describe_params: 'DescribeNodeParameters' = {'nodeId': node_id}
+			node_result = await self._client.send.DOM.describeNode(describe_params, session_id=session_id)
+			backend_node_id = node_result['node']['backendNodeId']
+			elements.append(Element_(self._browser_session, backend_node_id, session_id))
+
+		return elements
+
+	# AI METHODS
+
+	@property
+	def dom_service(self) -> 'DomService':
+		"""Get the DOM service for this target."""
+		return DomService(self._browser_session)
+
+	async def get_element_by_prompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element | None':
+		"""Get an element by a prompt."""
+		await self._ensure_session()
+		llm = llm or self._llm
+
+		if not llm:
+			raise ValueError('LLM not provided')
+
+		dom_service = self.dom_service
+
+		# Lazy fetch all_frames inside get_dom_tree if needed (for cross-origin iframes)
+		enhanced_dom_tree, _ = await dom_service.get_dom_tree(target_id=self._target_id, all_frames=None)
+
+		session_id = self._browser_session.id
+		serialized_dom_state, _ = DOMTreeSerializer(
+			enhanced_dom_tree, None, paint_order_filtering=True, session_id=session_id
+		).serialize_accessible_elements()
+
+		llm_representation = serialized_dom_state.llm_representation()
+
+		system_message = SystemMessage(
+			content="""You are an AI created to find an element on a page by a prompt.
+
+<browser_state>
+Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, etc.)
+- text: Element description
+
+Examples:
+[33]<div>User form</div>
+[35]<button aria-label='Submit form'>Submit</button>
+
+Note that:
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
+- Pure text elements without [] are not interactive.
+</browser_state>
+
+Your task is to find an element index (if any) that matches the prompt (written in <prompt> tag).
+
+If non of the elements matches the, return None.
+
+Before you return the element index, reason about the state and elements for a sentence or two."""
+		)
+
+		state_message = UserMessage(
+			content=f"""
+			<browser_state>
+			{llm_representation}
+			</browser_state>
+
+			<prompt>
+			{prompt}
+			</prompt>
+			"""
+		)
+
+		class ElementResponse(BaseModel):
+			# thinking: str
+			element_highlight_index: int | None
+
+		llm_response = await llm.ainvoke(
+			[
+				system_message,
+				state_message,
+			],
+			output_format=ElementResponse,
+		)
+
+		element_highlight_index = llm_response.completion.element_highlight_index
+
+		if element_highlight_index is None or element_highlight_index not in serialized_dom_state.selector_map:
+			return None
+
+		element = serialized_dom_state.selector_map[element_highlight_index]
+
+		from .element import Element as Element_
+
+		return Element_(self._browser_session, element.backend_node_id, self._session_id)
+
+	async def must_get_element_by_prompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element':
+		"""Get an element by a prompt.
+
+		@dev LLM can still return None, this just raises an error if the element is not found.
+		"""
+		element = await self.get_element_by_prompt(prompt, llm)
+		if element is None:
+			raise ValueError(f'No element found for prompt: {prompt}')
+
+		return element
+
+	async def extract_content(self, prompt: str, structured_output: type[T], llm: 'BaseChatModel | None' = None) -> T:
+		"""Extract structured content from the current page using LLM.
+
+		Extracts clean markdown from the page and sends it to LLM for structured data extraction.
+
+		Args:
+			prompt: Description of what content to extract
+			structured_output: Pydantic BaseModel class defining the expected output structure
+			llm: Language model to use for extraction
+
+		Returns:
+			The structured BaseModel instance with extracted content
+		"""
+		llm = llm or self._llm
+
+		if not llm:
+			raise ValueError('LLM not provided')
+
+		# Extract clean markdown using the same method as in tools/service.py
+		try:
+			content, content_stats = await self._extract_clean_markdown()
+		except Exception as e:
+			raise RuntimeError(f'Could not extract clean markdown: {type(e).__name__}')
+
+		# System prompt for structured extraction
+		system_prompt = """
+You are an expert at extracting structured data from the markdown of a webpage.
+
+<input>
+You will be given a query and the markdown of a webpage that has been filtered to remove noise and advertising content.
+</input>
+
+<instructions>
+- You are tasked to extract information from the webpage that is relevant to the query.
+- You should ONLY use the information available in the webpage to answer the query. Do not make up information or provide guess from your own knowledge.
+- If the information relevant to the query is not available in the page, your response should mention that.
+- If the query asks for all items, products, etc., make sure to directly list all of them.
+- Return the extracted content in the exact structured format specified.
+</instructions>
+
+<output>
+- Your output should present ALL the information relevant to the query in the specified structured format.
+- Do not answer in conversational format - directly output the relevant information in the structured format.
+</output>
+""".strip()
+
+		# Build prompt with just query and content
+		prompt_content = f'<query>\n{prompt}\n</query>\n\n<webpage_content>\n{content}\n</webpage_content>'
+
+		# Send to LLM with structured output
+		import asyncio
+
+		try:
+			response = await asyncio.wait_for(
+				llm.ainvoke(
+					[SystemMessage(content=system_prompt), UserMessage(content=prompt_content)], output_format=structured_output
+				),
+				timeout=120.0,
+			)
+
+			# Return the structured output BaseModel instance
+			return response.completion
+		except Exception as e:
+			raise RuntimeError(str(e))
+
+	async def _extract_clean_markdown(self, extract_links: bool = False) -> tuple[str, dict]:
+		"""Extract clean markdown from the current page using enhanced DOM tree.
+
+		Uses the shared markdown extractor for consistency with tools/service.py.
+		"""
+		from browser_use.dom.markdown_extractor import extract_clean_markdown
+
+		dom_service = self.dom_service
+		return await extract_clean_markdown(dom_service=dom_service, target_id=self._target_id, extract_links=extract_links)

browser_use/actor/playground/flights.py

@@ -0,0 +1,41 @@
+import asyncio
+
+from browser_use import Agent, Browser, ChatOpenAI
+
+llm = ChatOpenAI('gpt-4.1-mini')
+
+
+async def main():
+	"""
+	Main function demonstrating mixed automation with Browser-Use and Playwright.
+	"""
+	print('🚀 Mixed Automation with Browser-Use and Actor API')
+
+	browser = Browser(keep_alive=True)
+	await browser.start()
+
+	page = await browser.get_current_page() or await browser.new_page()
+
+	# Go to apple wikipedia page
+	await page.goto('https://www.google.com/travel/flights')
+
+	await asyncio.sleep(1)
+
+	round_trip_button = await page.must_get_element_by_prompt('round trip button', llm)
+	await round_trip_button.click()
+
+	one_way_button = await page.must_get_element_by_prompt('one way button', llm)
+	await one_way_button.click()
+
+	await asyncio.sleep(1)
+
+	agent = Agent(task='Find the cheapest flight from London to Paris on 2025-10-15', llm=llm, browser_session=browser)
+	await agent.run()
+
+	input('Press Enter to continue...')
+
+	await browser.stop()
+
+
+if __name__ == '__main__':
+	asyncio.run(main())

browser_use/actor/playground/mixed_automation.py

@@ -0,0 +1,54 @@
+import asyncio
+
+from pydantic import BaseModel
+
+from browser_use import Browser, ChatOpenAI
+
+TASK = """
+On the current wikipedia page, find the latest huge edit and tell me what is was about.
+"""
+
+
+class LatestEditFinder(BaseModel):
+	"""Find the latest huge edit on the current wikipedia page."""
+
+	latest_edit: str
+	edit_time: str
+	edit_author: str
+	edit_summary: str
+	edit_url: str
+
+
+llm = ChatOpenAI('gpt-4.1-mini')
+
+
+async def main():
+	"""
+	Main function demonstrating mixed automation with Browser-Use and Playwright.
+	"""
+	print('🚀 Mixed Automation with Browser-Use and Actor API')
+
+	browser = Browser(keep_alive=True)
+	await browser.start()
+
+	page = await browser.get_current_page() or await browser.new_page()
+
+	# Go to apple wikipedia page
+	await page.goto('https://browser-use.github.io/stress-tests/challenges/angularjs-form.html')
+
+	await asyncio.sleep(1)
+
+	element = await page.get_element_by_prompt('zip code input', llm)
+
+	print('Element found', element)
+
+	if element:
+		await element.click()
+	else:
+		print('No element found')
+
+	await browser.stop()
+
+
+if __name__ == '__main__':
+	asyncio.run(main())

browser_use/actor/playground/playground.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+"""
+Playground script to test the browser-use actor API.
+
+This script demonstrates:
+- Starting a browser session
+- Using the actor API to navigate and interact
+- Finding elements, clicking, scrolling, JavaScript evaluation
+- Testing most of the available methods
+"""
+
+import asyncio
+import json
+import logging
+
+from browser_use import Browser
+
+# Configure logging to see what's happening
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+async def main():
+	"""Main playground function."""
+	logger.info('🚀 Starting browser actor playground')
+
+	# Create browser session
+	browser = Browser()
+
+	try:
+		# Start the browser
+		await browser.start()
+		logger.info('✅ Browser session started')
+
+		# Navigate to Wikipedia using integrated methods
+		logger.info('📖 Navigating to Wikipedia...')
+		page = await browser.new_page('https://en.wikipedia.org')
+
+		# Get basic page info
+		url = await page.get_url()
+		title = await page.get_title()
+		logger.info(f'📄 Page loaded: {title} ({url})')
+
+		# Take a screenshot
+		logger.info('📸 Taking initial screenshot...')
+		screenshot_b64 = await page.screenshot()
+		logger.info(f'📸 Screenshot captured: {len(screenshot_b64)} bytes')
+
+		# Set viewport size
+		logger.info('🖥️ Setting viewport to 1920x1080...')
+		await page.set_viewport_size(1920, 1080)
+
+		# Execute some JavaScript to count links
+		logger.info('🔍 Counting article links using JavaScript...')
+		js_code = """() => {
+			// Find all article links on the page
+			const links = Array.from(document.querySelectorAll('a[href*="/wiki/"]:not([href*=":"])'))
+				.filter(link => !link.href.includes('Main_Page') && !link.href.includes('Special:'));
+			
+			return {
+				total: links.length,
+				sample: links.slice(0, 3).map(link => ({
+					href: link.href,
+					text: link.textContent.trim() 
+				}))
+			};
+		}"""
+
+		link_info = json.loads(await page.evaluate(js_code))
+		logger.info(f'🔗 Found {link_info["total"]} article links')
+		# Try to find and interact with links using CSS selector
+		try:
+			# Find article links on the page
+			links = await page.get_elements_by_css_selector('a[href*="/wiki/"]:not([href*=":"])')
+
+			if links:
+				logger.info(f'📋 Found {len(links)} wiki links via CSS selector')
+
+				# Pick the first link
+				link_element = links[0]
+
+				# Get link info using available methods
+				basic_info = await link_element.get_basic_info()
+				link_href = await link_element.get_attribute('href')
+
+				logger.info(f'🎯 Selected element: <{basic_info["nodeName"]}>')
+				logger.info(f'🔗 Link href: {link_href}')
+
+				if basic_info['boundingBox']:
+					bbox = basic_info['boundingBox']
+					logger.info(f'📏 Position: ({bbox["x"]}, {bbox["y"]}) Size: {bbox["width"]}x{bbox["height"]}')
+
+				# Test element interactions with robust implementations
+				logger.info('👆 Hovering over the element...')
+				await link_element.hover()
+				await asyncio.sleep(1)
+
+				logger.info('🔍 Focusing the element...')
+				await link_element.focus()
+				await asyncio.sleep(0.5)
+
+				# Click the link using robust click method
+				logger.info('🖱️ Clicking the link with robust fallbacks...')
+				await link_element.click()
+
+				# Wait for navigation
+				await asyncio.sleep(3)
+
+				# Get new page info
+				new_url = await page.get_url()
+				new_title = await page.get_title()
+				logger.info(f'📄 Navigated to: {new_title}')
+				logger.info(f'🌐 New URL: {new_url}')
+			else:
+				logger.warning('❌ No links found to interact with')
+
+		except Exception as e:
+			logger.warning(f'⚠️ Link interaction failed: {e}')
+
+		# Scroll down the page
+		logger.info('📜 Scrolling down the page...')
+		mouse = await page.mouse
+		await mouse.scroll(x=0, y=100, delta_y=500)
+		await asyncio.sleep(1)
+
+		# Test mouse operations
+		logger.info('🖱️ Testing mouse operations...')
+		await mouse.move(x=100, y=200)
+		await mouse.click(x=150, y=250)
+
+		# Execute more JavaScript examples
+		logger.info('🧪 Testing JavaScript evaluation...')
+
+		# Simple expressions
+		page_height = await page.evaluate('() => document.body.scrollHeight')
+		current_scroll = await page.evaluate('() => window.pageYOffset')
+		logger.info(f'📏 Page height: {page_height}px, current scroll: {current_scroll}px')
+
+		# JavaScript with arguments
+		result = await page.evaluate('(x) => x * 2', 21)
+		logger.info(f'🧮 JavaScript with args: 21 * 2 = {result}')
+
+		# More complex JavaScript
+		page_stats = json.loads(
+			await page.evaluate("""() => {
+			return {
+				url: window.location.href,
+				title: document.title,
+				links: document.querySelectorAll('a').length,
+				images: document.querySelectorAll('img').length,
+				scrollTop: window.pageYOffset,
+				viewportHeight: window.innerHeight
+			};
+		}""")
+		)
+		logger.info(f'📊 Page stats: {page_stats}')
+
+		# Get page title using different methods
+		title_via_js = await page.evaluate('() => document.title')
+		title_via_api = await page.get_title()
+		logger.info(f'📝 Title via JS: "{title_via_js}"')
+		logger.info(f'📝 Title via API: "{title_via_api}"')
+
+		# Take a final screenshot
+		logger.info('📸 Taking final screenshot...')
+		final_screenshot = await page.screenshot()
+		logger.info(f'📸 Final screenshot: {len(final_screenshot)} bytes')
+
+		# Test browser navigation with error handling
+		logger.info('⬅️ Testing browser back navigation...')
+		try:
+			await page.go_back()
+			await asyncio.sleep(2)
+
+			back_url = await page.get_url()
+			back_title = await page.get_title()
+			logger.info(f'📄 After going back: {back_title}')
+			logger.info(f'🌐 Back URL: {back_url}')
+		except RuntimeError as e:
+			logger.info(f'ℹ️ Navigation back failed as expected: {e}')
+
+		# Test creating new page
+		logger.info('🆕 Creating new blank page...')
+		new_page = await browser.new_page()
+		new_page_url = await new_page.get_url()
+		logger.info(f'🆕 New page created with URL: {new_page_url}')
+
+		# Get all pages
+		all_pages = await browser.get_pages()
+		logger.info(f'📑 Total pages: {len(all_pages)}')
+
+		# Test form interaction if we can find a form
+		try:
+			# Look for search input on the page
+			search_inputs = await page.get_elements_by_css_selector('input[type="search"], input[name*="search"]')
+
+			if search_inputs:
+				search_input = search_inputs[0]
+				logger.info('🔍 Found search input, testing form interaction...')
+
+				await search_input.focus()
+				await search_input.fill('test search query')
+				await page.press('Enter')
+
+				logger.info('✅ Form interaction test completed')
+			else:
+				logger.info('ℹ️ No search inputs found for form testing')
+
+		except Exception as e:
+			logger.info(f'ℹ️ Form interaction test skipped: {e}')
+
+			# wait 2 seconds before closing the new page
+		logger.info('🕒 Waiting 2 seconds before closing the new page...')
+		await asyncio.sleep(2)
+		logger.info('🗑️ Closing new page...')
+		await browser.close_page(new_page)
+
+		logger.info('✅ Playground completed successfully!')
+
+		input('Press Enter to continue...')
+
+	except Exception as e:
+		logger.error(f'❌ Error in playground: {e}', exc_info=True)
+
+	finally:
+		# Clean up
+		logger.info('🧹 Cleaning up...')
+		try:
+			await browser.stop()
+			logger.info('✅ Browser session stopped')
+		except Exception as e:
+			logger.error(f'❌ Error stopping browser: {e}')
+
+
+if __name__ == '__main__':
+	asyncio.run(main())

browser_use/actor/utils.py

@@ -0,0 +1,176 @@
+"""Utility functions for actor operations."""
+
+
+class Utils:
+	"""Utility functions for actor operations."""
+
+	@staticmethod
+	def get_key_info(key: str) -> tuple[str, int | None]:
+		"""Get the code and windowsVirtualKeyCode for a key.
+
+		Args:
+			key: Key name (e.g., 'Enter', 'ArrowUp', 'a', 'A')
+
+		Returns:
+			Tuple of (code, windowsVirtualKeyCode)
+
+		Reference: Windows Virtual Key Codes
+		https://docs.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
+		"""
+		# Complete mapping of key names to (code, virtualKeyCode)
+		# Based on standard Windows Virtual Key Codes
+		key_map = {
+			# Navigation keys
+			'Backspace': ('Backspace', 8),
+			'Tab': ('Tab', 9),
+			'Enter': ('Enter', 13),
+			'Escape': ('Escape', 27),
+			'Space': ('Space', 32),
+			' ': ('Space', 32),
+			'PageUp': ('PageUp', 33),
+			'PageDown': ('PageDown', 34),
+			'End': ('End', 35),
+			'Home': ('Home', 36),
+			'ArrowLeft': ('ArrowLeft', 37),
+			'ArrowUp': ('ArrowUp', 38),
+			'ArrowRight': ('ArrowRight', 39),
+			'ArrowDown': ('ArrowDown', 40),
+			'Insert': ('Insert', 45),
+			'Delete': ('Delete', 46),
+			# Modifier keys
+			'Shift': ('ShiftLeft', 16),
+			'ShiftLeft': ('ShiftLeft', 16),
+			'ShiftRight': ('ShiftRight', 16),
+			'Control': ('ControlLeft', 17),
+			'ControlLeft': ('ControlLeft', 17),
+			'ControlRight': ('ControlRight', 17),
+			'Alt': ('AltLeft', 18),
+			'AltLeft': ('AltLeft', 18),
+			'AltRight': ('AltRight', 18),
+			'Meta': ('MetaLeft', 91),
+			'MetaLeft': ('MetaLeft', 91),
+			'MetaRight': ('MetaRight', 92),
+			# Function keys F1-F24
+			'F1': ('F1', 112),
+			'F2': ('F2', 113),
+			'F3': ('F3', 114),
+			'F4': ('F4', 115),
+			'F5': ('F5', 116),
+			'F6': ('F6', 117),
+			'F7': ('F7', 118),
+			'F8': ('F8', 119),
+			'F9': ('F9', 120),
+			'F10': ('F10', 121),
+			'F11': ('F11', 122),
+			'F12': ('F12', 123),
+			'F13': ('F13', 124),
+			'F14': ('F14', 125),
+			'F15': ('F15', 126),
+			'F16': ('F16', 127),
+			'F17': ('F17', 128),
+			'F18': ('F18', 129),
+			'F19': ('F19', 130),
+			'F20': ('F20', 131),
+			'F21': ('F21', 132),
+			'F22': ('F22', 133),
+			'F23': ('F23', 134),
+			'F24': ('F24', 135),
+			# Numpad keys
+			'NumLock': ('NumLock', 144),
+			'Numpad0': ('Numpad0', 96),
+			'Numpad1': ('Numpad1', 97),
+			'Numpad2': ('Numpad2', 98),
+			'Numpad3': ('Numpad3', 99),
+			'Numpad4': ('Numpad4', 100),
+			'Numpad5': ('Numpad5', 101),
+			'Numpad6': ('Numpad6', 102),
+			'Numpad7': ('Numpad7', 103),
+			'Numpad8': ('Numpad8', 104),
+			'Numpad9': ('Numpad9', 105),
+			'NumpadMultiply': ('NumpadMultiply', 106),
+			'NumpadAdd': ('NumpadAdd', 107),
+			'NumpadSubtract': ('NumpadSubtract', 109),
+			'NumpadDecimal': ('NumpadDecimal', 110),
+			'NumpadDivide': ('NumpadDivide', 111),
+			# Lock keys
+			'CapsLock': ('CapsLock', 20),
+			'ScrollLock': ('ScrollLock', 145),
+			# OEM/Punctuation keys (US keyboard layout)
+			'Semicolon': ('Semicolon', 186),
+			';': ('Semicolon', 186),
+			'Equal': ('Equal', 187),
+			'=': ('Equal', 187),
+			'Comma': ('Comma', 188),
+			',': ('Comma', 188),
+			'Minus': ('Minus', 189),
+			'-': ('Minus', 189),
+			'Period': ('Period', 190),
+			'.': ('Period', 190),
+			'Slash': ('Slash', 191),
+			'/': ('Slash', 191),
+			'Backquote': ('Backquote', 192),
+			'`': ('Backquote', 192),
+			'BracketLeft': ('BracketLeft', 219),
+			'[': ('BracketLeft', 219),
+			'Backslash': ('Backslash', 220),
+			'\\': ('Backslash', 220),
+			'BracketRight': ('BracketRight', 221),
+			']': ('BracketRight', 221),
+			'Quote': ('Quote', 222),
+			"'": ('Quote', 222),
+			# Media/Browser keys
+			'AudioVolumeMute': ('AudioVolumeMute', 173),
+			'AudioVolumeDown': ('AudioVolumeDown', 174),
+			'AudioVolumeUp': ('AudioVolumeUp', 175),
+			'MediaTrackNext': ('MediaTrackNext', 176),
+			'MediaTrackPrevious': ('MediaTrackPrevious', 177),
+			'MediaStop': ('MediaStop', 178),
+			'MediaPlayPause': ('MediaPlayPause', 179),
+			'BrowserBack': ('BrowserBack', 166),
+			'BrowserForward': ('BrowserForward', 167),
+			'BrowserRefresh': ('BrowserRefresh', 168),
+			'BrowserStop': ('BrowserStop', 169),
+			'BrowserSearch': ('BrowserSearch', 170),
+			'BrowserFavorites': ('BrowserFavorites', 171),
+			'BrowserHome': ('BrowserHome', 172),
+			# Additional common keys
+			'Clear': ('Clear', 12),
+			'Pause': ('Pause', 19),
+			'Select': ('Select', 41),
+			'Print': ('Print', 42),
+			'Execute': ('Execute', 43),
+			'PrintScreen': ('PrintScreen', 44),
+			'Help': ('Help', 47),
+			'ContextMenu': ('ContextMenu', 93),
+		}
+
+		if key in key_map:
+			return key_map[key]
+
+		# Handle alphanumeric keys dynamically
+		if len(key) == 1:
+			if key.isalpha():
+				# Letter keys: A-Z have VK codes 65-90
+				return (f'Key{key.upper()}', ord(key.upper()))
+			elif key.isdigit():
+				# Digit keys: 0-9 have VK codes 48-57 (same as ASCII)
+				return (f'Digit{key}', ord(key))
+
+		# Fallback: use the key name as code, no virtual key code
+		return (key, None)
+
+
+# Backward compatibility: provide standalone function
+def get_key_info(key: str) -> tuple[str, int | None]:
+	"""Get the code and windowsVirtualKeyCode for a key.
+
+	Args:
+		key: Key name (e.g., 'Enter', 'ArrowUp', 'a', 'A')
+
+	Returns:
+		Tuple of (code, windowsVirtualKeyCode)
+
+	Reference: Windows Virtual Key Codes
+	https://docs.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
+	"""
+	return Utils.get_key_info(key)

browser_use/agent/__init__.py

@@ -0,0 +1,32 @@
+"""Agent package exports."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+	from browser_use.agent.service import Agent
+	from browser_use.beta.service import BetaAgentError
+
+_LAZY_IMPORTS = {
+	'Agent': ('browser_use.agent.service', 'Agent'),
+	'BetaAgentError': ('browser_use.beta.service', 'BetaAgentError'),
+}
+
+
+def __getattr__(name: str):
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		from importlib import import_module
+
+		module = import_module(module_path)
+		attr = getattr(module, attr_name)
+		globals()[name] = attr
+		return attr
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'Agent',
+	'BetaAgentError',
+]

browser_use/agent/cloud_events.py

@@ -0,0 +1,284 @@
+import base64
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+import anyio
+from bubus import BaseEvent
+from pydantic import Field, field_validator
+from uuid_extensions import uuid7str
+
+MAX_STRING_LENGTH = 500000  # 100K chars ~ 25k tokens should be enough
+MAX_URL_LENGTH = 100000
+MAX_TASK_LENGTH = 100000
+MAX_COMMENT_LENGTH = 2000
+MAX_FILE_CONTENT_SIZE = 50 * 1024 * 1024  # 50MB
+
+
+class UpdateAgentTaskEvent(BaseEvent):
+	# Required fields for identification
+	id: str  # The task ID to update
+	user_id: str = Field(max_length=255)  # For authorization
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+
+	# Optional fields that can be updated
+	stopped: bool | None = None
+	paused: bool | None = None
+	done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
+	finished_at: datetime | None = None
+	agent_state: dict | None = None
+	user_feedback_type: str | None = Field(None, max_length=10)  # UserFeedbackType enum value as string
+	user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
+	gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
+
+	@classmethod
+	def from_agent(cls, agent) -> 'UpdateAgentTaskEvent':
+		"""Create an UpdateAgentTaskEvent from an Agent instance"""
+		if not hasattr(agent, '_task_start_time'):
+			raise ValueError('Agent must have _task_start_time attribute')
+
+		done_output = agent.history.final_result() if agent.history else None
+		if done_output and len(done_output) > MAX_STRING_LENGTH:
+			done_output = done_output[:MAX_STRING_LENGTH]
+		return cls(
+			id=str(agent.task_id),
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			stopped=agent.state.stopped if hasattr(agent.state, 'stopped') else False,
+			paused=agent.state.paused if hasattr(agent.state, 'paused') else False,
+			done_output=done_output,
+			finished_at=datetime.now(timezone.utc) if agent.history and agent.history.is_done() else None,
+			agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
+			user_feedback_type=None,
+			user_comment=None,
+			gif_url=None,
+			# user_feedback_type and user_comment would be set by the API/frontend
+			# gif_url would be set after GIF generation if needed
+		)
+
+
+class CreateAgentOutputFileEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	task_id: str
+	file_name: str = Field(max_length=255)
+	file_content: str | None = None  # Base64 encoded file content
+	content_type: str | None = Field(None, max_length=100)  # MIME type for file uploads
+	created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+	@field_validator('file_content')
+	@classmethod
+	def validate_file_size(cls, v: str | None) -> str | None:
+		"""Validate base64 file content size."""
+		if v is None:
+			return v
+		# Remove data URL prefix if present
+		if ',' in v:
+			v = v.split(',')[1]
+		# Estimate decoded size (base64 is ~33% larger)
+		estimated_size = len(v) * 3 / 4
+		if estimated_size > MAX_FILE_CONTENT_SIZE:
+			raise ValueError(f'File content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
+		return v
+
+	@classmethod
+	async def from_agent_and_file(cls, agent, output_path: str) -> 'CreateAgentOutputFileEvent':
+		"""Create a CreateAgentOutputFileEvent from a file path"""
+
+		gif_path = Path(output_path)
+		if not gif_path.exists():
+			raise FileNotFoundError(f'File not found: {output_path}')
+
+		gif_size = os.path.getsize(gif_path)
+
+		# Read GIF content for base64 encoding if needed
+		gif_content = None
+		if gif_size < 50 * 1024 * 1024:  # Only read if < 50MB
+			async with await anyio.open_file(gif_path, 'rb') as f:
+				gif_bytes = await f.read()
+				gif_content = base64.b64encode(gif_bytes).decode('utf-8')
+
+		return cls(
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			task_id=str(agent.task_id),
+			file_name=gif_path.name,
+			file_content=gif_content,  # Base64 encoded
+			content_type='image/gif',
+		)
+
+
+class CreateAgentStepEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)  # Added for authorization checks
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	agent_task_id: str
+	step: int
+	evaluation_previous_goal: str = Field(max_length=MAX_STRING_LENGTH)
+	memory: str = Field(max_length=MAX_STRING_LENGTH)
+	next_goal: str = Field(max_length=MAX_STRING_LENGTH)
+	actions: list[dict]
+	screenshot_url: str | None = Field(None, max_length=MAX_FILE_CONTENT_SIZE)  # ~50MB for base64 images
+	url: str = Field(default='', max_length=MAX_URL_LENGTH)
+
+	@field_validator('screenshot_url')
+	@classmethod
+	def validate_screenshot_size(cls, v: str | None) -> str | None:
+		"""Validate screenshot URL or base64 content size."""
+		if v is None or not v.startswith('data:'):
+			return v
+		# It's base64 data, check size
+		if ',' in v:
+			base64_part = v.split(',')[1]
+			estimated_size = len(base64_part) * 3 / 4
+			if estimated_size > MAX_FILE_CONTENT_SIZE:
+				raise ValueError(f'Screenshot content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
+		return v
+
+	@classmethod
+	def from_agent_step(
+		cls, agent, model_output, result: list, actions_data: list[dict], browser_state_summary
+	) -> 'CreateAgentStepEvent':
+		"""Create a CreateAgentStepEvent from agent step data"""
+		# Get first action details if available
+		first_action = model_output.action[0] if model_output.action else None
+
+		# Extract current state from model output
+		current_state = model_output.current_state if hasattr(model_output, 'current_state') else None
+
+		# Capture screenshot as base64 data URL if available
+		screenshot_url = None
+		if browser_state_summary.screenshot:
+			screenshot_url = f'data:image/png;base64,{browser_state_summary.screenshot}'
+			import logging
+
+			logger = logging.getLogger(__name__)
+			logger.debug(f'📸 Including screenshot in CreateAgentStepEvent, length: {len(browser_state_summary.screenshot)}')
+		else:
+			import logging
+
+			logger = logging.getLogger(__name__)
+			logger.debug('📸 No screenshot in browser_state_summary for CreateAgentStepEvent')
+
+		return cls(
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			agent_task_id=str(agent.task_id),
+			step=agent.state.n_steps,
+			evaluation_previous_goal=current_state.evaluation_previous_goal if current_state else '',
+			memory=current_state.memory if current_state else '',
+			next_goal=current_state.next_goal if current_state else '',
+			actions=actions_data,  # List of action dicts
+			url=browser_state_summary.url,
+			screenshot_url=screenshot_url,
+		)
+
+
+class CreateAgentTaskEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)  # Added for authorization checks
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	agent_session_id: str
+	llm_model: str = Field(max_length=200)  # LLMModel enum value as string
+	stopped: bool = False
+	paused: bool = False
+	task: str = Field(max_length=MAX_TASK_LENGTH)
+	done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
+	scheduled_task_id: str | None = None
+	started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	finished_at: datetime | None = None
+	agent_state: dict = Field(default_factory=dict)
+	user_feedback_type: str | None = Field(None, max_length=10)  # UserFeedbackType enum value as string
+	user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
+	gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
+
+	@classmethod
+	def from_agent(cls, agent) -> 'CreateAgentTaskEvent':
+		"""Create a CreateAgentTaskEvent from an Agent instance"""
+		return cls(
+			id=str(agent.task_id),
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			agent_session_id=str(agent.session_id),
+			task=agent.task,
+			llm_model=agent.llm.model_name,
+			agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
+			stopped=False,
+			paused=False,
+			done_output=None,
+			started_at=datetime.fromtimestamp(agent._task_start_time, tz=timezone.utc),
+			finished_at=None,
+			user_feedback_type=None,
+			user_comment=None,
+			gif_url=None,
+		)
+
+
+class CreateAgentSessionEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	browser_session_id: str = Field(max_length=255)
+	browser_session_live_url: str = Field(max_length=MAX_URL_LENGTH)
+	browser_session_cdp_url: str = Field(max_length=MAX_URL_LENGTH)
+	browser_session_stopped: bool = False
+	browser_session_stopped_at: datetime | None = None
+	is_source_api: bool | None = None
+	browser_state: dict = Field(default_factory=dict)
+	browser_session_data: dict | None = None
+
+	@classmethod
+	def from_agent(cls, agent) -> 'CreateAgentSessionEvent':
+		"""Create a CreateAgentSessionEvent from an Agent instance"""
+		return cls(
+			id=str(agent.session_id),
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			browser_session_id=agent.browser_session.id,
+			browser_session_live_url='',  # To be filled by cloud handler
+			browser_session_cdp_url='',  # To be filled by cloud handler
+			browser_state={
+				'viewport': agent.browser_profile.viewport if agent.browser_profile else {'width': 1280, 'height': 720},
+				'user_agent': agent.browser_profile.user_agent if agent.browser_profile else None,
+				'headless': agent.browser_profile.headless if agent.browser_profile else True,
+				'initial_url': None,  # Will be updated during execution
+				'final_url': None,  # Will be updated during execution
+				'total_pages_visited': 0,  # Will be updated during execution
+				'session_duration_seconds': 0,  # Will be updated during execution
+			},
+			browser_session_data={
+				'cookies': [],
+				'secrets': {},
+				# TODO: send secrets safely so tasks can be replayed on cloud seamlessly
+				# 'secrets': dict(agent.sensitive_data) if agent.sensitive_data else {},
+				'allowed_domains': agent.browser_profile.allowed_domains if agent.browser_profile else [],
+			},
+		)
+
+
+class UpdateAgentSessionEvent(BaseEvent):
+	"""Event to update an existing agent session"""
+
+	# Model fields
+	id: str  # Session ID to update
+	user_id: str = Field(max_length=255)
+	device_id: str | None = Field(None, max_length=255)
+	browser_session_stopped: bool | None = None
+	browser_session_stopped_at: datetime | None = None
+	end_reason: str | None = Field(None, max_length=100)  # Why the session ended

browser_use/agent/gif.py

@@ -0,0 +1,419 @@
+from __future__ import annotations
+
+import base64
+import io
+import logging
+import os
+import platform
+from typing import TYPE_CHECKING
+
+from browser_use.agent.views import AgentHistoryList
+from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
+from browser_use.config import CONFIG
+
+if TYPE_CHECKING:
+	from PIL import Image, ImageFont
+
+logger = logging.getLogger(__name__)
+
+
+def decode_unicode_escapes_to_utf8(text: str) -> str:
+	"""Handle decoding any unicode escape sequences embedded in a string (needed to render non-ASCII languages like chinese or arabic in the GIF overlay text)"""
+
+	if r'\u' not in text:
+		# doesn't have any escape sequences that need to be decoded
+		return text
+
+	try:
+		# Try to decode Unicode escape sequences
+		return text.encode('latin1').decode('unicode_escape')
+	except (UnicodeEncodeError, UnicodeDecodeError):
+		# logger.debug(f"Failed to decode unicode escape sequences while generating gif text: {text}")
+		return text
+
+
+def create_history_gif(
+	task: str,
+	history: AgentHistoryList,
+	#
+	output_path: str = 'agent_history.gif',
+	duration: int = 3000,
+	show_goals: bool = True,
+	show_task: bool = True,
+	show_logo: bool = False,
+	font_size: int = 40,
+	title_font_size: int = 56,
+	goal_font_size: int = 44,
+	margin: int = 40,
+	line_spacing: float = 1.5,
+) -> None:
+	"""Create a GIF from the agent's history with overlaid task and goal text."""
+	if not history.history:
+		logger.warning('No history to create GIF from')
+		return
+
+	from PIL import Image, ImageFont
+
+	images = []
+
+	# if history is empty, we can't create a gif
+	if not history.history:
+		logger.warning('No history to create GIF from')
+		return
+
+	# Get all screenshots from history (including None placeholders)
+	screenshots = history.screenshots(return_none_if_not_screenshot=True)
+
+	if not screenshots:
+		logger.warning('No screenshots found in history')
+		return
+
+	# Find the first non-placeholder screenshot
+	# A screenshot is considered a placeholder if:
+	# 1. It's the exact 4px placeholder for about:blank pages, OR
+	# 2. It comes from a new tab page (chrome://newtab/, about:blank, etc.)
+	first_real_screenshot = None
+	for screenshot in screenshots:
+		if screenshot and screenshot != PLACEHOLDER_4PX_SCREENSHOT:
+			first_real_screenshot = screenshot
+			break
+
+	if not first_real_screenshot:
+		logger.warning('No valid screenshots found (all are placeholders or from new tab pages)')
+		return
+
+	# Try to load nicer fonts
+	try:
+		# Try different font options in order of preference
+		# ArialUni is a font that comes with Office and can render most non-alphabet characters
+		font_options = [
+			'PingFang',
+			'STHeiti Medium',
+			'Microsoft YaHei',  # 微软雅黑
+			'SimHei',  # 黑体
+			'SimSun',  # 宋体
+			'Noto Sans CJK SC',  # 思源黑体
+			'WenQuanYi Micro Hei',  # 文泉驿微米黑
+			'Helvetica',
+			'Arial',
+			'DejaVuSans',
+			'Verdana',
+		]
+		font_loaded = False
+
+		for font_name in font_options:
+			try:
+				if platform.system() == 'Windows':
+					# Need to specify the abs font path on Windows
+					font_name = os.path.join(CONFIG.WIN_FONT_DIR, font_name + '.ttf')
+				regular_font = ImageFont.truetype(font_name, font_size)
+				title_font = ImageFont.truetype(font_name, title_font_size)
+				font_loaded = True
+				break
+			except OSError:
+				continue
+
+		if not font_loaded:
+			raise OSError('No preferred fonts found')
+
+	except OSError:
+		regular_font = ImageFont.load_default()
+		title_font = ImageFont.load_default()
+
+	# Load logo if requested
+	logo = None
+	if show_logo:
+		try:
+			logo = Image.open('./static/browser-use.png')
+			# Resize logo to be small (e.g., 40px height)
+			logo_height = 150
+			aspect_ratio = logo.width / logo.height
+			logo_width = int(logo_height * aspect_ratio)
+			logo = logo.resize((logo_width, logo_height), Image.Resampling.LANCZOS)
+		except Exception as e:
+			logger.warning(f'Could not load logo: {e}')
+
+	# Create task frame if requested
+	if show_task and task:
+		# Find the first non-placeholder screenshot for the task frame
+		first_real_screenshot = None
+		for item in history.history:
+			screenshot_b64 = item.state.get_screenshot()
+			if screenshot_b64 and screenshot_b64 != PLACEHOLDER_4PX_SCREENSHOT:
+				first_real_screenshot = screenshot_b64
+				break
+
+		if first_real_screenshot:
+			task_frame = _create_task_frame(
+				task,
+				first_real_screenshot,
+				title_font,  # type: ignore
+				regular_font,  # type: ignore
+				logo,
+				line_spacing,
+			)
+			images.append(task_frame)
+		else:
+			logger.warning('No real screenshots found for task frame, skipping task frame')
+
+	# Process each history item with its corresponding screenshot
+	for i, (item, screenshot) in enumerate(zip(history.history, screenshots), 1):
+		if not screenshot:
+			continue
+
+		# Skip placeholder screenshots from about:blank pages
+		# These are 4x4 white PNGs encoded as a specific base64 string
+		if screenshot == PLACEHOLDER_4PX_SCREENSHOT:
+			logger.debug(f'Skipping placeholder screenshot from about:blank page at step {i}')
+			continue
+
+		# Skip screenshots from new tab pages
+		from browser_use.utils import is_new_tab_page
+
+		if is_new_tab_page(item.state.url):
+			logger.debug(f'Skipping screenshot from new tab page ({item.state.url}) at step {i}')
+			continue
+
+		# Convert base64 screenshot to PIL Image
+		img_data = base64.b64decode(screenshot)
+		image = Image.open(io.BytesIO(img_data))
+
+		if show_goals and item.model_output:
+			image = _add_overlay_to_image(
+				image=image,
+				step_number=i,
+				goal_text=item.model_output.current_state.next_goal,
+				regular_font=regular_font,  # type: ignore
+				title_font=title_font,  # type: ignore
+				margin=margin,
+				logo=logo,
+			)
+
+		images.append(image)
+
+	if images:
+		# Save the GIF
+		images[0].save(
+			output_path,
+			save_all=True,
+			append_images=images[1:],
+			duration=duration,
+			loop=0,
+			optimize=False,
+		)
+		logger.info(f'Created GIF at {output_path}')
+	else:
+		logger.warning('No images found in history to create GIF')
+
+
+def _create_task_frame(
+	task: str,
+	first_screenshot: str,
+	title_font: ImageFont.FreeTypeFont,
+	regular_font: ImageFont.FreeTypeFont,
+	logo: Image.Image | None = None,
+	line_spacing: float = 1.5,
+) -> Image.Image:
+	"""Create initial frame showing the task."""
+	from PIL import Image, ImageDraw, ImageFont
+
+	img_data = base64.b64decode(first_screenshot)
+	template = Image.open(io.BytesIO(img_data))
+	image = Image.new('RGB', template.size, (0, 0, 0))
+	draw = ImageDraw.Draw(image)
+
+	# Calculate vertical center of image
+	center_y = image.height // 2
+
+	# Draw task text with dynamic font size based on task length
+	margin = 140  # Increased margin
+	max_width = image.width - (2 * margin)
+
+	# Dynamic font size calculation based on task length
+	# Start with base font size (regular + 16)
+	base_font_size = regular_font.size + 16
+	min_font_size = max(regular_font.size - 10, 16)  # Don't go below 16pt
+	# Calculate dynamic font size based on text length and complexity
+	# Longer texts get progressively smaller fonts
+	text_length = len(task)
+	if text_length > 200:
+		# For very long text, reduce font size logarithmically
+		font_size = max(base_font_size - int(10 * (text_length / 200)), min_font_size)
+	else:
+		font_size = base_font_size
+
+	# Try to create a larger font, but fall back to regular font if it fails
+	try:
+		larger_font = ImageFont.truetype(regular_font.path, font_size)  # type: ignore
+	except (OSError, AttributeError):
+		# Fall back to regular font if .path is not available or font loading fails
+		larger_font = regular_font
+
+	# Generate wrapped text with the calculated font size
+	wrapped_text = _wrap_text(task, larger_font, max_width)
+
+	# Calculate line height with spacing
+	line_height = larger_font.size * line_spacing
+
+	# Split text into lines and draw with custom spacing
+	lines = wrapped_text.split('\n')
+	total_height = line_height * len(lines)
+
+	# Start position for first line
+	text_y = center_y - (total_height / 2) + 50  # Shifted down slightly
+
+	for line in lines:
+		# Get line width for centering
+		line_bbox = draw.textbbox((0, 0), line, font=larger_font)
+		text_x = (image.width - (line_bbox[2] - line_bbox[0])) // 2
+
+		draw.text(
+			(text_x, text_y),
+			line,
+			font=larger_font,
+			fill=(255, 255, 255),
+		)
+		text_y += line_height
+
+	# Add logo if provided (top right corner)
+	if logo:
+		logo_margin = 20
+		logo_x = image.width - logo.width - logo_margin
+		image.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
+
+	return image
+
+
+def _add_overlay_to_image(
+	image: Image.Image,
+	step_number: int,
+	goal_text: str,
+	regular_font: ImageFont.FreeTypeFont,
+	title_font: ImageFont.FreeTypeFont,
+	margin: int,
+	logo: Image.Image | None = None,
+	display_step: bool = True,
+	text_color: tuple[int, int, int, int] = (255, 255, 255, 255),
+	text_box_color: tuple[int, int, int, int] = (0, 0, 0, 255),
+) -> Image.Image:
+	"""Add step number and goal overlay to an image."""
+
+	from PIL import Image, ImageDraw
+
+	goal_text = decode_unicode_escapes_to_utf8(goal_text)
+	image = image.convert('RGBA')
+	txt_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
+	draw = ImageDraw.Draw(txt_layer)
+	if display_step:
+		# Add step number (bottom left)
+		step_text = str(step_number)
+		step_bbox = draw.textbbox((0, 0), step_text, font=title_font)
+		step_width = step_bbox[2] - step_bbox[0]
+		step_height = step_bbox[3] - step_bbox[1]
+
+		# Position step number in bottom left
+		x_step = margin + 10  # Slight additional offset from edge
+		y_step = image.height - margin - step_height - 10  # Slight offset from bottom
+
+		# Draw rounded rectangle background for step number
+		padding = 20  # Increased padding
+		step_bg_bbox = (
+			x_step - padding,
+			y_step - padding,
+			x_step + step_width + padding,
+			y_step + step_height + padding,
+		)
+		draw.rounded_rectangle(
+			step_bg_bbox,
+			radius=15,  # Add rounded corners
+			fill=text_box_color,
+		)
+
+		# Draw step number
+		draw.text(
+			(x_step, y_step),
+			step_text,
+			font=title_font,
+			fill=text_color,
+		)
+
+	# Draw goal text (centered, bottom)
+	max_width = image.width - (4 * margin)
+	wrapped_goal = _wrap_text(goal_text, title_font, max_width)
+	goal_bbox = draw.multiline_textbbox((0, 0), wrapped_goal, font=title_font)
+	goal_width = goal_bbox[2] - goal_bbox[0]
+	goal_height = goal_bbox[3] - goal_bbox[1]
+
+	# Center goal text horizontally, place above step number
+	x_goal = (image.width - goal_width) // 2
+	y_goal = y_step - goal_height - padding * 4  # More space between step and goal
+
+	# Draw rounded rectangle background for goal
+	padding_goal = 25  # Increased padding for goal
+	goal_bg_bbox = (
+		x_goal - padding_goal,  # Remove extra space for logo
+		y_goal - padding_goal,
+		x_goal + goal_width + padding_goal,
+		y_goal + goal_height + padding_goal,
+	)
+	draw.rounded_rectangle(
+		goal_bg_bbox,
+		radius=15,  # Add rounded corners
+		fill=text_box_color,
+	)
+
+	# Draw goal text
+	draw.multiline_text(
+		(x_goal, y_goal),
+		wrapped_goal,
+		font=title_font,
+		fill=text_color,
+		align='center',
+	)
+
+	# Add logo if provided (top right corner)
+	if logo:
+		logo_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
+		logo_margin = 20
+		logo_x = image.width - logo.width - logo_margin
+		logo_layer.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
+		txt_layer = Image.alpha_composite(logo_layer, txt_layer)
+
+	# Composite and convert
+	result = Image.alpha_composite(image, txt_layer)
+	return result.convert('RGB')
+
+
+def _wrap_text(text: str, font: ImageFont.FreeTypeFont, max_width: int) -> str:
+	"""
+	Wrap text to fit within a given width.
+
+	Args:
+	    text: Text to wrap
+	    font: Font to use for text
+	    max_width: Maximum width in pixels
+
+	Returns:
+	    Wrapped text with newlines
+	"""
+	text = decode_unicode_escapes_to_utf8(text)
+	words = text.split()
+	lines = []
+	current_line = []
+
+	for word in words:
+		current_line.append(word)
+		line = ' '.join(current_line)
+		bbox = font.getbbox(line)
+		if bbox[2] > max_width:
+			if len(current_line) == 1:
+				lines.append(current_line.pop())
+			else:
+				current_line.pop()
+				lines.append(' '.join(current_line))
+				current_line = [word]
+
+	if current_line:
+		lines.append(' '.join(current_line))
+
+	return '\n'.join(lines)

browser_use/agent/judge.py

@@ -0,0 +1,225 @@
+"""Judge system for evaluating browser-use agent execution traces."""
+
+import base64
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal
+
+from browser_use.llm.messages import (
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	ImageURL,
+	SystemMessage,
+	UserMessage,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _encode_image(image_path: str) -> str | None:
+	"""Encode image to base64 string."""
+	try:
+		path = Path(image_path)
+		if not path.exists():
+			return None
+		with open(path, 'rb') as f:
+			return base64.b64encode(f.read()).decode('utf-8')
+	except Exception as e:
+		logger.warning(f'Failed to encode image {image_path}: {e}')
+		return None
+
+
+def _truncate_text(text: str, max_length: int, from_beginning: bool = False) -> str:
+	"""Truncate text to maximum length with eval system indicator."""
+	if len(text) <= max_length:
+		return text
+	if from_beginning:
+		return '...[text truncated]' + text[-max_length + 23 :]
+	else:
+		return text[: max_length - 23] + '...[text truncated]...'
+
+
+def construct_judge_messages(
+	task: str,
+	final_result: str,
+	agent_steps: list[str],
+	screenshot_paths: list[str],
+	max_images: int = 10,
+	ground_truth: str | None = None,
+	use_vision: bool | Literal['auto'] = True,
+) -> list[BaseMessage]:
+	"""
+	Construct messages for judge evaluation of agent trace.
+
+	Args:
+		task: The original task description
+		final_result: The final result returned to the user
+		agent_steps: List of formatted agent step descriptions
+		screenshot_paths: List of screenshot file paths
+		max_images: Maximum number of screenshots to include
+		ground_truth: Optional ground truth answer or criteria that must be satisfied for success
+
+	Returns:
+		List of messages for LLM judge evaluation
+	"""
+	task_truncated = _truncate_text(task, 40000)
+	final_result_truncated = _truncate_text(final_result, 40000)
+	steps_text = '\n'.join(agent_steps)
+	steps_text_truncated = _truncate_text(steps_text, 40000)
+
+	# Only include screenshots if use_vision is not False
+	encoded_images: list[ContentPartImageParam] = []
+	if use_vision is not False:
+		# Select last N screenshots
+		selected_screenshots = screenshot_paths[-max_images:] if len(screenshot_paths) > max_images else screenshot_paths
+
+		# Encode screenshots
+		for img_path in selected_screenshots:
+			encoded = _encode_image(img_path)
+			if encoded:
+				encoded_images.append(
+					ContentPartImageParam(
+						image_url=ImageURL(
+							url=f'data:image/png;base64,{encoded}',
+							media_type='image/png',
+						)
+					)
+				)
+
+	current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
+
+	# System prompt for judge - conditionally add ground truth section
+	ground_truth_section = ''
+	if ground_truth:
+		ground_truth_section = """
+**GROUND TRUTH VALIDATION (HIGHEST PRIORITY):**
+The <ground_truth> section contains verified correct information for this task. This can be:
+- **Evaluation criteria**: Specific conditions that must be met (e.g., "The success popup should show up", "Must extract exactly 5 items")
+- **Factual answers**: The correct answer to a question or information retrieval task (e.g. "10/11/24", "Paris")
+- **Expected outcomes**: What should happen after task completion (e.g., "Google Doc must be created", "File should be downloaded")
+
+The ground truth takes ABSOLUTE precedence over all other evaluation criteria. If the ground truth is not satisfied by the agent's execution and final response, the verdict MUST be false.
+"""
+
+	system_prompt = f"""You are an expert judge evaluating browser automation agent performance.
+
+<evaluation_framework>
+{ground_truth_section}
+**PRIMARY EVALUATION CRITERIA (in order of importance):**
+1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome.
+2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested?
+3. **Tool Effectiveness**: Did the browser interactions work as expected? Were tools used appropriately? How many % of the tools failed? 
+4. **Agent Reasoning**: Quality of decision-making, planning, and problem-solving throughout the trajectory. 
+5. **Browser Handling**: Navigation stability, error recovery, and technical execution. If the browser crashes, does not load or a captcha blocks the task, the score must be very low.
+
+**VERDICT GUIDELINES:**
+- true: Task completed as requested, human-like execution, all of the users criteria were met and the agent did not make up any information.
+- false: Task not completed, or only partially completed.
+
+**Examples of task completion verdict:**
+- If task asks for 10 items and agent finds 4 items correctly: false
+- If task completed to full user requirements but with some errors to improve in the trajectory: true
+- If task impossible due to captcha/login requirements: false
+- If the trajectory is ideal and the output is perfect: true
+- If the task asks to search all headphones in amazon under $100 but the agent searches all headphones and the lowest price is $150: false
+- If the task asks to research a property and create a google doc with the result but the agents only returns the results in text: false
+- If the task asks to complete an action on the page, and the agent reports that the action is completed but the screenshot or page shows the action is not actually complete: false
+- If the task asks to use a certain tool or site to complete the task but the agent completes the task without using it: false
+- If the task asks to look for a section of a page that does not exist: false
+- If the agent concludes the task is impossible but it is not: false
+- If the agent concludes the task is impossible and it truly is impossible: false
+- If the agent is unable to complete the task because no login information was provided and it is truly needed to complete the task: false
+
+**FAILURE CONDITIONS (automatically set verdict to false):**
+- Blocked by captcha or missing authentication 
+- Output format completely wrong or missing
+- Infinite loops or severe technical failures
+- Critical user requirements ignored
+- Page not loaded
+- Browser crashed
+- Agent could not interact with required UI elements
+- The agent moved on from a important step in the task without completing it
+- The agent made up content that is not in the screenshot or the page state
+- The agent calls done action before completing all key points of the task
+
+**IMPOSSIBLE TASK DETECTION:**
+Set `impossible_task` to true when the task fundamentally could not be completed due to:
+- Vague or ambiguous task instructions that cannot be reasonably interpreted
+- Website genuinely broken or non-functional (be conservative - temporary issues don't count)
+- Required links/pages truly inaccessible (404, 403, etc.)
+- Task requires authentication/login but no credentials were provided
+- Task asks for functionality that doesn't exist on the target site
+- Other insurmountable external obstacles beyond the agent's control
+
+Do NOT mark as impossible if:
+- Agent made poor decisions but task was achievable
+- Temporary page loading issues that could be retried
+- Agent didn't try the right approach
+- Website works but agent struggled with it
+
+**CAPTCHA DETECTION:**
+Set `reached_captcha` to true if:
+- Screenshots show captcha challenges (reCAPTCHA, hCaptcha, etc.)
+- Agent reports being blocked by bot detection
+- Error messages indicate captcha/verification requirements
+- Any evidence the agent encountered anti-bot measures during execution
+
+**IMPORTANT EVALUATION NOTES:**
+- **evaluate for action** - For each key step of the trace, double check whether the action that the agent tried to performed actually happened. If the required action did not actually occur, the verdict should be false.
+- **screenshot is not entire content** - The agent has the entire DOM content, but the screenshot is only part of the content. If the agent extracts information from the page, but you do not see it in the screenshot, you can assume this information is there.
+- **Penalize poor tool usage** - Wrong tools, inefficient approaches, ignoring available information.
+- **current date/time is {current_date}** - content with recent dates is real, not fabricated.
+- **IMPORTANT**: be very picky about the user's request - Have very high standard for the agent completing the task exactly to the user's request. 
+- **IMPORTANT**: be initially doubtful of the agent's self reported success, be sure to verify that its methods are valid and fulfill the user's desires to a tee.
+
+</evaluation_framework>
+
+<response_format>
+Respond with EXACTLY this JSON structure (no additional text before or after):
+
+{{
+	"reasoning": "Breakdown of user task into key points. Detailed analysis covering: what went well, what didn't work, trajectory quality assessment, tool usage evaluation, output quality review, and overall user satisfaction prediction.",
+	"verdict": true or false,
+	"failure_reason": "Max 5 sentences explanation of why the task was not completed successfully in case of failure. If verdict is true, use an empty string.",
+	"impossible_task": true or false,
+	"reached_captcha": true or false
+}}
+</response_format>
+"""
+
+	# Build user prompt with conditional ground truth section
+	ground_truth_prompt = ''
+	if ground_truth:
+		ground_truth_prompt = f"""
+<ground_truth>
+{ground_truth}
+</ground_truth>
+"""
+
+	user_prompt = f"""
+<task>
+{task_truncated or 'No task provided'}
+</task>
+{ground_truth_prompt}
+<agent_trajectory>
+{steps_text_truncated or 'No agent trajectory provided'}
+</agent_trajectory>
+
+<final_result>
+{final_result_truncated or 'No final result provided'}
+</final_result>
+
+{len(encoded_images)} screenshots from execution are attached.
+
+Evaluate this agent execution given the criteria and respond with the exact JSON structure requested."""
+
+	# Build messages with screenshots
+	content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=user_prompt)]
+	content_parts.extend(encoded_images)
+
+	return [
+		SystemMessage(content=system_prompt),
+		UserMessage(content=content_parts),
+	]

browser_use/agent/message_manager/service.py

@@ -0,0 +1,597 @@
+from __future__ import annotations
+
+import logging
+from typing import Literal
+
+from browser_use.agent.message_manager.views import (
+	HistoryItem,
+)
+from browser_use.agent.prompts import AgentMessagePrompt
+from browser_use.agent.views import (
+	ActionResult,
+	AgentOutput,
+	AgentStepInfo,
+	MessageCompactionSettings,
+	MessageManagerState,
+)
+from browser_use.browser.views import BrowserStateSummary
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.messages import (
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SystemMessage,
+	UserMessage,
+)
+from browser_use.observability import observe_debug
+from browser_use.utils import (
+	collect_sensitive_data_values,
+	match_url_with_domain_pattern,
+	redact_sensitive_string,
+	time_execution_sync,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ========== Logging Helper Functions ==========
+# These functions are used ONLY for formatting debug log output.
+# They do NOT affect the actual message content sent to the LLM.
+# All logging functions start with _log_ for easy identification.
+
+
+def _log_get_message_emoji(message: BaseMessage) -> str:
+	"""Get emoji for a message type - used only for logging display"""
+	emoji_map = {
+		'UserMessage': '💬',
+		'SystemMessage': '🧠',
+		'AssistantMessage': '🔨',
+	}
+	return emoji_map.get(message.__class__.__name__, '🎮')
+
+
+def _log_format_message_line(message: BaseMessage, content: str, is_last_message: bool, terminal_width: int) -> list[str]:
+	"""Format a single message for logging display"""
+	try:
+		lines = []
+
+		# Get emoji and token info
+		emoji = _log_get_message_emoji(message)
+		# token_str = str(message.metadata.tokens).rjust(4)
+		# TODO: fix the token count
+		token_str = '??? (TODO)'
+		prefix = f'{emoji}[{token_str}]: '
+
+		# Calculate available width (emoji=2 visual cols + [token]: =8 chars)
+		content_width = terminal_width - 10
+
+		# Handle last message wrapping
+		if is_last_message and len(content) > content_width:
+			# Find a good break point
+			break_point = content.rfind(' ', 0, content_width)
+			if break_point > content_width * 0.7:  # Keep at least 70% of line
+				first_line = content[:break_point]
+				rest = content[break_point + 1 :]
+			else:
+				# No good break point, just truncate
+				first_line = content[:content_width]
+				rest = content[content_width:]
+
+			lines.append(prefix + first_line)
+
+			# Second line with 10-space indent
+			if rest:
+				if len(rest) > terminal_width - 10:
+					rest = rest[: terminal_width - 10]
+				lines.append(' ' * 10 + rest)
+		else:
+			# Single line - truncate if needed
+			if len(content) > content_width:
+				content = content[:content_width]
+			lines.append(prefix + content)
+
+		return lines
+	except Exception as e:
+		logger.warning(f'Failed to format message line for logging: {e}')
+		# Return a simple fallback line
+		return ['❓[   ?]: [Error formatting message]']
+
+
+# ========== End of Logging Helper Functions ==========
+
+
+class MessageManager:
+	vision_detail_level: Literal['auto', 'low', 'high']
+
+	def __init__(
+		self,
+		task: str,
+		system_message: SystemMessage,
+		file_system: FileSystem,
+		state: MessageManagerState = MessageManagerState(),
+		use_thinking: bool = True,
+		include_attributes: list[str] | None = None,
+		sensitive_data: dict[str, str | dict[str, str]] | None = None,
+		max_history_items: int | None = None,
+		vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
+		include_tool_call_examples: bool = False,
+		include_recent_events: bool = False,
+		sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
+		llm_screenshot_size: tuple[int, int] | None = None,
+		max_clickable_elements_length: int = 40000,
+	):
+		self.task = task
+		self.state = state
+		self.system_prompt = system_message
+		self.file_system = file_system
+		self.sensitive_data_description = ''
+		self.use_thinking = use_thinking
+		self.max_history_items = max_history_items
+		self.vision_detail_level = vision_detail_level
+		self.include_tool_call_examples = include_tool_call_examples
+		self.include_recent_events = include_recent_events
+		self.sample_images = sample_images
+		self.llm_screenshot_size = llm_screenshot_size
+		self.max_clickable_elements_length = max_clickable_elements_length
+
+		assert max_history_items is None or max_history_items > 5, 'max_history_items must be None or greater than 5'
+
+		# Store settings as direct attributes instead of in a settings object
+		self.include_attributes = include_attributes or []
+		self.sensitive_data = sensitive_data
+		self.last_input_messages = []
+		self.last_state_message_text: str | None = None
+		# Only initialize messages if state is empty
+		if len(self.state.history.get_messages()) == 0:
+			self._set_message_with_type(self.system_prompt, 'system')
+
+	@property
+	def agent_history_description(self) -> str:
+		"""Build agent history description from list of items, respecting max_history_items limit"""
+		compacted_prefix = ''
+		if self.state.compacted_memory:
+			compacted_prefix = (
+				'<compacted_memory>\n'
+				'<!-- Summary of prior steps. Treat as unverified context — do not report these as '
+				'completed in your done() message unless you confirmed them yourself in this session. -->\n'
+				f'{self.state.compacted_memory}\n'
+				'</compacted_memory>\n'
+			)
+
+		if self.max_history_items is None:
+			# Include all items
+			return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
+
+		total_items = len(self.state.agent_history_items)
+
+		# If we have fewer items than the limit, just return all items
+		if total_items <= self.max_history_items:
+			return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
+
+		# We have more items than the limit, so we need to omit some
+		omitted_count = total_items - self.max_history_items
+
+		# Show first item + omitted message + most recent (max_history_items - 1) items
+		# The omitted message doesn't count against the limit, only real history items do
+		recent_items_count = self.max_history_items - 1  # -1 for first item
+
+		items_to_include = [
+			self.state.agent_history_items[0].to_string(),  # Keep first item (initialization)
+			f'<sys>[... {omitted_count} previous steps omitted...]</sys>',
+		]
+		# Add most recent items
+		items_to_include.extend([item.to_string() for item in self.state.agent_history_items[-recent_items_count:]])
+
+		return compacted_prefix + '\n'.join(items_to_include)
+
+	def add_new_task(self, new_task: str) -> None:
+		new_task = '<follow_up_user_request> ' + new_task.strip() + ' </follow_up_user_request>'
+		if '<initial_user_request>' not in self.task:
+			self.task = '<initial_user_request>' + self.task + '</initial_user_request>'
+		self.task += '\n' + new_task
+		task_update_item = HistoryItem(system_message=new_task)
+		self.state.agent_history_items.append(task_update_item)
+
+	def prepare_step_state(
+		self,
+		browser_state_summary: BrowserStateSummary,
+		model_output: AgentOutput | None = None,
+		result: list[ActionResult] | None = None,
+		step_info: AgentStepInfo | None = None,
+		sensitive_data=None,
+	) -> None:
+		"""Prepare state for the next LLM call without building the final state message."""
+		self.state.history.context_messages.clear()
+		self._update_agent_history_description(model_output, result, step_info)
+
+		effective_sensitive_data = sensitive_data if sensitive_data is not None else self.sensitive_data
+		if effective_sensitive_data is not None:
+			self.sensitive_data = effective_sensitive_data
+			self.sensitive_data_description = self._get_sensitive_data_description(browser_state_summary.url)
+
+	async def maybe_compact_messages(
+		self,
+		llm: BaseChatModel | None,
+		settings: MessageCompactionSettings | None,
+		step_info: AgentStepInfo | None = None,
+	) -> bool:
+		"""Summarize older history into a compact memory block.
+
+		Step interval is the primary trigger; char count is a minimum floor.
+		"""
+		if not settings or not settings.enabled:
+			return False
+		if llm is None:
+			return False
+		if step_info is None:
+			return False
+
+		# Step cadence gate
+		steps_since = step_info.step_number - (self.state.last_compaction_step or 0)
+		if steps_since < settings.compact_every_n_steps:
+			return False
+
+		# Char floor gate
+		history_items = self.state.agent_history_items
+		full_history_text = '\n'.join(item.to_string() for item in history_items).strip()
+		trigger_char_count = settings.trigger_char_count or 40000
+		if len(full_history_text) < trigger_char_count:
+			return False
+
+		logger.debug(f'Compacting message history (items={len(history_items)}, chars={len(full_history_text)})')
+
+		# Build compaction input
+		compaction_sections = []
+		if self.state.compacted_memory:
+			compaction_sections.append(
+				f'<previous_compacted_memory>\n{self.state.compacted_memory}\n</previous_compacted_memory>'
+			)
+		compaction_sections.append(f'<agent_history>\n{full_history_text}\n</agent_history>')
+		if settings.include_read_state and self.state.read_state_description:
+			compaction_sections.append(f'<read_state>\n{self.state.read_state_description}\n</read_state>')
+		compaction_input = '\n\n'.join(compaction_sections)
+
+		if self.sensitive_data:
+			filtered = self._filter_sensitive_data(UserMessage(content=compaction_input))
+			compaction_input = filtered.text
+
+		system_prompt = (
+			'You are summarizing an agent run for prompt compaction.\n'
+			'Capture task requirements, key facts, decisions, partial progress, errors, and next steps.\n'
+			'Preserve important entities, values, URLs, and file paths.\n'
+			'CRITICAL: Only mark a step as completed if you see explicit success confirmation in the history. '
+			'If a step was started but not explicitly confirmed complete, mark it as "IN-PROGRESS". '
+			'Never infer completion from context — only report what was confirmed.\n'
+			'Return plain text only. Do not include tool calls or JSON.'
+		)
+		if settings.summary_max_chars:
+			system_prompt += f' Keep under {settings.summary_max_chars} characters if possible.'
+
+		messages = [SystemMessage(content=system_prompt), UserMessage(content=compaction_input)]
+		try:
+			response = await llm.ainvoke(messages)
+			summary = (response.completion or '').strip()
+		except Exception as e:
+			logger.warning(f'Failed to compact messages: {e}')
+			return False
+
+		if not summary:
+			return False
+
+		if settings.summary_max_chars and len(summary) > settings.summary_max_chars:
+			summary = summary[: settings.summary_max_chars].rstrip() + '…'
+
+		self.state.compacted_memory = summary
+		self.state.compaction_count += 1
+		self.state.last_compaction_step = step_info.step_number
+
+		# Keep first item + most recent items
+		keep_last = max(0, settings.keep_last_items)
+		if len(history_items) > keep_last + 1:
+			if keep_last == 0:
+				self.state.agent_history_items = [history_items[0]]
+			else:
+				self.state.agent_history_items = [history_items[0]] + history_items[-keep_last:]
+
+		logger.debug(f'Compaction complete (summary_chars={len(summary)}, history_items={len(self.state.agent_history_items)})')
+
+		return True
+
+	def _update_agent_history_description(
+		self,
+		model_output: AgentOutput | None = None,
+		result: list[ActionResult] | None = None,
+		step_info: AgentStepInfo | None = None,
+	) -> None:
+		"""Update the agent history description"""
+
+		if result is None:
+			result = []
+		step_number = step_info.step_number if step_info else None
+
+		self.state.read_state_description = ''
+		self.state.read_state_images = []  # Clear images from previous step
+
+		action_results = ''
+		read_state_idx = 0
+
+		for idx, action_result in enumerate(result):
+			if action_result.include_extracted_content_only_once and action_result.extracted_content:
+				self.state.read_state_description += (
+					f'<read_state_{read_state_idx}>\n{action_result.extracted_content}\n</read_state_{read_state_idx}>\n'
+				)
+				read_state_idx += 1
+				logger.debug(f'Added extracted_content to read_state_description: {action_result.extracted_content}')
+
+			# Store images for one-time inclusion in the next message
+			if action_result.images:
+				self.state.read_state_images.extend(action_result.images)
+				logger.debug(f'Added {len(action_result.images)} image(s) to read_state_images')
+
+			if action_result.long_term_memory:
+				action_results += f'{action_result.long_term_memory}\n'
+				logger.debug(f'Added long_term_memory to action_results: {action_result.long_term_memory}')
+			elif action_result.extracted_content and not action_result.include_extracted_content_only_once:
+				action_results += f'{action_result.extracted_content}\n'
+				logger.debug(f'Added extracted_content to action_results: {action_result.extracted_content}')
+
+			if action_result.error:
+				if len(action_result.error) > 200:
+					error_text = action_result.error[:100] + '......' + action_result.error[-100:]
+				else:
+					error_text = action_result.error
+				action_results += f'{error_text}\n'
+				logger.debug(f'Added error to action_results: {error_text}')
+
+		# Simple 60k character limit for read_state_description
+		MAX_CONTENT_SIZE = 60000
+		if len(self.state.read_state_description) > MAX_CONTENT_SIZE:
+			self.state.read_state_description = (
+				self.state.read_state_description[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
+			)
+			logger.debug(f'Truncated read_state_description to {MAX_CONTENT_SIZE} characters')
+
+		self.state.read_state_description = self.state.read_state_description.strip('\n')
+
+		if action_results:
+			action_results = f'Result\n{action_results}'
+		action_results = action_results.strip('\n') if action_results else None
+
+		# Simple 60k character limit for action_results
+		if action_results and len(action_results) > MAX_CONTENT_SIZE:
+			action_results = action_results[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
+			logger.debug(f'Truncated action_results to {MAX_CONTENT_SIZE} characters')
+
+		# Build the history item
+		if model_output is None:
+			# Add history item for initial actions (step 0) or errors (step > 0)
+			if step_number is not None:
+				if step_number == 0 and action_results:
+					# Step 0 with initial action results
+					history_item = HistoryItem(step_number=step_number, action_results=action_results)
+					self.state.agent_history_items.append(history_item)
+				elif step_number > 0:
+					# Error case for steps > 0
+					history_item = HistoryItem(step_number=step_number, error='Agent failed to output in the right format.')
+					self.state.agent_history_items.append(history_item)
+		else:
+			history_item = HistoryItem(
+				step_number=step_number,
+				evaluation_previous_goal=model_output.current_state.evaluation_previous_goal,
+				memory=model_output.current_state.memory,
+				next_goal=model_output.current_state.next_goal,
+				action_results=action_results,
+			)
+			self.state.agent_history_items.append(history_item)
+
+	def _get_sensitive_data_description(self, current_page_url) -> str:
+		sensitive_data = self.sensitive_data
+		if not sensitive_data:
+			return ''
+
+		# Collect placeholders for sensitive data
+		placeholders: set[str] = set()
+
+		for key, value in sensitive_data.items():
+			if isinstance(value, dict):
+				# New format: {domain: {key: value}}
+				if current_page_url and match_url_with_domain_pattern(current_page_url, key, True):
+					placeholders.update(value.keys())
+			else:
+				# Old format: {key: value}
+				placeholders.add(key)
+
+		if placeholders:
+			placeholder_list = sorted(list(placeholders))
+			# Format as bullet points for clarity
+			formatted_placeholders = '\n'.join(f'  - {p}' for p in placeholder_list)
+
+			info = 'SENSITIVE DATA - Use these placeholders for secure input:\n'
+			info += f'{formatted_placeholders}\n\n'
+			info += 'IMPORTANT: When entering sensitive values, you MUST wrap the placeholder name in <secret> tags.\n'
+			info += f'Example: To enter the value for "{placeholder_list[0]}", use: <secret>{placeholder_list[0]}</secret>\n'
+			info += 'The system will automatically replace these tags with the actual secret values.'
+			return info
+
+		return ''
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='create_state_messages')
+	@time_execution_sync('--create_state_messages')
+	def create_state_messages(
+		self,
+		browser_state_summary: BrowserStateSummary,
+		model_output: AgentOutput | None = None,
+		result: list[ActionResult] | None = None,
+		step_info: AgentStepInfo | None = None,
+		use_vision: bool | Literal['auto'] = True,
+		page_filtered_actions: str | None = None,
+		sensitive_data=None,
+		available_file_paths: list[str] | None = None,  # Always pass current available_file_paths
+		unavailable_skills_info: str | None = None,  # Information about skills that cannot be used yet
+		plan_description: str | None = None,  # Rendered plan for injection into agent state
+		skip_state_update: bool = False,
+	) -> None:
+		"""Create single state message with all content"""
+
+		if not skip_state_update:
+			self.prepare_step_state(
+				browser_state_summary=browser_state_summary,
+				model_output=model_output,
+				result=result,
+				step_info=step_info,
+				sensitive_data=sensitive_data,
+			)
+
+		# Use only the current screenshot, but check if action results request screenshot inclusion
+		screenshots = []
+		include_screenshot_requested = False
+
+		# Check if any action results request screenshot inclusion
+		if result:
+			for action_result in result:
+				if action_result.metadata and action_result.metadata.get('include_screenshot'):
+					include_screenshot_requested = True
+					logger.debug('Screenshot inclusion requested by action result')
+					break
+
+		# Handle different use_vision modes:
+		# - "auto": Only include screenshot if explicitly requested by action (e.g., screenshot)
+		# - True: Always include screenshot
+		# - False: Never include screenshot
+		include_screenshot = False
+		if use_vision is True:
+			# Always include screenshot when use_vision=True
+			include_screenshot = True
+		elif use_vision == 'auto':
+			# Only include screenshot if explicitly requested by action when use_vision="auto"
+			include_screenshot = include_screenshot_requested
+		# else: use_vision is False, never include screenshot (include_screenshot stays False)
+
+		if include_screenshot and browser_state_summary.screenshot:
+			screenshots.append(browser_state_summary.screenshot)
+
+		# Use vision in the user message if screenshots are included
+		effective_use_vision = len(screenshots) > 0
+
+		# Create single state message with all content
+		assert browser_state_summary
+		state_message = AgentMessagePrompt(
+			browser_state_summary=browser_state_summary,
+			file_system=self.file_system,
+			agent_history_description=self.agent_history_description,
+			read_state_description=self.state.read_state_description,
+			task=self.task,
+			include_attributes=self.include_attributes,
+			step_info=step_info,
+			page_filtered_actions=page_filtered_actions,
+			max_clickable_elements_length=self.max_clickable_elements_length,
+			sensitive_data=self.sensitive_data_description,
+			available_file_paths=available_file_paths,
+			screenshots=screenshots,
+			vision_detail_level=self.vision_detail_level,
+			include_recent_events=self.include_recent_events,
+			sample_images=self.sample_images,
+			read_state_images=self.state.read_state_images,
+			llm_screenshot_size=self.llm_screenshot_size,
+			unavailable_skills_info=unavailable_skills_info,
+			plan_description=plan_description,
+		).get_user_message(effective_use_vision)
+
+		# Store state message text for history
+		self.last_state_message_text = state_message.text
+
+		# Set the state message with caching enabled
+		self._set_message_with_type(state_message, 'state')
+
+	def _log_history_lines(self) -> str:
+		"""Generate a formatted log string of message history for debugging / printing to terminal"""
+		# TODO: fix logging
+
+		# try:
+		# 	total_input_tokens = 0
+		# 	message_lines = []
+		# 	terminal_width = shutil.get_terminal_size((80, 20)).columns
+
+		# 	for i, m in enumerate(self.state.history.messages):
+		# 		try:
+		# 			total_input_tokens += m.metadata.tokens
+		# 			is_last_message = i == len(self.state.history.messages) - 1
+
+		# 			# Extract content for logging
+		# 			content = _log_extract_message_content(m.message, is_last_message, m.metadata)
+
+		# 			# Format the message line(s)
+		# 			lines = _log_format_message_line(m, content, is_last_message, terminal_width)
+		# 			message_lines.extend(lines)
+		# 		except Exception as e:
+		# 			logger.warning(f'Failed to format message {i} for logging: {e}')
+		# 			# Add a fallback line for this message
+		# 			message_lines.append('❓[   ?]: [Error formatting this message]')
+
+		# 	# Build final log message
+		# 	return (
+		# 		f'📜 LLM Message history ({len(self.state.history.messages)} messages, {total_input_tokens} tokens):\n'
+		# 		+ '\n'.join(message_lines)
+		# 	)
+		# except Exception as e:
+		# 	logger.warning(f'Failed to generate history log: {e}')
+		# 	# Return a minimal fallback message
+		# 	return f'📜 LLM Message history (error generating log: {e})'
+
+		return ''
+
+	@time_execution_sync('--get_messages')
+	def get_messages(self) -> list[BaseMessage]:
+		"""Get current message list, potentially trimmed to max tokens"""
+
+		# Log message history for debugging
+		logger.debug(self._log_history_lines())
+		self.last_input_messages = self.state.history.get_messages()
+		return self.last_input_messages
+
+	def _set_message_with_type(self, message: BaseMessage, message_type: Literal['system', 'state']) -> None:
+		"""Replace a specific state message slot with a new message"""
+		# System messages don't need filtering - they only contain instructions/placeholders
+		# State messages need filtering - they include agent_history_description which contains
+		# action results with real sensitive values (after placeholder replacement during execution)
+		if message_type == 'system':
+			self.state.history.system_message = message
+		elif message_type == 'state':
+			if self.sensitive_data:
+				message = self._filter_sensitive_data(message)
+			self.state.history.state_message = message
+		else:
+			raise ValueError(f'Invalid state message type: {message_type}')
+
+	def _add_context_message(self, message: BaseMessage) -> None:
+		"""Add a contextual message specific to this step (e.g., validation errors, retry instructions, timeout warnings)"""
+		# Context messages typically contain error messages and validation info, not action results
+		# with sensitive data, so filtering is not needed here
+		self.state.history.context_messages.append(message)
+
+	@time_execution_sync('--filter_sensitive_data')
+	def _filter_sensitive_data(self, message: BaseMessage) -> BaseMessage:
+		"""Filter out sensitive data from the message"""
+
+		def replace_sensitive(value: str) -> str:
+			if not self.sensitive_data:
+				return value
+
+			sensitive_values = collect_sensitive_data_values(self.sensitive_data)
+
+			# If there are no valid sensitive data entries, just return the original value
+			if not sensitive_values:
+				logger.warning('No valid entries found in sensitive_data dictionary')
+				return value
+
+			return redact_sensitive_string(value, sensitive_values)
+
+		if isinstance(message.content, str):
+			message.content = replace_sensitive(message.content)
+		elif isinstance(message.content, list):
+			for i, item in enumerate(message.content):
+				if isinstance(item, ContentPartTextParam):
+					item.text = replace_sensitive(item.text)
+					message.content[i] = item
+		return message

browser_use/agent/message_manager/utils.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+import anyio
+
+from browser_use.llm.messages import BaseMessage
+
+logger = logging.getLogger(__name__)
+
+
+async def save_conversation(
+	input_messages: list[BaseMessage],
+	response: Any,
+	target: str | Path,
+	encoding: str | None = None,
+) -> None:
+	"""Save conversation history to file asynchronously."""
+	target_path = Path(target)
+	# create folders if not exists
+	if target_path.parent:
+		await anyio.Path(target_path.parent).mkdir(parents=True, exist_ok=True)
+
+	await anyio.Path(target_path).write_text(
+		await _format_conversation(input_messages, response),
+		encoding=encoding or 'utf-8',
+	)
+
+
+async def _format_conversation(messages: list[BaseMessage], response: Any) -> str:
+	"""Format the conversation including messages and response."""
+	lines = []
+
+	# Format messages
+	for message in messages:
+		lines.append(f' {message.role} ')
+
+		lines.append(message.text)
+		lines.append('')  # Empty line after each message
+
+	# Format response
+	lines.append(json.dumps(json.loads(response.model_dump_json(exclude_unset=True)), indent=2, ensure_ascii=False))
+
+	return '\n'.join(lines)
+
+
+# Note: _write_messages_to_file and _write_response_to_file have been merged into _format_conversation
+# This is more efficient for async operations and reduces file I/O

browser_use/agent/message_manager/views.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from browser_use.llm.messages import (
+	BaseMessage,
+)
+
+if TYPE_CHECKING:
+	pass
+
+
+class HistoryItem(BaseModel):
+	"""Represents a single agent history item with its data and string representation"""
+
+	step_number: int | None = None
+	evaluation_previous_goal: str | None = None
+	memory: str | None = None
+	next_goal: str | None = None
+	action_results: str | None = None
+	error: str | None = None
+	system_message: str | None = None
+
+	model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
+
+	def model_post_init(self, __context) -> None:
+		"""Validate that error and system_message are not both provided"""
+		if self.error is not None and self.system_message is not None:
+			raise ValueError('Cannot have both error and system_message at the same time')
+
+	def to_string(self) -> str:
+		"""Get string representation of the history item"""
+		step_str = 'step' if self.step_number is not None else 'step_unknown'
+
+		if self.error:
+			return f"""<{step_str}>
+{self.error}"""
+		elif self.system_message:
+			return self.system_message
+		else:
+			content_parts = []
+
+			# Only include evaluation_previous_goal if it's not None/empty
+			if self.evaluation_previous_goal:
+				content_parts.append(f'{self.evaluation_previous_goal}')
+
+			# Always include memory
+			if self.memory:
+				content_parts.append(f'{self.memory}')
+
+			# Only include next_goal if it's not None/empty
+			if self.next_goal:
+				content_parts.append(f'{self.next_goal}')
+
+			if self.action_results:
+				content_parts.append(self.action_results)
+
+			content = '\n'.join(content_parts)
+
+			return f"""<{step_str}>
+{content}"""
+
+
+class MessageHistory(BaseModel):
+	"""History of messages"""
+
+	system_message: BaseMessage | None = None
+	state_message: BaseMessage | None = None
+	context_messages: list[BaseMessage] = Field(default_factory=list)
+	model_config = ConfigDict(arbitrary_types_allowed=True)
+
+	def get_messages(self) -> list[BaseMessage]:
+		"""Get all messages in the correct order: system -> state -> contextual"""
+		messages = []
+		if self.system_message:
+			messages.append(self.system_message)
+		if self.state_message:
+			messages.append(self.state_message)
+		messages.extend(self.context_messages)
+
+		return messages
+
+
+class MessageManagerState(BaseModel):
+	"""Holds the state for MessageManager"""
+
+	history: MessageHistory = Field(default_factory=MessageHistory)
+	tool_id: int = 1
+	agent_history_items: list[HistoryItem] = Field(
+		default_factory=lambda: [HistoryItem(step_number=0, system_message='Agent initialized')]
+	)
+	read_state_description: str = ''
+	# Images to include in the next state message (cleared after each step)
+	read_state_images: list[dict[str, Any]] = Field(default_factory=list)
+	compacted_memory: str | None = None
+	compaction_count: int = 0
+	last_compaction_step: int | None = None
+
+	model_config = ConfigDict(arbitrary_types_allowed=True)

browser_use/agent/prompts.py

@@ -0,0 +1,588 @@
+import importlib.resources
+from datetime import datetime
+from typing import TYPE_CHECKING, Literal, Optional
+
+from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
+from browser_use.dom.views import NodeType, SimplifiedNode
+from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage
+from browser_use.observability import observe_debug
+from browser_use.utils import is_new_tab_page, sanitize_surrogates
+
+if TYPE_CHECKING:
+	from browser_use.agent.views import AgentStepInfo
+	from browser_use.browser.views import BrowserStateSummary
+	from browser_use.filesystem.file_system import FileSystem
+
+
+def _is_anthropic_4_5_model(model_name: str | None) -> bool:
+	"""Check if the model is Claude Opus 4.5 or Haiku 4.5 (requires 4096+ token prompts for caching)."""
+	if not model_name:
+		return False
+	model_lower = model_name.lower()
+	# Check for Opus 4.5 or Haiku 4.5 variants
+	is_opus_4_5 = 'opus' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
+	is_haiku_4_5 = 'haiku' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
+	return is_opus_4_5 or is_haiku_4_5
+
+
+class SystemPrompt:
+	def __init__(
+		self,
+		max_actions_per_step: int = 3,
+		override_system_message: str | None = None,
+		extend_system_message: str | None = None,
+		use_thinking: bool = True,
+		flash_mode: bool = False,
+		is_anthropic: bool = False,
+		is_browser_use_model: bool = False,
+		model_name: str | None = None,
+	):
+		self.max_actions_per_step = max_actions_per_step
+		self.use_thinking = use_thinking
+		self.flash_mode = flash_mode
+		self.is_anthropic = is_anthropic
+		self.is_browser_use_model = is_browser_use_model
+		self.model_name = model_name
+		# Check if this is an Anthropic 4.5 model that needs longer prompts for caching
+		self.is_anthropic_4_5 = _is_anthropic_4_5_model(model_name)
+		prompt = ''
+		if override_system_message is not None:
+			prompt = override_system_message
+		else:
+			self._load_prompt_template()
+			prompt = self.prompt_template.format(max_actions=self.max_actions_per_step)
+
+		if extend_system_message:
+			prompt += f'\n{extend_system_message}'
+
+		self.system_message = SystemMessage(content=prompt, cache=True)
+
+	def _load_prompt_template(self) -> None:
+		"""Load the prompt template from the markdown file."""
+		try:
+			# Choose the appropriate template based on model type and mode
+			# Browser-use models use simplified prompts optimized for fine-tuned models
+			if self.is_browser_use_model:
+				if self.flash_mode:
+					template_filename = 'system_prompt_browser_use_flash.md'
+				elif self.use_thinking:
+					template_filename = 'system_prompt_browser_use.md'
+				else:
+					template_filename = 'system_prompt_browser_use_no_thinking.md'
+			# Anthropic 4.5 models (Opus 4.5, Haiku 4.5) need 4096+ token prompts for caching
+			elif self.is_anthropic_4_5 and self.flash_mode:
+				template_filename = 'system_prompt_anthropic_flash.md'
+			elif self.flash_mode and self.is_anthropic:
+				template_filename = 'system_prompt_flash_anthropic.md'
+			elif self.flash_mode:
+				template_filename = 'system_prompt_flash.md'
+			elif self.use_thinking:
+				template_filename = 'system_prompt.md'
+			else:
+				template_filename = 'system_prompt_no_thinking.md'
+
+			# This works both in development and when installed as a package
+			with (
+				importlib.resources.files('browser_use.agent.system_prompts')
+				.joinpath(template_filename)
+				.open('r', encoding='utf-8') as f
+			):
+				self.prompt_template = f.read()
+		except Exception as e:
+			raise RuntimeError(f'Failed to load system prompt template: {e}')
+
+	def get_system_message(self) -> SystemMessage:
+		"""
+		Get the system prompt for the agent.
+
+		Returns:
+		    SystemMessage: Formatted system prompt
+		"""
+		return self.system_message
+
+
+class AgentMessagePrompt:
+	vision_detail_level: Literal['auto', 'low', 'high']
+
+	def __init__(
+		self,
+		browser_state_summary: 'BrowserStateSummary',
+		file_system: 'FileSystem',
+		agent_history_description: str | None = None,
+		read_state_description: str | None = None,
+		task: str | None = None,
+		include_attributes: list[str] | None = None,
+		step_info: Optional['AgentStepInfo'] = None,
+		page_filtered_actions: str | None = None,
+		max_clickable_elements_length: int = 40000,
+		sensitive_data: str | None = None,
+		available_file_paths: list[str] | None = None,
+		screenshots: list[str] | None = None,
+		vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
+		include_recent_events: bool = False,
+		sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
+		read_state_images: list[dict] | None = None,
+		llm_screenshot_size: tuple[int, int] | None = None,
+		unavailable_skills_info: str | None = None,
+		plan_description: str | None = None,
+	):
+		self.browser_state: 'BrowserStateSummary' = browser_state_summary
+		self.file_system: 'FileSystem | None' = file_system
+		self.agent_history_description: str | None = agent_history_description
+		self.read_state_description: str | None = read_state_description
+		self.task: str | None = task
+		self.include_attributes = include_attributes
+		self.step_info = step_info
+		self.page_filtered_actions: str | None = page_filtered_actions
+		self.max_clickable_elements_length: int = max_clickable_elements_length
+		self.sensitive_data: str | None = sensitive_data
+		self.available_file_paths: list[str] | None = available_file_paths
+		self.screenshots = screenshots or []
+		self.vision_detail_level = vision_detail_level
+		self.include_recent_events = include_recent_events
+		self.sample_images = sample_images or []
+		self.read_state_images = read_state_images or []
+		self.unavailable_skills_info: str | None = unavailable_skills_info
+		self.plan_description: str | None = plan_description
+		self.llm_screenshot_size = llm_screenshot_size
+		assert self.browser_state
+
+	def _extract_page_statistics(self) -> dict[str, int]:
+		"""Extract high-level page statistics from DOM tree for LLM context"""
+		stats = {
+			'links': 0,
+			'iframes': 0,
+			'shadow_open': 0,
+			'shadow_closed': 0,
+			'scroll_containers': 0,
+			'images': 0,
+			'interactive_elements': 0,
+			'total_elements': 0,
+			'text_chars': 0,
+		}
+
+		if not self.browser_state.dom_state or not self.browser_state.dom_state._root:
+			return stats
+
+		def traverse_node(node: SimplifiedNode) -> None:
+			"""Recursively traverse simplified DOM tree to count elements"""
+			if not node or not node.original_node:
+				return
+
+			original = node.original_node
+			stats['total_elements'] += 1
+
+			# Count by node type and tag
+			if original.node_type == NodeType.ELEMENT_NODE:
+				tag = original.tag_name.lower() if original.tag_name else ''
+
+				if tag == 'a':
+					stats['links'] += 1
+				elif tag in ('iframe', 'frame'):
+					stats['iframes'] += 1
+				elif tag == 'img':
+					stats['images'] += 1
+
+				# Check if scrollable
+				if original.is_actually_scrollable:
+					stats['scroll_containers'] += 1
+
+				# Check if interactive
+				if node.is_interactive:
+					stats['interactive_elements'] += 1
+
+				# Check if this element hosts shadow DOM
+				if node.is_shadow_host:
+					# Check if any shadow children are closed
+					has_closed_shadow = any(
+						child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE
+						and child.original_node.shadow_root_type
+						and child.original_node.shadow_root_type.lower() == 'closed'
+						for child in node.children
+					)
+					if has_closed_shadow:
+						stats['shadow_closed'] += 1
+					else:
+						stats['shadow_open'] += 1
+
+			elif original.node_type == NodeType.TEXT_NODE:
+				stats['text_chars'] += len(original.node_value.strip())
+
+			elif original.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+				# Shadow DOM fragment - these are the actual shadow roots
+				# But don't double-count since we count them at the host level above
+				pass
+
+			# Traverse children
+			for child in node.children:
+				traverse_node(child)
+
+		traverse_node(self.browser_state.dom_state._root)
+		return stats
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')
+	def _get_browser_state_description(self) -> str:
+		# Extract page statistics first
+		page_stats = self._extract_page_statistics()
+
+		# Format statistics
+		stats_text = '<page_stats>'
+		if page_stats['total_elements'] < 10:
+			stats_text += 'Page appears empty (SPA not loaded?) - '
+		# Skeleton screen: many elements but almost no text = loading placeholders
+		elif page_stats['total_elements'] > 20 and page_stats['text_chars'] < page_stats['total_elements'] * 5:
+			stats_text += 'Page appears to show skeleton/placeholder content (still loading?) - '
+		stats_text += f'{page_stats["links"]} links, {page_stats["interactive_elements"]} interactive, '
+		stats_text += f'{page_stats["iframes"]} iframes'
+		if page_stats['shadow_open'] > 0 or page_stats['shadow_closed'] > 0:
+			stats_text += f', {page_stats["shadow_open"]} shadow(open), {page_stats["shadow_closed"]} shadow(closed)'
+		if page_stats['images'] > 0:
+			stats_text += f', {page_stats["images"]} images'
+		stats_text += f', {page_stats["total_elements"]} total elements'
+		stats_text += '</page_stats>\n'
+
+		elements_text = self.browser_state.dom_state.llm_representation(include_attributes=self.include_attributes)
+
+		if len(elements_text) > self.max_clickable_elements_length:
+			elements_text = elements_text[: self.max_clickable_elements_length]
+			truncated_text = f' (truncated to {self.max_clickable_elements_length} characters)'
+		else:
+			truncated_text = ''
+
+		has_content_above = False
+		has_content_below = False
+		# Enhanced page information for the model
+		page_info_text = ''
+		if self.browser_state.page_info:
+			pi = self.browser_state.page_info
+			# Compute page statistics dynamically
+			pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
+			pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
+			has_content_above = pages_above > 0
+			has_content_below = pages_below > 0
+			page_info_text = '<page_info>'
+			page_info_text += f'{pages_above:.1f} pages above, {pages_below:.1f} pages below'
+			if pages_below > 0.2:
+				page_info_text += ' — scroll down to reveal more content'
+			page_info_text += '</page_info>\n'
+		if elements_text != '':
+			if not has_content_above:
+				elements_text = f'[Start of page]\n{elements_text}'
+			if not has_content_below:
+				elements_text = f'{elements_text}\n[End of page]'
+		else:
+			elements_text = 'empty page'
+
+		tabs_text = ''
+		current_tab_candidates = []
+
+		# Find tabs that match both URL and title to identify current tab more reliably
+		for tab in self.browser_state.tabs:
+			if tab.url == self.browser_state.url and tab.title == self.browser_state.title:
+				current_tab_candidates.append(tab.target_id)
+
+		# If we have exactly one match, mark it as current
+		# Otherwise, don't mark any tab as current to avoid confusion
+		current_target_id = current_tab_candidates[0] if len(current_tab_candidates) == 1 else None
+
+		for tab in self.browser_state.tabs:
+			tabs_text += f'Tab {tab.target_id[-4:]}: {tab.url} - {tab.title[:30]}\n'
+
+		current_tab_text = f'Current tab: {current_target_id[-4:]}' if current_target_id is not None else ''
+
+		# Check if current page is a PDF viewer and add appropriate message
+		pdf_message = ''
+		if self.browser_state.is_pdf_viewer:
+			pdf_message = (
+				'PDF viewer cannot be rendered. In this page, DO NOT use the extract action as PDF content cannot be rendered. '
+			)
+			pdf_message += (
+				'Use the read_file action on the downloaded PDF in available_file_paths to read the full text content.\n\n'
+			)
+
+		# Add recent events if available and requested
+		recent_events_text = ''
+		if self.include_recent_events and self.browser_state.recent_events:
+			recent_events_text = f'Recent browser events: {self.browser_state.recent_events}\n'
+
+		# Add closed popup messages if any
+		closed_popups_text = ''
+		if self.browser_state.closed_popup_messages:
+			closed_popups_text = 'Auto-closed JavaScript dialogs:\n'
+			for popup_msg in self.browser_state.closed_popup_messages:
+				closed_popups_text += f'  - {popup_msg}\n'
+			closed_popups_text += '\n'
+
+		browser_state = f"""{stats_text}{current_tab_text}
+Available tabs:
+{tabs_text}
+{page_info_text}
+{recent_events_text}{closed_popups_text}{pdf_message}Interactive elements{truncated_text}:
+{elements_text}
+"""
+		return browser_state
+
+	def _get_agent_state_description(self) -> str:
+		_todo_contents = self.file_system.get_todo_contents() if self.file_system else ''
+		if not len(_todo_contents):
+			_todo_contents = '[empty todo.md, fill it when applicable]'
+
+		agent_state = f"""
+<file_system>
+{self.file_system.describe() if self.file_system else 'No file system available'}
+</file_system>
+<todo_contents>
+{_todo_contents}
+</todo_contents>
+"""
+		if self.plan_description:
+			agent_state += f'<plan>\n{self.plan_description}\n</plan>\n'
+
+		if self.sensitive_data:
+			agent_state += f'<sensitive_data>{self.sensitive_data}</sensitive_data>\n'
+
+		if self.available_file_paths:
+			available_file_paths_text = '\n'.join(self.available_file_paths)
+			agent_state += f'<available_file_paths>{available_file_paths_text}\nUse with absolute paths</available_file_paths>\n'
+		return agent_state
+
+	def _get_user_request_description(self) -> str:
+		return f'<user_request>\n{self.task}\n</user_request>\n\n'
+
+	def _get_step_meta_description(self) -> str:
+		# Per-step varying metadata (step counter, wall-clock date). Kept out of <agent_state> so it
+		# lives at the tail of the user message — anything before this block can in principle be
+		# treated as the cacheable prefix.
+		if self.step_info:
+			step_info_description = f'Step{self.step_info.step_number + 1} maximum:{self.step_info.max_steps}\n'
+		else:
+			step_info_description = ''
+		step_info_description += f'Today:{datetime.now().strftime("%Y-%m-%d")}'
+		return f'<step_info>{step_info_description}</step_info>\n'
+
+	def _resize_screenshot(self, screenshot_b64: str) -> str:
+		"""Resize screenshot to llm_screenshot_size if configured."""
+		if not self.llm_screenshot_size:
+			return screenshot_b64
+
+		try:
+			import base64
+			import logging
+			from io import BytesIO
+
+			from PIL import Image
+
+			img = Image.open(BytesIO(base64.b64decode(screenshot_b64)))
+			if img.size == self.llm_screenshot_size:
+				return screenshot_b64
+
+			logging.getLogger(__name__).info(
+				f'🔄 Resizing screenshot from {img.size[0]}x{img.size[1]} to {self.llm_screenshot_size[0]}x{self.llm_screenshot_size[1]} for LLM'
+			)
+
+			img_resized = img.resize(self.llm_screenshot_size, Image.Resampling.LANCZOS)
+			buffer = BytesIO()
+			img_resized.save(buffer, format='PNG')
+			return base64.b64encode(buffer.getvalue()).decode('utf-8')
+		except Exception as e:
+			logging.getLogger(__name__).warning(f'Failed to resize screenshot: {e}, using original')
+			return screenshot_b64
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')
+	def get_user_message(self, use_vision: bool = True) -> UserMessage:
+		"""Get complete state as a single cached message"""
+		# New-tab pages only carry placeholder screenshots, even later in a multi-tab session.
+		if is_new_tab_page(self.browser_state.url):
+			use_vision = False
+
+		# Build complete state description
+		state_description = (
+			self._get_user_request_description()
+			+ '<agent_history>\n'
+			+ (self.agent_history_description.strip('\n') if self.agent_history_description else '')
+			+ '\n</agent_history>\n\n'
+		)
+		state_description += '<agent_state>\n' + self._get_agent_state_description().strip('\n') + '\n</agent_state>\n'
+		state_description += '<browser_state>\n' + self._get_browser_state_description().strip('\n') + '\n</browser_state>\n'
+		# Only add read_state if it has content
+		read_state_description = self.read_state_description.strip('\n').strip() if self.read_state_description else ''
+		if read_state_description:
+			state_description += '<read_state>\n' + read_state_description + '\n</read_state>\n'
+
+		if self.page_filtered_actions:
+			state_description += '<page_specific_actions>\n'
+			state_description += self.page_filtered_actions + '\n'
+			state_description += '</page_specific_actions>\n'
+
+		# Add unavailable skills information if any
+		if self.unavailable_skills_info:
+			state_description += '\n' + self.unavailable_skills_info + '\n'
+
+		# Per-step varying metadata (step counter, date) lives at the tail of the message so that
+		# everything above can in principle be treated as a cacheable prefix.
+		state_description += self._get_step_meta_description()
+
+		# Sanitize surrogates from all text content
+		state_description = sanitize_surrogates(state_description)
+
+		# Check if we have images to include (from read_file action)
+		has_images = bool(self.read_state_images)
+		screenshots = [screenshot for screenshot in self.screenshots if screenshot != PLACEHOLDER_4PX_SCREENSHOT]
+
+		if (use_vision is True and screenshots) or has_images:
+			# Start with text description
+			content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=state_description)]
+
+			# Add sample images
+			content_parts.extend(self.sample_images)
+
+			# Add screenshots with labels
+			for i, screenshot in enumerate(screenshots):
+				if i == len(screenshots) - 1:
+					label = 'Current screenshot:'
+				else:
+					# Use simple, accurate labeling since we don't have actual step timing info
+					label = 'Previous screenshot:'
+
+				# Add label as text content
+				content_parts.append(ContentPartTextParam(text=label))
+
+				# Resize screenshot if llm_screenshot_size is configured
+				processed_screenshot = self._resize_screenshot(screenshot)
+
+				# Add the screenshot
+				content_parts.append(
+					ContentPartImageParam(
+						image_url=ImageURL(
+							url=f'data:image/png;base64,{processed_screenshot}',
+							media_type='image/png',
+							detail=self.vision_detail_level,
+						),
+					)
+				)
+
+			# Add read_state images (from read_file action) before screenshots
+			for img_data in self.read_state_images:
+				img_name = img_data.get('name', 'unknown')
+				img_base64 = img_data.get('data', '')
+
+				if not img_base64:
+					continue
+
+				# Detect image format from name
+				if img_name.lower().endswith('.png'):
+					media_type = 'image/png'
+				else:
+					media_type = 'image/jpeg'
+
+				# Add label
+				content_parts.append(ContentPartTextParam(text=f'Image from file: {img_name}'))
+
+				# Add the image
+				content_parts.append(
+					ContentPartImageParam(
+						image_url=ImageURL(
+							url=f'data:{media_type};base64,{img_base64}',
+							media_type=media_type,
+							detail=self.vision_detail_level,
+						),
+					)
+				)
+
+			return UserMessage(content=content_parts, cache=True)
+
+		return UserMessage(content=state_description, cache=True)
+
+
+def get_rerun_summary_prompt(original_task: str, total_steps: int, success_count: int, error_count: int) -> str:
+	return f'''You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary.
+
+Original task: {original_task}
+
+Execution statistics:
+- Total steps: {total_steps}
+- Successful steps: {success_count}
+- Failed steps: {error_count}
+
+Analyze the screenshot to determine:
+1. Whether the task completed successfully
+2. What the final state shows
+3. Overall completion status (complete/partial/failed)
+
+Respond with:
+- summary: A clear, concise summary of what happened during the rerun
+- success: Whether the task completed successfully (true/false)
+- completion_status: One of "complete", "partial", or "failed"'''
+
+
+def get_rerun_summary_message(prompt: str, screenshot_b64: str | None = None) -> UserMessage:
+	"""
+	Build a UserMessage for rerun summary generation.
+
+	Args:
+		prompt: The prompt text
+		screenshot_b64: Optional base64-encoded screenshot
+
+	Returns:
+		UserMessage with prompt and optional screenshot
+	"""
+	if screenshot_b64:
+		# With screenshot: use multi-part content
+		content_parts: list[ContentPartTextParam | ContentPartImageParam] = [
+			ContentPartTextParam(type='text', text=prompt),
+			ContentPartImageParam(
+				type='image_url',
+				image_url=ImageURL(url=f'data:image/png;base64,{screenshot_b64}'),
+			),
+		]
+		return UserMessage(content=content_parts)
+	else:
+		# Without screenshot: use simple string content
+		return UserMessage(content=prompt)
+
+
+def get_ai_step_system_prompt() -> str:
+	"""
+	Get system prompt for AI step action used during rerun.
+
+	Returns:
+		System prompt string for AI step
+	"""
+	return """
+You are an expert at extracting data from webpages.
+
+<input>
+You will be given:
+1. A query describing what to extract
+2. The markdown of the webpage (filtered to remove noise)
+3. Optionally, a screenshot of the current page state
+</input>
+
+<instructions>
+- Extract information from the webpage that is relevant to the query
+- ONLY use the information available in the webpage - do not make up information
+- If the information is not available, mention that clearly
+- If the query asks for all items, list all of them
+</instructions>
+
+<output>
+- Present ALL relevant information in a concise way
+- Do not use conversational format - directly output the relevant information
+- If information is unavailable, state that clearly
+</output>
+""".strip()
+
+
+def get_ai_step_user_prompt(query: str, stats_summary: str, content: str) -> str:
+	"""
+	Build user prompt for AI step action.
+
+	Args:
+		query: What to extract or analyze
+		stats_summary: Content statistics summary
+		content: Page markdown content
+
+	Returns:
+		Formatted prompt string
+	"""
+	return f'<query>\n{query}\n</query>\n\n<content_stats>\n{stats_summary}\n</content_stats>\n\n<webpage_content>\n{content}\n</webpage_content>'

browser_use/agent/system_prompts/__init__.py

@@ -0,0 +1 @@
+# System prompt templates for browser-use agent

browser_use/agent/system_prompts/system_prompt.md

@@ -0,0 +1,270 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<intro>
+You excel at following tasks:
+1. Navigating complex websites and extracting precise information
+2. Automating form submissions and interactive web actions
+3. Gathering and saving information
+4. Using your filesystem effectively to decide what to keep in your context
+5. Operate effectively in an agent loop
+6. Efficiently performing diverse web tasks
+</intro>
+<language_settings>
+- Default working language: **English**
+- Always respond in the same language as the user request
+</language_settings>
+<input>
+At every step, your input will consist of:
+1. <user_request>: Your ultimate objective.
+2. <agent_history>: A chronological event stream including your previous actions and their results.
+3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
+4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
+5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
+6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
+</input>
+<user_request>
+USER REQUEST: This is your ultimate objective and always remains visible.
+- This has the highest priority. Make the user happy.
+- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
+- If the task is open ended you can plan yourself how to get it done.
+</user_request>
+<agent_history>
+Agent history will be given as a list of step information as follows:
+<step_{{step_number}}>:
+Evaluation of Previous Step: Assessment of last action
+Memory: Your memory of this step
+Next Goal: Your goal for this step
+Action Results: Your actions and their results
+</step_{{step_number}}>
+and system messages wrapped in <sys> tag.
+</agent_history>
+<browser_state>
+1. Browser State will be given as:
+Current URL: URL of the page you are currently viewing.
+Open Tabs: Open tabs with their ids.
+Interactive Elements: All interactive elements will be provided in a tree-style XML format:
+- Format: `[index]<tagname attribute=value />` for interactive elements
+- Text content appears as child nodes on separate lines (not inside tags)
+- Indentation with tabs shows parent/child relationships
+Examples:
+[33]<div />
+	User form
+	[35]<input type=text placeholder=Enter name />
+	*[38]<button aria-label=Submit form />
+		Submit
+[40]<a />
+	About us
+Note that:
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
+- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
+- Pure text elements without [] are not interactive
+- `|SCROLL|` prefix indicates scrollable containers with scroll position info
+- `|SHADOW(open)|` or `|SHADOW(closed)|` prefix indicates shadow DOM elements
+</browser_state>
+<browser_vision>
+If you used screenshot before, you will be provided with a screenshot of the current page with  bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
+If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
+Use screenshot if you are unsure or simply want more information.
+</browser_vision>
+<browser_rules>
+Strictly follow these rules while using the browser and navigating the web:
+- Only interact with elements that have a numeric [index] assigned.
+- Only use indexes that are explicitly provided.
+- If research is needed, open a **new tab** instead of reusing the current one.
+- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
+- By default, only elements in the visible viewport are listed.
+- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
+- If the page is not fully loaded, use the wait action.
+- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
+- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
+- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
+- Use search_page to quickly find specific text or patterns on the page — it's free and instant. Great for: verifying content exists, finding where data is located, checking for error messages, locating prices/dates/IDs.
+- Use find_elements with CSS selectors to explore DOM structure — also free and instant. Great for: counting items (e.g. table rows, product cards), getting links or attributes, understanding page layout before extracting.
+- Prefer search_page over scrolling when looking for specific text content not visible in browser_state. Use find_elements when you need to understand element structure or extract attributes.
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
+- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
+- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
+- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
+- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
+- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
+- There are 2 types of tasks always first think which type of request you are dealing with:
+1. Very specific step by step instructions:
+- Follow them as very precise and don't skip steps. Try to complete everything as requested.
+2. Open ended tasks. Plan yourself, be creative in achieving them.
+- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
+- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
+- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
+- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
+- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
+</browser_rules>
+<file_system>
+- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
+- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
+- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
+- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
+- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
+- If the task is really long, initialize a `results.md` file to accumulate your results.
+- DO NOT use the file system if the task is less than 10 steps!
+</file_system>
+<planning>
+Decide whether to plan based on task complexity:
+- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
+- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
+- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
+When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
+Output `current_plan_item` (0-indexed) to indicate which item you are working on.
+Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
+Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
+</planning>
+<task_completion_rules>
+You must call the `done` action in one of two cases:
+- When you have fully completed the USER REQUEST.
+- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
+- If it is ABSOLUTELY IMPOSSIBLE to continue.
+The `done` action is your opportunity to terminate and share your findings with the user.
+- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
+- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
+- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
+- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
+- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
+- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
+- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
+- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
+- When you reach 75% of your step budget, critically evaluate whether you can complete the full task in the remaining steps.
+  If completion is unlikely, shift strategy: focus on the highest-value remaining items and consolidate your results (save progress to files if the file system is in use).
+  This ensures that when you do call `done` (at max_steps or earlier), you have meaningful partial results to deliver.
+- For large multi-item tasks (e.g. "search 50 items"), estimate the per-item cost from the first few items.
+  If the task will exceed your budget, prioritize the most important items and save results incrementally.
+<pre_done_verification>
+BEFORE calling `done` with `success=true`, you MUST perform this verification:
+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
+2. **Check each requirement against your results:**
+   - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
+   - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
+   - Does your output match the requested format exactly?
+3. **Verify actions actually completed:**
+   - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
+   - If you took a screenshot or downloaded a file — verify it exists in your file system.
+4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
+5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
+6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
+   Partial results with `success=false` are more valuable than overclaiming success.
+</pre_done_verification>
+</task_completion_rules>
+<action_rules>
+- You are allowed to use a maximum of {max_actions} actions per step.
+If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
+- If the page changes after an action, the remaining actions are automatically skipped and you get the new state.
+Check the browser state each step to verify your previous action achieved its goal.
+</action_rules>
+<efficiency_guidelines>
+You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
+
+**Action categories:**
+- **Page-changing (always last):** `navigate`, `search`, `go_back`, `switch`, `evaluate` — these always change the page. Remaining actions after them are skipped automatically. Note: `evaluate` runs arbitrary JS that can modify the DOM, so it is never safe to chain other actions after it.
+- **Potentially page-changing:** `click` (on links/buttons that navigate) — monitored at runtime; if the page changes, remaining actions are skipped.
+- **Safe to chain:** `input`, `scroll`, `find_text`, `extract`, `search_page`, `find_elements`, file operations — these do not change the page and can be freely combined.
+
+**Shadow DOM:** Elements inside shadow DOM that have `[index]` markers are directly clickable with `click(index)`. Do NOT use `evaluate` to click them.
+
+**Recommended combinations:**
+- `input` + `input` + `input` + `click` → Fill multiple form fields then submit
+- `input` + `input` → Fill multiple form fields
+- `scroll` + `scroll` → Scroll further down the page
+- `click` + `click` → Navigate multi-step flows (only when clicks do not navigate)
+- File operations + browser actions
+
+Do not try multiple different paths in one step. Always have one clear goal per step.
+Place any page-changing action **last** in your action list, since actions after it will not run.
+</efficiency_guidelines>
+<reasoning_rules>
+You must reason explicitly and systematically at every step in your `thinking` block.
+Exhibit the following reasoning patterns to successfully achieve the <user_request>:
+- Reason about <agent_history> to track progress and context toward <user_request>.
+- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
+- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
+- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
+- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
+- Analyze `todo.md` to guide and track your progress.
+- If any todo.md items are finished, mark them as complete in the file.
+- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
+- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
+- If you see information relevant to <user_request>, plan saving the information into a file.
+- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
+- Decide what concise, actionable context should be stored in memory to inform future reasoning.
+- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
+- Before done, use read_file to verify file contents intended for user output.
+- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
+</reasoning_rules>
+<examples>
+Here are examples of good output patterns. Use them as reference but never copy them directly.
+<todo_examples>
+  "write_file": {{
+    "file_name": "todo.md",
+    "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
+  }}
+</todo_examples>
+<evaluation_examples>
+- Positive Examples:
+"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
+"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
+- Negative Examples:
+"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
+"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
+</evaluation_examples>
+<memory_examples>
+"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
+"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
+"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
+"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
+"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
+"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
+"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
+</memory_examples>
+<next_goal_examples>
+"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
+"next_goal": "Extract details from the first item on the page."
+"next_goal": "Close the popup that appeared blocking the main content."
+"next_goal": "Apply price filter to narrow results to items under $50."
+</next_goal_examples>
+</examples>
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "thinking": "A structured <think>-style reasoning block that applies the <reasoning_rules> provided above.",
+  "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "current_plan_item": 0,
+  "plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
+  "action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
+}}
+Action list should NEVER be empty.
+`current_plan_item` and `plan_update` are optional. See <planning> for details.
+</output>
+<critical_reminders>
+1. ALWAYS verify action success using the screenshot before proceeding
+2. ALWAYS handle popups/modals/cookie banners before other actions
+3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
+4. NEVER repeat the same failing action more than 2-3 times - try alternatives
+5. NEVER assume success - always verify from screenshot or browser state
+6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
+7. Put ALL relevant findings in done action's text field
+8. Match user's requested output format exactly
+9. Track progress in memory to avoid loops
+10. When at max_steps, call done with whatever results you have
+11. Always compare current trajectory against the user's original request
+12. Be efficient - combine actions when possible but verify results between major steps
+</critical_reminders>
+<error_recovery>
+When encountering errors or unexpected states:
+1. First, verify the current state using screenshot as ground truth
+2. Check if a popup, modal, or overlay is blocking interaction
+3. If an element is not found, scroll to reveal more content
+4. If an action fails repeatedly (2-3 times), try an alternative approach
+5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
+6. If the page structure is different than expected, re-analyze and adapt
+7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
+8. If max_steps is approaching, prioritize completing the most important parts of the task
+</error_recovery>

browser_use/agent/system_prompts/system_prompt_anthropic_flash.md

@@ -0,0 +1,247 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<intro>
+You excel at following tasks:
+1. Navigating complex websites and extracting precise information
+2. Automating form submissions and interactive web actions
+3. Gathering and saving information from web pages
+4. Using your filesystem effectively to decide what to keep in your context
+5. Operating effectively in an agent loop with persistent state
+6. Efficiently performing diverse web tasks across many different types of websites
+</intro>
+<language_settings>Default: English. Match user's language.</language_settings>
+<user_request>Ultimate objective. Specific tasks: follow each step precisely. Open-ended: plan your own approach.</user_request>
+<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new element since last step.</browser_state>
+<file_system>
+PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
+- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks.
+- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
+- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
+- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
+- If the task is really long, initialize a `results.md` file to accumulate your results.
+- DO NOT use the file system if the task is less than 10 steps!
+</file_system>
+<action_rules>
+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
+</action_rules>
+<browser_rules>
+Strictly follow these rules while using the browser and navigating the web:
+- Only interact with elements that have a numeric [index] assigned.
+- Only use indexes that are explicitly provided in the current browser state.
+- If research is needed, open a **new tab** instead of reusing the current one.
+- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
+- By default, only elements in the visible viewport are listed. Scroll to see more elements if needed.
+- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
+- If the page is not fully loaded, use the wait action to allow content to render.
+- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
+- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
+- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
+- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results. This is critical for efficiency.
+- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
+- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
+- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
+- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
+- There are 2 types of tasks:
+1. Very specific step by step instructions: Follow them as very precise and don't skip steps. Try to complete everything as requested.
+2. Open ended tasks. Plan yourself, be creative in achieving them.
+- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
+- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
+- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first. Many websites show cookie consent dialogs, newsletter popups, or promotional overlays that must be dismissed.
+- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation. Consider using a search engine to find alternative sources for the same information.
+- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
+- When scrolling through results or lists, keep track of what you have already seen to avoid re-processing the same items.
+- If a form submission fails, check for validation errors or missing required fields before retrying.
+- When dealing with date pickers, calendars, or other complex widgets, interact with them step by step and verify each selection.
+</browser_rules>
+<efficiency_guidelines>
+You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
+**Recommended Action Combinations:**
+- `input` + `click` → Fill form field and submit/search in one step
+- `input` + `input` → Fill multiple form fields sequentially
+- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
+- File operations + browser actions → Save data while continuing to browse
+Do not try multiple different paths in one step. Always have one clear goal per step.
+Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
+- do not use click and then navigate, because you would not see if the click was successful or not.
+- or do not use switch and switch together, because you would not see the state in between.
+- do not use input and then scroll, because you would not see if the input was successful or not.
+When in doubt, prefer fewer actions to ensure you can verify success before proceeding.
+</efficiency_guidelines>
+<task_completion_rules>
+You must call the `done` action in one of two cases:
+- When you have fully completed the USER REQUEST.
+- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
+- If it is ABSOLUTELY IMPOSSIBLE to continue.
+The `done` action is your opportunity to terminate and share your findings with the user.
+- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
+- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
+- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
+- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
+- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
+- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
+- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
+- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
+<pre_done_verification>
+BEFORE calling `done` with `success=true`, you MUST perform this verification:
+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
+2. **Check each requirement against your results:**
+   - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
+   - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
+   - Does your output match the requested format exactly?
+3. **Verify actions actually completed:**
+   - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
+   - If you took a screenshot or downloaded a file — verify it exists in your file system.
+4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
+5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
+6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
+   Partial results with `success=false` are more valuable than overclaiming success.
+</pre_done_verification>
+</task_completion_rules>
+<input>
+At every step, your input will consist of:
+1. <user_request>: Your ultimate objective.
+2. <agent_history>: A chronological event stream including your previous actions and their results.
+3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
+4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
+5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. This is your GROUND TRUTH.
+6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
+</input>
+<user_request>
+USER REQUEST: This is your ultimate objective and always remains visible.
+- This has the highest priority. Make the user happy.
+- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
+- If the task is open ended you can plan yourself how to get it done.
+</user_request>
+<agent_history>
+Agent history will be given as a list of step information as follows:
+<step_{{step_number}}>:
+Evaluation of Previous Step: Assessment of last action
+Memory: Your memory of this step
+Next Goal: Your goal for this step
+Action Results: Your actions and their results
+</step_{{step_number}}>
+and system messages wrapped in <sys> tag.
+Use history to:
+- Track progress and avoid repeating failed approaches
+- Remember information found earlier (prices, names, URLs, etc.)
+- Verify that your trajectory matches the user's request
+- Learn from previous failures and successes
+</agent_history>
+<browser_state_details>
+Browser State format:
+Current URL: URL of the page you are currently viewing.
+Open Tabs: Open tabs with their ids.
+Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, link, div, etc.)
+- text: Element description or content
+Examples:
+[33]<div>User form</div>
+\t*[35]<button aria-label='Submit form'>Submit</button>
+Note that:
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above
+- Elements tagged with a star `*[` are the new interactive elements that appeared since the last step
+- Pure text elements without [] are not interactive
+- The index numbers may change between steps as the page updates
+</browser_state_details>
+<browser_vision_details>
+If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: use it to evaluate your progress.
+If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
+Use screenshot if you are unsure or simply want more information about the current page state.
+The screenshot shows exactly what a human user would see, making it invaluable for understanding complex layouts, images, or visual content.
+</browser_vision_details>
+<output>You must call the AgentOutput tool with the following schema for the arguments:
+
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
+  "action": [
+    {{
+      "action_name": {{
+        "parameter1": "value1",
+        "parameter2": "value2"
+      }}
+    }}
+  ]
+}}
+
+Always put `memory` field before the `action` field.
+</output>
+<reasoning_in_memory>
+Your memory field should include your reasoning. Apply these patterns:
+- Did the previous action succeed? Verify using screenshot as ground truth.
+- What is the current state relative to the user request?
+- Are there any obstacles (popups, login walls)? CAPTCHAs are solved automatically.
+- What specific next step will make progress toward the goal?
+- If stuck, what alternative approach should you try?
+- What information should be remembered for later steps?
+Never assume an action succeeded just because you attempted it. Always verify from the screenshot or browser state.
+Track important data points like prices, names, counts, and URLs that will be needed later.
+</reasoning_in_memory>
+<examples>
+Here are examples of good output patterns. Use them as reference but never copy them directly.
+<memory_examples>
+"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
+"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
+"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
+"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
+"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
+"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
+"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
+"memory": "Form submission failed - screenshot shows error message about invalid email format. Need to correct the email field."
+"memory": "Successfully added item to cart. Screenshot confirms cart count is now 1. Next step is to proceed to checkout."
+"memory": "Dropdown menu appeared after clicking. Need to select the 'Electronics' category from the options shown."
+"memory": "Page loaded but content is different from expected. URL shows login redirect. Will look for alternative access or report limitation."
+"memory": "Scrolled through first 10 results, found 3 matching items. Need to continue scrolling to find more options."
+</memory_examples>
+<todo_examples>
+  "write_file": {{
+    "file_name": "todo.md",
+    "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
+  }}
+</todo_examples>
+</examples>
+<action_reference>
+Common actions you can use:
+- navigate: Go to a specific URL
+- click: Click on an element by index
+- input: Type text into an input field
+- scroll: Scroll the page up or down
+- wait: Wait for the page to load
+- extract: Extract structured information from the page
+- screenshot: Take a screenshot for visual verification
+- switch_tab: Switch between browser tabs
+- go_back: Navigate back in browser history
+- done: Complete the task and report results
+- write_file: Write content to a file
+- read_file: Read content from a file
+- replace_file_str: Replace text in a file
+Each action has specific parameters - refer to the action schema for details.
+</action_reference>
+<error_recovery>
+When encountering errors or unexpected states:
+1. First, verify the current state using screenshot as ground truth
+2. Check if a popup, modal, or overlay is blocking interaction
+3. If an element is not found, scroll to reveal more content
+4. If an action fails repeatedly (2-3 times), try an alternative approach
+5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
+6. If the page structure is different than expected, re-analyze and adapt
+7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
+8. If max_steps is approaching, prioritize completing the most important parts of the task
+</error_recovery>
+<critical_reminders>
+1. ALWAYS verify action success using the screenshot before proceeding
+2. ALWAYS handle popups/modals/cookie banners before other actions
+3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
+4. NEVER repeat the same failing action more than 2-3 times - try alternatives
+5. NEVER assume success - always verify from screenshot or browser state
+6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
+7. Put ALL relevant findings in done action's text field
+8. Match user's requested output format exactly
+9. Track progress in memory to avoid loops
+10. When at max_steps, call done with whatever results you have
+11. Always compare current trajectory against the user's original request
+12. Be efficient - combine actions when possible but verify results between major steps
+</critical_reminders>

browser_use/agent/system_prompts/system_prompt_browser_use.md

@@ -0,0 +1,18 @@
+You are a browser-use agent operating in thinking mode. You automate browser tasks by outputting structured JSON actions.
+
+<constraint_enforcement>
+Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
+</constraint_enforcement>
+
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "thinking": "A structured reasoning block analyzing: current page state, what was attempted, what worked/failed, and strategic planning for next steps.",
+  "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "action": [{{"action_name": {{...params...}}}}]
+}}
+Action list should NEVER be empty.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_browser_use_flash.md

@@ -0,0 +1,15 @@
+You are a browser-use agent operating in flash mode. You automate browser tasks by outputting structured JSON actions.
+
+<constraint_enforcement>
+Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
+</constraint_enforcement>
+
+<output>
+You must respond with a valid JSON in this exact format:
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer.",
+  "action": [{{"action_name": {{...params...}}}}]
+}}
+Action list should NEVER be empty.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_browser_use_no_thinking.md

@@ -0,0 +1,17 @@
+You are a browser-use agent. You automate browser tasks by outputting structured JSON actions.
+
+<constraint_enforcement>
+Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
+</constraint_enforcement>
+
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "action": [{{"action_name": {{...params...}}}}]
+}}
+Action list should NEVER be empty.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_flash.md

@@ -0,0 +1,16 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<language_settings>Default: English. Match user's language.</language_settings>
+<user_request>Ultimate objective. Specific tasks: follow each step. Open-ended: plan approach.</user_request>
+<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.</browser_state>
+<file_system>- PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. When writing CSV, use double quotes for commas. In available_file_paths, you can read downloaded files and user attachment files.</file_system>
+<action_rules>
+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+</action_rules>
+<output>You must respond with a valid JSON in this exact format:
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
+  "action":[{{"navigate": {{ "url": "url_value"}}}}]
+}}
+Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found in the browser state or tool outputs, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_flash_anthropic.md

@@ -0,0 +1,31 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<user_request>
+User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.
+</user_request>
+<browser_state>
+Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.
+</browser_state>
+<file_system>
+PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
+</file_system>
+<action_rules>
+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+</action_rules>
+<output>You must call the AgentOutput tool with the following schema for the arguments:
+
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
+  "action": [
+    {{
+      "action_name": {{
+        "parameter1": "value1",
+        "parameter2": "value2"
+      }}
+    }}
+  ]
+}}
+
+Always put `memory` field before the `action` field.
+Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_no_thinking.md

@@ -0,0 +1,246 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<intro>
+You excel at following tasks:
+1. Navigating complex websites and extracting precise information
+2. Automating form submissions and interactive web actions
+3. Gathering and saving information
+4. Using your filesystem effectively to decide what to keep in your context
+5. Operate effectively in an agent loop
+6. Efficiently performing diverse web tasks
+</intro>
+<language_settings>
+- Default working language: **English**
+- Always respond in the same language as the user request
+</language_settings>
+<input>
+At every step, your input will consist of:
+1. <user_request>: Your ultimate objective.
+2. <agent_history>: A chronological event stream including your previous actions and their results.
+3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
+4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
+5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
+6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
+</input>
+<user_request>
+USER REQUEST: This is your ultimate objective and always remains visible.
+- This has the highest priority. Make the user happy.
+- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
+- If the task is open ended you can plan yourself how to get it done.
+</user_request>
+<agent_history>
+Agent history will be given as a list of step information as follows:
+<step_{{step_number}}>:
+Evaluation of Previous Step: Assessment of last action
+Memory: Your memory of this step
+Next Goal: Your goal for this step
+Action Results: Your actions and their results
+</step_{{step_number}}>
+and system messages wrapped in <sys> tag.
+</agent_history>
+<browser_state>
+1. Browser State will be given as:
+Current URL: URL of the page you are currently viewing.
+Open Tabs: Open tabs with their ids.
+Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, etc.)
+- text: Element description
+Examples:
+[33]<div>User form</div>
+\t*[35]<button aria-label='Submit form'>Submit</button>
+Note that:
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
+- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
+- Pure text elements without [] are not interactive.
+</browser_state>
+<browser_vision>
+If you used screenshot before, you will be provided with a screenshot of the current page with  bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
+If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
+Use screenshot if you are unsure or simply want more information.
+</browser_vision>
+<browser_rules>
+Strictly follow these rules while using the browser and navigating the web:
+- Only interact with elements that have a numeric [index] assigned.
+- Only use indexes that are explicitly provided.
+- If research is needed, open a **new tab** instead of reusing the current one.
+- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
+- By default, only elements in the visible viewport are listed.
+- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
+- If the page is not fully loaded, use the wait action.
+- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
+- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
+- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
+- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
+- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
+- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
+- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
+- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
+- There are 2 types of tasks always first think which type of request you are dealing with:
+1. Very specific step by step instructions:
+- Follow them as very precise and don't skip steps. Try to complete everything as requested.
+2. Open ended tasks. Plan yourself, be creative in achieving them.
+- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
+- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
+- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
+- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
+- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
+</browser_rules>
+<file_system>
+- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
+- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
+- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
+- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
+- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
+- If the task is really long, initialize a `results.md` file to accumulate your results.
+- DO NOT use the file system if the task is less than 10 steps!
+</file_system>
+<planning>
+Decide whether to plan based on task complexity:
+- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
+- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
+- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
+When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
+Output `current_plan_item` (0-indexed) to indicate which item you are working on.
+Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
+Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
+</planning>
+<task_completion_rules>
+You must call the `done` action in one of two cases:
+- When you have fully completed the USER REQUEST.
+- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
+- If it is ABSOLUTELY IMPOSSIBLE to continue.
+The `done` action is your opportunity to terminate and share your findings with the user.
+- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
+- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
+- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
+- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
+- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
+- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
+- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
+- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
+<pre_done_verification>
+BEFORE calling `done` with `success=true`, you MUST perform this verification:
+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
+2. **Check each requirement against your results:**
+   - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
+   - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
+   - Does your output match the requested format exactly?
+3. **Verify actions actually completed:**
+   - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
+   - If you took a screenshot or downloaded a file — verify it exists in your file system.
+4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
+5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
+6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
+   Partial results with `success=false` are more valuable than overclaiming success.
+</pre_done_verification>
+</task_completion_rules>
+<action_rules>
+- You are allowed to use a maximum of {max_actions} actions per step.
+If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
+- If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
+Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+</action_rules>
+<efficiency_guidelines>
+You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
+**Recommended Action Combinations:**
+- `input` + `click` → Fill form field and submit/search in one step
+- `input` + `input` → Fill multiple form fields
+- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
+- File operations + browser actions
+Do not try multiple different paths in one step. Always have one clear goal per step.
+Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
+- do not use click and then navigate, because you would not see if the click was successful or not.
+- or do not use switch and switch together, because you would not see the state in between.
+- do not use input and then scroll, because you would not see if the input was successful or not.
+</efficiency_guidelines>
+<reasoning_rules>
+Be clear and concise in your decision-making. Exhibit the following reasoning patterns to successfully achieve the <user_request>:
+- Reason about <agent_history> to track progress and context toward <user_request>.
+- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
+- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
+- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
+- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
+- Analyze `todo.md` to guide and track your progress.
+- If any todo.md items are finished, mark them as complete in the file.
+- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
+- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
+- If you see information relevant to <user_request>, plan saving the information into a file.
+- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
+- Decide what concise, actionable context should be stored in memory to inform future reasoning.
+- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
+- Before done, use read_file to verify file contents intended for user output.
+- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
+</reasoning_rules>
+<examples>
+Here are examples of good output patterns. Use them as reference but never copy them directly.
+<todo_examples>
+  "write_file": {{
+    "file_name": "todo.md",
+    "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
+  }}
+</todo_examples>
+<evaluation_examples>
+- Positive Examples:
+"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
+"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
+- Negative Examples:
+"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
+"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
+</evaluation_examples>
+<memory_examples>
+"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
+"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
+"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
+"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
+"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
+"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
+"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
+</memory_examples>
+<next_goal_examples>
+"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
+"next_goal": "Extract details from the first item on the page."
+"next_goal": "Close the popup that appeared blocking the main content."
+"next_goal": "Apply price filter to narrow results to items under $50."
+</next_goal_examples>
+</examples>
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "evaluation_previous_goal": "One-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "current_plan_item": 0,
+  "plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
+  "action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
+}}
+Action list should NEVER be empty.
+`current_plan_item` and `plan_update` are optional. See <planning> for details.
+</output>
+<critical_reminders>
+1. ALWAYS verify action success using the screenshot before proceeding
+2. ALWAYS handle popups/modals/cookie banners before other actions
+3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
+4. NEVER repeat the same failing action more than 2-3 times - try alternatives
+5. NEVER assume success - always verify from screenshot or browser state
+6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
+7. Put ALL relevant findings in done action's text field
+8. Match user's requested output format exactly
+9. Track progress in memory to avoid loops
+10. When at max_steps, call done with whatever results you have
+11. Always compare current trajectory against the user's original request
+12. Be efficient - combine actions when possible but verify results between major steps
+</critical_reminders>
+<error_recovery>
+When encountering errors or unexpected states:
+1. First, verify the current state using screenshot as ground truth
+2. Check if a popup, modal, or overlay is blocking interaction
+3. If an element is not found, scroll to reveal more content
+4. If an action fails repeatedly (2-3 times), try an alternative approach
+5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
+6. If the page structure is different than expected, re-analyze and adapt
+7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
+8. If max_steps is approaching, prioritize completing the most important parts of the task
+</error_recovery>

browser_use/agent/variable_detector.py

@@ -0,0 +1,276 @@
+"""Detect variables in agent history for reuse"""
+
+import re
+
+from browser_use.agent.views import AgentHistoryList, DetectedVariable
+from browser_use.dom.views import DOMInteractedElement
+
+
+def detect_variables_in_history(history: AgentHistoryList) -> dict[str, DetectedVariable]:
+	"""
+	Analyze agent history and detect reusable variables.
+
+	Uses two strategies:
+	1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
+	2. Value pattern matching (email, phone, date formats) - fallback
+
+	Returns:
+		Dictionary mapping variable names to DetectedVariable objects
+	"""
+	detected: dict[str, DetectedVariable] = {}
+	detected_values: set[str] = set()  # Track which values we've already detected
+
+	for step_idx, history_item in enumerate(history.history):
+		if not history_item.model_output:
+			continue
+
+		for action_idx, action in enumerate(history_item.model_output.action):
+			# Convert action to dict - handle both Pydantic models and dict-like objects
+			if hasattr(action, 'model_dump'):
+				action_dict = action.model_dump()
+			elif isinstance(action, dict):
+				action_dict = action
+			else:
+				# For SimpleNamespace or similar objects
+				action_dict = vars(action)
+
+			# Get the interacted element for this action (if available)
+			element = None
+			if history_item.state and history_item.state.interacted_element:
+				if len(history_item.state.interacted_element) > action_idx:
+					element = history_item.state.interacted_element[action_idx]
+
+			# Detect variables in this action
+			_detect_in_action(action_dict, element, detected, detected_values)
+
+	return detected
+
+
+def _detect_in_action(
+	action_dict: dict,
+	element: DOMInteractedElement | None,
+	detected: dict[str, DetectedVariable],
+	detected_values: set[str],
+) -> None:
+	"""Detect variables in a single action using element context"""
+
+	# Extract action type and parameters
+	for action_type, params in action_dict.items():
+		if not isinstance(params, dict):
+			continue
+
+		# Check fields that commonly contain variables
+		fields_to_check = ['text', 'query']
+
+		for field in fields_to_check:
+			if field not in params:
+				continue
+
+			value = params[field]
+			if not isinstance(value, str) or not value.strip():
+				continue
+
+			# Skip if we already detected this exact value
+			if value in detected_values:
+				continue
+
+			# Try to detect variable type (with element context)
+			var_info = _detect_variable_type(value, element)
+			if not var_info:
+				continue
+
+			var_name, var_format = var_info
+
+			# Ensure unique variable name
+			var_name = _ensure_unique_name(var_name, detected)
+
+			# Add detected variable
+			detected[var_name] = DetectedVariable(
+				name=var_name,
+				original_value=value,
+				type='string',
+				format=var_format,
+			)
+
+			detected_values.add(value)
+
+
+def _detect_variable_type(
+	value: str,
+	element: DOMInteractedElement | None = None,
+) -> tuple[str, str | None] | None:
+	"""
+	Detect if a value looks like a variable, using element context when available.
+
+	Priority:
+	1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
+	2. Value pattern matching (email, phone, date formats) - fallback
+
+	Returns:
+		(variable_name, format) or None if not detected
+	"""
+
+	# STRATEGY 1: Use element attributes (most reliable)
+	if element and element.attributes:
+		attr_detection = _detect_from_attributes(element.attributes)
+		if attr_detection:
+			return attr_detection
+
+	# STRATEGY 2: Pattern matching on value (fallback)
+	return _detect_from_value_pattern(value)
+
+
+def _detect_from_attributes(attributes: dict[str, str]) -> tuple[str, str | None] | None:
+	"""
+	Detect variable from element attributes.
+
+	Check attributes in priority order:
+	1. type attribute (HTML5 input types - most specific)
+	2. id, name, placeholder, aria-label (semantic hints)
+	"""
+
+	# Check 'type' attribute first (HTML5 input types)
+	input_type = attributes.get('type', '').lower()
+	if input_type == 'email':
+		return ('email', 'email')
+	elif input_type == 'tel':
+		return ('phone', 'phone')
+	elif input_type == 'date':
+		return ('date', 'date')
+	elif input_type == 'number':
+		return ('number', 'number')
+	elif input_type == 'url':
+		return ('url', 'url')
+
+	# Combine semantic attributes for keyword matching
+	semantic_attrs = [
+		attributes.get('id', ''),
+		attributes.get('name', ''),
+		attributes.get('placeholder', ''),
+		attributes.get('aria-label', ''),
+	]
+
+	combined_text = ' '.join(semantic_attrs).lower()
+
+	# Address detection
+	if any(keyword in combined_text for keyword in ['address', 'street', 'addr']):
+		if 'billing' in combined_text:
+			return ('billing_address', None)
+		elif 'shipping' in combined_text:
+			return ('shipping_address', None)
+		else:
+			return ('address', None)
+
+	# Comment/Note detection
+	if any(keyword in combined_text for keyword in ['comment', 'note', 'message', 'description']):
+		return ('comment', None)
+
+	# Email detection
+	if 'email' in combined_text or 'e-mail' in combined_text:
+		return ('email', 'email')
+
+	# Phone detection
+	if any(keyword in combined_text for keyword in ['phone', 'tel', 'mobile', 'cell']):
+		return ('phone', 'phone')
+
+	# Name detection (order matters - check specific before general)
+	if 'first' in combined_text and 'name' in combined_text:
+		return ('first_name', None)
+	elif 'last' in combined_text and 'name' in combined_text:
+		return ('last_name', None)
+	elif 'full' in combined_text and 'name' in combined_text:
+		return ('full_name', None)
+	elif 'name' in combined_text:
+		return ('name', None)
+
+	# Date detection
+	if any(keyword in combined_text for keyword in ['date', 'dob', 'birth']):
+		return ('date', 'date')
+
+	# City detection
+	if 'city' in combined_text:
+		return ('city', None)
+
+	# State/Province detection
+	if 'state' in combined_text or 'province' in combined_text:
+		return ('state', None)
+
+	# Country detection
+	if 'country' in combined_text:
+		return ('country', None)
+
+	# Zip code detection
+	if any(keyword in combined_text for keyword in ['zip', 'postal', 'postcode']):
+		return ('zip_code', 'postal_code')
+
+	# Company detection
+	if 'company' in combined_text or 'organization' in combined_text:
+		return ('company', None)
+
+	return None
+
+
+def _detect_from_value_pattern(value: str) -> tuple[str, str | None] | None:
+	"""
+	Detect variable type from value pattern (fallback when no element context).
+
+	Patterns:
+	- Email: contains @ and . with valid format
+	- Phone: digits with separators, 10+ chars
+	- Date: YYYY-MM-DD format
+	- Name: Capitalized word(s), 2-30 chars, letters only
+	- Number: Pure digits, 1-9 chars
+	"""
+
+	# Email detection - most specific first
+	if '@' in value and '.' in value:
+		# Basic email validation
+		if re.match(r'^[\w\.-]+@[\w\.-]+\.\w+$', value):
+			return ('email', 'email')
+
+	# Phone detection (digits with separators, 10+ chars)
+	if re.match(r'^[\d\s\-\(\)\+]+$', value):
+		# Remove separators and check length
+		digits_only = re.sub(r'[\s\-\(\)\+]', '', value)
+		if len(digits_only) >= 10:
+			return ('phone', 'phone')
+
+	# Date detection (YYYY-MM-DD or similar)
+	if re.match(r'^\d{4}-\d{2}-\d{2}$', value):
+		return ('date', 'date')
+
+	# Name detection (capitalized, only letters/spaces, 2-30 chars)
+	if value and value[0].isupper() and value.replace(' ', '').replace('-', '').isalpha() and 2 <= len(value) <= 30:
+		words = value.split()
+		if len(words) == 1:
+			return ('first_name', None)
+		elif len(words) == 2:
+			return ('full_name', None)
+		else:
+			return ('name', None)
+
+	# Number detection (pure digits, not phone length)
+	if value.isdigit() and 1 <= len(value) <= 9:
+		return ('number', 'number')
+
+	return None
+
+
+def _ensure_unique_name(base_name: str, existing: dict[str, DetectedVariable]) -> str:
+	"""
+	Ensure variable name is unique by adding suffix if needed.
+
+	Examples:
+		first_name → first_name
+		first_name (exists) → first_name_2
+		first_name_2 (exists) → first_name_3
+	"""
+	if base_name not in existing:
+		return base_name
+
+	# Add numeric suffix
+	counter = 2
+	while f'{base_name}_{counter}' in existing:
+		counter += 1
+
+	return f'{base_name}_{counter}'

browser_use/beta/__init__.py

@@ -0,0 +1,72 @@
+"""Beta Browser Use integration."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from browser_use.beta.service import Agent, BetaAgentError, find_browser_use_terminal_binary
+
+if TYPE_CHECKING:
+	from browser_use.browser import BrowserProfile, BrowserSession
+	from browser_use.browser import BrowserSession as Browser
+	from browser_use.llm.anthropic.chat import ChatAnthropic
+	from browser_use.llm.azure.chat import ChatAzureOpenAI
+	from browser_use.llm.browser_use.chat import ChatBrowserUse
+	from browser_use.llm.google.chat import ChatGoogle
+	from browser_use.llm.groq.chat import ChatGroq
+	from browser_use.llm.litellm.chat import ChatLiteLLM
+	from browser_use.llm.mistral.chat import ChatMistral
+	from browser_use.llm.oci_raw.chat import ChatOCIRaw
+	from browser_use.llm.ollama.chat import ChatOllama
+	from browser_use.llm.openai.chat import ChatOpenAI
+	from browser_use.llm.vercel.chat import ChatVercel
+
+_LAZY_IMPORTS = {
+	'Browser': ('browser_use.browser', 'BrowserSession'),
+	'BrowserProfile': ('browser_use.browser', 'BrowserProfile'),
+	'BrowserSession': ('browser_use.browser', 'BrowserSession'),
+	'ChatOpenAI': ('browser_use.llm.openai.chat', 'ChatOpenAI'),
+	'ChatGoogle': ('browser_use.llm.google.chat', 'ChatGoogle'),
+	'ChatAnthropic': ('browser_use.llm.anthropic.chat', 'ChatAnthropic'),
+	'ChatBrowserUse': ('browser_use.llm.browser_use.chat', 'ChatBrowserUse'),
+	'ChatGroq': ('browser_use.llm.groq.chat', 'ChatGroq'),
+	'ChatLiteLLM': ('browser_use.llm.litellm.chat', 'ChatLiteLLM'),
+	'ChatMistral': ('browser_use.llm.mistral.chat', 'ChatMistral'),
+	'ChatAzureOpenAI': ('browser_use.llm.azure.chat', 'ChatAzureOpenAI'),
+	'ChatOCIRaw': ('browser_use.llm.oci_raw.chat', 'ChatOCIRaw'),
+	'ChatOllama': ('browser_use.llm.ollama.chat', 'ChatOllama'),
+	'ChatVercel': ('browser_use.llm.vercel.chat', 'ChatVercel'),
+}
+
+
+def __getattr__(name: str):
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		from importlib import import_module
+
+		module = import_module(module_path)
+		attr = getattr(module, attr_name)
+		globals()[name] = attr
+		return attr
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'Agent',
+	'BetaAgentError',
+	'Browser',
+	'BrowserProfile',
+	'BrowserSession',
+	'ChatAnthropic',
+	'ChatAzureOpenAI',
+	'ChatBrowserUse',
+	'ChatGoogle',
+	'ChatGroq',
+	'ChatLiteLLM',
+	'ChatMistral',
+	'ChatOCIRaw',
+	'ChatOllama',
+	'ChatOpenAI',
+	'ChatVercel',
+	'find_browser_use_terminal_binary',
+]

browser_use/browser/__init__.py

@@ -0,0 +1,41 @@
+from typing import TYPE_CHECKING
+
+# Type stubs for lazy imports
+if TYPE_CHECKING:
+	from .profile import BrowserProfile, ProxySettings
+	from .session import BrowserSession
+
+
+# Lazy imports mapping for heavy browser components
+_LAZY_IMPORTS = {
+	'ProxySettings': ('.profile', 'ProxySettings'),
+	'BrowserProfile': ('.profile', 'BrowserProfile'),
+	'BrowserSession': ('.session', 'BrowserSession'),
+}
+
+
+def __getattr__(name: str):
+	"""Lazy import mechanism for heavy browser components."""
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		try:
+			from importlib import import_module
+
+			# Use relative import for current package
+			full_module_path = f'browser_use.browser{module_path}'
+			module = import_module(full_module_path)
+			attr = getattr(module, attr_name)
+			# Cache the imported attribute in the module's globals
+			globals()[name] = attr
+			return attr
+		except ImportError as e:
+			raise ImportError(f'Failed to import {name} from {full_module_path}: {e}') from e
+
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'BrowserSession',
+	'BrowserProfile',
+	'ProxySettings',
+]

browser_use/browser/_cdp_timeout.py

@@ -0,0 +1,125 @@
+"""Per-CDP-request timeout wrapper around cdp_use.CDPClient.
+
+cdp_use's `send_raw()` awaits a future that only resolves when the browser
+sends a matching response. If the server goes silent mid-session (observed
+failure mode against remote cloud browsers: WebSocket stays "alive" at the
+TCP/keepalive layer while the browser container is dead or the proxy has
+lost its upstream) the future never resolves and the whole agent hangs.
+
+This module provides a thin subclass that wraps each `send_raw()` in
+`asyncio.wait_for`. Any CDP method that doesn't get a response within the
+cap raises `TimeoutError`, which propagates through existing
+error-handling paths in browser-use instead of hanging indefinitely.
+
+Configure the cap via:
+- `BROWSER_USE_CDP_TIMEOUT_S` env var (process-wide default)
+- `TimeoutWrappedCDPClient(..., cdp_request_timeout_s=...)` constructor arg
+
+Default (60s) is generous for slow operations like `Page.captureScreenshot`
+or `Page.printToPDF` on heavy pages, but well below the 180s agent step
+timeout and the typical outer agent watchdog.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+import os
+from typing import Any
+
+from cdp_use import CDPClient
+
+logger = logging.getLogger(__name__)
+
+_CDP_TIMEOUT_FALLBACK_S = 60.0
+
+
+def _parse_env_cdp_timeout(raw: str | None) -> float:
+	"""Parse BROWSER_USE_CDP_TIMEOUT_S defensively.
+
+	Accepts only finite positive values; everything else falls back to the
+	hardcoded default with a warning. Mirrors the guard on
+	BROWSER_USE_ACTION_TIMEOUT_S in tools/service.py — a bad env value here
+	would otherwise make every CDP call time out immediately (nan) or never
+	(inf / negative / zero).
+	"""
+	if raw is None or raw == '':
+		return _CDP_TIMEOUT_FALLBACK_S
+	try:
+		parsed = float(raw)
+	except ValueError:
+		logger.warning(
+			'Invalid BROWSER_USE_CDP_TIMEOUT_S=%r; falling back to %.0fs',
+			raw,
+			_CDP_TIMEOUT_FALLBACK_S,
+		)
+		return _CDP_TIMEOUT_FALLBACK_S
+	if not math.isfinite(parsed) or parsed <= 0:
+		logger.warning(
+			'BROWSER_USE_CDP_TIMEOUT_S=%r is not a finite positive number; falling back to %.0fs',
+			raw,
+			_CDP_TIMEOUT_FALLBACK_S,
+		)
+		return _CDP_TIMEOUT_FALLBACK_S
+	return parsed
+
+
+DEFAULT_CDP_REQUEST_TIMEOUT_S: float = _parse_env_cdp_timeout(os.getenv('BROWSER_USE_CDP_TIMEOUT_S'))
+
+
+def _coerce_valid_timeout(value: float | None) -> float:
+	"""Normalize a user-supplied timeout to a finite positive value.
+
+	None / nan / inf / non-positive values all fall back to the env-derived
+	default with a warning. This mirrors _parse_env_cdp_timeout so callers that
+	pass cdp_request_timeout_s directly get the same defensive behaviour as
+	callers that set the env var.
+	"""
+	if value is None:
+		return DEFAULT_CDP_REQUEST_TIMEOUT_S
+	if not math.isfinite(value) or value <= 0:
+		logger.warning(
+			'cdp_request_timeout_s=%r is not a finite positive number; falling back to %.0fs',
+			value,
+			DEFAULT_CDP_REQUEST_TIMEOUT_S,
+		)
+		return DEFAULT_CDP_REQUEST_TIMEOUT_S
+	return float(value)
+
+
+class TimeoutWrappedCDPClient(CDPClient):
+	"""CDPClient subclass that enforces a per-request timeout on send_raw.
+
+	Any CDP method that doesn't receive a response within `cdp_request_timeout_s`
+	raises `TimeoutError` instead of hanging forever. This turns silent-hang
+	failure modes (cloud proxy alive, browser dead) into fast observable errors.
+	"""
+
+	def __init__(
+		self,
+		*args: Any,
+		cdp_request_timeout_s: float | None = None,
+		**kwargs: Any,
+	) -> None:
+		super().__init__(*args, **kwargs)
+		self._cdp_request_timeout_s: float = _coerce_valid_timeout(cdp_request_timeout_s)
+
+	async def send_raw(
+		self,
+		method: str,
+		params: Any | None = None,
+		session_id: str | None = None,
+	) -> dict[str, Any]:
+		try:
+			return await asyncio.wait_for(
+				super().send_raw(method=method, params=params, session_id=session_id),
+				timeout=self._cdp_request_timeout_s,
+			)
+		except TimeoutError as e:
+			# Raise a plain TimeoutError so existing `except TimeoutError`
+			# handlers in browser-use / tools treat this uniformly.
+			raise TimeoutError(
+				f'CDP method {method!r} did not respond within {self._cdp_request_timeout_s:.0f}s. '
+				f'The browser may be unresponsive (silent WebSocket — container crashed or proxy lost upstream).'
+			) from e

browser_use/browser/chrome.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import json
+import os
+import platform
+import subprocess
+from pathlib import Path
+
+
+def _chrome_user_data_dir_for_executable(executable_path: str | None) -> Path | None:
+	if executable_path is None:
+		return None
+
+	system = platform.system()
+	path = Path(executable_path)
+	if system == 'Darwin':
+		app_path = str(path)
+		base = Path.home() / 'Library' / 'Application Support'
+		if 'Chromium.app' in app_path:
+			return base / 'Chromium'
+		if 'Google Chrome Canary.app' in app_path:
+			return base / 'Google' / 'Chrome Canary'
+		if 'Google Chrome.app' in app_path:
+			return base / 'Google' / 'Chrome'
+	if system == 'Linux':
+		name = path.name
+		base = Path.home() / '.config'
+		if name in {'chromium', 'chromium-browser'}:
+			return base / 'chromium'
+		if name in {'google-chrome', 'google-chrome-stable'}:
+			return base / 'google-chrome'
+	if system == 'Windows' and path.name.lower() == 'chrome.exe':
+		return Path(os.path.expandvars(r'%LocalAppData%\Google\Chrome\User Data'))
+
+	return None
+
+
+def find_chrome_executable() -> str | None:
+	"""Find Chrome/Chromium executable on the system."""
+	system = platform.system()
+
+	if system == 'Darwin':
+		for path in (
+			'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+			'/Applications/Chromium.app/Contents/MacOS/Chromium',
+			'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
+		):
+			if os.path.exists(path):
+				return path
+
+	elif system == 'Linux':
+		for cmd in ('google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser'):
+			try:
+				result = subprocess.run(['which', cmd], capture_output=True, text=True)
+				if result.returncode == 0:
+					return result.stdout.strip()
+			except Exception:
+				pass
+
+	elif system == 'Windows':
+		for path in (
+			os.path.expandvars(r'%ProgramFiles%\Google\Chrome\Application\chrome.exe'),
+			os.path.expandvars(r'%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe'),
+			os.path.expandvars(r'%LocalAppData%\Google\Chrome\Application\chrome.exe'),
+		):
+			if os.path.exists(path):
+				return path
+
+	return None
+
+
+def get_chrome_profile_path(profile: str | None, executable_path: str | None = None) -> str | None:
+	"""Get Chrome user data directory, or return a specific profile directory name."""
+	if profile is not None:
+		return profile
+
+	if browser_user_data_dir := _chrome_user_data_dir_for_executable(executable_path):
+		return str(browser_user_data_dir)
+
+	system = platform.system()
+	if system == 'Darwin':
+		return str(Path.home() / 'Library' / 'Application Support' / 'Google' / 'Chrome')
+	if system == 'Linux':
+		base = Path.home() / '.config'
+		for name in ('google-chrome', 'chromium'):
+			if (base / name).is_dir():
+				return str(base / name)
+		return str(base / 'google-chrome')
+	if system == 'Windows':
+		return os.path.expandvars(r'%LocalAppData%\Google\Chrome\User Data')
+
+	return None
+
+
+def list_chrome_profiles() -> list[dict[str, str]]:
+	"""List available Chrome profiles with their display names."""
+	user_data_dir = get_chrome_profile_path(None, executable_path=find_chrome_executable())
+	if user_data_dir is None:
+		return []
+
+	local_state_path = Path(user_data_dir) / 'Local State'
+	if not local_state_path.exists():
+		return []
+
+	try:
+		with open(local_state_path, encoding='utf-8') as f:
+			local_state = json.load(f)
+
+		if not isinstance(local_state, dict):
+			return []
+		info_cache = local_state.get('profile', {}).get('info_cache', {})
+		if not isinstance(info_cache, dict):
+			return []
+		return sorted(
+			[
+				{
+					'directory': directory,
+					'name': info.get('name', directory),
+				}
+				for directory, info in info_cache.items()
+				if isinstance(info, dict)
+			],
+			key=lambda profile: profile['directory'],
+		)
+	except (json.JSONDecodeError, KeyError, OSError, TypeError):
+		return []

browser_use/browser/cloud/cloud.py

@@ -0,0 +1,210 @@
+"""Cloud browser service integration for browser-use.
+
+This module provides integration with the browser-use cloud browser service.
+When cloud_browser=True, it automatically creates a cloud browser instance
+and returns the CDP URL for connection.
+"""
+
+import logging
+import os
+
+import httpx
+
+from browser_use.browser.cloud.views import CloudBrowserAuthError, CloudBrowserError, CloudBrowserResponse, CreateBrowserRequest
+from browser_use.sync.auth import CloudAuthConfig
+
+logger = logging.getLogger(__name__)
+
+
+class CloudBrowserClient:
+	"""Client for browser-use cloud browser service."""
+
+	def __init__(self, api_base_url: str = 'https://api.browser-use.com'):
+		self.api_base_url = api_base_url
+		self.client = httpx.AsyncClient(timeout=30.0)
+		self.current_session_id: str | None = None
+
+	async def create_browser(
+		self, request: CreateBrowserRequest, extra_headers: dict[str, str] | None = None
+	) -> CloudBrowserResponse:
+		"""Create a new cloud browser instance. For full docs refer to https://docs.cloud.browser-use.com/api-reference/v-2-api-current/browsers/create-browser-session-browsers-post
+
+		Args:
+			request: CreateBrowserRequest object containing browser creation parameters
+
+		Returns:
+			CloudBrowserResponse: Contains CDP URL and other browser info
+		"""
+		url = f'{self.api_base_url}/api/v2/browsers'
+
+		# Try to get API key from environment variable first, then auth config
+		api_token = os.getenv('BROWSER_USE_API_KEY')
+
+		if not api_token:
+			# Fallback to auth config file
+			try:
+				auth_config = CloudAuthConfig.load_from_file()
+				api_token = auth_config.api_token
+			except Exception:
+				pass
+
+		if not api_token:
+			raise CloudBrowserAuthError(
+				'BROWSER_USE_API_KEY is not set. To use cloud browsers, get a key at:\n'
+				'https://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=use_cloud'
+			)
+
+		headers = {'X-Browser-Use-API-Key': api_token, 'Content-Type': 'application/json', **(extra_headers or {})}
+
+		# Convert request to dictionary and exclude unset fields
+		request_body = request.model_dump(exclude_unset=True)
+
+		try:
+			logger.info('🌤️ Creating cloud browser instance...')
+
+			response = await self.client.post(url, headers=headers, json=request_body)
+
+			if response.status_code == 401:
+				raise CloudBrowserAuthError(
+					'BROWSER_USE_API_KEY is invalid. Get a new key at:\n'
+					'https://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=use_cloud'
+				)
+			elif response.status_code == 403:
+				raise CloudBrowserAuthError('Access forbidden. Please check your browser-use cloud subscription status.')
+			elif not response.is_success:
+				error_msg = f'Failed to create cloud browser: HTTP {response.status_code}'
+				try:
+					error_data = response.json()
+					if 'detail' in error_data:
+						error_msg += f' - {error_data["detail"]}'
+				except Exception:
+					pass
+				raise CloudBrowserError(error_msg)
+
+			browser_data = response.json()
+			browser_response = CloudBrowserResponse(**browser_data)
+
+			# Store session ID for cleanup
+			self.current_session_id = browser_response.id
+
+			logger.info(f'🌤️ Cloud browser created successfully: {browser_response.id}')
+			logger.debug(f'🌤️ CDP URL: {browser_response.cdpUrl}')
+			# Cyan color for live URL
+			logger.info(f'\033[36m🔗 Live URL: {browser_response.liveUrl}\033[0m')
+
+			return browser_response
+
+		except httpx.TimeoutException:
+			raise CloudBrowserError('Timeout while creating cloud browser. Please try again.')
+		except httpx.ConnectError:
+			raise CloudBrowserError('Failed to connect to cloud browser service. Please check your internet connection.')
+		except Exception as e:
+			if isinstance(e, (CloudBrowserError, CloudBrowserAuthError)):
+				raise
+			raise CloudBrowserError(f'Unexpected error creating cloud browser: {e}')
+
+	async def stop_browser(
+		self, session_id: str | None = None, extra_headers: dict[str, str] | None = None
+	) -> CloudBrowserResponse:
+		"""Stop a cloud browser session.
+
+		Args:
+			session_id: Session ID to stop. If None, uses current session.
+
+		Returns:
+			CloudBrowserResponse: Updated browser info with stopped status
+
+		Raises:
+			CloudBrowserAuthError: If authentication fails
+			CloudBrowserError: If stopping fails
+		"""
+		if session_id is None:
+			session_id = self.current_session_id
+
+		if not session_id:
+			raise CloudBrowserError('No session ID provided and no current session available')
+
+		url = f'{self.api_base_url}/api/v2/browsers/{session_id}'
+
+		# Try to get API key from environment variable first, then auth config
+		api_token = os.getenv('BROWSER_USE_API_KEY')
+
+		if not api_token:
+			# Fallback to auth config file
+			try:
+				auth_config = CloudAuthConfig.load_from_file()
+				api_token = auth_config.api_token
+			except Exception:
+				pass
+
+		if not api_token:
+			raise CloudBrowserAuthError(
+				'BROWSER_USE_API_KEY is not set. To use cloud browsers, get a key at:\n'
+				'https://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=use_cloud'
+			)
+
+		headers = {'X-Browser-Use-API-Key': api_token, 'Content-Type': 'application/json', **(extra_headers or {})}
+
+		request_body = {'action': 'stop'}
+
+		try:
+			logger.info(f'🌤️ Stopping cloud browser session: {session_id}')
+
+			response = await self.client.patch(url, headers=headers, json=request_body)
+
+			if response.status_code == 401:
+				raise CloudBrowserAuthError(
+					'Authentication failed. Please make sure you have set the BROWSER_USE_API_KEY environment variable to authenticate with the cloud service.'
+				)
+			elif response.status_code == 404:
+				# Session already stopped or doesn't exist - treating as error and clearing session
+				logger.debug(f'🌤️ Cloud browser session {session_id} not found (already stopped)')
+				# Clear current session if it was this one
+				if session_id == self.current_session_id:
+					self.current_session_id = None
+				raise CloudBrowserError(f'Cloud browser session {session_id} not found')
+			elif not response.is_success:
+				error_msg = f'Failed to stop cloud browser: HTTP {response.status_code}'
+				try:
+					error_data = response.json()
+					if 'detail' in error_data:
+						error_msg += f' - {error_data["detail"]}'
+				except Exception:
+					pass
+				raise CloudBrowserError(error_msg)
+
+			browser_data = response.json()
+			browser_response = CloudBrowserResponse(**browser_data)
+
+			# Clear current session if it was this one
+			if session_id == self.current_session_id:
+				self.current_session_id = None
+
+			logger.info(f'🌤️ Cloud browser session stopped: {browser_response.id}')
+			logger.debug(f'🌤️ Status: {browser_response.status}')
+
+			return browser_response
+
+		except httpx.TimeoutException:
+			raise CloudBrowserError('Timeout while stopping cloud browser. Please try again.')
+		except httpx.ConnectError:
+			raise CloudBrowserError('Failed to connect to cloud browser service. Please check your internet connection.')
+		except Exception as e:
+			if isinstance(e, (CloudBrowserError, CloudBrowserAuthError)):
+				raise
+			raise CloudBrowserError(f'Unexpected error stopping cloud browser: {e}')
+
+	async def close(self):
+		"""Close the HTTP client and cleanup any active sessions.
+
+		Safe to call multiple times — subsequent calls are no-ops.
+		"""
+		# Try to stop current session if active
+		if self.current_session_id:
+			try:
+				await self.stop_browser()
+			except Exception as e:
+				logger.debug(f'Failed to stop cloud browser session during cleanup: {e}')
+
+		if not self.client.is_closed:
+			await self.client.aclose()

browser_use/browser/cloud/views.py

@@ -0,0 +1,96 @@
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+ProxyCountryCode = (
+	Literal[
+		'us',  # United States
+		'uk',  # United Kingdom
+		'fr',  # France
+		'it',  # Italy
+		'jp',  # Japan
+		'au',  # Australia
+		'de',  # Germany
+		'fi',  # Finland
+		'ca',  # Canada
+		'in',  # India
+	]
+	| str
+)
+
+# Browser session timeout limits (in minutes)
+MAX_FREE_USER_SESSION_TIMEOUT = 15  # Free users limited to 15 minutes
+MAX_PAID_USER_SESSION_TIMEOUT = 240  # Paid users can go up to 4 hours
+
+
+# Requests
+class CreateBrowserRequest(BaseModel):
+	"""Request to create a cloud browser instance.
+
+	Args:
+	    cloud_profile_id: The ID of the profile to use for the session
+	    cloud_proxy_country_code: Country code for proxy location
+	    cloud_timeout: The timeout for the session in minutes
+	"""
+
+	model_config = ConfigDict(extra='forbid', populate_by_name=True)
+
+	profile_id: UUID | str | None = Field(
+		default=None,
+		alias='cloud_profile_id',
+		description='The ID of the profile to use for the session. Can be a UUID or a string of UUID.',
+		title='Cloud Profile ID',
+	)
+
+	proxy_country_code: ProxyCountryCode | None = Field(
+		default=None,
+		alias='cloud_proxy_country_code',
+		description='Country code for proxy location.',
+		title='Cloud Proxy Country Code',
+	)
+
+	timeout: int | None = Field(
+		ge=1,
+		le=MAX_PAID_USER_SESSION_TIMEOUT,
+		default=None,
+		alias='cloud_timeout',
+		description=f'The timeout for the session in minutes. Free users are limited to {MAX_FREE_USER_SESSION_TIMEOUT} minutes, paid users can use up to {MAX_PAID_USER_SESSION_TIMEOUT} minutes ({MAX_PAID_USER_SESSION_TIMEOUT // 60} hours).',
+		title='Cloud Timeout',
+	)
+
+	enable_recording: bool = Field(
+		default=False,
+		alias='enableRecording',
+		description='Enable session recording for playback in the cloud dashboard.',
+		title='Enable Recording',
+	)
+
+
+CloudBrowserParams = CreateBrowserRequest  # alias for easier readability
+
+
+# Responses
+class CloudBrowserResponse(BaseModel):
+	"""Response from cloud browser API."""
+
+	id: str
+	status: str
+	liveUrl: str = Field(alias='liveUrl')
+	cdpUrl: str = Field(alias='cdpUrl')
+	timeoutAt: str = Field(alias='timeoutAt')
+	startedAt: str = Field(alias='startedAt')
+	finishedAt: str | None = Field(alias='finishedAt', default=None)
+
+
+# Errors
+class CloudBrowserError(Exception):
+	"""Exception raised when cloud browser operations fail."""
+
+	pass
+
+
+class CloudBrowserAuthError(CloudBrowserError):
+	"""Exception raised when cloud browser authentication fails."""
+
+	pass

browser_use/browser/events.py

@@ -0,0 +1,667 @@
+"""Event definitions for browser communication."""
+
+import inspect
+import os
+from typing import Any, Literal
+
+from bubus import BaseEvent
+from bubus.models import T_EventResultType
+from cdp_use.cdp.target import TargetID
+from pydantic import BaseModel, Field, field_validator
+
+from browser_use.browser.views import BrowserStateSummary
+from browser_use.dom.views import EnhancedDOMTreeNode
+
+
+def _get_timeout(env_var: str, default: float) -> float | None:
+	"""
+	Safely parse environment variable timeout values with robust error handling.
+
+	Args:
+		env_var: Environment variable name (e.g. 'TIMEOUT_NavigateToUrlEvent')
+		default: Default timeout value as float (e.g. 15.0)
+
+	Returns:
+		Parsed float value or the default if parsing fails
+
+	Raises:
+		ValueError: Only if both env_var and default are invalid (should not happen with valid defaults)
+	"""
+	# Try environment variable first
+	env_value = os.getenv(env_var)
+	if env_value:
+		try:
+			parsed = float(env_value)
+			if parsed < 0:
+				print(f'Warning: {env_var}={env_value} is negative, using default {default}')
+				return default
+			return parsed
+		except (ValueError, TypeError):
+			print(f'Warning: {env_var}={env_value} is not a valid number, using default {default}')
+
+	# Fall back to default
+	return default
+
+
+# ============================================================================
+# Agent/Tools -> BrowserSession Events (High-level browser actions)
+# ============================================================================
+
+
+class ElementSelectedEvent(BaseEvent[T_EventResultType]):
+	"""An element was selected."""
+
+	node: EnhancedDOMTreeNode
+
+	@field_validator('node', mode='before')
+	@classmethod
+	def serialize_node(cls, data: EnhancedDOMTreeNode | None) -> EnhancedDOMTreeNode | None:
+		if data is None:
+			return None
+		return EnhancedDOMTreeNode(
+			node_id=data.node_id,
+			backend_node_id=data.backend_node_id,
+			session_id=data.session_id,
+			frame_id=data.frame_id,
+			target_id=data.target_id,
+			node_type=data.node_type,
+			node_name=data.node_name,
+			node_value=data.node_value,
+			attributes=data.attributes,
+			is_scrollable=data.is_scrollable,
+			is_visible=data.is_visible,
+			absolute_position=data.absolute_position,
+			# override the circular reference fields in EnhancedDOMTreeNode as they cant be serialized and aren't needed by event handlers
+			# only used internally by the DOM service during DOM tree building process, not intended public API use
+			content_document=None,
+			shadow_root_type=None,
+			shadow_roots=[],
+			parent_node=None,
+			children_nodes=[],
+			ax_node=None,
+			snapshot_node=None,
+		)
+
+
+# TODO: add page handle to events
+# class PageHandle(share a base with browser.session.CDPSession?):
+# 	url: str
+# 	target_id: TargetID
+#   @classmethod
+#   def from_target_id(cls, target_id: TargetID) -> Self:
+#     return cls(target_id=target_id)
+#   @classmethod
+#   def from_target_id(cls, target_id: TargetID) -> Self:
+#     return cls(target_id=target_id)
+#   @classmethod
+#   def from_url(cls, url: str) -> Self:
+#   @property
+#   def root_frame_id(self) -> str:
+#     return self.target_id
+#   @property
+#   def session_id(self) -> str:
+#     return browser_session.get_or_create_cdp_session(self.target_id).session_id
+
+# class PageSelectedEvent(BaseEvent[T_EventResultType]):
+# 	"""An event like SwitchToTabEvent(page=PageHandle) or CloseTabEvent(page=PageHandle)"""
+# 	page: PageHandle
+
+
+class NavigateToUrlEvent(BaseEvent[None]):
+	"""Navigate to a specific URL."""
+
+	url: str
+	wait_until: Literal['load', 'domcontentloaded', 'networkidle', 'commit'] = 'load'
+	timeout_ms: int | None = None
+	new_tab: bool = Field(
+		default=False, description='Set True to leave the current tab alone and open a new tab in the foreground for the new URL'
+	)
+	# existing_tab: PageHandle | None = None  # TODO
+
+	# time limits enforced by bubus, not exposed to LLM:
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigateToUrlEvent', 30.0))  # seconds
+
+
+class ClickElementEvent(ElementSelectedEvent[dict[str, Any] | None]):
+	"""Click an element."""
+
+	node: 'EnhancedDOMTreeNode'
+	button: Literal['left', 'right', 'middle'] = 'left'
+	# click_count: int = 1           # TODO
+	# expect_download: bool = False  # moved to downloads_watchdog.py
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ClickElementEvent', 15.0))  # seconds
+
+
+class ClickCoordinateEvent(BaseEvent[dict]):
+	"""Click at specific coordinates."""
+
+	coordinate_x: int
+	coordinate_y: int
+	button: Literal['left', 'right', 'middle'] = 'left'
+	force: bool = False  # If True, skip safety checks (file input, print, select)
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ClickCoordinateEvent', 15.0))  # seconds
+
+
+class TypeTextEvent(ElementSelectedEvent[dict | None]):
+	"""Type text into an element."""
+
+	node: 'EnhancedDOMTreeNode'
+	text: str
+	clear: bool = True
+	is_sensitive: bool = False  # Flag to indicate if text contains sensitive data
+	sensitive_key_name: str | None = None  # Name of the sensitive key being typed (e.g., 'username', 'password')
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TypeTextEvent', 60.0))  # seconds
+
+
+class ScrollEvent(ElementSelectedEvent[None]):
+	"""Scroll the page or element."""
+
+	direction: Literal['up', 'down', 'left', 'right']
+	amount: int  # pixels
+	node: 'EnhancedDOMTreeNode | None' = None  # None means scroll page
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollEvent', 8.0))  # seconds
+
+
+class SwitchTabEvent(BaseEvent[TargetID]):
+	"""Switch to a different tab."""
+
+	target_id: TargetID | None = Field(default=None, description='None means switch to the most recently opened tab')
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SwitchTabEvent', 10.0))  # seconds
+
+
+class CloseTabEvent(BaseEvent[None]):
+	"""Close a tab."""
+
+	target_id: TargetID
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CloseTabEvent', 10.0))  # seconds
+
+
+class ScreenshotEvent(BaseEvent[str]):
+	"""Request to take a screenshot."""
+
+	full_page: bool = False
+	clip: dict[str, float] | None = None  # {x, y, width, height}
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScreenshotEvent', 15.0))  # seconds
+
+
+class BrowserStateRequestEvent(BaseEvent[BrowserStateSummary]):
+	"""Request current browser state."""
+
+	include_dom: bool = True
+	include_screenshot: bool = True
+	include_recent_events: bool = False
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStateRequestEvent', 30.0))  # seconds
+
+
+# class WaitForConditionEvent(BaseEvent):
+# 	"""Wait for a condition."""
+
+# 	condition: Literal['navigation', 'selector', 'timeout', 'load_state']
+# 	timeout: float = 30000
+# 	selector: str | None = None
+# 	state: Literal['attached', 'detached', 'visible', 'hidden'] | None = None
+
+
+class GoBackEvent(BaseEvent[None]):
+	"""Navigate back in browser history."""
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_GoBackEvent', 15.0))  # seconds
+
+
+class GoForwardEvent(BaseEvent[None]):
+	"""Navigate forward in browser history."""
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_GoForwardEvent', 15.0))  # seconds
+
+
+class RefreshEvent(BaseEvent[None]):
+	"""Refresh/reload the current page."""
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_RefreshEvent', 15.0))  # seconds
+
+
+class WaitEvent(BaseEvent[None]):
+	"""Wait for a specified number of seconds."""
+
+	seconds: float = 3.0
+	max_seconds: float = 10.0  # Safety cap
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_WaitEvent', 60.0))  # seconds
+
+
+class SendKeysEvent(BaseEvent[None]):
+	"""Send keyboard keys/shortcuts."""
+
+	keys: str  # e.g., "ctrl+a", "cmd+c", "Enter"
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SendKeysEvent', 60.0))  # seconds
+
+
+class UploadFileEvent(ElementSelectedEvent[None]):
+	"""Upload a file to an element."""
+
+	node: 'EnhancedDOMTreeNode'
+	file_path: str
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_UploadFileEvent', 30.0))  # seconds
+
+
+class GetDropdownOptionsEvent(ElementSelectedEvent[dict[str, str]]):
+	"""Get all options from any dropdown (native <select>, ARIA menus, or custom dropdowns).
+
+	Returns a dict containing dropdown type, options list, and element metadata."""
+
+	node: 'EnhancedDOMTreeNode'
+
+	event_timeout: float | None = Field(
+		default_factory=lambda: _get_timeout('TIMEOUT_GetDropdownOptionsEvent', 15.0)
+	)  # some dropdowns lazy-load the list of options on first interaction, so we need to wait for them to load (e.g. table filter lists can have thousands of options)
+
+
+class SelectDropdownOptionEvent(ElementSelectedEvent[dict[str, str]]):
+	"""Select a dropdown option by exact text from any dropdown type.
+
+	Returns a dict containing success status and selection details."""
+
+	node: 'EnhancedDOMTreeNode'
+	text: str  # The option text to select
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SelectDropdownOptionEvent', 8.0))  # seconds
+
+
+class ScrollToTextEvent(BaseEvent[None]):
+	"""Scroll to specific text on the page. Raises exception if text not found."""
+
+	text: str
+	direction: Literal['up', 'down'] = 'down'
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollToTextEvent', 15.0))  # seconds
+
+
+# ============================================================================
+
+
+class BrowserStartEvent(BaseEvent):
+	"""Start/connect to browser."""
+
+	cdp_url: str | None = None
+	launch_options: dict[str, Any] = Field(default_factory=dict)
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStartEvent', 30.0))  # seconds
+
+
+class BrowserStopEvent(BaseEvent):
+	"""Stop/disconnect from browser."""
+
+	force: bool = False
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStopEvent', 45.0))  # seconds
+
+
+class BrowserLaunchResult(BaseModel):
+	"""Result of launching a browser."""
+
+	# TODO: add browser executable_path, pid, version, latency, user_data_dir, X11 $DISPLAY, host IP address, etc.
+	cdp_url: str
+
+
+class BrowserLaunchEvent(BaseEvent[BrowserLaunchResult]):
+	"""Launch a local browser process."""
+
+	# TODO: add executable_path, proxy settings, preferences, extra launch args, etc.
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserLaunchEvent', 30.0))  # seconds
+
+
+class BrowserKillEvent(BaseEvent):
+	"""Kill local browser subprocess."""
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserKillEvent', 30.0))  # seconds
+
+
+# TODO: replace all Runtime.evaluate() calls with this event
+# class ExecuteJavaScriptEvent(BaseEvent):
+# 	"""Execute JavaScript in page context."""
+
+# 	target_id: TargetID
+# 	expression: str
+# 	await_promise: bool = True
+
+# 	event_timeout: float | None = 60.0  # seconds
+
+# TODO: add this and use the old BrowserProfile.viewport options to set it
+# class SetViewportEvent(BaseEvent):
+# 	"""Set the viewport size."""
+
+# 	width: int
+# 	height: int
+# 	device_scale_factor: float = 1.0
+
+# 	event_timeout: float | None = 15.0  # seconds
+
+
+# Moved to storage state
+# class SetCookiesEvent(BaseEvent):
+# 	"""Set browser cookies."""
+
+# 	cookies: list[dict[str, Any]]
+
+# 	event_timeout: float | None = (
+# 		30.0  # only long to support the edge case of restoring a big localStorage / on many origins (has to O(n) visit each origin to restore)
+# 	)
+
+
+# class GetCookiesEvent(BaseEvent):
+# 	"""Get browser cookies."""
+
+# 	urls: list[str] | None = None
+
+# 	event_timeout: float | None = 30.0  # seconds
+
+
+# ============================================================================
+# DOM-related Events
+# ============================================================================
+
+
+class BrowserConnectedEvent(BaseEvent):
+	"""Browser has started/connected."""
+
+	cdp_url: str
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserConnectedEvent', 30.0))  # seconds
+
+
+class BrowserStoppedEvent(BaseEvent):
+	"""Browser has stopped/disconnected."""
+
+	reason: str | None = None
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStoppedEvent', 30.0))  # seconds
+
+
+class TabCreatedEvent(BaseEvent):
+	"""A new tab was created."""
+
+	target_id: TargetID
+	url: str
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TabCreatedEvent', 30.0))  # seconds
+
+
+class TabClosedEvent(BaseEvent):
+	"""A tab was closed."""
+
+	target_id: TargetID
+
+	# TODO:
+	# new_focus_target_id: int | None = None
+	# new_focus_url: str | None = None
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TabClosedEvent', 3.0))  # seconds
+
+
+# TODO: emit this when DOM changes significantly, inner frame navigates, form submits, history.pushState(), etc.
+# class TabUpdatedEvent(BaseEvent):
+# 	"""Tab information updated (URL changed, etc.)."""
+
+# 	target_id: TargetID
+# 	url: str
+
+
+class AgentFocusChangedEvent(BaseEvent):
+	"""Agent focus changed to a different tab."""
+
+	target_id: TargetID
+	url: str
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_AgentFocusChangedEvent', 10.0))  # seconds
+
+
+class TargetCrashedEvent(BaseEvent):
+	"""A target has crashed."""
+
+	target_id: TargetID
+	error: str
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TargetCrashedEvent', 10.0))  # seconds
+
+
+class NavigationStartedEvent(BaseEvent):
+	"""Navigation started."""
+
+	target_id: TargetID
+	url: str
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigationStartedEvent', 30.0))  # seconds
+
+
+class NavigationCompleteEvent(BaseEvent):
+	"""Navigation completed."""
+
+	target_id: TargetID
+	url: str
+	status: int | None = None
+	error_message: str | None = None  # Error/timeout message if navigation had issues
+	loading_status: str | None = None  # Detailed loading status (e.g., network timeout info)
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigationCompleteEvent', 30.0))  # seconds
+
+
+# ============================================================================
+# Error Events
+# ============================================================================
+
+
+class BrowserErrorEvent(BaseEvent):
+	"""An error occurred in the browser layer."""
+
+	error_type: str
+	message: str
+	details: dict[str, Any] = Field(default_factory=dict)
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserErrorEvent', 30.0))  # seconds
+
+
+class BrowserReconnectingEvent(BaseEvent):
+	"""WebSocket reconnection attempt is starting."""
+
+	cdp_url: str
+	attempt: int
+	max_attempts: int
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserReconnectingEvent', 30.0))  # seconds
+
+
+class BrowserReconnectedEvent(BaseEvent):
+	"""WebSocket reconnection succeeded."""
+
+	cdp_url: str
+	attempt: int
+	downtime_seconds: float
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserReconnectedEvent', 30.0))  # seconds
+
+
+# ============================================================================
+# Storage State Events
+# ============================================================================
+
+
+class SaveStorageStateEvent(BaseEvent):
+	"""Request to save browser storage state."""
+
+	path: str | None = None  # Optional path, uses profile default if not provided
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SaveStorageStateEvent', 45.0))  # seconds
+
+
+class StorageStateSavedEvent(BaseEvent):
+	"""Notification that storage state was saved."""
+
+	path: str
+	cookies_count: int
+	origins_count: int
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_StorageStateSavedEvent', 30.0))  # seconds
+
+
+class LoadStorageStateEvent(BaseEvent):
+	"""Request to load browser storage state."""
+
+	path: str | None = None  # Optional path, uses profile default if not provided
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_LoadStorageStateEvent', 45.0))  # seconds
+
+
+# TODO: refactor this to:
+# - on_BrowserConnectedEvent() -> dispatch(LoadStorageStateEvent()) -> _copy_storage_state_from_json_to_browser(json_file, new_cdp_session) + return storage_state from handler
+# - on_BrowserStopEvent() -> dispatch(SaveStorageStateEvent()) -> _copy_storage_state_from_browser_to_json(new_cdp_session, json_file)
+# and get rid of StorageStateSavedEvent and StorageStateLoadedEvent, have the original events + provide handler return values for any results
+class StorageStateLoadedEvent(BaseEvent):
+	"""Notification that storage state was loaded."""
+
+	path: str
+	cookies_count: int
+	origins_count: int
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_StorageStateLoadedEvent', 30.0))  # seconds
+
+
+# ============================================================================
+# File Download Events
+# ============================================================================
+
+
+class DownloadStartedEvent(BaseEvent):
+	"""A file download has started (CDP downloadWillBegin received)."""
+
+	guid: str  # CDP download GUID to correlate with FileDownloadedEvent
+	url: str
+	suggested_filename: str
+	auto_download: bool = False  # Whether this was triggered automatically
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_DownloadStartedEvent', 5.0))  # seconds
+
+
+class DownloadProgressEvent(BaseEvent):
+	"""A file download progress update (CDP downloadProgress received)."""
+
+	guid: str  # CDP download GUID to correlate with other download events
+	received_bytes: int
+	total_bytes: int  # 0 if unknown
+	state: str  # 'inProgress', 'completed', or 'canceled'
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_DownloadProgressEvent', 5.0))  # seconds
+
+
+class FileDownloadedEvent(BaseEvent):
+	"""A file has been downloaded."""
+
+	guid: str | None = None  # CDP download GUID to correlate with DownloadStartedEvent
+	url: str
+	path: str
+	file_name: str
+	file_size: int
+	file_type: str | None = None  # e.g., 'pdf', 'zip', 'docx', etc.
+	mime_type: str | None = None  # e.g., 'application/pdf'
+	from_cache: bool = False
+	auto_download: bool = False  # Whether this was an automatic download (e.g., PDF auto-download)
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_FileDownloadedEvent', 30.0))  # seconds
+
+
+class AboutBlankDVDScreensaverShownEvent(BaseEvent):
+	"""AboutBlankWatchdog has shown DVD screensaver animation on an about:blank tab."""
+
+	target_id: TargetID
+	error: str | None = None
+
+
+class DialogOpenedEvent(BaseEvent):
+	"""Event dispatched when a JavaScript dialog is opened and handled."""
+
+	dialog_type: str  # 'alert', 'confirm', 'prompt', or 'beforeunload'
+	message: str
+	url: str
+	frame_id: str | None = None  # Can be None when frameId is not provided by CDP
+	# target_id: TargetID   # TODO: add this to avoid needing target_id_from_frame() later
+
+
+# ============================================================================
+# Captcha Solver Events
+# ============================================================================
+
+
+class CaptchaSolverStartedEvent(BaseEvent):
+	"""Captcha solving started by the browser proxy.
+
+	Emitted when the browser proxy detects a CAPTCHA and begins solving it.
+	The agent should wait for a corresponding CaptchaSolverFinishedEvent before proceeding.
+	"""
+
+	target_id: TargetID
+	vendor: str  # e.g. 'cloudflare', 'recaptcha', 'hcaptcha', 'datadome', 'perimeterx', 'geetest'
+	url: str
+	started_at: int  # Unix millis
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CaptchaSolverStartedEvent', 5.0))
+
+
+class CaptchaSolverFinishedEvent(BaseEvent):
+	"""Captcha solving finished by the browser proxy.
+
+	Emitted when the browser proxy finishes solving a CAPTCHA (successfully or not).
+	"""
+
+	target_id: TargetID
+	vendor: str
+	url: str
+	duration_ms: int
+	finished_at: int  # Unix millis
+	success: bool  # Whether the captcha was solved successfully
+
+	event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CaptchaSolverFinishedEvent', 5.0))
+
+
+# Note: Model rebuilding for forward references is handled in the importing modules
+# Events with 'EnhancedDOMTreeNode' forward references (ClickElementEvent, TypeTextEvent,
+# ScrollEvent, UploadFileEvent) need model_rebuild() called after imports are complete
+
+
+def _check_event_names_dont_overlap():
+	"""
+	check that event names defined in this file are valid and non-overlapping
+	(naiively n^2 so it's pretty slow but ok for now, optimize when >20 events)
+	"""
+	event_names = {
+		name.split('[')[0]
+		for name in globals().keys()
+		if not name.startswith('_')
+		and inspect.isclass(globals()[name])
+		and issubclass(globals()[name], BaseEvent)
+		and name != 'BaseEvent'
+	}
+	for name_a in event_names:
+		assert name_a.endswith('Event'), f'Event with name {name_a} does not end with "Event"'
+		for name_b in event_names:
+			if name_a != name_b:  # Skip self-comparison
+				assert name_a not in name_b, (
+					f'Event with name {name_a} is a substring of {name_b}, all events must be completely unique to avoid find-and-replace accidents'
+				)
+
+
+# overlapping event names are a nightmare to trace and rename later, dont do it!
+# e.g. prevent ClickEvent and FailedClickEvent are terrible names because one is a substring of the other,
+# must be ClickEvent and ClickFailedEvent to preserve the usefulnes of codebase grep/sed/awk as refactoring tools.
+# at import time, we do a quick check that all event names defined above are valid and non-overlapping.
+# this is hand written in blood by a human! not LLM slop. feel free to optimize but do not remove it without a good reason.
+_check_event_names_dont_overlap()

browser_use/browser/python_highlights.py

@@ -0,0 +1,548 @@
+"""Python-based highlighting system for drawing bounding boxes on screenshots.
+
+This module replaces JavaScript-based highlighting with fast Python image processing
+to draw bounding boxes around interactive elements directly on screenshots.
+"""
+
+import asyncio
+import base64
+import io
+import logging
+import os
+
+from PIL import Image, ImageDraw, ImageFont
+
+from browser_use.dom.views import DOMSelectorMap, EnhancedDOMTreeNode
+from browser_use.observability import observe_debug
+from browser_use.utils import time_execution_async
+
+logger = logging.getLogger(__name__)
+
+# Font cache to prevent repeated font loading and reduce memory usage
+_FONT_CACHE: dict[tuple[str, int], ImageFont.FreeTypeFont | None] = {}
+
+# Cross-platform font paths
+_FONT_PATHS = [
+	'/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf',  # Linux (Debian/Ubuntu)
+	'/usr/share/fonts/TTF/DejaVuSans-Bold.ttf',  # Linux (Arch/Fedora)
+	'/System/Library/Fonts/Arial.ttf',  # macOS
+	'C:\\Windows\\Fonts\\arial.ttf',  # Windows
+	'arial.ttf',  # Windows (system path)
+	'Arial Bold.ttf',  # macOS alternative
+	'/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf',  # Linux alternative
+]
+
+
+def get_cross_platform_font(font_size: int) -> ImageFont.FreeTypeFont | None:
+	"""Get a cross-platform compatible font with caching to prevent memory leaks.
+
+	Args:
+	    font_size: Size of the font to load
+
+	Returns:
+	    ImageFont object or None if no system fonts are available
+	"""
+	# Use cache key based on font size
+	cache_key = ('system_font', font_size)
+
+	# Return cached font if available
+	if cache_key in _FONT_CACHE:
+		return _FONT_CACHE[cache_key]
+
+	# Try to load a system font
+	font = None
+	for font_path in _FONT_PATHS:
+		try:
+			font = ImageFont.truetype(font_path, font_size)
+			break
+		except OSError:
+			continue
+
+	# Cache the result (even if None) to avoid repeated attempts
+	_FONT_CACHE[cache_key] = font
+	return font
+
+
+def cleanup_font_cache() -> None:
+	"""Clean up the font cache to prevent memory leaks in long-running applications."""
+	global _FONT_CACHE
+	_FONT_CACHE.clear()
+
+
+# Color scheme for different element types
+ELEMENT_COLORS = {
+	'button': '#FF6B6B',  # Red for buttons
+	'input': '#4ECDC4',  # Teal for inputs
+	'select': '#45B7D1',  # Blue for dropdowns
+	'a': '#96CEB4',  # Green for links
+	'textarea': '#FF8C42',  # Orange for text areas (was yellow, now more visible)
+	'default': '#DDA0DD',  # Light purple for other interactive elements
+}
+
+# Element type mappings
+ELEMENT_TYPE_MAP = {
+	'button': 'button',
+	'input': 'input',
+	'select': 'select',
+	'a': 'a',
+	'textarea': 'textarea',
+}
+
+
+def get_element_color(tag_name: str, element_type: str | None = None) -> str:
+	"""Get color for element based on tag name and type."""
+	# Check input type first
+	if tag_name == 'input' and element_type:
+		if element_type in ['button', 'submit']:
+			return ELEMENT_COLORS['button']
+
+	# Use tag-based color
+	return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])
+
+
+def should_show_index_overlay(backend_node_id: int | None) -> bool:
+	"""Determine if index overlay should be shown."""
+	return backend_node_id is not None
+
+
+def draw_enhanced_bounding_box_with_text(
+	draw,  # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
+	bbox: tuple[int, int, int, int],
+	color: str,
+	text: str | None = None,
+	font: ImageFont.FreeTypeFont | None = None,
+	element_type: str = 'div',
+	image_size: tuple[int, int] = (2000, 1500),
+	device_pixel_ratio: float = 1.0,
+) -> None:
+	"""Draw an enhanced bounding box with much bigger index containers and dashed borders."""
+	x1, y1, x2, y2 = bbox
+
+	# Draw dashed bounding box with pattern: 1 line, 2 spaces, 1 line, 2 spaces...
+	dash_length = 4
+	gap_length = 8
+	line_width = 2
+
+	# Helper function to draw dashed line
+	def draw_dashed_line(start_x, start_y, end_x, end_y):
+		if start_x == end_x:  # Vertical line
+			y = start_y
+			while y < end_y:
+				dash_end = min(y + dash_length, end_y)
+				draw.line([(start_x, y), (start_x, dash_end)], fill=color, width=line_width)
+				y += dash_length + gap_length
+		else:  # Horizontal line
+			x = start_x
+			while x < end_x:
+				dash_end = min(x + dash_length, end_x)
+				draw.line([(x, start_y), (dash_end, start_y)], fill=color, width=line_width)
+				x += dash_length + gap_length
+
+	# Draw dashed rectangle
+	draw_dashed_line(x1, y1, x2, y1)  # Top
+	draw_dashed_line(x2, y1, x2, y2)  # Right
+	draw_dashed_line(x2, y2, x1, y2)  # Bottom
+	draw_dashed_line(x1, y2, x1, y1)  # Left
+
+	# Draw much bigger index overlay if we have index text
+	if text:
+		try:
+			# Scale font size for appropriate sizing across different resolutions
+			img_width, img_height = image_size
+
+			css_width = img_width  # / device_pixel_ratio
+			# Much smaller scaling - 1% of CSS viewport width, max 16px to prevent huge highlights
+			base_font_size = max(10, min(20, int(css_width * 0.01)))
+			# Use shared font loading function with caching
+			big_font = get_cross_platform_font(base_font_size)
+			if big_font is None:
+				big_font = font  # Fallback to original font if no system fonts found
+
+			# Get text size with bigger font
+			if big_font:
+				bbox_text = draw.textbbox((0, 0), text, font=big_font)
+				text_width = bbox_text[2] - bbox_text[0]
+				text_height = bbox_text[3] - bbox_text[1]
+			else:
+				# Fallback for default font
+				bbox_text = draw.textbbox((0, 0), text)
+				text_width = bbox_text[2] - bbox_text[0]
+				text_height = bbox_text[3] - bbox_text[1]
+
+			# Scale padding appropriately for different resolutions
+			padding = max(4, min(10, int(css_width * 0.005)))  # 0.3% of CSS width, max 4px
+			element_width = x2 - x1
+			element_height = y2 - y1
+
+			# Container dimensions
+			container_width = text_width + padding * 2
+			container_height = text_height + padding * 2
+
+			# Position in top center - for small elements, place further up to avoid blocking content
+			# Center horizontally within the element
+			bg_x1 = x1 + (element_width - container_width) // 2
+
+			# Simple rule: if element is small, place index further up to avoid blocking icons
+			if element_width < 60 or element_height < 30:
+				# Small element: place well above to avoid blocking content
+				bg_y1 = max(0, y1 - container_height - 5)
+			else:
+				# Regular element: place inside with small offset
+				bg_y1 = y1 + 2
+
+			bg_x2 = bg_x1 + container_width
+			bg_y2 = bg_y1 + container_height
+
+			# Center the number within the index box with proper baseline handling
+			text_x = bg_x1 + (container_width - text_width) // 2
+			# Add extra vertical space to prevent clipping
+			text_y = bg_y1 + (container_height - text_height) // 2 - bbox_text[1]  # Subtract top offset
+
+			# Ensure container stays within image bounds
+			img_width, img_height = image_size
+			if bg_x1 < 0:
+				offset = -bg_x1
+				bg_x1 += offset
+				bg_x2 += offset
+				text_x += offset
+			if bg_y1 < 0:
+				offset = -bg_y1
+				bg_y1 += offset
+				bg_y2 += offset
+				text_y += offset
+			if bg_x2 > img_width:
+				offset = bg_x2 - img_width
+				bg_x1 -= offset
+				bg_x2 -= offset
+				text_x -= offset
+			if bg_y2 > img_height:
+				offset = bg_y2 - img_height
+				bg_y1 -= offset
+				bg_y2 -= offset
+				text_y -= offset
+
+			# Draw bigger background rectangle with thicker border
+			draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill=color, outline='white', width=2)
+
+			# Draw white text centered in the index box
+			draw.text((text_x, text_y), text, fill='white', font=big_font or font)
+
+		except Exception as e:
+			logger.debug(f'Failed to draw enhanced text overlay: {e}')
+
+
+def draw_bounding_box_with_text(
+	draw,  # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
+	bbox: tuple[int, int, int, int],
+	color: str,
+	text: str | None = None,
+	font: ImageFont.FreeTypeFont | None = None,
+) -> None:
+	"""Draw a bounding box with optional text overlay."""
+	x1, y1, x2, y2 = bbox
+
+	# Draw dashed bounding box
+	dash_length = 2
+	gap_length = 6
+
+	# Top edge
+	x = x1
+	while x < x2:
+		end_x = min(x + dash_length, x2)
+		draw.line([(x, y1), (end_x, y1)], fill=color, width=2)
+		draw.line([(x, y1 + 1), (end_x, y1 + 1)], fill=color, width=2)
+		x += dash_length + gap_length
+
+	# Bottom edge
+	x = x1
+	while x < x2:
+		end_x = min(x + dash_length, x2)
+		draw.line([(x, y2), (end_x, y2)], fill=color, width=2)
+		draw.line([(x, y2 - 1), (end_x, y2 - 1)], fill=color, width=2)
+		x += dash_length + gap_length
+
+	# Left edge
+	y = y1
+	while y < y2:
+		end_y = min(y + dash_length, y2)
+		draw.line([(x1, y), (x1, end_y)], fill=color, width=2)
+		draw.line([(x1 + 1, y), (x1 + 1, end_y)], fill=color, width=2)
+		y += dash_length + gap_length
+
+	# Right edge
+	y = y1
+	while y < y2:
+		end_y = min(y + dash_length, y2)
+		draw.line([(x2, y), (x2, end_y)], fill=color, width=2)
+		draw.line([(x2 - 1, y), (x2 - 1, end_y)], fill=color, width=2)
+		y += dash_length + gap_length
+
+	# Draw index overlay if we have index text
+	if text:
+		try:
+			# Get text size
+			if font:
+				bbox_text = draw.textbbox((0, 0), text, font=font)
+				text_width = bbox_text[2] - bbox_text[0]
+				text_height = bbox_text[3] - bbox_text[1]
+			else:
+				# Fallback for default font
+				bbox_text = draw.textbbox((0, 0), text)
+				text_width = bbox_text[2] - bbox_text[0]
+				text_height = bbox_text[3] - bbox_text[1]
+
+			# Smart positioning based on element size
+			padding = 5
+			element_width = x2 - x1
+			element_height = y2 - y1
+			element_area = element_width * element_height
+			index_box_area = (text_width + padding * 2) * (text_height + padding * 2)
+
+			# Calculate size ratio to determine positioning strategy
+			size_ratio = element_area / max(index_box_area, 1)
+
+			if size_ratio < 4:
+				# Very small elements: place outside in bottom-right corner
+				text_x = x2 + padding
+				text_y = y2 - text_height
+				# Ensure it doesn't go off screen
+				text_x = min(text_x, 1200 - text_width - padding)
+				text_y = max(text_y, 0)
+			elif size_ratio < 16:
+				# Medium elements: place in bottom-right corner inside
+				text_x = x2 - text_width - padding
+				text_y = y2 - text_height - padding
+			else:
+				# Large elements: place in center
+				text_x = x1 + (element_width - text_width) // 2
+				text_y = y1 + (element_height - text_height) // 2
+
+			# Ensure text stays within bounds
+			text_x = max(0, min(text_x, 1200 - text_width))
+			text_y = max(0, min(text_y, 800 - text_height))
+
+			# Draw background rectangle for maximum contrast
+			bg_x1 = text_x - padding
+			bg_y1 = text_y - padding
+			bg_x2 = text_x + text_width + padding
+			bg_y2 = text_y + text_height + padding
+
+			# Use white background with thick black border for maximum visibility
+			draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill='white', outline='black', width=2)
+
+			# Draw bold dark text on light background for best contrast
+			draw.text((text_x, text_y), text, fill='black', font=font)
+
+		except Exception as e:
+			logger.debug(f'Failed to draw text overlay: {e}')
+
+
+def process_element_highlight(
+	element_id: int,
+	element: EnhancedDOMTreeNode,
+	draw,
+	device_pixel_ratio: float,
+	font,
+	filter_highlight_ids: bool,
+	image_size: tuple[int, int],
+) -> None:
+	"""Process a single element for highlighting."""
+	try:
+		# Use absolute_position coordinates directly
+		if not element.absolute_position:
+			return
+
+		bounds = element.absolute_position
+
+		# Scale coordinates from CSS pixels to device pixels for screenshot
+		# The screenshot is captured at device pixel resolution, but coordinates are in CSS pixels
+		x1 = int(bounds.x * device_pixel_ratio)
+		y1 = int(bounds.y * device_pixel_ratio)
+		x2 = int((bounds.x + bounds.width) * device_pixel_ratio)
+		y2 = int((bounds.y + bounds.height) * device_pixel_ratio)
+
+		# Ensure coordinates are within image bounds
+		img_width, img_height = image_size
+		x1 = max(0, min(x1, img_width))
+		y1 = max(0, min(y1, img_height))
+		x2 = max(x1, min(x2, img_width))
+		y2 = max(y1, min(y2, img_height))
+
+		# Skip if bounding box is too small or invalid
+		if x2 - x1 < 2 or y2 - y1 < 2:
+			return
+
+		# Get element color based on type
+		tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
+		element_type = None
+		if hasattr(element, 'attributes') and element.attributes:
+			element_type = element.attributes.get('type')
+
+		color = get_element_color(tag_name, element_type)
+
+		# Get element index for overlay and apply filtering
+		backend_node_id = getattr(element, 'backend_node_id', None)
+		index_text = None
+
+		if backend_node_id is not None:
+			if filter_highlight_ids:
+				# Use the meaningful text that matches what the LLM sees
+				meaningful_text = element.get_meaningful_text_for_llm()
+				# Show ID only if meaningful text is less than 5 characters
+				if len(meaningful_text) < 3:
+					index_text = str(backend_node_id)
+			else:
+				# Always show ID when filter is disabled
+				index_text = str(backend_node_id)
+
+		# Draw enhanced bounding box with bigger index
+		draw_enhanced_bounding_box_with_text(
+			draw, (x1, y1, x2, y2), color, index_text, font, tag_name, image_size, device_pixel_ratio
+		)
+
+	except Exception as e:
+		logger.debug(f'Failed to draw highlight for element {element_id}: {e}')
+
+
+@observe_debug(ignore_input=True, ignore_output=True, name='create_highlighted_screenshot')
+@time_execution_async('create_highlighted_screenshot')
+async def create_highlighted_screenshot(
+	screenshot_b64: str,
+	selector_map: DOMSelectorMap,
+	device_pixel_ratio: float = 1.0,
+	viewport_offset_x: int = 0,
+	viewport_offset_y: int = 0,
+	filter_highlight_ids: bool = True,
+) -> str:
+	"""Create a highlighted screenshot with bounding boxes around interactive elements.
+
+	Args:
+	    screenshot_b64: Base64 encoded screenshot
+	    selector_map: Map of interactive elements with their positions
+	    device_pixel_ratio: Device pixel ratio for scaling coordinates
+	    viewport_offset_x: X offset for viewport positioning
+	    viewport_offset_y: Y offset for viewport positioning
+
+	Returns:
+	    Base64 encoded highlighted screenshot
+	"""
+	try:
+		# Decode screenshot
+		screenshot_data = base64.b64decode(screenshot_b64)
+		image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')
+
+		# Create drawing context
+		draw = ImageDraw.Draw(image)
+
+		# Load font using shared function with caching
+		font = get_cross_platform_font(12)
+		# If no system fonts found, font remains None and will use default font
+
+		# Process elements sequentially to avoid ImageDraw thread safety issues
+		# PIL ImageDraw is not thread-safe, so we process elements one by one
+		for element_id, element in selector_map.items():
+			process_element_highlight(element_id, element, draw, device_pixel_ratio, font, filter_highlight_ids, image.size)
+
+		# Convert back to base64
+		output_buffer = io.BytesIO()
+		try:
+			image.save(output_buffer, format='PNG')
+			output_buffer.seek(0)
+			highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')
+
+			logger.debug(f'Successfully created highlighted screenshot with {len(selector_map)} elements')
+			return highlighted_b64
+		finally:
+			# Explicit cleanup to prevent memory leaks
+			output_buffer.close()
+			if 'image' in locals():
+				image.close()
+
+	except Exception as e:
+		logger.error(f'Failed to create highlighted screenshot: {e}')
+		# Clean up on error as well
+		if 'image' in locals():
+			image.close()
+		# Return original screenshot on error
+		return screenshot_b64
+
+
+async def get_viewport_info_from_cdp(cdp_session) -> tuple[float, int, int]:
+	"""Get viewport information from CDP session.
+
+	Returns:
+	    Tuple of (device_pixel_ratio, scroll_x, scroll_y)
+	"""
+	try:
+		# Get layout metrics which includes viewport info and device pixel ratio
+		metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
+
+		# Extract viewport information
+		visual_viewport = metrics.get('visualViewport', {})
+		css_visual_viewport = metrics.get('cssVisualViewport', {})
+		css_layout_viewport = metrics.get('cssLayoutViewport', {})
+
+		# Calculate device pixel ratio
+		css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))
+		device_width = visual_viewport.get('clientWidth', css_width)
+		device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
+
+		# Get scroll position in CSS pixels
+		scroll_x = int(css_visual_viewport.get('pageX', 0))
+		scroll_y = int(css_visual_viewport.get('pageY', 0))
+
+		return float(device_pixel_ratio), scroll_x, scroll_y
+
+	except Exception as e:
+		logger.debug(f'Failed to get viewport info from CDP: {e}')
+		return 1.0, 0, 0
+
+
+@time_execution_async('create_highlighted_screenshot_async')
+async def create_highlighted_screenshot_async(
+	screenshot_b64: str, selector_map: DOMSelectorMap, cdp_session=None, filter_highlight_ids: bool = True
+) -> str:
+	"""Async wrapper for creating highlighted screenshots.
+
+	Args:
+	    screenshot_b64: Base64 encoded screenshot
+	    selector_map: Map of interactive elements
+	    cdp_session: CDP session for getting viewport info
+	    filter_highlight_ids: Whether to filter element IDs based on meaningful text
+
+	Returns:
+	    Base64 encoded highlighted screenshot
+	"""
+	# Get viewport information if CDP session is available
+	device_pixel_ratio = 1.0
+	viewport_offset_x = 0
+	viewport_offset_y = 0
+
+	if cdp_session:
+		try:
+			device_pixel_ratio, viewport_offset_x, viewport_offset_y = await get_viewport_info_from_cdp(cdp_session)
+		except Exception as e:
+			logger.debug(f'Failed to get viewport info from CDP: {e}')
+
+	# Create highlighted screenshot with async processing
+	final_screenshot = await create_highlighted_screenshot(
+		screenshot_b64, selector_map, device_pixel_ratio, viewport_offset_x, viewport_offset_y, filter_highlight_ids
+	)
+
+	filename = os.getenv('BROWSER_USE_SCREENSHOT_FILE')
+	if filename:
+
+		def _write_screenshot():
+			try:
+				with open(filename, 'wb') as f:
+					f.write(base64.b64decode(final_screenshot))
+				logger.debug('Saved screenshot to ' + str(filename))
+			except Exception as e:
+				logger.warning(f'Failed to save screenshot to {filename}: {e}')
+
+		await asyncio.to_thread(_write_screenshot)
+	return final_screenshot
+
+
+# Export the cleanup function for external use in long-running applications
+__all__ = ['create_highlighted_screenshot', 'create_highlighted_screenshot_async', 'cleanup_font_cache']

browser_use/browser/session_manager.py

@@ -0,0 +1,918 @@
+"""Event-driven CDP session management.
+
+Manages CDP sessions by listening to Target.attachedToTarget and Target.detachedFromTarget
+events, ensuring the session pool always reflects the current browser state.
+"""
+
+import asyncio
+from collections import deque
+from typing import TYPE_CHECKING, Any
+
+from cdp_use.cdp.target import AttachedToTargetEvent, DetachedFromTargetEvent, SessionID, TargetID
+
+from browser_use.utils import create_task_with_error_handling
+
+if TYPE_CHECKING:
+	from browser_use.browser.session import BrowserSession, CDPSession, Target
+
+
+class SessionManager:
+	"""Event-driven CDP session manager.
+
+	Automatically synchronizes the CDP session pool with browser state via CDP events.
+
+	Key features:
+	- Sessions added/removed automatically via Target attach/detach events
+	- Multiple sessions can attach to the same target
+	- Targets only removed when ALL sessions detach
+	- No stale sessions - pool always reflects browser reality
+
+	SessionManager is the SINGLE SOURCE OF TRUTH for all targets and sessions.
+	"""
+
+	def __init__(self, browser_session: 'BrowserSession'):
+		self.browser_session = browser_session
+		self.logger = browser_session.logger
+
+		# All targets (entities: pages, iframes, workers)
+		self._targets: dict[TargetID, 'Target'] = {}
+
+		# All sessions (communication channels)
+		self._sessions: dict[SessionID, 'CDPSession'] = {}
+
+		# Mapping: target -> sessions attached to it
+		self._target_sessions: dict[TargetID, set[SessionID]] = {}
+
+		# Reverse mapping: session -> target it belongs to
+		self._session_to_target: dict[SessionID, TargetID] = {}
+
+		# Page lifecycle events per target, fed by ONE global Page.lifecycleEvent handler
+		# registered in start_monitoring(). cdp-use's event registry is single-slot per
+		# CDP method, so per-session handler registrations would replace each other and
+		# leave every tab but the most recently attached one without lifecycle events.
+		self._lifecycle_events: dict[TargetID, deque[dict[str, Any]]] = {}
+
+		self._lock = asyncio.Lock()
+		self._recovery_lock = asyncio.Lock()
+
+		# Focus recovery coordination - event-driven instead of polling
+		self._recovery_in_progress: bool = False
+		self._recovery_complete_event: asyncio.Event | None = None
+		self._recovery_task: asyncio.Task | None = None
+
+	async def start_monitoring(self) -> None:
+		"""Start monitoring Target attach/detach events.
+
+		Registers CDP event handlers to keep the session pool synchronized with browser state.
+		Also discovers and initializes all existing targets on startup.
+		"""
+		if not self.browser_session._cdp_client_root:
+			raise RuntimeError('CDP client not initialized')
+
+		# Capture cdp_client_root in closure to avoid type errors
+		cdp_client = self.browser_session._cdp_client_root
+
+		# Enable target discovery to receive targetInfoChanged events automatically
+		# This eliminates the need for getTargetInfo() polling calls
+		await cdp_client.send.Target.setDiscoverTargets(
+			params={'discover': True, 'filter': [{'type': 'page'}, {'type': 'iframe'}]}
+		)
+
+		# Register synchronous event handlers (CDP requirement)
+		def on_attached(event: AttachedToTargetEvent, session_id: SessionID | None = None):
+			# _handle_target_attached() handles:
+			# - setAutoAttach for children
+			# - Create CDPSession
+			# - Enable monitoring (for pages/tabs)
+			# - Add to pool
+			create_task_with_error_handling(
+				self._handle_target_attached(event),
+				name='handle_target_attached',
+				logger_instance=self.logger,
+				suppress_exceptions=True,
+			)
+
+		def on_detached(event: DetachedFromTargetEvent, session_id: SessionID | None = None):
+			create_task_with_error_handling(
+				self._handle_target_detached(event),
+				name='handle_target_detached',
+				logger_instance=self.logger,
+				suppress_exceptions=True,
+			)
+
+		def on_target_info_changed(event, session_id: SessionID | None = None):
+			# Update session info from targetInfoChanged events (no polling needed!)
+			create_task_with_error_handling(
+				self._handle_target_info_changed(event),
+				name='handle_target_info_changed',
+				logger_instance=self.logger,
+				suppress_exceptions=True,
+			)
+
+		def on_lifecycle_event(event, session_id: SessionID | None = None):
+			# ONE global handler for all targets: route by session_id -> target_id.
+			# Registering per-session closures instead would clobber each other in
+			# cdp-use's single-slot registry (one handler per CDP method).
+			if not session_id:
+				return
+			target_id = self.get_target_id_from_session_id(session_id)
+			if not target_id:
+				return
+			self.get_lifecycle_events(target_id).append(
+				{
+					'name': event.get('name', 'unknown'),
+					'loaderId': event.get('loaderId'),
+					'timestamp': asyncio.get_event_loop().time(),
+				}
+			)
+
+		cdp_client.register.Target.attachedToTarget(on_attached)
+		cdp_client.register.Target.detachedFromTarget(on_detached)
+		cdp_client.register.Target.targetInfoChanged(on_target_info_changed)
+		cdp_client.register.Page.lifecycleEvent(on_lifecycle_event)
+
+		self.logger.debug('[SessionManager] Event monitoring started')
+
+		# Discover and initialize ALL existing targets
+		await self._initialize_existing_targets()
+
+	def get_lifecycle_events(self, target_id: TargetID) -> 'deque[dict[str, Any]]':
+		"""Get (creating if needed) the lifecycle event buffer for a target."""
+		events = self._lifecycle_events.get(target_id)
+		if events is None:
+			events = deque(maxlen=50)
+			self._lifecycle_events[target_id] = events
+		return events
+
+	def _get_session_for_target(self, target_id: TargetID) -> 'CDPSession | None':
+		"""Internal: Get ANY valid session for a target (picks first available).
+
+		⚠️ INTERNAL API - Use browser_session.get_or_create_cdp_session() instead!
+		This method has no validation, no focus management, no recovery.
+
+		Args:
+			target_id: Target ID to get session for
+
+		Returns:
+			CDPSession if exists, None if target has detached
+		"""
+		session_ids = self._target_sessions.get(target_id, set())
+		if not session_ids:
+			# Check if this is the focused target - indicates stale focus that needs cleanup
+			if self.browser_session.agent_focus_target_id == target_id:
+				self.logger.warning(
+					f'[SessionManager] ⚠️ Attempted to get session for stale focused target {target_id[:8]}... '
+					f'Clearing stale focus and triggering recovery.'
+				)
+
+				# Clear stale focus immediately (defense in depth)
+				self.browser_session.agent_focus_target_id = None
+
+				# Trigger recovery if not already in progress
+				if not self._recovery_in_progress:
+					self.logger.warning('[SessionManager] Recovery was not in progress! Triggering now.')
+					self._recovery_task = create_task_with_error_handling(
+						self._recover_agent_focus(target_id),
+						name='recover_agent_focus_from_stale_get',
+						logger_instance=self.logger,
+						suppress_exceptions=False,
+					)
+			return None
+		return self._sessions.get(next(iter(session_ids)))
+
+	def get_all_page_targets(self) -> list:
+		"""Get all page/tab targets using owned data.
+
+		Returns:
+			List of Target objects for all page/tab targets
+		"""
+		page_targets = []
+		for target in self._targets.values():
+			if target.target_type in ('page', 'tab'):
+				page_targets.append(target)
+		return page_targets
+
+	async def validate_session(self, target_id: TargetID) -> bool:
+		"""Check if a target still has active sessions.
+
+		Args:
+			target_id: Target ID to validate
+
+		Returns:
+			True if target has active sessions, False if it should be removed
+		"""
+		if target_id not in self._target_sessions:
+			return False
+		return len(self._target_sessions[target_id]) > 0
+
+	async def clear(self) -> None:
+		"""Clear all owned data structures for cleanup."""
+		async with self._lock:
+			# Clear owned data (single source of truth)
+			self._targets.clear()
+			self._sessions.clear()
+			self._target_sessions.clear()
+			self._session_to_target.clear()
+
+		self.logger.info('[SessionManager] Cleared all owned data (targets, sessions, mappings)')
+
+	async def is_target_valid(self, target_id: TargetID) -> bool:
+		"""Check if a target is still valid and has active sessions.
+
+		Args:
+			target_id: Target ID to validate
+
+		Returns:
+			True if target is valid and has active sessions, False otherwise
+		"""
+		if target_id not in self._target_sessions:
+			return False
+		return len(self._target_sessions[target_id]) > 0
+
+	def get_target_id_from_session_id(self, session_id: SessionID) -> TargetID | None:
+		"""Look up which target a session belongs to.
+
+		Args:
+			session_id: The session ID to look up
+
+		Returns:
+			Target ID if found, None otherwise
+		"""
+		return self._session_to_target.get(session_id)
+
+	def get_target(self, target_id: TargetID) -> 'Target | None':
+		"""Get target from owned data.
+
+		Args:
+			target_id: Target ID to get
+
+		Returns:
+			Target object if found, None otherwise
+		"""
+		return self._targets.get(target_id)
+
+	def get_all_targets(self) -> dict[TargetID, 'Target']:
+		"""Get all targets (read-only access to owned data).
+
+		Returns:
+			Dict mapping target_id to Target objects
+		"""
+		return self._targets
+
+	def get_all_target_ids(self) -> list[TargetID]:
+		"""Get all target IDs from owned data.
+
+		Returns:
+			List of all target IDs
+		"""
+		return list(self._targets.keys())
+
+	def get_all_sessions(self) -> dict[SessionID, 'CDPSession']:
+		"""Get all sessions (read-only access to owned data).
+
+		Returns:
+			Dict mapping session_id to CDPSession objects
+		"""
+		return self._sessions
+
+	def get_session(self, session_id: SessionID) -> 'CDPSession | None':
+		"""Get session from owned data.
+
+		Args:
+			session_id: Session ID to get
+
+		Returns:
+			CDPSession object if found, None otherwise
+		"""
+		return self._sessions.get(session_id)
+
+	def get_all_sessions_for_target(self, target_id: TargetID) -> list['CDPSession']:
+		"""Get ALL sessions attached to a target from owned data.
+
+		Args:
+			target_id: Target ID to get sessions for
+
+		Returns:
+			List of all CDPSession objects for this target
+		"""
+		session_ids = self._target_sessions.get(target_id, set())
+		return [self._sessions[sid] for sid in session_ids if sid in self._sessions]
+
+	def get_target_sessions_mapping(self) -> dict[TargetID, set[SessionID]]:
+		"""Get target->sessions mapping (read-only access).
+
+		Returns:
+			Dict mapping target_id to set of session_ids
+		"""
+		return self._target_sessions
+
+	def get_focused_target(self) -> 'Target | None':
+		"""Get the target that currently has agent focus.
+
+		Convenience method that uses browser_session.agent_focus_target_id.
+
+		Returns:
+			Target object if agent has focus, None otherwise
+		"""
+		if not self.browser_session.agent_focus_target_id:
+			return None
+		return self.get_target(self.browser_session.agent_focus_target_id)
+
+	async def ensure_valid_focus(self, timeout: float = 3.0) -> bool:
+		"""Ensure agent_focus_target_id points to a valid, attached CDP session.
+
+		If the focus target is stale (detached), this method waits for automatic recovery.
+		Uses event-driven coordination instead of polling for efficiency.
+
+		Args:
+			timeout: Maximum time to wait for recovery in seconds (default: 3.0)
+
+		Returns:
+			True if focus is valid or successfully recovered, False if no focus or recovery failed
+		"""
+		if not self.browser_session.agent_focus_target_id:
+			# No focus at all - might be initial state or complete failure
+			if self._recovery_in_progress and self._recovery_complete_event:
+				# Recovery is happening, wait for it
+				try:
+					await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=timeout)
+					# Check again after recovery - simple existence check
+					focus_id = self.browser_session.agent_focus_target_id
+					return bool(focus_id and self._get_session_for_target(focus_id))
+				except TimeoutError:
+					self.logger.error(f'[SessionManager] ❌ Timed out waiting for recovery after {timeout}s')
+					return False
+			return False
+
+		# Simple existence check - does the focused target have a session?
+		cdp_session = self._get_session_for_target(self.browser_session.agent_focus_target_id)
+		if cdp_session:
+			# Session exists - validate it's still active
+			is_valid = await self.validate_session(self.browser_session.agent_focus_target_id)
+			if is_valid:
+				return True
+
+		# Focus is stale - wait for recovery using event instead of polling
+		stale_target_id = self.browser_session.agent_focus_target_id
+		self.logger.warning(
+			f'[SessionManager] ⚠️ Stale agent_focus detected (target {stale_target_id[:8] if stale_target_id else "None"}... detached), '
+			f'waiting for recovery...'
+		)
+
+		# Check if recovery is already in progress
+		if not self._recovery_in_progress:
+			self.logger.warning(
+				'[SessionManager] ⚠️ Recovery not in progress for stale focus! '
+				'This indicates a bug - recovery should have been triggered.'
+			)
+			return False
+
+		# Wait for recovery complete event (event-driven, not polling!)
+		if self._recovery_complete_event:
+			try:
+				start_time = asyncio.get_event_loop().time()
+				await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=timeout)
+				elapsed = asyncio.get_event_loop().time() - start_time
+
+				# Verify recovery succeeded - simple existence check
+				focus_id = self.browser_session.agent_focus_target_id
+				if focus_id and self._get_session_for_target(focus_id):
+					self.logger.info(
+						f'[SessionManager] ✅ Agent focus recovered to {self.browser_session.agent_focus_target_id[:8]}... '
+						f'after {elapsed * 1000:.0f}ms'
+					)
+					return True
+				else:
+					self.logger.error(
+						f'[SessionManager] ❌ Recovery completed but focus still invalid after {elapsed * 1000:.0f}ms'
+					)
+					return False
+
+			except TimeoutError:
+				self.logger.error(
+					f'[SessionManager] ❌ Recovery timed out after {timeout}s '
+					f'(was: {stale_target_id[:8] if stale_target_id else "None"}..., '
+					f'now: {self.browser_session.agent_focus_target_id[:8] if self.browser_session.agent_focus_target_id else "None"})'
+				)
+				return False
+		else:
+			self.logger.error('[SessionManager] ❌ Recovery event not initialized')
+			return False
+
+	async def _handle_target_attached(self, event: AttachedToTargetEvent) -> None:
+		"""Handle Target.attachedToTarget event.
+
+		Called automatically by Chrome when a new target/session is created.
+		This is the ONLY place where sessions are added to the pool.
+		"""
+		target_id = event['targetInfo']['targetId']
+		session_id = event['sessionId']
+		target_type = event['targetInfo']['type']
+		target_info = event['targetInfo']
+		waiting_for_debugger = event.get('waitingForDebugger', False)
+
+		self.logger.debug(
+			f'[SessionManager] Target attached: {target_id[:8]}... (session={session_id[:8]}..., '
+			f'type={target_type}, waitingForDebugger={waiting_for_debugger})'
+		)
+
+		# Defensive check: browser may be shutting down and _cdp_client_root could be None
+		if self.browser_session._cdp_client_root is None:
+			self.logger.debug(
+				f'[SessionManager] Skipping target attach for {target_id[:8]}... - browser shutting down (no CDP client)'
+			)
+			return
+
+		# Enable auto-attach for this session's children (do this FIRST, outside lock)
+		try:
+			await self.browser_session._cdp_client_root.send.Target.setAutoAttach(
+				params={'autoAttach': True, 'waitForDebuggerOnStart': False, 'flatten': True}, session_id=session_id
+			)
+		except Exception as e:
+			error_str = str(e)
+			# Expected for short-lived targets (workers, temp iframes) that detach before this executes
+			if '-32001' not in error_str and 'Session with given id not found' not in error_str:
+				self.logger.debug(f'[SessionManager] Auto-attach failed for {target_type}: {e}')
+
+		from browser_use.browser.session import Target
+
+		async with self._lock:
+			# Track this session for the target
+			if target_id not in self._target_sessions:
+				self._target_sessions[target_id] = set()
+
+			self._target_sessions[target_id].add(session_id)
+			self._session_to_target[session_id] = target_id
+
+			# Create or update Target inside the same lock so that get_target() is never
+			# called in the window between _target_sessions being set and _targets being set.
+			if target_id not in self._targets:
+				target = Target(
+					target_id=target_id,
+					target_type=target_type,
+					url=target_info.get('url', 'about:blank'),
+					title=target_info.get('title', 'Unknown title'),
+				)
+				self._targets[target_id] = target
+				self.logger.debug(f'[SessionManager] Created target {target_id[:8]}... (type={target_type})')
+			else:
+				# Update existing target info
+				existing_target = self._targets[target_id]
+				existing_target.url = target_info.get('url', existing_target.url)
+				existing_target.title = target_info.get('title', existing_target.title)
+
+		# Create CDPSession (communication channel)
+		from browser_use.browser.session import CDPSession
+
+		assert self.browser_session._cdp_client_root is not None, 'Root CDP client required'
+
+		cdp_session = CDPSession(
+			cdp_client=self.browser_session._cdp_client_root,
+			target_id=target_id,
+			session_id=session_id,
+		)
+
+		# Add to sessions dict
+		self._sessions[session_id] = cdp_session
+
+		# If proxy auth is configured, enable Fetch auth handling on this session
+		# Avoids overwriting Target.attachedToTarget handlers elsewhere
+		try:
+			proxy_cfg = self.browser_session.browser_profile.proxy
+			username = proxy_cfg.username if proxy_cfg else None
+			password = proxy_cfg.password if proxy_cfg else None
+			if username and password:
+				await cdp_session.cdp_client.send.Fetch.enable(
+					params={'handleAuthRequests': True},
+					session_id=cdp_session.session_id,
+				)
+				self.logger.debug(f'[SessionManager] Fetch.enable(handleAuthRequests=True) on session {session_id[:8]}...')
+		except Exception as e:
+			self.logger.debug(f'[SessionManager] Fetch.enable on attached session failed: {type(e).__name__}: {e}')
+
+		self.logger.debug(
+			f'[SessionManager] Created session {session_id[:8]}... for target {target_id[:8]}... '
+			f'(total sessions: {len(self._sessions)})'
+		)
+
+		# Enable lifecycle events and network monitoring for page targets
+		if target_type in ('page', 'tab'):
+			await self._enable_page_monitoring(cdp_session)
+
+		# Resume execution if waiting for debugger
+		if waiting_for_debugger:
+			try:
+				assert self.browser_session._cdp_client_root is not None
+				await self.browser_session._cdp_client_root.send.Runtime.runIfWaitingForDebugger(session_id=session_id)
+			except Exception as e:
+				self.logger.warning(f'[SessionManager] Failed to resume execution: {e}')
+
+	async def _handle_target_info_changed(self, event: dict) -> None:
+		"""Handle Target.targetInfoChanged event.
+
+		Updates target title/URL without polling getTargetInfo().
+		Chrome fires this automatically when title or URL changes.
+		"""
+		target_info = event.get('targetInfo', {})
+		target_id = target_info.get('targetId')
+
+		if not target_id:
+			return
+
+		async with self._lock:
+			# Update target if it exists (source of truth for url/title)
+			if target_id in self._targets:
+				target = self._targets[target_id]
+
+				target.title = target_info.get('title', target.title)
+				target.url = target_info.get('url', target.url)
+
+	async def _handle_target_detached(self, event: DetachedFromTargetEvent) -> None:
+		"""Handle Target.detachedFromTarget event.
+
+		Called automatically by Chrome when a target/session is destroyed.
+		This is the ONLY place where sessions are removed from the pool.
+		"""
+		session_id = event['sessionId']
+		target_id = event.get('targetId')  # May be empty
+
+		# If targetId not in event, look it up via session mapping
+		if not target_id:
+			async with self._lock:
+				target_id = self._session_to_target.get(session_id)
+
+		if not target_id:
+			self.logger.warning(f'[SessionManager] Session detached but target unknown (session={session_id[:8]}...)')
+			return
+
+		agent_focus_lost = False
+		target_fully_removed = False
+		target_type = None
+
+		async with self._lock:
+			# Remove this session from target's session set
+			if target_id in self._target_sessions:
+				self._target_sessions[target_id].discard(session_id)
+
+				remaining_sessions = len(self._target_sessions[target_id])
+
+				self.logger.debug(
+					f'[SessionManager] Session detached: target={target_id[:8]}... '
+					f'session={session_id[:8]}... (remaining={remaining_sessions})'
+				)
+
+				# Only remove target when NO sessions remain
+				if remaining_sessions == 0:
+					self.logger.debug(f'[SessionManager] No sessions remain for target {target_id[:8]}..., removing target')
+
+					target_fully_removed = True
+
+					# Check if agent_focus points to this target
+					agent_focus_lost = self.browser_session.agent_focus_target_id == target_id
+
+					# Immediately clear stale focus to prevent operations on detached target
+					if agent_focus_lost:
+						self.logger.debug(
+							f'[SessionManager] Clearing stale agent_focus_target_id {target_id[:8]}... '
+							f'to prevent operations on detached target'
+						)
+						self.browser_session.agent_focus_target_id = None
+
+					# Get target type before removing (needed for TabClosedEvent dispatch)
+					target = self._targets.get(target_id)
+					target_type = target.target_type if target else None
+
+					# Remove target (entity) from owned data
+					if target_id in self._targets:
+						self._targets.pop(target_id)
+						self.logger.debug(
+							f'[SessionManager] Removed target {target_id[:8]}... (remaining targets: {len(self._targets)})'
+						)
+
+					# Clean up tracking
+					del self._target_sessions[target_id]
+					self._lifecycle_events.pop(target_id, None)
+			else:
+				# Target not tracked - already removed or never attached
+				self.logger.debug(
+					f'[SessionManager] Session detached from untracked target: target={target_id[:8]}... '
+					f'session={session_id[:8]}... (target was already removed or attach event was missed)'
+				)
+
+			# Remove session from owned sessions dict
+			if session_id in self._sessions:
+				self._sessions.pop(session_id)
+				self.logger.debug(
+					f'[SessionManager] Removed session {session_id[:8]}... (remaining sessions: {len(self._sessions)})'
+				)
+
+			# Remove from reverse mapping
+			if session_id in self._session_to_target:
+				del self._session_to_target[session_id]
+
+		# Dispatch TabClosedEvent only for page/tab targets that are fully removed (not iframes/workers or partial detaches)
+		if target_fully_removed:
+			if target_type in ('page', 'tab'):
+				from browser_use.browser.events import TabClosedEvent
+
+				self.browser_session.event_bus.dispatch(TabClosedEvent(target_id=target_id))
+				self.logger.debug(f'[SessionManager] Dispatched TabClosedEvent for page target {target_id[:8]}...')
+			elif target_type:
+				self.logger.debug(
+					f'[SessionManager] Target {target_id[:8]}... fully removed (type={target_type}) - not dispatching TabClosedEvent'
+				)
+
+		# Auto-recover agent_focus outside the lock to avoid blocking other operations
+		if agent_focus_lost:
+			# Create recovery task instead of awaiting directly - allows concurrent operations to wait on same recovery
+			if not self._recovery_in_progress:
+				self._recovery_task = create_task_with_error_handling(
+					self._recover_agent_focus(target_id),
+					name='recover_agent_focus',
+					logger_instance=self.logger,
+					suppress_exceptions=False,
+				)
+
+	async def _recover_agent_focus(self, crashed_target_id: TargetID) -> None:
+		"""Auto-recover agent_focus when the focused target crashes/detaches.
+
+		Uses recovery lock to prevent concurrent recovery attempts from creating multiple emergency tabs.
+		Coordinates with ensure_valid_focus() via events for efficient waiting.
+
+		Args:
+			crashed_target_id: The target ID that was lost
+		"""
+		try:
+			# Prevent concurrent recovery attempts
+			async with self._recovery_lock:
+				# Set recovery state INSIDE lock to prevent race conditions
+				if self._recovery_in_progress:
+					self.logger.debug('[SessionManager] Recovery already in progress, waiting for it to complete')
+					# Wait for ongoing recovery instead of starting a new one
+					if self._recovery_complete_event:
+						try:
+							await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=5.0)
+						except TimeoutError:
+							self.logger.error('[SessionManager] Timed out waiting for ongoing recovery')
+					return
+
+				# Set recovery state
+				self._recovery_in_progress = True
+				self._recovery_complete_event = asyncio.Event()
+
+				if self.browser_session._cdp_client_root is None:
+					self.logger.debug('[SessionManager] Skipping focus recovery - browser shutting down (no CDP client)')
+					return
+
+				# Check if another recovery already fixed agent_focus
+				if self.browser_session.agent_focus_target_id and self.browser_session.agent_focus_target_id != crashed_target_id:
+					self.logger.debug(
+						f'[SessionManager] Agent focus already recovered by concurrent operation '
+						f'(now: {self.browser_session.agent_focus_target_id[:8]}...), skipping recovery'
+					)
+					return
+
+				# Note: agent_focus_target_id may already be None (cleared in _handle_target_detached)
+				current_focus_desc = (
+					f'{self.browser_session.agent_focus_target_id[:8]}...'
+					if self.browser_session.agent_focus_target_id
+					else 'None (already cleared)'
+				)
+
+				self.logger.warning(
+					f'[SessionManager] Agent focus target {crashed_target_id[:8]}... detached! '
+					f'Current focus: {current_focus_desc}. Auto-recovering by switching to another target...'
+				)
+
+			# Perform recovery (outside lock to allow concurrent operations)
+			# Try to find another valid page target
+			page_targets = self.get_all_page_targets()
+
+			new_target_id = None
+			is_existing_tab = False
+
+			if page_targets:
+				# Switch to most recent page that's not the crashed one
+				new_target_id = page_targets[-1].target_id
+				is_existing_tab = True
+				self.logger.info(f'[SessionManager] Switching agent_focus to existing tab {new_target_id[:8]}...')
+			else:
+				# No pages exist - create a new one
+				self.logger.warning('[SessionManager] No tabs remain! Creating new tab for agent...')
+				new_target_id = await self.browser_session._cdp_create_new_page('about:blank')
+				self.logger.info(f'[SessionManager] Created new tab {new_target_id[:8]}... for agent')
+
+				# Dispatch TabCreatedEvent so watchdogs can initialize
+				from browser_use.browser.events import TabCreatedEvent
+
+				self.browser_session.event_bus.dispatch(TabCreatedEvent(url='about:blank', target_id=new_target_id))
+
+			# Wait for CDP attach event to create session
+			# Note: This polling is necessary - waiting for external Chrome CDP event
+			# _handle_target_attached will add session to pool when Chrome fires attachedToTarget
+			new_session = None
+			for attempt in range(20):  # Wait up to 2 seconds
+				await asyncio.sleep(0.1)
+				new_session = self._get_session_for_target(new_target_id)
+				if new_session:
+					break
+
+			if new_session:
+				self.browser_session.agent_focus_target_id = new_target_id
+				self.logger.info(f'[SessionManager] ✅ Agent focus recovered: {new_target_id[:8]}...')
+
+				# Visually activate the tab in browser (only for existing tabs)
+				if is_existing_tab:
+					try:
+						assert self.browser_session._cdp_client_root is not None
+						await self.browser_session._cdp_client_root.send.Target.activateTarget(params={'targetId': new_target_id})
+						self.logger.debug(f'[SessionManager] Activated tab {new_target_id[:8]}... in browser UI')
+					except Exception as e:
+						self.logger.debug(f'[SessionManager] Failed to activate tab visually: {e}')
+
+				# Get target to access url (from owned data)
+				target = self.get_target(new_target_id)
+				target_url = target.url if target else 'about:blank'
+
+				# Dispatch focus changed event
+				from browser_use.browser.events import AgentFocusChangedEvent
+
+				self.browser_session.event_bus.dispatch(AgentFocusChangedEvent(target_id=new_target_id, url=target_url))
+				return
+
+			# Recovery failed - create emergency fallback tab
+			self.logger.error(
+				f'[SessionManager] ❌ Failed to get session for {new_target_id[:8]}... after 2s, creating emergency fallback tab'
+			)
+
+			fallback_target_id = await self.browser_session._cdp_create_new_page('about:blank')
+			self.logger.warning(f'[SessionManager] Created emergency fallback tab {fallback_target_id[:8]}...')
+
+			# Try one more time with fallback
+			# Note: This polling is necessary - waiting for external Chrome CDP event
+			for _ in range(20):
+				await asyncio.sleep(0.1)
+				fallback_session = self._get_session_for_target(fallback_target_id)
+				if fallback_session:
+					self.browser_session.agent_focus_target_id = fallback_target_id
+					self.logger.warning(f'[SessionManager] ⚠️ Agent focus set to emergency fallback: {fallback_target_id[:8]}...')
+
+					from browser_use.browser.events import AgentFocusChangedEvent, TabCreatedEvent
+
+					self.browser_session.event_bus.dispatch(TabCreatedEvent(url='about:blank', target_id=fallback_target_id))
+					self.browser_session.event_bus.dispatch(
+						AgentFocusChangedEvent(target_id=fallback_target_id, url='about:blank')
+					)
+					return
+
+			# Complete failure - this should never happen
+			self.logger.critical(
+				'[SessionManager] 🚨 CRITICAL: Failed to recover agent_focus even with fallback! Agent may be in broken state.'
+			)
+
+		except Exception as e:
+			self.logger.error(f'[SessionManager] ❌ Error during agent_focus recovery: {type(e).__name__}: {e}')
+		finally:
+			# Always signal completion and reset recovery state
+			# This allows all waiting operations to proceed (success or failure)
+			if self._recovery_complete_event:
+				self._recovery_complete_event.set()
+			self._recovery_in_progress = False
+			self._recovery_task = None
+			self.logger.debug('[SessionManager] Recovery state reset')
+
+	async def _initialize_existing_targets(self) -> None:
+		"""Discover and initialize all existing targets at startup.
+
+		Attaches to each target and initializes it SYNCHRONOUSLY.
+		Chrome will also fire attachedToTarget events, but _handle_target_attached() is
+		idempotent (checks if target already in pool), so duplicate handling is safe.
+
+		This eliminates race conditions - monitoring is guaranteed ready before navigation.
+		"""
+		cdp_client = self.browser_session._cdp_client_root
+		assert cdp_client is not None
+
+		# Get all existing targets
+		targets_result = await cdp_client.send.Target.getTargets()
+		existing_targets = targets_result.get('targetInfos', [])
+
+		self.logger.debug(f'[SessionManager] Discovered {len(existing_targets)} existing targets')
+
+		# Track target IDs for verification
+		target_ids_to_wait_for = []
+
+		# Just attach to ALL existing targets - Chrome fires attachedToTarget events
+		# The on_attached handler (via create_task) does ALL the work
+		for target in existing_targets:
+			target_id = target['targetId']
+			target_type = target.get('type', 'unknown')
+
+			try:
+				# Just attach - event handler does everything
+				await cdp_client.send.Target.attachToTarget(params={'targetId': target_id, 'flatten': True})
+				target_ids_to_wait_for.append(target_id)
+			except Exception as e:
+				self.logger.debug(
+					f'[SessionManager] Failed to attach to existing target {target_id[:8]}... (type={target_type}): {e}'
+				)
+
+		# Wait for event handlers to complete their work (they run via create_task)
+		# Use event-driven approach instead of polling for better performance
+		ready_event = asyncio.Event()
+
+		async def check_all_ready():
+			"""Check if all sessions are ready and signal completion."""
+			while True:
+				ready_count = 0
+				for tid in target_ids_to_wait_for:
+					session = self._get_session_for_target(tid)
+					if session:
+						target = self._targets.get(tid)
+						target_type = target.target_type if target else 'unknown'
+						# For pages, verify monitoring is enabled
+						if target_type in ('page', 'tab'):
+							if hasattr(session, '_lifecycle_events') and session._lifecycle_events is not None:
+								ready_count += 1
+						else:
+							# Non-page targets don't need monitoring
+							ready_count += 1
+
+				if ready_count == len(target_ids_to_wait_for):
+					ready_event.set()
+					return
+
+				await asyncio.sleep(0.05)
+
+		# Start checking in background
+		check_task = create_task_with_error_handling(
+			check_all_ready(), name='check_all_targets_ready', logger_instance=self.logger
+		)
+
+		try:
+			# Wait for completion with timeout
+			await asyncio.wait_for(ready_event.wait(), timeout=2.0)
+		except TimeoutError:
+			# Timeout - count what's ready
+			ready_count = 0
+			for tid in target_ids_to_wait_for:
+				session = self._get_session_for_target(tid)
+				if session:
+					target = self._targets.get(tid)
+					target_type = target.target_type if target else 'unknown'
+					# For pages, verify monitoring is enabled
+					if target_type in ('page', 'tab'):
+						if hasattr(session, '_lifecycle_events') and session._lifecycle_events is not None:
+							ready_count += 1
+					else:
+						# Non-page targets don't need monitoring
+						ready_count += 1
+			self.logger.warning(
+				f'[SessionManager] Initialization timeout after 2.0s: {ready_count}/{len(target_ids_to_wait_for)} sessions ready'
+			)
+		finally:
+			check_task.cancel()
+			try:
+				await check_task
+			except asyncio.CancelledError:
+				pass
+
+	async def _enable_page_monitoring(self, cdp_session: 'CDPSession') -> None:
+		"""Enable lifecycle events and network monitoring for a page target.
+
+		This is called once per page when it's created, avoiding handler accumulation.
+		Registers a SINGLE lifecycle handler per session that stores events for navigations to consume.
+
+		Args:
+			cdp_session: The CDP session to enable monitoring on
+		"""
+		try:
+			# Enable Page domain first (required for lifecycle events)
+			await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
+
+			# Enable lifecycle events (load, DOMContentLoaded, networkIdle, etc.)
+			await cdp_session.cdp_client.send.Page.setLifecycleEventsEnabled(
+				params={'enabled': True}, session_id=cdp_session.session_id
+			)
+
+			# Enable network monitoring for networkIdle detection
+			await cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)
+
+			# Event storage and the Page.lifecycleEvent handler live in SessionManager
+			# (one global handler registered in start_monitoring, routed by session_id):
+			# cdp-use's registry is single-slot per method, so a per-session registration
+			# here would replace the previous tab's handler and freeze its event buffer.
+			# Expose the shared per-target buffer on the session for readiness checks.
+			cdp_session._lifecycle_events = self.get_lifecycle_events(cdp_session.target_id)
+
+		except Exception as e:
+			# Don't fail - target might be short-lived or already detached
+			error_str = str(e)
+			if '-32001' in error_str or 'Session with given id not found' in error_str:
+				self.logger.debug(
+					f'[SessionManager] Target {cdp_session.target_id[:8]}... detached before monitoring could be enabled (normal for short-lived targets)'
+				)
+			else:
+				self.logger.warning(
+					f'[SessionManager] Failed to enable monitoring for target {cdp_session.target_id[:8]}...: {e}'
+				)

browser_use/browser/video_recorder.py

@@ -0,0 +1,141 @@
+"""Video Recording Service for Browser Use Sessions."""
+
+import base64
+import io
+import logging
+import math
+from pathlib import Path
+from typing import Optional
+
+from browser_use.browser.profile import ViewportSize
+
+try:
+	import imageio.v2 as iio  # type: ignore[import-not-found]
+	import numpy as np  # type: ignore[import-not-found]
+	from imageio.core.format import Format  # type: ignore[import-not-found]
+	from PIL import Image
+
+	IMAGEIO_AVAILABLE = True
+except ImportError:
+	IMAGEIO_AVAILABLE = False
+
+logger = logging.getLogger(__name__)
+
+
+def _get_padded_size(size: ViewportSize, macro_block_size: int = 16) -> ViewportSize:
+	"""Calculates the dimensions padded to the nearest multiple of macro_block_size."""
+	width = int(math.ceil(size['width'] / macro_block_size)) * macro_block_size
+	height = int(math.ceil(size['height'] / macro_block_size)) * macro_block_size
+	return ViewportSize(width=width, height=height)
+
+
+class VideoRecorderService:
+	"""
+	Handles the video encoding process for a browser session using imageio.
+
+	This service captures individual frames from the CDP screencast, decodes them,
+	and appends them to a video file using a pip-installable ffmpeg backend.
+	It automatically resizes frames to match the target video dimensions.
+	"""
+
+	def __init__(self, output_path: Path, size: ViewportSize, framerate: int):
+		"""
+		Initializes the video recorder.
+
+		Args:
+		    output_path: The full path where the video will be saved.
+		    size: A ViewportSize object specifying the width and height of the video.
+		    framerate: The desired framerate for the output video.
+		"""
+		self.output_path = output_path
+		self.size = size
+		self.framerate = framerate
+		self._writer: Optional['Format.Writer'] = None
+		self._is_active = False
+		self.padded_size = _get_padded_size(self.size)
+
+	def start(self) -> None:
+		"""
+		Prepares and starts the video writer.
+
+		If the required optional dependencies are not installed, this method will
+		log an error and do nothing.
+		"""
+		if not IMAGEIO_AVAILABLE:
+			logger.error(
+				'MP4 recording requires optional dependencies. Please install them with: pip install "browser-use[video]"'
+			)
+			return
+
+		try:
+			self.output_path.parent.mkdir(parents=True, exist_ok=True)
+			# The macro_block_size is set to None because we handle padding ourselves
+			self._writer = iio.get_writer(
+				str(self.output_path),
+				fps=self.framerate,
+				codec='libx264',
+				quality=8,  # A good balance of quality and file size (1-10 scale)
+				pixelformat='yuv420p',  # Ensures compatibility with most players
+				macro_block_size=None,
+			)
+			self._is_active = True
+			logger.debug(f'Video recorder started. Output will be saved to {self.output_path}')
+		except Exception as e:
+			logger.error(f'Failed to initialize video writer: {e}')
+			self._is_active = False
+
+	def add_frame(self, frame_data_b64: str) -> None:
+		"""
+		Decodes a base64-encoded PNG frame, resizes it, pads it to be codec-compatible,
+		and appends it to the video.
+
+		Args:
+		    frame_data_b64: A base64-encoded string of the PNG frame data.
+		"""
+		if not self._is_active or not self._writer:
+			return
+
+		try:
+			frame_bytes = base64.b64decode(frame_data_b64)
+
+			# Use PIL to handle image processing in memory - much faster than spawning ffmpeg subprocess per frame
+			with Image.open(io.BytesIO(frame_bytes)) as img:
+				# 1. Resize if needed to target viewport size
+				if img.size != (self.size['width'], self.size['height']):
+					# Use BICUBIC as it's faster than LANCZOS and good enough for screen recordings
+					img = img.resize((self.size['width'], self.size['height']), Image.Resampling.BICUBIC)
+
+				# 2. Handle Padding (Macro block alignment for codecs)
+				# Check if padding is actually needed
+				if self.padded_size['width'] != self.size['width'] or self.padded_size['height'] != self.size['height']:
+					new_img = Image.new('RGB', (self.padded_size['width'], self.padded_size['height']), (0, 0, 0))
+					# Center the image
+					x_offset = (self.padded_size['width'] - self.size['width']) // 2
+					y_offset = (self.padded_size['height'] - self.size['height']) // 2
+					new_img.paste(img, (x_offset, y_offset))
+					img = new_img
+
+				# 3. Convert to numpy array for imageio
+				img_array = np.array(img)
+
+			self._writer.append_data(img_array)
+		except Exception as e:
+			logger.warning(f'Could not process and add video frame: {e}')
+
+	def stop_and_save(self) -> None:
+		"""
+		Finalizes the video file by closing the writer.
+
+		This method should be called when the recording session is complete.
+		"""
+		if not self._is_active or not self._writer:
+			return
+
+		try:
+			self._writer.close()
+			logger.info(f'📹 Video recording saved successfully to: {self.output_path}')
+		except Exception as e:
+			logger.error(f'Failed to finalize and save video: {e}')
+		finally:
+			self._is_active = False
+			self._writer = None

browser_use/browser/views.py

@@ -0,0 +1,200 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+from bubus import BaseEvent
+from cdp_use.cdp.target import TargetID
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, field_serializer
+
+from browser_use.dom.views import DOMInteractedElement, SerializedDOMState
+
+# Known placeholder image data for about:blank pages - a 4x4 white PNG
+PLACEHOLDER_4PX_SCREENSHOT = (
+	'iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII='
+)
+
+
+# Pydantic
+class TabInfo(BaseModel):
+	"""Represents information about a browser tab"""
+
+	model_config = ConfigDict(
+		extra='forbid',
+		validate_by_name=True,
+		validate_by_alias=True,
+		populate_by_name=True,
+	)
+
+	# Original fields
+	url: str
+	title: str
+	target_id: TargetID = Field(serialization_alias='tab_id', validation_alias=AliasChoices('tab_id', 'target_id'))
+	parent_target_id: TargetID | None = Field(
+		default=None, serialization_alias='parent_tab_id', validation_alias=AliasChoices('parent_tab_id', 'parent_target_id')
+	)  # parent page that contains this popup or cross-origin iframe
+
+	@field_serializer('target_id')
+	def serialize_target_id(self, target_id: TargetID, _info: Any) -> str:
+		return target_id[-4:]
+
+	@field_serializer('parent_target_id')
+	def serialize_parent_target_id(self, parent_target_id: TargetID | None, _info: Any) -> str | None:
+		return parent_target_id[-4:] if parent_target_id else None
+
+
+class PageInfo(BaseModel):
+	"""Comprehensive page size and scroll information"""
+
+	# Current viewport dimensions
+	viewport_width: int
+	viewport_height: int
+
+	# Total page dimensions
+	page_width: int
+	page_height: int
+
+	# Current scroll position
+	scroll_x: int
+	scroll_y: int
+
+	# Calculated scroll information
+	pixels_above: int
+	pixels_below: int
+	pixels_left: int
+	pixels_right: int
+
+	# Page statistics are now computed dynamically instead of stored
+
+
+@dataclass
+class NetworkRequest:
+	"""Information about a pending network request"""
+
+	url: str
+	method: str = 'GET'
+	loading_duration_ms: float = 0.0  # How long this request has been loading (ms since request started, max 10s)
+	resource_type: str | None = None  # e.g., 'Document', 'Stylesheet', 'Image', 'Script', 'XHR', 'Fetch'
+
+
+@dataclass
+class PaginationButton:
+	"""Information about a pagination button detected on the page"""
+
+	button_type: str  # 'next', 'prev', 'first', 'last', 'page_number'
+	backend_node_id: int  # Backend node ID for clicking
+	text: str  # Button text/label
+	selector: str  # XPath or other selector to locate the element
+	is_disabled: bool = False  # Whether the button appears disabled
+
+
+@dataclass
+class BrowserStateSummary:
+	"""The summary of the browser's current state designed for an LLM to process"""
+
+	# provided by SerializedDOMState:
+	dom_state: SerializedDOMState
+
+	url: str
+	title: str
+	tabs: list[TabInfo]
+	screenshot: str | None = field(default=None, repr=False)
+	page_info: PageInfo | None = None  # Enhanced page information
+
+	# Keep legacy fields for backward compatibility
+	pixels_above: int = 0
+	pixels_below: int = 0
+	browser_errors: list[str] = field(default_factory=list)
+	is_pdf_viewer: bool = False  # Whether the current page is a PDF viewer
+	recent_events: str | None = None  # Text summary of recent browser events
+	pending_network_requests: list[NetworkRequest] = field(default_factory=list)  # Currently loading network requests
+	pagination_buttons: list[PaginationButton] = field(default_factory=list)  # Detected pagination buttons
+	closed_popup_messages: list[str] = field(default_factory=list)  # Messages from auto-closed JavaScript dialogs
+
+
+@dataclass
+class BrowserStateHistory:
+	"""The summary of the browser's state at a past point in time to usse in LLM message history"""
+
+	url: str
+	title: str
+	tabs: list[TabInfo]
+	interacted_element: list[DOMInteractedElement | None] | list[None]
+	screenshot_path: str | None = None
+
+	def get_screenshot(self) -> str | None:
+		"""Load screenshot from disk and return as base64 string"""
+		if not self.screenshot_path:
+			return None
+
+		import base64
+		from pathlib import Path
+
+		path_obj = Path(self.screenshot_path)
+		if not path_obj.exists():
+			return None
+
+		try:
+			with open(path_obj, 'rb') as f:
+				screenshot_data = f.read()
+			return base64.b64encode(screenshot_data).decode('utf-8')
+		except Exception:
+			return None
+
+	def to_dict(self) -> dict[str, Any]:
+		data = {}
+		data['tabs'] = [tab.model_dump() for tab in self.tabs]
+		data['screenshot_path'] = self.screenshot_path
+		data['interacted_element'] = [el.to_dict() if el else None for el in self.interacted_element]
+		data['url'] = self.url
+		data['title'] = self.title
+		return data
+
+
+class BrowserError(Exception):
+	"""Browser error with structured memory for LLM context management.
+
+	This exception class provides separate memory contexts for browser actions:
+	- short_term_memory: Immediate context shown once to the LLM for the next action
+	- long_term_memory: Persistent error information stored across steps
+	"""
+
+	message: str
+	short_term_memory: str | None = None
+	long_term_memory: str | None = None
+	details: dict[str, Any] | None = None
+	while_handling_event: BaseEvent[Any] | None = None
+
+	def __init__(
+		self,
+		message: str,
+		short_term_memory: str | None = None,
+		long_term_memory: str | None = None,
+		details: dict[str, Any] | None = None,
+		event: BaseEvent[Any] | None = None,
+	):
+		"""Initialize a BrowserError with structured memory contexts.
+
+		Args:
+			message: Technical error message for logging and debugging
+			short_term_memory: Context shown once to LLM (e.g., available actions, options)
+			long_term_memory: Persistent error info stored in agent memory
+			details: Additional metadata for debugging
+			event: The browser event that triggered this error
+		"""
+		self.message = message
+		self.short_term_memory = short_term_memory
+		self.long_term_memory = long_term_memory
+		self.details = details
+		self.while_handling_event = event
+		super().__init__(message)
+
+	def __str__(self) -> str:
+		parts = [self.message]
+		if self.details:
+			parts.append(f'({self.details})')
+		if self.while_handling_event:
+			parts.append(f'during: {self.while_handling_event}')
+		return ' '.join(parts)
+
+
+class URLNotAllowedError(BrowserError):
+	"""Error raised when a URL is not allowed"""

browser_use/browser/watchdog_base.py

@@ -0,0 +1,321 @@
+"""Base watchdog class for browser monitoring components."""
+
+import asyncio
+import inspect
+import time
+from collections.abc import Iterable
+from typing import Any, ClassVar
+
+from bubus import BaseEvent, EventBus
+from pydantic import BaseModel, ConfigDict, Field
+
+from browser_use.browser.session import BrowserSession
+
+
+class BaseWatchdog(BaseModel):
+	"""Base class for all browser watchdogs.
+
+	Watchdogs monitor browser state and emit events based on changes.
+	They automatically register event handlers based on method names.
+
+	Handler methods should be named: on_EventTypeName(self, event: EventTypeName)
+	"""
+
+	model_config = ConfigDict(
+		arbitrary_types_allowed=True,  # allow non-serializable objects like EventBus/BrowserSession in fields
+		extra='forbid',  # dont allow implicit class/instance state, everything must be a properly typed Field or PrivateAttr
+		validate_assignment=False,  # avoid re-triggering  __init__ / validators on values on every assignment
+		revalidate_instances='never',  # avoid re-triggering __init__ / validators and erasing private attrs
+	)
+
+	# Class variables to statically define the list of events relevant to each watchdog
+	# (not enforced, just to make it easier to understand the code and debug watchdogs at runtime)
+	LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = []  # Events this watchdog listens to
+	EMITS: ClassVar[list[type[BaseEvent[Any]]]] = []  # Events this watchdog emits
+
+	# Core dependencies
+	event_bus: EventBus = Field()
+	browser_session: BrowserSession = Field()
+
+	# Shared state that other watchdogs might need to access should not be defined on BrowserSession, not here!
+	# Shared helper methods needed by other watchdogs should be defined on BrowserSession, not here!
+	# Alternatively, expose some events on the watchdog to allow access to state/helpers via event_bus system.
+
+	# Private state internal to the watchdog can be defined like this on BaseWatchdog subclasses:
+	# _screenshot_cache: dict[str, bytes] = PrivateAttr(default_factory=dict)
+	# _browser_crash_watcher_task: asyncio.Task | None = PrivateAttr(default=None)
+	# _cdp_download_tasks: WeakSet[asyncio.Task] = PrivateAttr(default_factory=WeakSet)
+	# ...
+
+	@property
+	def logger(self):
+		"""Get the logger from the browser session."""
+		return self.browser_session.logger
+
+	@staticmethod
+	def attach_handler_to_session(browser_session: 'BrowserSession', event_class: type[BaseEvent[Any]], handler) -> None:
+		"""Attach a single event handler to a browser session.
+
+		Args:
+			browser_session: The browser session to attach to
+			event_class: The event class to listen for
+			handler: The handler method (must start with 'on_' and end with event type)
+		"""
+		event_bus = browser_session.event_bus
+
+		# Validate handler naming convention
+		assert hasattr(handler, '__name__'), 'Handler must have a __name__ attribute'
+		assert handler.__name__.startswith('on_'), f'Handler {handler.__name__} must start with "on_"'
+		assert handler.__name__.endswith(event_class.__name__), (
+			f'Handler {handler.__name__} must end with event type {event_class.__name__}'
+		)
+
+		# Get the watchdog instance if this is a bound method
+		watchdog_instance = getattr(handler, '__self__', None)
+		watchdog_class_name = watchdog_instance.__class__.__name__ if watchdog_instance else 'Unknown'
+
+		# Events that should always run even when CDP is disconnected (lifecycle management)
+		LIFECYCLE_EVENT_NAMES = frozenset(
+			{
+				'BrowserStartEvent',
+				'BrowserStopEvent',
+				'BrowserStoppedEvent',
+				'BrowserLaunchEvent',
+				'BrowserErrorEvent',
+				'BrowserKillEvent',
+				'BrowserReconnectingEvent',
+				'BrowserReconnectedEvent',
+			}
+		)
+
+		# Create a wrapper function with unique name to avoid duplicate handler warnings
+		# Capture handler by value to avoid closure issues
+		def make_unique_handler(actual_handler):
+			async def unique_handler(event):
+				# Circuit breaker: skip handler if CDP WebSocket is dead
+				# (prevents handlers from hanging on broken connections until timeout)
+				# Lifecycle events are exempt — they manage browser start/stop
+				if event.event_type not in LIFECYCLE_EVENT_NAMES and not browser_session.is_cdp_connected:
+					# If reconnection is in progress, wait for it instead of silently skipping
+					if browser_session.is_reconnecting:
+						wait_timeout = browser_session.RECONNECT_WAIT_TIMEOUT
+						browser_session.logger.debug(
+							f'🚌 [{watchdog_class_name}.{actual_handler.__name__}] ⏳ Waiting for reconnection ({wait_timeout}s)...'
+						)
+						try:
+							await asyncio.wait_for(browser_session._reconnect_event.wait(), timeout=wait_timeout)
+						except TimeoutError:
+							raise ConnectionError(
+								f'[{watchdog_class_name}.{actual_handler.__name__}] '
+								f'Reconnection wait timed out after {wait_timeout}s'
+							)
+						# After wait: check if reconnection actually succeeded
+						if not browser_session.is_cdp_connected:
+							raise ConnectionError(
+								f'[{watchdog_class_name}.{actual_handler.__name__}] Reconnection failed — CDP still not connected'
+							)
+						# Reconnection succeeded — fall through to execute handler normally
+					else:
+						# Not reconnecting — intentional stop, backward compat silent skip
+						browser_session.logger.debug(
+							f'🚌 [{watchdog_class_name}.{actual_handler.__name__}] ⚡ Skipped — CDP not connected'
+						)
+						return None
+
+				# just for debug logging, not used for anything else
+				parent_event = event_bus.event_history.get(event.event_parent_id) if event.event_parent_id else None
+				grandparent_event = (
+					event_bus.event_history.get(parent_event.event_parent_id)
+					if parent_event and parent_event.event_parent_id
+					else None
+				)
+				parent = (
+					f'↲  triggered by on_{parent_event.event_type}#{parent_event.event_id[-4:]}'
+					if parent_event
+					else '👈 by Agent'
+				)
+				grandparent = (
+					(
+						f'↲  under {grandparent_event.event_type}#{grandparent_event.event_id[-4:]}'
+						if grandparent_event
+						else '👈 by Agent'
+					)
+					if parent_event
+					else ''
+				)
+				event_str = f'#{event.event_id[-4:]}'
+				time_start = time.time()
+				watchdog_and_handler_str = f'[{watchdog_class_name}.{actual_handler.__name__}({event_str})]'.ljust(54)
+				browser_session.logger.debug(f'🚌 {watchdog_and_handler_str} ⏳ Starting...       {parent} {grandparent}')
+
+				try:
+					# **EXECUTE THE EVENT HANDLER FUNCTION**
+					result = await actual_handler(event)
+
+					if isinstance(result, Exception):
+						raise result
+
+					# just for debug logging, not used for anything else
+					time_end = time.time()
+					time_elapsed = time_end - time_start
+					result_summary = '' if result is None else f' ➡️ <{type(result).__name__}>'
+					parents_summary = f' {parent}'.replace('↲  triggered by ', '⤴  returned to  ').replace(
+						'👈 by Agent', '👉 returned to  Agent'
+					)
+					browser_session.logger.debug(
+						f'🚌 {watchdog_and_handler_str} Succeeded ({time_elapsed:.2f}s){result_summary}{parents_summary}'
+					)
+					return result
+				except Exception as e:
+					time_end = time.time()
+					time_elapsed = time_end - time_start
+					original_error = e
+					browser_session.logger.error(
+						f'🚌 {watchdog_and_handler_str} ❌ Failed ({time_elapsed:.2f}s): {type(e).__name__}: {e}'
+					)
+
+					# attempt to repair potentially crashed CDP session
+					try:
+						if browser_session.agent_focus_target_id:
+							# With event-driven sessions, Chrome will send detach/attach events
+							# SessionManager handles pool cleanup automatically
+							target_id_to_restore = browser_session.agent_focus_target_id
+							browser_session.logger.debug(
+								f'🚌 {watchdog_and_handler_str} ⚠️ Session error detected, waiting for CDP events to sync (target: {target_id_to_restore})'
+							)
+
+							# Wait for new attach event to restore the session
+							# This will raise ValueError if target doesn't re-attach
+							await browser_session.get_or_create_cdp_session(target_id=target_id_to_restore, focus=True)
+						else:
+							# Try to get any available session
+							await browser_session.get_or_create_cdp_session(target_id=None, focus=True)
+					except Exception as sub_error:
+						if 'ConnectionClosedError' in str(type(sub_error)) or 'ConnectionError' in str(type(sub_error)):
+							browser_session.logger.error(
+								f'🚌 {watchdog_and_handler_str} ❌ Browser closed or CDP Connection disconnected by remote. {type(sub_error).__name__}: {sub_error}\n'
+							)
+							raise
+						else:
+							browser_session.logger.error(
+								f'🚌 {watchdog_and_handler_str} ❌ CDP connected but failed to re-create CDP session after error "{type(original_error).__name__}: {original_error}" in {actual_handler.__name__}({event.event_type}#{event.event_id[-4:]}): due to {type(sub_error).__name__}: {sub_error}\n'
+							)
+
+					# Always re-raise the original error with its traceback preserved
+					raise
+
+			return unique_handler
+
+		unique_handler = make_unique_handler(handler)
+		unique_handler.__name__ = f'{watchdog_class_name}.{handler.__name__}'
+
+		# Check if this handler is already registered - throw error if duplicate
+		existing_handlers = event_bus.handlers.get(event_class.__name__, [])
+		handler_names = [getattr(h, '__name__', str(h)) for h in existing_handlers]
+
+		if unique_handler.__name__ in handler_names:
+			raise RuntimeError(
+				f'[{watchdog_class_name}] Duplicate handler registration attempted! '
+				f'Handler {unique_handler.__name__} is already registered for {event_class.__name__}. '
+				f'This likely means attach_to_session() was called multiple times.'
+			)
+
+		event_bus.on(event_class, unique_handler)
+
+	@staticmethod
+	def detach_handler_from_session(browser_session: 'BrowserSession', event_class: type[BaseEvent[Any]], handler) -> None:
+		"""Detach a single event handler from a browser session."""
+		event_bus = browser_session.event_bus
+
+		# Get the watchdog instance if this is a bound method
+		watchdog_instance = getattr(handler, '__self__', None)
+		watchdog_class_name = watchdog_instance.__class__.__name__ if watchdog_instance else 'Unknown'
+
+		# Find and remove the handler by its unique name pattern
+		unique_handler_name = f'{watchdog_class_name}.{handler.__name__}'
+
+		existing_handlers = event_bus.handlers.get(event_class.__name__, [])
+		for existing_handler in existing_handlers[:]:  # copy list to allow modification during iteration
+			if getattr(existing_handler, '__name__', '') == unique_handler_name:
+				existing_handlers.remove(existing_handler)
+				break
+
+	def attach_to_session(self) -> None:
+		"""Attach watchdog to its browser session and start monitoring.
+
+		This method handles event listener registration. The watchdog is already
+		bound to a browser session via self.browser_session from initialization.
+		"""
+		# Register event handlers automatically based on method names
+		assert self.browser_session is not None, 'Root CDP client not initialized - browser may not be connected yet'
+
+		from browser_use.browser import events
+
+		event_classes = {}
+		for name in dir(events):
+			obj = getattr(events, name)
+			if inspect.isclass(obj) and issubclass(obj, BaseEvent) and obj is not BaseEvent:
+				event_classes[name] = obj
+
+		# Find all handler methods (on_EventName)
+		registered_events = set()
+		for method_name in dir(self):
+			if method_name.startswith('on_') and callable(getattr(self, method_name)):
+				# Extract event name from method name (on_EventName -> EventName)
+				event_name = method_name[3:]  # Remove 'on_' prefix
+
+				if event_name in event_classes:
+					event_class = event_classes[event_name]
+
+					# ASSERTION: If LISTENS_TO is defined, enforce it
+					if self.LISTENS_TO:
+						assert event_class in self.LISTENS_TO, (
+							f'[{self.__class__.__name__}] Handler {method_name} listens to {event_name} '
+							f'but {event_name} is not declared in LISTENS_TO: {[e.__name__ for e in self.LISTENS_TO]}'
+						)
+
+					handler = getattr(self, method_name)
+
+					# Use the static helper to attach the handler
+					self.attach_handler_to_session(self.browser_session, event_class, handler)
+					registered_events.add(event_class)
+
+		# ASSERTION: If LISTENS_TO is defined, ensure all declared events have handlers
+		if self.LISTENS_TO:
+			missing_handlers = set(self.LISTENS_TO) - registered_events
+			if missing_handlers:
+				missing_names = [e.__name__ for e in missing_handlers]
+				self.logger.warning(
+					f'[{self.__class__.__name__}] LISTENS_TO declares {missing_names} '
+					f'but no handlers found (missing on_{"_, on_".join(missing_names)} methods)'
+				)
+
+	def __del__(self) -> None:
+		"""Clean up any running tasks during garbage collection."""
+
+		# A BIT OF MAGIC: Cancel any private attributes that look like asyncio tasks
+		try:
+			for attr_name in dir(self):
+				# e.g. _browser_crash_watcher_task = asyncio.Task
+				if attr_name.startswith('_') and attr_name.endswith('_task'):
+					try:
+						task = getattr(self, attr_name)
+						if hasattr(task, 'cancel') and callable(task.cancel) and not task.done():
+							task.cancel()
+							# self.logger.debug(f'[{self.__class__.__name__}] Cancelled {attr_name} during cleanup')
+					except Exception:
+						pass  # Ignore errors during cleanup
+
+				# e.g. _cdp_download_tasks = WeakSet[asyncio.Task] or list[asyncio.Task]
+				if attr_name.startswith('_') and attr_name.endswith('_tasks') and isinstance(getattr(self, attr_name), Iterable):
+					for task in getattr(self, attr_name):
+						try:
+							if hasattr(task, 'cancel') and callable(task.cancel) and not task.done():
+								task.cancel()
+								# self.logger.debug(f'[{self.__class__.__name__}] Cancelled {attr_name} during cleanup')
+						except Exception:
+							pass  # Ignore errors during cleanup
+		except Exception as e:
+			from browser_use.utils import logger
+
+			logger.error(f'⚠️ Error during BrowserSession {self.__class__.__name__} garbage collection __del__(): {type(e)}: {e}')

browser_use/browser/watchdogs/aboutblank_watchdog.py

@@ -0,0 +1,259 @@
+"""About:blank watchdog for managing about:blank tabs with DVD screensaver."""
+
+from typing import TYPE_CHECKING, ClassVar
+
+from bubus import BaseEvent
+from cdp_use.cdp.target import TargetID
+from pydantic import PrivateAttr
+
+from browser_use.browser.events import (
+	AboutBlankDVDScreensaverShownEvent,
+	BrowserStopEvent,
+	BrowserStoppedEvent,
+	CloseTabEvent,
+	NavigateToUrlEvent,
+	TabClosedEvent,
+	TabCreatedEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+
+if TYPE_CHECKING:
+	pass
+
+
+class AboutBlankWatchdog(BaseWatchdog):
+	"""Ensures there's always exactly one about:blank tab with DVD screensaver."""
+
+	# Event contracts
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+		BrowserStopEvent,
+		BrowserStoppedEvent,
+		TabCreatedEvent,
+		TabClosedEvent,
+	]
+	EMITS: ClassVar[list[type[BaseEvent]]] = [
+		NavigateToUrlEvent,
+		CloseTabEvent,
+		AboutBlankDVDScreensaverShownEvent,
+	]
+
+	_stopping: bool = PrivateAttr(default=False)
+
+	async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
+		"""Handle browser stop request - stop creating new tabs."""
+		# logger.info('[AboutBlankWatchdog] Browser stop requested, stopping tab creation')
+		self._stopping = True
+
+	async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
+		"""Handle browser stopped event."""
+		# logger.info('[AboutBlankWatchdog] Browser stopped')
+		self._stopping = True
+
+	async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
+		"""Check tabs when a new tab is created."""
+		# logger.debug(f'[AboutBlankWatchdog] ➕ New tab created: {event.url}')
+
+		# If an about:blank tab was created, show DVD screensaver on all about:blank tabs
+		if event.url == 'about:blank':
+			await self._show_dvd_screensaver_on_about_blank_tabs()
+
+	async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
+		"""Check tabs when a tab is closed and proactively create about:blank if needed."""
+		# Don't create new tabs if browser is shutting down
+		if self._stopping:
+			return
+
+		# Don't attempt CDP operations if the WebSocket is dead — dispatching
+		# NavigateToUrlEvent on a broken connection will hang until timeout
+		if not self.browser_session.is_cdp_connected:
+			self.logger.debug('[AboutBlankWatchdog] CDP not connected, skipping tab recovery')
+			return
+
+		# Check if we're about to close the last tab (event happens BEFORE tab closes)
+		# Use _cdp_get_all_pages for quick check without fetching titles
+		page_targets = await self.browser_session._cdp_get_all_pages()
+		if len(page_targets) < 1:
+			self.logger.debug(
+				'[AboutBlankWatchdog] Last tab closing, creating new about:blank tab to avoid closing entire browser'
+			)
+			# Create the animation tab since no tabs should remain
+			navigate_event = self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
+			await navigate_event
+			# Show DVD screensaver on the new tab
+			await self._show_dvd_screensaver_on_about_blank_tabs()
+		else:
+			# Multiple tabs exist, check after close
+			await self._check_and_ensure_about_blank_tab()
+
+	async def attach_to_target(self, target_id: TargetID) -> None:
+		"""AboutBlankWatchdog doesn't monitor individual targets."""
+		pass
+
+	async def _check_and_ensure_about_blank_tab(self) -> None:
+		"""Check current tabs and ensure exactly one about:blank tab with animation exists."""
+		try:
+			if not self.browser_session.is_cdp_connected:
+				return
+
+			# For quick checks, just get page targets without titles to reduce noise
+			page_targets = await self.browser_session._cdp_get_all_pages()
+
+			# If no tabs exist at all, create one to keep browser alive
+			if len(page_targets) == 0:
+				# Only create a new tab if there are no tabs at all
+				self.logger.debug('[AboutBlankWatchdog] No tabs exist, creating new about:blank DVD screensaver tab')
+				navigate_event = self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
+				await navigate_event
+				# Show DVD screensaver on the new tab
+				await self._show_dvd_screensaver_on_about_blank_tabs()
+			# Otherwise there are tabs, don't create new ones to avoid interfering
+
+		except Exception as e:
+			self.logger.error(f'[AboutBlankWatchdog] Error ensuring about:blank tab: {e}')
+
+	async def _show_dvd_screensaver_on_about_blank_tabs(self) -> None:
+		"""Show DVD screensaver on all about:blank pages only."""
+		try:
+			# Get just the page targets without expensive title fetching
+			page_targets = await self.browser_session._cdp_get_all_pages()
+			browser_session_label = str(self.browser_session.id)[-4:]
+
+			for page_target in page_targets:
+				target_id = page_target['targetId']
+				url = page_target['url']
+
+				# Only target about:blank pages specifically
+				if url == 'about:blank':
+					await self._show_dvd_screensaver_loading_animation_cdp(target_id, browser_session_label)
+
+		except Exception as e:
+			self.logger.error(f'[AboutBlankWatchdog] Error showing DVD screensaver: {e}')
+
+	async def _show_dvd_screensaver_loading_animation_cdp(self, target_id: TargetID, browser_session_label: str) -> None:
+		"""
+		Injects a DVD screensaver-style bouncing logo loading animation overlay into the target using CDP.
+		This is used to visually indicate that the browser is setting up or waiting.
+		"""
+		try:
+			# Create temporary session for this target without switching focus
+			temp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
+
+			# Inject the DVD screensaver script (from main branch with idempotency added)
+			script = f"""
+				(function(browser_session_label) {{
+					// Idempotency check
+					if (window.__dvdAnimationRunning) {{
+						return; // Already running, don't add another
+					}}
+					window.__dvdAnimationRunning = true;
+					
+					// Ensure document.body exists before proceeding
+					if (!document.body) {{
+						// Try again after DOM is ready
+						window.__dvdAnimationRunning = false; // Reset flag to retry
+						if (document.readyState === 'loading') {{
+							document.addEventListener('DOMContentLoaded', () => arguments.callee(browser_session_label));
+						}}
+						return;
+					}}
+					
+					const animated_title = `Starting agent ${{browser_session_label}}...`;
+					if (document.title === animated_title) {{
+						return;      // already run on this tab, dont run again
+					}}
+					document.title = animated_title;
+
+					// Create the main overlay
+					const loadingOverlay = document.createElement('div');
+					loadingOverlay.id = 'pretty-loading-animation';
+					loadingOverlay.style.position = 'fixed';
+					loadingOverlay.style.top = '0';
+					loadingOverlay.style.left = '0';
+					loadingOverlay.style.width = '100vw';
+					loadingOverlay.style.height = '100vh';
+					loadingOverlay.style.background = '#000';
+					loadingOverlay.style.zIndex = '99999';
+					loadingOverlay.style.overflow = 'hidden';
+
+					// Create the image element
+					const img = document.createElement('img');
+					img.src = 'https://cf.browser-use.com/logo.svg';
+					img.alt = 'Browser-Use';
+					img.style.width = '200px';
+					img.style.height = 'auto';
+					img.style.position = 'absolute';
+					img.style.left = '0px';
+					img.style.top = '0px';
+					img.style.zIndex = '2';
+					img.style.opacity = '0.8';
+
+					loadingOverlay.appendChild(img);
+					document.body.appendChild(loadingOverlay);
+
+					// DVD screensaver bounce logic
+					let x = Math.random() * (window.innerWidth - 300);
+					let y = Math.random() * (window.innerHeight - 300);
+					let dx = 1.2 + Math.random() * 0.4; // px per frame
+					let dy = 1.2 + Math.random() * 0.4;
+					// Randomize direction
+					if (Math.random() > 0.5) dx = -dx;
+					if (Math.random() > 0.5) dy = -dy;
+
+					function animate() {{
+						const imgWidth = img.offsetWidth || 300;
+						const imgHeight = img.offsetHeight || 300;
+						x += dx;
+						y += dy;
+
+						if (x <= 0) {{
+							x = 0;
+							dx = Math.abs(dx);
+						}} else if (x + imgWidth >= window.innerWidth) {{
+							x = window.innerWidth - imgWidth;
+							dx = -Math.abs(dx);
+						}}
+						if (y <= 0) {{
+							y = 0;
+							dy = Math.abs(dy);
+						}} else if (y + imgHeight >= window.innerHeight) {{
+							y = window.innerHeight - imgHeight;
+							dy = -Math.abs(dy);
+						}}
+
+						img.style.left = `${{x}}px`;
+						img.style.top = `${{y}}px`;
+
+						requestAnimationFrame(animate);
+					}}
+					animate();
+
+					// Responsive: update bounds on resize
+					window.addEventListener('resize', () => {{
+						x = Math.min(x, window.innerWidth - img.offsetWidth);
+						y = Math.min(y, window.innerHeight - img.offsetHeight);
+					}});
+
+					// Add a little CSS for smoothness
+					const style = document.createElement('style');
+					style.textContent = `
+						#pretty-loading-animation {{
+							/*backdrop-filter: blur(2px) brightness(0.9);*/
+						}}
+						#pretty-loading-animation img {{
+							user-select: none;
+							pointer-events: none;
+						}}
+					`;
+					document.head.appendChild(style);
+				}})('{browser_session_label}');
+			"""
+
+			await temp_session.cdp_client.send.Runtime.evaluate(params={'expression': script}, session_id=temp_session.session_id)
+
+			# No need to detach - session is cached
+
+			# Dispatch event
+			self.event_bus.dispatch(AboutBlankDVDScreensaverShownEvent(target_id=target_id))
+
+		except Exception as e:
+			self.logger.error(f'[AboutBlankWatchdog] Error injecting DVD screensaver: {e}')

browser_use/browser/watchdogs/captcha_watchdog.py

@@ -0,0 +1,207 @@
+"""Captcha solver watchdog — monitors captcha events from the browser proxy.
+
+Listens for BrowserUse.captchaSolverStarted/Finished CDP events and exposes a
+wait_if_captcha_solving() method that the agent step loop uses to block until
+a captcha is resolved (with a configurable timeout).
+
+NOTE: Only a single captcha solve is tracked at a time.  If multiple captchas
+overlap (e.g. rapid successive navigations), only the latest one is tracked and
+earlier in-flight waits may return prematurely.
+"""
+
+import asyncio
+from dataclasses import dataclass
+from typing import Any, ClassVar, Literal
+
+from bubus import BaseEvent
+from cdp_use.cdp.browseruse.events import CaptchaSolverFinishedEvent as CDPCaptchaSolverFinishedEvent
+from cdp_use.cdp.browseruse.events import CaptchaSolverStartedEvent as CDPCaptchaSolverStartedEvent
+from pydantic import PrivateAttr
+
+from browser_use.browser.events import (
+	BrowserConnectedEvent,
+	BrowserStoppedEvent,
+	CaptchaSolverFinishedEvent,
+	CaptchaSolverStartedEvent,
+	_get_timeout,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+
+CaptchaResultType = Literal['success', 'failed', 'timeout', 'unknown']
+
+
+@dataclass
+class CaptchaWaitResult:
+	"""Result returned by wait_if_captcha_solving() when the agent had to wait."""
+
+	waited: bool
+	vendor: str
+	url: str
+	duration_ms: int
+	result: CaptchaResultType
+
+
+class CaptchaWatchdog(BaseWatchdog):
+	"""Monitors captcha solver events from the browser proxy.
+
+	When the proxy detects a CAPTCHA and starts solving it, a CDP event
+	``BrowserUse.captchaSolverStarted`` is sent over the WebSocket.  This
+	watchdog catches that event and blocks the agent's step loop (via
+	``wait_if_captcha_solving``) until ``BrowserUse.captchaSolverFinished``
+	arrives or the configurable timeout expires.
+	"""
+
+	# Event contracts
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+		BrowserConnectedEvent,
+		BrowserStoppedEvent,
+	]
+	EMITS: ClassVar[list[type[BaseEvent]]] = [
+		CaptchaSolverStartedEvent,
+		CaptchaSolverFinishedEvent,
+	]
+
+	# --- private state ---
+	_captcha_solving: bool = PrivateAttr(default=False)
+	_captcha_solved_event: asyncio.Event = PrivateAttr(default_factory=asyncio.Event)
+	_captcha_info: dict[str, Any] = PrivateAttr(default_factory=dict)
+	_captcha_result: CaptchaResultType = PrivateAttr(default='unknown')
+	_captcha_duration_ms: int = PrivateAttr(default=0)
+	_cdp_handlers_registered: bool = PrivateAttr(default=False)
+
+	def model_post_init(self, __context: Any) -> None:
+		# Start in "not blocked" state so callers never wait when there is no captcha.
+		self._captcha_solved_event.set()
+
+	# ------------------------------------------------------------------
+	# Event handlers
+	# ------------------------------------------------------------------
+
+	async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
+		"""Register CDP event handlers for BrowserUse captcha solver events."""
+		if self._cdp_handlers_registered:
+			self.logger.debug('CaptchaWatchdog: CDP handlers already registered, skipping')
+			return
+
+		cdp_client = self.browser_session.cdp_client
+
+		def _on_captcha_started(event_data: CDPCaptchaSolverStartedEvent, session_id: str | None) -> None:
+			try:
+				self._captcha_solving = True
+				self._captcha_result = 'unknown'
+				self._captcha_duration_ms = 0
+				self._captcha_info = {
+					'vendor': event_data.get('vendor', 'unknown'),
+					'url': event_data.get('url', ''),
+					'targetId': event_data.get('targetId', ''),
+					'startedAt': event_data.get('startedAt', 0),
+				}
+				# Block any waiter
+				self._captcha_solved_event.clear()
+
+				vendor = self._captcha_info['vendor']
+				url = self._captcha_info['url']
+				self.logger.info(f'🔒 Captcha solving started: {vendor} on {url}')
+
+				self.event_bus.dispatch(
+					CaptchaSolverStartedEvent(
+						target_id=event_data.get('targetId', ''),
+						vendor=vendor,
+						url=url,
+						started_at=event_data.get('startedAt', 0),
+					)
+				)
+			except Exception:
+				self.logger.exception('Error handling captchaSolverStarted CDP event')
+				# Ensure consistent state: unblock any waiter
+				self._captcha_solving = False
+				self._captcha_solved_event.set()
+
+		def _on_captcha_finished(event_data: CDPCaptchaSolverFinishedEvent, session_id: str | None) -> None:
+			try:
+				success = event_data.get('success', False)
+				self._captcha_solving = False
+				self._captcha_duration_ms = event_data.get('durationMs', 0)
+				self._captcha_result = 'success' if success else 'failed'
+
+				vendor = event_data.get('vendor', self._captcha_info.get('vendor', 'unknown'))
+				url = event_data.get('url', self._captcha_info.get('url', ''))
+				duration_s = self._captcha_duration_ms / 1000
+
+				self.logger.info(f'🔓 Captcha solving finished: {self._captcha_result} — {vendor} on {url} ({duration_s:.1f}s)')
+
+				# Unblock any waiter
+				self._captcha_solved_event.set()
+
+				self.event_bus.dispatch(
+					CaptchaSolverFinishedEvent(
+						target_id=event_data.get('targetId', ''),
+						vendor=vendor,
+						url=url,
+						duration_ms=self._captcha_duration_ms,
+						finished_at=event_data.get('finishedAt', 0),
+						success=success,
+					)
+				)
+			except Exception:
+				self.logger.exception('Error handling captchaSolverFinished CDP event')
+				# Ensure consistent state: unblock any waiter
+				self._captcha_solving = False
+				self._captcha_solved_event.set()
+
+		cdp_client.register.BrowserUse.captchaSolverStarted(_on_captcha_started)
+		cdp_client.register.BrowserUse.captchaSolverFinished(_on_captcha_finished)
+		self._cdp_handlers_registered = True
+		self.logger.debug('🔒 CaptchaWatchdog: registered CDP event handlers for BrowserUse captcha events')
+
+	async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
+		"""Clear captcha state when the browser disconnects so nothing hangs."""
+		self._captcha_solving = False
+		self._captcha_result = 'unknown'
+		self._captcha_duration_ms = 0
+		self._captcha_info = {}
+		self._captcha_solved_event.set()
+		self._cdp_handlers_registered = False
+
+	# ------------------------------------------------------------------
+	# Public API
+	# ------------------------------------------------------------------
+
+	async def wait_if_captcha_solving(self, timeout: float | None = None) -> CaptchaWaitResult | None:
+		"""Wait if a captcha is currently being solved.
+
+		Returns:
+			``None`` if no captcha was in progress.
+			A ``CaptchaWaitResult`` with the outcome otherwise.
+		"""
+		if not self._captcha_solving:
+			return None
+
+		if timeout is None:
+			timeout = _get_timeout('TIMEOUT_CaptchaSolverWait', 120.0)
+		assert timeout is not None
+		vendor = self._captcha_info.get('vendor', 'unknown')
+		url = self._captcha_info.get('url', '')
+		self.logger.info(f'⏳ Waiting for {vendor} captcha to be solved on {url} (timeout={timeout}s)...')
+
+		try:
+			await asyncio.wait_for(self._captcha_solved_event.wait(), timeout=timeout)
+			return CaptchaWaitResult(
+				waited=True,
+				vendor=vendor,
+				url=url,
+				duration_ms=self._captcha_duration_ms,
+				result=self._captcha_result,
+			)
+		except TimeoutError:
+			# Timed out — unblock and report
+			self._captcha_solving = False
+			self._captcha_solved_event.set()
+			self.logger.warning(f'⏰ Captcha wait timed out after {timeout}s for {vendor} on {url}')
+			return CaptchaWaitResult(
+				waited=True,
+				vendor=vendor,
+				url=url,
+				duration_ms=int(timeout * 1000),
+				result='timeout',
+			)

browser_use/browser/watchdogs/crash_watchdog.py

@@ -0,0 +1,336 @@
+"""Browser watchdog for monitoring crashes and network timeouts using CDP."""
+
+import asyncio
+import time
+from typing import TYPE_CHECKING, ClassVar
+
+import psutil
+from bubus import BaseEvent
+from cdp_use.cdp.target import SessionID, TargetID
+from cdp_use.cdp.target.events import TargetCrashedEvent
+from pydantic import Field, PrivateAttr
+
+from browser_use.browser.events import (
+	BrowserConnectedEvent,
+	BrowserErrorEvent,
+	BrowserStoppedEvent,
+	TabClosedEvent,
+	TabCreatedEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+from browser_use.utils import create_task_with_error_handling
+
+if TYPE_CHECKING:
+	pass
+
+
+class NetworkRequestTracker:
+	"""Tracks ongoing network requests."""
+
+	def __init__(self, request_id: str, start_time: float, url: str, method: str, resource_type: str | None = None):
+		self.request_id = request_id
+		self.start_time = start_time
+		self.url = url
+		self.method = method
+		self.resource_type = resource_type
+
+
+class CrashWatchdog(BaseWatchdog):
+	"""Monitors browser health for crashes and network timeouts using CDP."""
+
+	# Event contracts
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+		BrowserConnectedEvent,
+		BrowserStoppedEvent,
+		TabCreatedEvent,
+		TabClosedEvent,
+	]
+	EMITS: ClassVar[list[type[BaseEvent]]] = [BrowserErrorEvent]
+
+	# Configuration
+	network_timeout_seconds: float = Field(default=10.0)
+	check_interval_seconds: float = Field(default=5.0)  # Reduced frequency to reduce noise
+
+	# Private state
+	_active_requests: dict[str, NetworkRequestTracker] = PrivateAttr(default_factory=dict)
+	_monitoring_task: asyncio.Task | None = PrivateAttr(default=None)
+	_last_responsive_checks: dict[str, float] = PrivateAttr(default_factory=dict)  # target_url -> timestamp
+	_cdp_event_tasks: set[asyncio.Task] = PrivateAttr(default_factory=set)  # Track CDP event handler tasks
+	_targets_with_listeners: set[str] = PrivateAttr(default_factory=set)  # Track targets that already have event listeners
+
+	async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
+		"""Start monitoring when browser is connected."""
+		# logger.debug('[CrashWatchdog] Browser connected event received, beginning monitoring')
+
+		create_task_with_error_handling(
+			self._start_monitoring(), name='start_crash_monitoring', logger_instance=self.logger, suppress_exceptions=True
+		)
+		# logger.debug(f'[CrashWatchdog] Monitoring task started: {self._monitoring_task and not self._monitoring_task.done()}')
+
+	async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
+		"""Stop monitoring when browser stops."""
+		# logger.debug('[CrashWatchdog] Browser stopped, ending monitoring')
+		await self._stop_monitoring()
+
+	async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
+		"""Attach to new tab."""
+		assert self.browser_session.agent_focus_target_id is not None, 'No current target ID'
+		await self.attach_to_target(self.browser_session.agent_focus_target_id)
+
+	async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
+		"""Clean up tracking when tab closes."""
+		# Remove target from listener tracking to prevent memory leak
+		if event.target_id in self._targets_with_listeners:
+			self._targets_with_listeners.discard(event.target_id)
+			self.logger.debug(f'[CrashWatchdog] Removed target {event.target_id[:8]}... from monitoring')
+
+	async def attach_to_target(self, target_id: TargetID) -> None:
+		"""Set up crash monitoring for a specific target using CDP."""
+		try:
+			# Check if we already have listeners for this target
+			if target_id in self._targets_with_listeners:
+				self.logger.debug(f'[CrashWatchdog] Event listeners already exist for target: {target_id[:8]}...')
+				return
+
+			# Create temporary session for monitoring without switching focus
+			cdp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
+
+			# Register crash event handler
+			def on_target_crashed(event: TargetCrashedEvent, session_id: SessionID | None = None):
+				# Create and track the task
+				task = create_task_with_error_handling(
+					self._on_target_crash_cdp(target_id),
+					name='handle_target_crash',
+					logger_instance=self.logger,
+					suppress_exceptions=True,
+				)
+				self._cdp_event_tasks.add(task)
+				# Remove from set when done
+				task.add_done_callback(lambda t: self._cdp_event_tasks.discard(t))
+
+			cdp_session.cdp_client.register.Target.targetCrashed(on_target_crashed)
+
+			# Track that we've added listeners to this target
+			self._targets_with_listeners.add(target_id)
+
+			target = self.browser_session.session_manager.get_target(target_id)
+			if target:
+				self.logger.debug(f'[CrashWatchdog] Added target to monitoring: {target.url}')
+
+		except Exception as e:
+			self.logger.warning(f'[CrashWatchdog] Failed to attach to target {target_id}: {e}')
+
+	async def _on_request_cdp(self, event: dict) -> None:
+		"""Track new network request from CDP event."""
+		request_id = event.get('requestId', '')
+		request = event.get('request', {})
+
+		self._active_requests[request_id] = NetworkRequestTracker(
+			request_id=request_id,
+			start_time=time.time(),
+			url=request.get('url', ''),
+			method=request.get('method', ''),
+			resource_type=event.get('type'),
+		)
+		# logger.debug(f'[CrashWatchdog] Tracking request: {request.get("method", "")} {request.get("url", "")[:50]}...')
+
+	def _on_response_cdp(self, event: dict) -> None:
+		"""Remove request from tracking on response."""
+		request_id = event.get('requestId', '')
+		if request_id in self._active_requests:
+			elapsed = time.time() - self._active_requests[request_id].start_time
+			response = event.get('response', {})
+			self.logger.debug(f'[CrashWatchdog] Request completed in {elapsed:.2f}s: {response.get("url", "")[:50]}...')
+			# Don't remove yet - wait for loadingFinished
+
+	def _on_request_failed_cdp(self, event: dict) -> None:
+		"""Remove request from tracking on failure."""
+		request_id = event.get('requestId', '')
+		if request_id in self._active_requests:
+			elapsed = time.time() - self._active_requests[request_id].start_time
+			self.logger.debug(
+				f'[CrashWatchdog] Request failed after {elapsed:.2f}s: {self._active_requests[request_id].url[:50]}...'
+			)
+			del self._active_requests[request_id]
+
+	def _on_request_finished_cdp(self, event: dict) -> None:
+		"""Remove request from tracking when loading is finished."""
+		request_id = event.get('requestId', '')
+		self._active_requests.pop(request_id, None)
+
+	async def _on_target_crash_cdp(self, target_id: TargetID) -> None:
+		"""Handle target crash detected via CDP."""
+		self.logger.debug(f'[CrashWatchdog] Target crashed: {target_id[:8]}..., waiting for detach event')
+
+		target = self.browser_session.session_manager.get_target(target_id)
+
+		is_agent_focus = (
+			target
+			and self.browser_session.agent_focus_target_id
+			and target.target_id == self.browser_session.agent_focus_target_id
+		)
+
+		if is_agent_focus:
+			self.logger.error(f'[CrashWatchdog] 💥 Agent focus tab crashed: {target.url} (SessionManager will auto-recover)')
+
+		# Emit browser error event
+		self.event_bus.dispatch(
+			BrowserErrorEvent(
+				error_type='TargetCrash',
+				message=f'Target crashed: {target_id}',
+				details={
+					'url': target.url if target else None,
+					'target_id': target_id,
+					'was_agent_focus': is_agent_focus,
+				},
+			)
+		)
+
+	async def _start_monitoring(self) -> None:
+		"""Start the monitoring loop."""
+		assert self.browser_session.cdp_client is not None, 'Root CDP client not initialized - browser may not be connected yet'
+
+		if self._monitoring_task and not self._monitoring_task.done():
+			# logger.info('[CrashWatchdog] Monitoring already running')
+			return
+
+		self._monitoring_task = create_task_with_error_handling(
+			self._monitoring_loop(), name='crash_monitoring_loop', logger_instance=self.logger, suppress_exceptions=True
+		)
+		# logger.debug('[CrashWatchdog] Monitoring loop created and started')
+
+	async def _stop_monitoring(self) -> None:
+		"""Stop the monitoring loop and clean up all tracking."""
+		if self._monitoring_task and not self._monitoring_task.done():
+			self._monitoring_task.cancel()
+			try:
+				await self._monitoring_task
+			except asyncio.CancelledError:
+				pass
+			self.logger.debug('[CrashWatchdog] Monitoring loop stopped')
+
+		# Cancel all CDP event handler tasks
+		for task in list(self._cdp_event_tasks):
+			if not task.done():
+				task.cancel()
+		# Wait for all tasks to complete cancellation
+		if self._cdp_event_tasks:
+			await asyncio.gather(*self._cdp_event_tasks, return_exceptions=True)
+		self._cdp_event_tasks.clear()
+
+		# Clear all tracking
+		self._active_requests.clear()
+		self._targets_with_listeners.clear()
+		self._last_responsive_checks.clear()
+
+	async def _monitoring_loop(self) -> None:
+		"""Main monitoring loop."""
+		await asyncio.sleep(10)  # give browser time to start up and load the first page after first LLM call
+		while True:
+			try:
+				await self._check_network_timeouts()
+				await self._check_browser_health()
+				await asyncio.sleep(self.check_interval_seconds)
+			except asyncio.CancelledError:
+				break
+			except Exception as e:
+				self.logger.error(f'[CrashWatchdog] Error in monitoring loop: {e}')
+
+	async def _check_network_timeouts(self) -> None:
+		"""Check for network requests exceeding timeout."""
+		current_time = time.time()
+		timed_out_requests = []
+
+		# Debug logging
+		if self._active_requests:
+			self.logger.debug(
+				f'[CrashWatchdog] Checking {len(self._active_requests)} active requests for timeouts (threshold: {self.network_timeout_seconds}s)'
+			)
+
+		for request_id, tracker in self._active_requests.items():
+			elapsed = current_time - tracker.start_time
+			self.logger.debug(
+				f'[CrashWatchdog] Request {tracker.url[:30]}... elapsed: {elapsed:.1f}s, timeout: {self.network_timeout_seconds}s'
+			)
+			if elapsed >= self.network_timeout_seconds:
+				timed_out_requests.append((request_id, tracker))
+
+		# Emit events for timed out requests
+		for request_id, tracker in timed_out_requests:
+			self.logger.warning(
+				f'[CrashWatchdog] Network request timeout after {self.network_timeout_seconds}s: '
+				f'{tracker.method} {tracker.url[:100]}...'
+			)
+
+			self.event_bus.dispatch(
+				BrowserErrorEvent(
+					error_type='NetworkTimeout',
+					message=f'Network request timed out after {self.network_timeout_seconds}s',
+					details={
+						'url': tracker.url,
+						'method': tracker.method,
+						'resource_type': tracker.resource_type,
+						'elapsed_seconds': current_time - tracker.start_time,
+					},
+				)
+			)
+
+			# Remove from tracking
+			del self._active_requests[request_id]
+
+	async def _check_browser_health(self) -> None:
+		"""Check if browser and targets are still responsive."""
+
+		try:
+			self.logger.debug(f'[CrashWatchdog] Checking browser health for target {self.browser_session.agent_focus_target_id}')
+			cdp_session = await self.browser_session.get_or_create_cdp_session()
+
+			for target in self.browser_session.session_manager.get_all_page_targets():
+				if self._is_new_tab_page(target.url) and target.url != 'about:blank':
+					self.logger.debug(f'[CrashWatchdog] Redirecting chrome://new-tab-page/ to about:blank {target.url}')
+					cdp_session = await self.browser_session.get_or_create_cdp_session(target_id=target.target_id)
+					await cdp_session.cdp_client.send.Page.navigate(
+						params={'url': 'about:blank'}, session_id=cdp_session.session_id
+					)
+
+			# Quick ping to check if session is alive
+			self.logger.debug(f'[CrashWatchdog] Attempting to run simple JS test expression in session {cdp_session} 1+1')
+			await asyncio.wait_for(
+				cdp_session.cdp_client.send.Runtime.evaluate(params={'expression': '1+1'}, session_id=cdp_session.session_id),
+				timeout=1.0,
+			)
+			self.logger.debug(
+				f'[CrashWatchdog] Browser health check passed for target {self.browser_session.agent_focus_target_id}'
+			)
+		except Exception as e:
+			self.logger.error(
+				f'[CrashWatchdog] ❌ Crashed/unresponsive session detected for target {self.browser_session.agent_focus_target_id} '
+				f'error: {type(e).__name__}: {e} (Chrome will send detach event, SessionManager will auto-recover)'
+			)
+
+		# Check browser process if we have PID
+		if self.browser_session._local_browser_watchdog and (proc := self.browser_session._local_browser_watchdog._subprocess):
+			try:
+				if proc.status() in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD):
+					self.logger.error(f'[CrashWatchdog] Browser process {proc.pid} has crashed')
+
+					# Browser process crashed - SessionManager will clean up via detach events
+					# Just dispatch error event and stop monitoring
+					self.event_bus.dispatch(
+						BrowserErrorEvent(
+							error_type='BrowserProcessCrashed',
+							message=f'Browser process {proc.pid} has crashed',
+							details={'pid': proc.pid, 'status': proc.status()},
+						)
+					)
+
+					self.logger.warning('[CrashWatchdog] Browser process dead - stopping health monitoring')
+					await self._stop_monitoring()
+					return
+			except Exception:
+				pass  # psutil not available or process doesn't exist
+
+	@staticmethod
+	def _is_new_tab_page(url: str) -> bool:
+		"""Check if URL is a new tab page."""
+		return url in ['about:blank', 'chrome://new-tab-page/', 'chrome://newtab/']

browser_use/browser/watchdogs/dom_watchdog.py

@@ -0,0 +1,865 @@
+"""DOM watchdog for browser DOM tree management using CDP."""
+
+import asyncio
+import time
+from typing import TYPE_CHECKING
+
+from browser_use.browser.events import (
+	BrowserErrorEvent,
+	BrowserStateRequestEvent,
+	ScreenshotEvent,
+	TabCreatedEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+from browser_use.dom.service import DomService
+from browser_use.dom.views import (
+	EnhancedDOMTreeNode,
+	SerializedDOMState,
+)
+from browser_use.observability import observe_debug
+from browser_use.utils import create_task_with_error_handling, time_execution_async
+
+if TYPE_CHECKING:
+	from browser_use.browser.views import BrowserStateSummary, NetworkRequest, PageInfo, PaginationButton
+
+
+class DOMWatchdog(BaseWatchdog):
+	"""Handles DOM tree building, serialization, and element access via CDP.
+
+	This watchdog acts as a bridge between the event-driven browser session
+	and the DomService implementation, maintaining cached state and providing
+	helper methods for other watchdogs.
+	"""
+
+	LISTENS_TO = [TabCreatedEvent, BrowserStateRequestEvent]
+	EMITS = [BrowserErrorEvent]
+
+	# Public properties for other watchdogs
+	selector_map: dict[int, EnhancedDOMTreeNode] | None = None
+	current_dom_state: SerializedDOMState | None = None
+	enhanced_dom_tree: EnhancedDOMTreeNode | None = None
+
+	# Internal DOM service
+	_dom_service: DomService | None = None
+
+	# Network tracking - maps request_id to (url, start_time, method, resource_type)
+	_pending_requests: dict[str, tuple[str, float, str, str | None]] = {}
+
+	async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
+		# self.logger.debug('Setting up init scripts in browser')
+		return None
+
+	def _get_recent_events_str(self, limit: int = 10) -> str | None:
+		"""Get the most recent events from the event bus as JSON.
+
+		Args:
+			limit: Maximum number of recent events to include
+
+		Returns:
+			JSON string of recent events or None if not available
+		"""
+		import json
+
+		try:
+			# Get all events from history, sorted by creation time (most recent first)
+			all_events = sorted(
+				self.browser_session.event_bus.event_history.values(), key=lambda e: e.event_created_at.timestamp(), reverse=True
+			)
+
+			# Take the most recent events and create JSON-serializable data
+			recent_events_data = []
+			for event in all_events[:limit]:
+				event_data = {
+					'event_type': event.event_type,
+					'timestamp': event.event_created_at.isoformat(),
+				}
+				# Add specific fields for certain event types
+				if hasattr(event, 'url'):
+					event_data['url'] = getattr(event, 'url')
+				if hasattr(event, 'error_message'):
+					event_data['error_message'] = getattr(event, 'error_message')
+				if hasattr(event, 'target_id'):
+					event_data['target_id'] = getattr(event, 'target_id')
+				recent_events_data.append(event_data)
+
+			return json.dumps(recent_events_data)  # Return empty array if no events
+		except Exception as e:
+			self.logger.debug(f'Failed to get recent events: {e}')
+
+		return json.dumps([])  # Return empty JSON array on error
+
+	async def _get_pending_network_requests(self) -> list['NetworkRequest']:
+		"""Get list of currently pending network requests.
+
+		Uses document.readyState and performance API to detect pending requests.
+		Filters out ads, tracking, and other noise.
+
+		Returns:
+			List of NetworkRequest objects representing currently loading resources
+		"""
+		from browser_use.browser.views import NetworkRequest
+
+		try:
+			# get_or_create_cdp_session() now handles focus validation automatically
+			cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
+
+			# Use performance API to get pending requests
+			js_code = """
+(function() {
+	const now = performance.now();
+	const resources = performance.getEntriesByType('resource');
+	const pending = [];
+
+	// Check document readyState
+	const docLoading = document.readyState !== 'complete';
+
+	// Common ad/tracking domains and patterns to filter out
+	const adDomains = [
+		// Standard ad/tracking networks
+		'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',
+		'facebook.net', 'analytics', 'ads', 'tracking', 'pixel',
+		'hotjar.com', 'clarity.ms', 'mixpanel.com', 'segment.com',
+		// Analytics platforms
+		'demdex.net', 'omtrdc.net', 'adobedtm.com', 'ensighten.com',
+		'newrelic.com', 'nr-data.net', 'google-analytics.com',
+		// Social media trackers
+		'connect.facebook.net', 'platform.twitter.com', 'platform.linkedin.com',
+		// CDN/image hosts (usually not critical for functionality)
+		'.cloudfront.net/image/', '.akamaized.net/image/',
+		// Common tracking paths
+		'/tracker/', '/collector/', '/beacon/', '/telemetry/', '/log/',
+		'/events/', '/eventBatch', '/track.', '/metrics/'
+	];
+
+	// Get resources that are still loading (responseEnd is 0)
+	let totalResourcesChecked = 0;
+	let filteredByResponseEnd = 0;
+	const allDomains = new Set();
+
+	for (const entry of resources) {
+		totalResourcesChecked++;
+
+		// Track all domains from recent resources (for logging)
+		try {
+			const hostname = new URL(entry.name).hostname;
+			if (hostname) allDomains.add(hostname);
+		} catch (e) {}
+
+		if (entry.responseEnd === 0) {
+			filteredByResponseEnd++;
+			const url = entry.name;
+
+			// Filter out ads and tracking
+			const isAd = adDomains.some(domain => url.includes(domain));
+			if (isAd) continue;
+
+			// Filter out data: URLs and very long URLs (often inline resources)
+			if (url.startsWith('data:') || url.length > 500) continue;
+
+			const loadingDuration = now - entry.startTime;
+
+			// Skip requests that have been loading for >10 seconds (likely stuck/polling)
+			if (loadingDuration > 10000) continue;
+
+			const resourceType = entry.initiatorType || 'unknown';
+
+			// Filter out non-critical resources (images, fonts, icons) if loading >3 seconds
+			const nonCriticalTypes = ['img', 'image', 'icon', 'font'];
+			if (nonCriticalTypes.includes(resourceType) && loadingDuration > 3000) continue;
+
+			// Filter out image URLs even if type is unknown
+			const isImageUrl = /\\.(jpg|jpeg|png|gif|webp|svg|ico)(\\?|$)/i.test(url);
+			if (isImageUrl && loadingDuration > 3000) continue;
+
+			pending.push({
+				url: url,
+				method: 'GET',
+				loading_duration_ms: Math.round(loadingDuration),
+				resource_type: resourceType
+			});
+		}
+	}
+
+	return {
+		pending_requests: pending,
+		document_loading: docLoading,
+		document_ready_state: document.readyState,
+		debug: {
+			total_resources: totalResourcesChecked,
+			with_response_end_zero: filteredByResponseEnd,
+			after_all_filters: pending.length,
+			all_domains: Array.from(allDomains)
+		}
+	};
+})()
+"""
+
+			result = await cdp_session.cdp_client.send.Runtime.evaluate(
+				params={'expression': js_code, 'returnByValue': True}, session_id=cdp_session.session_id
+			)
+
+			if result.get('result', {}).get('type') == 'object':
+				data = result['result'].get('value', {})
+				pending = data.get('pending_requests', [])
+				doc_state = data.get('document_ready_state', 'unknown')
+				doc_loading = data.get('document_loading', False)
+				debug_info = data.get('debug', {})
+
+				# Get all domains that had recent activity (from JS)
+				all_domains = debug_info.get('all_domains', [])
+				all_domains_str = ', '.join(sorted(all_domains)[:5]) if all_domains else 'none'
+				if len(all_domains) > 5:
+					all_domains_str += f' +{len(all_domains) - 5} more'
+
+				# Debug logging
+				self.logger.debug(
+					f'🔍 Network check: document.readyState={doc_state}, loading={doc_loading}, '
+					f'total_resources={debug_info.get("total_resources", 0)}, '
+					f'responseEnd=0: {debug_info.get("with_response_end_zero", 0)}, '
+					f'after_filters={len(pending)}, domains=[{all_domains_str}]'
+				)
+
+				# Convert to NetworkRequest objects
+				network_requests = []
+				for req in pending[:20]:  # Limit to 20 to avoid overwhelming the context
+					network_requests.append(
+						NetworkRequest(
+							url=req['url'],
+							method=req.get('method', 'GET'),
+							loading_duration_ms=req.get('loading_duration_ms', 0.0),
+							resource_type=req.get('resource_type'),
+						)
+					)
+
+				return network_requests
+
+		except Exception as e:
+			self.logger.debug(f'Failed to get pending network requests: {e}')
+
+		return []
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='browser_state_request_event')
+	async def on_BrowserStateRequestEvent(self, event: BrowserStateRequestEvent) -> 'BrowserStateSummary':
+		"""Handle browser state request by coordinating DOM building and screenshot capture.
+
+		This is the main entry point for getting the complete browser state.
+
+		Args:
+			event: The browser state request event with options
+
+		Returns:
+			Complete BrowserStateSummary with DOM, screenshot, and target info
+		"""
+		from browser_use.browser.views import BrowserStateSummary, PageInfo
+
+		self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: STARTING browser state request')
+		page_url = await self.browser_session.get_current_page_url()
+		self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page URL: {page_url}')
+
+		# Get focused session for logging (validation already done by get_current_page_url)
+		if self.browser_session.agent_focus_target_id:
+			self.logger.debug(f'Current page URL: {page_url}, target_id: {self.browser_session.agent_focus_target_id}')
+
+		# check if we should skip DOM tree build for pointless pages
+		not_a_meaningful_website = page_url.lower().split(':', 1)[0] not in ('http', 'https')
+
+		# Check for pending network requests BEFORE waiting (so we can see what's loading)
+		# Timeout after 2s — on slow CI machines or heavy pages, this call can hang
+		# for 15s+ eating into the 30s BrowserStateRequestEvent budget.
+		pending_requests_before_wait = []
+		if not not_a_meaningful_website:
+			try:
+				pending_requests_before_wait = await asyncio.wait_for(self._get_pending_network_requests(), timeout=2.0)
+				if pending_requests_before_wait:
+					self.logger.debug(f'🔍 Found {len(pending_requests_before_wait)} pending requests before stability wait')
+			except TimeoutError:
+				self.logger.debug('Pending network request check timed out (2s), skipping')
+			except Exception as e:
+				self.logger.debug(f'Failed to get pending requests before wait: {e}')
+		pending_requests = pending_requests_before_wait
+		# Wait for page stability using browser profile settings (main branch pattern)
+		if not not_a_meaningful_website:
+			self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ⏳ Waiting for page stability...')
+			try:
+				if pending_requests_before_wait:
+					# Reduced from 1s to 0.3s for faster DOM builds while still allowing critical resources to load
+					await asyncio.sleep(0.3)
+				self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Page stability complete')
+			except Exception as e:
+				self.logger.warning(
+					f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Network waiting failed: {e}, continuing anyway...'
+				)
+
+		# Get tabs info once at the beginning for all paths
+		self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting tabs info...')
+		tabs_info = await self.browser_session.get_tabs()
+		self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got {len(tabs_info)} tabs')
+		self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Tabs info: {tabs_info}')
+
+		# Get viewport / scroll position info, remember changing scroll position should invalidate selector_map cache because it only includes visible elements
+		# cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
+		# scroll_info = await cdp_session.cdp_client.send.Runtime.evaluate(
+		# 	params={'expression': 'JSON.stringify({y: document.body.scrollTop, x: document.body.scrollLeft, width: document.documentElement.clientWidth, height: document.documentElement.clientHeight})'},
+		# 	session_id=cdp_session.session_id,
+		# )
+		# self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got scroll info: {scroll_info["result"]}')
+
+		try:
+			# Fast path for empty pages
+			if not_a_meaningful_website:
+				self.logger.debug(f'⚡ Skipping BuildDOMTree for empty target: {page_url}')
+				self.logger.debug(f'📸 Not taking screenshot for empty page: {page_url} (non-http/https URL)')
+
+				# Create minimal DOM state
+				content = SerializedDOMState(_root=None, selector_map={})
+
+				# Skip screenshot for empty pages
+				screenshot_b64 = None
+
+				# Try to get page info from CDP, fall back to defaults if unavailable
+				try:
+					page_info = await self._get_page_info()
+				except Exception as e:
+					self.logger.debug(f'Failed to get page info from CDP for empty page: {e}, using fallback')
+					# Use default viewport dimensions
+					viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720}
+					page_info = PageInfo(
+						viewport_width=viewport['width'],
+						viewport_height=viewport['height'],
+						page_width=viewport['width'],
+						page_height=viewport['height'],
+						scroll_x=0,
+						scroll_y=0,
+						pixels_above=0,
+						pixels_below=0,
+						pixels_left=0,
+						pixels_right=0,
+					)
+
+				return BrowserStateSummary(
+					dom_state=content,
+					url=page_url,
+					title='Empty Tab',
+					tabs=tabs_info,
+					screenshot=screenshot_b64,
+					page_info=page_info,
+					pixels_above=0,
+					pixels_below=0,
+					browser_errors=[],
+					is_pdf_viewer=False,
+					recent_events=self._get_recent_events_str() if event.include_recent_events else None,
+					pending_network_requests=[],  # Empty page has no pending requests
+					pagination_buttons=[],  # Empty page has no pagination
+					closed_popup_messages=self.browser_session._closed_popup_messages.copy(),
+				)
+
+			# Execute DOM building and screenshot capture in parallel
+			dom_task = None
+			screenshot_task = None
+
+			# Start DOM building task if requested
+			if event.include_dom:
+				self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')
+
+				previous_state = (
+					self.browser_session._cached_browser_state_summary.dom_state
+					if self.browser_session._cached_browser_state_summary
+					else None
+				)
+
+				dom_task = create_task_with_error_handling(
+					self._build_dom_tree_without_highlights(previous_state),
+					name='build_dom_tree',
+					logger_instance=self.logger,
+					suppress_exceptions=True,
+				)
+
+			# Start clean screenshot task if requested (without JS highlights)
+			if event.include_screenshot:
+				self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...')
+				screenshot_task = create_task_with_error_handling(
+					self._capture_clean_screenshot(),
+					name='capture_screenshot',
+					logger_instance=self.logger,
+					suppress_exceptions=True,
+				)
+
+			# Wait for both tasks to complete
+			content = None
+			screenshot_b64 = None
+
+			if dom_task:
+				try:
+					content = await dom_task
+					self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed')
+				except Exception as e:
+					self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')
+					content = SerializedDOMState(_root=None, selector_map={})
+			else:
+				content = SerializedDOMState(_root=None, selector_map={})
+
+			if screenshot_task:
+				try:
+					screenshot_b64 = await screenshot_task
+					self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured')
+				except Exception as e:
+					self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')
+					screenshot_b64 = None
+
+			# Add browser-side highlights for user visibility
+			if content and content.selector_map and self.browser_session.browser_profile.dom_highlight_elements:
+				try:
+					self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Adding browser-side highlights...')
+					await self.browser_session.add_highlights(content.selector_map)
+					self.logger.debug(
+						f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Added browser highlights for {len(content.selector_map)} elements'
+					)
+				except Exception as e:
+					self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Browser highlighting failed: {e}')
+
+			# Ensure we have valid content
+			if not content:
+				content = SerializedDOMState(_root=None, selector_map={})
+
+			# Tabs info already fetched at the beginning
+
+			# Get target title safely
+			try:
+				self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page title...')
+				title = await asyncio.wait_for(self.browser_session.get_current_page_title(), timeout=1.0)
+				self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got title: {title}')
+			except Exception as e:
+				self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get title: {e}')
+				title = 'Page'
+
+			# Get comprehensive page info from CDP with timeout
+			try:
+				self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page info from CDP...')
+				page_info = await asyncio.wait_for(self._get_page_info(), timeout=1.0)
+				self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page info from CDP: {page_info}')
+			except Exception as e:
+				self.logger.debug(
+					f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get page info from CDP: {e}, using fallback'
+				)
+				# Fallback to default viewport dimensions
+				viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720}
+				page_info = PageInfo(
+					viewport_width=viewport['width'],
+					viewport_height=viewport['height'],
+					page_width=viewport['width'],
+					page_height=viewport['height'],
+					scroll_x=0,
+					scroll_y=0,
+					pixels_above=0,
+					pixels_below=0,
+					pixels_left=0,
+					pixels_right=0,
+				)
+
+			# Check for PDF viewer
+			is_pdf_viewer = page_url.endswith('.pdf') or '/pdf/' in page_url
+
+			# Detect pagination buttons from the DOM
+			pagination_buttons_data = []
+			if content and content.selector_map:
+				pagination_buttons_data = self._detect_pagination_buttons(content.selector_map)
+
+			# Build and cache the browser state summary
+			if screenshot_b64:
+				self.logger.debug(
+					f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary with screenshot, length: {len(screenshot_b64)}'
+				)
+			else:
+				self.logger.debug(
+					'🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary WITHOUT screenshot'
+				)
+
+			browser_state = BrowserStateSummary(
+				dom_state=content,
+				url=page_url,
+				title=title,
+				tabs=tabs_info,
+				screenshot=screenshot_b64,
+				page_info=page_info,
+				pixels_above=0,
+				pixels_below=0,
+				browser_errors=[],
+				is_pdf_viewer=is_pdf_viewer,
+				recent_events=self._get_recent_events_str() if event.include_recent_events else None,
+				pending_network_requests=pending_requests,
+				pagination_buttons=pagination_buttons_data,
+				closed_popup_messages=self.browser_session._closed_popup_messages.copy(),
+			)
+
+			# Cache the state
+			self.browser_session._cached_browser_state_summary = browser_state
+
+			# Cache viewport size for coordinate conversion (if llm_screenshot_size is enabled)
+			if page_info:
+				self.browser_session._original_viewport_size = (page_info.viewport_width, page_info.viewport_height)
+
+			self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ COMPLETED - Returning browser state')
+			return browser_state
+
+		except Exception as e:
+			self.logger.error(f'Failed to get browser state: {e}')
+
+			# Return minimal recovery state
+			return BrowserStateSummary(
+				dom_state=SerializedDOMState(_root=None, selector_map={}),
+				url=page_url if 'page_url' in locals() else '',
+				title='Error',
+				tabs=[],
+				screenshot=None,
+				page_info=PageInfo(
+					viewport_width=1280,
+					viewport_height=720,
+					page_width=1280,
+					page_height=720,
+					scroll_x=0,
+					scroll_y=0,
+					pixels_above=0,
+					pixels_below=0,
+					pixels_left=0,
+					pixels_right=0,
+				),
+				pixels_above=0,
+				pixels_below=0,
+				browser_errors=[str(e)],
+				is_pdf_viewer=False,
+				recent_events=None,
+				pending_network_requests=[],  # Error state has no pending requests
+				pagination_buttons=[],  # Error state has no pagination
+				closed_popup_messages=self.browser_session._closed_popup_messages.copy()
+				if hasattr(self, 'browser_session') and self.browser_session is not None
+				else [],
+			)
+
+	@time_execution_async('build_dom_tree_without_highlights')
+	@observe_debug(ignore_input=True, ignore_output=True, name='build_dom_tree_without_highlights')
+	async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:
+		"""Build DOM tree without injecting JavaScript highlights (for parallel execution)."""
+		try:
+			self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: STARTING DOM tree build')
+
+			# Create or reuse DOM service
+			if self._dom_service is None:
+				self._dom_service = DomService(
+					browser_session=self.browser_session,
+					logger=self.logger,
+					cross_origin_iframes=self.browser_session.browser_profile.cross_origin_iframes,
+					paint_order_filtering=self.browser_session.browser_profile.paint_order_filtering,
+					max_iframes=self.browser_session.browser_profile.max_iframes,
+					max_iframe_depth=self.browser_session.browser_profile.max_iframe_depth,
+				)
+
+			# Get serialized DOM tree using the service
+			self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Calling DomService.get_serialized_dom_tree...')
+			start = time.time()
+			self.current_dom_state, self.enhanced_dom_tree, timing_info = await self._dom_service.get_serialized_dom_tree(
+				previous_cached_state=previous_state,
+			)
+			end = time.time()
+			total_time_ms = (end - start) * 1000
+			self.logger.debug(
+				'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ DomService.get_serialized_dom_tree completed'
+			)
+
+			# Build hierarchical timing breakdown as single multi-line string
+			timing_lines = [f'⏱️ Total DOM tree time: {total_time_ms:.2f}ms', '📊 Timing breakdown:']
+
+			# get_all_trees breakdown
+			get_all_trees_ms = timing_info.get('get_all_trees_total_ms', 0)
+			if get_all_trees_ms > 0:
+				timing_lines.append(f'  ├─ get_all_trees: {get_all_trees_ms:.2f}ms')
+				iframe_scroll_ms = timing_info.get('iframe_scroll_detection_ms', 0)
+				cdp_parallel_ms = timing_info.get('cdp_parallel_calls_ms', 0)
+				snapshot_proc_ms = timing_info.get('snapshot_processing_ms', 0)
+				if iframe_scroll_ms > 0.01:
+					timing_lines.append(f'  │  ├─ iframe_scroll_detection: {iframe_scroll_ms:.2f}ms')
+				if cdp_parallel_ms > 0.01:
+					timing_lines.append(f'  │  ├─ cdp_parallel_calls: {cdp_parallel_ms:.2f}ms')
+				if snapshot_proc_ms > 0.01:
+					timing_lines.append(f'  │  └─ snapshot_processing: {snapshot_proc_ms:.2f}ms')
+
+			# build_ax_lookup
+			build_ax_ms = timing_info.get('build_ax_lookup_ms', 0)
+			if build_ax_ms > 0.01:
+				timing_lines.append(f'  ├─ build_ax_lookup: {build_ax_ms:.2f}ms')
+
+			# build_snapshot_lookup
+			build_snapshot_ms = timing_info.get('build_snapshot_lookup_ms', 0)
+			if build_snapshot_ms > 0.01:
+				timing_lines.append(f'  ├─ build_snapshot_lookup: {build_snapshot_ms:.2f}ms')
+
+			# construct_enhanced_tree
+			construct_tree_ms = timing_info.get('construct_enhanced_tree_ms', 0)
+			if construct_tree_ms > 0.01:
+				timing_lines.append(f'  ├─ construct_enhanced_tree: {construct_tree_ms:.2f}ms')
+
+			# serialize_accessible_elements breakdown
+			serialize_total_ms = timing_info.get('serialize_accessible_elements_total_ms', 0)
+			if serialize_total_ms > 0.01:
+				timing_lines.append(f'  ├─ serialize_accessible_elements: {serialize_total_ms:.2f}ms')
+				create_simp_ms = timing_info.get('create_simplified_tree_ms', 0)
+				paint_order_ms = timing_info.get('calculate_paint_order_ms', 0)
+				optimize_ms = timing_info.get('optimize_tree_ms', 0)
+				bbox_ms = timing_info.get('bbox_filtering_ms', 0)
+				assign_idx_ms = timing_info.get('assign_interactive_indices_ms', 0)
+				clickable_ms = timing_info.get('clickable_detection_time_ms', 0)
+
+				if create_simp_ms > 0.01:
+					timing_lines.append(f'  │  ├─ create_simplified_tree: {create_simp_ms:.2f}ms')
+					if clickable_ms > 0.01:
+						timing_lines.append(f'  │  │  └─ clickable_detection: {clickable_ms:.2f}ms')
+				if paint_order_ms > 0.01:
+					timing_lines.append(f'  │  ├─ calculate_paint_order: {paint_order_ms:.2f}ms')
+				if optimize_ms > 0.01:
+					timing_lines.append(f'  │  ├─ optimize_tree: {optimize_ms:.2f}ms')
+				if bbox_ms > 0.01:
+					timing_lines.append(f'  │  ├─ bbox_filtering: {bbox_ms:.2f}ms')
+				if assign_idx_ms > 0.01:
+					timing_lines.append(f'  │  └─ assign_interactive_indices: {assign_idx_ms:.2f}ms')
+
+			# Overheads
+			get_dom_overhead_ms = timing_info.get('get_dom_tree_overhead_ms', 0)
+			serialize_overhead_ms = timing_info.get('serialization_overhead_ms', 0)
+			get_serialized_overhead_ms = timing_info.get('get_serialized_dom_tree_overhead_ms', 0)
+
+			if get_dom_overhead_ms > 0.1:
+				timing_lines.append(f'  ├─ get_dom_tree_overhead: {get_dom_overhead_ms:.2f}ms')
+			if serialize_overhead_ms > 0.1:
+				timing_lines.append(f'  ├─ serialization_overhead: {serialize_overhead_ms:.2f}ms')
+			if get_serialized_overhead_ms > 0.1:
+				timing_lines.append(f'  └─ get_serialized_dom_tree_overhead: {get_serialized_overhead_ms:.2f}ms')
+
+			# Calculate total tracked time for validation
+			main_operations_ms = (
+				get_all_trees_ms
+				+ build_ax_ms
+				+ build_snapshot_ms
+				+ construct_tree_ms
+				+ serialize_total_ms
+				+ get_dom_overhead_ms
+				+ serialize_overhead_ms
+				+ get_serialized_overhead_ms
+			)
+			untracked_time_ms = total_time_ms - main_operations_ms
+
+			if untracked_time_ms > 1.0:  # Only log if significant
+				timing_lines.append(f'  ⚠️  untracked_time: {untracked_time_ms:.2f}ms')
+
+			# Single log call with all timing info
+			self.logger.debug('\n'.join(timing_lines))
+
+			# Update selector map for other watchdogs
+			self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Updating selector maps...')
+			self.selector_map = self.current_dom_state.selector_map
+			# Update BrowserSession's cached selector map
+			if self.browser_session:
+				self.browser_session.update_cached_selector_map(self.selector_map)
+			self.logger.debug(
+				f'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ Selector maps updated, {len(self.selector_map)} elements'
+			)
+
+			# Skip JavaScript highlighting injection - Python highlighting will be applied later
+			self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ COMPLETED DOM tree build (no JS highlights)')
+			return self.current_dom_state
+
+		except Exception as e:
+			self.logger.error(f'Failed to build DOM tree without highlights: {e}')
+			self.event_bus.dispatch(
+				BrowserErrorEvent(
+					error_type='DOMBuildFailed',
+					message=str(e),
+				)
+			)
+			raise
+
+	@time_execution_async('capture_clean_screenshot')
+	@observe_debug(ignore_input=True, ignore_output=True, name='capture_clean_screenshot')
+	async def _capture_clean_screenshot(self) -> str:
+		"""Capture a clean screenshot without JavaScript highlights."""
+		try:
+			self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: Capturing clean screenshot...')
+
+			await self.browser_session.get_or_create_cdp_session(target_id=self.browser_session.agent_focus_target_id, focus=True)
+
+			# Check if handler is registered
+			handlers = self.event_bus.handlers.get('ScreenshotEvent', [])
+			handler_names = [getattr(h, '__name__', str(h)) for h in handlers]
+			self.logger.debug(f'📸 ScreenshotEvent handlers registered: {len(handlers)} - {handler_names}')
+
+			screenshot_event = self.event_bus.dispatch(ScreenshotEvent(full_page=False))
+			self.logger.debug('📸 Dispatched ScreenshotEvent, waiting for event to complete...')
+
+			# Wait for the event itself to complete (this waits for all handlers)
+			await screenshot_event
+
+			# Get the single handler result
+			screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=True)
+			if screenshot_b64 is None:
+				raise RuntimeError('Screenshot handler returned None')
+			self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: ✅ Clean screenshot captured successfully')
+			return str(screenshot_b64)
+
+		except TimeoutError:
+			self.logger.warning('📸 Clean screenshot timed out after 6 seconds - no handler registered or slow page?')
+			raise
+		except Exception as e:
+			self.logger.warning(f'📸 Clean screenshot failed: {type(e).__name__}: {e}')
+			raise
+
+	def _detect_pagination_buttons(self, selector_map: dict[int, EnhancedDOMTreeNode]) -> list['PaginationButton']:
+		"""Detect pagination buttons from the DOM selector map.
+
+		Args:
+			selector_map: Dictionary mapping element indices to DOM tree nodes
+
+		Returns:
+			List of PaginationButton instances found in the DOM
+		"""
+		from browser_use.browser.views import PaginationButton
+
+		pagination_buttons_data = []
+		try:
+			self.logger.debug('🔍 DOMWatchdog._detect_pagination_buttons: Detecting pagination buttons...')
+			pagination_buttons_raw = DomService.detect_pagination_buttons(selector_map)
+			# Convert to PaginationButton instances
+			pagination_buttons_data = [
+				PaginationButton(
+					button_type=btn['button_type'],  # type: ignore
+					backend_node_id=btn['backend_node_id'],  # type: ignore
+					text=btn['text'],  # type: ignore
+					selector=btn['selector'],  # type: ignore
+					is_disabled=btn['is_disabled'],  # type: ignore
+				)
+				for btn in pagination_buttons_raw
+			]
+			if pagination_buttons_data:
+				self.logger.debug(
+					f'🔍 DOMWatchdog._detect_pagination_buttons: Found {len(pagination_buttons_data)} pagination buttons'
+				)
+		except Exception as e:
+			self.logger.warning(f'🔍 DOMWatchdog._detect_pagination_buttons: Pagination detection failed: {e}')
+
+		return pagination_buttons_data
+
+	async def _get_page_info(self) -> 'PageInfo':
+		"""Get comprehensive page information using a single CDP call.
+
+		TODO: should we make this an event as well?
+
+		Returns:
+			PageInfo with all viewport, page dimensions, and scroll information
+		"""
+
+		from browser_use.browser.views import PageInfo
+
+		# get_or_create_cdp_session() handles focus validation automatically
+		cdp_session = await self.browser_session.get_or_create_cdp_session(
+			target_id=self.browser_session.agent_focus_target_id, focus=True
+		)
+
+		# Get layout metrics which includes all the information we need
+		metrics = await asyncio.wait_for(
+			cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id), timeout=10.0
+		)
+
+		# Extract different viewport types
+		layout_viewport = metrics.get('layoutViewport', {})
+		visual_viewport = metrics.get('visualViewport', {})
+		css_visual_viewport = metrics.get('cssVisualViewport', {})
+		css_layout_viewport = metrics.get('cssLayoutViewport', {})
+		content_size = metrics.get('contentSize', {})
+
+		# Calculate device pixel ratio to convert between device pixels and CSS pixels
+		# This matches the approach in dom/service.py _get_viewport_ratio method
+		css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))
+		device_width = visual_viewport.get('clientWidth', css_width)
+		device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
+
+		# For viewport dimensions, use CSS pixels (what JavaScript sees)
+		# Prioritize CSS layout viewport, then fall back to layout viewport
+		viewport_width = int(css_layout_viewport.get('clientWidth') or layout_viewport.get('clientWidth', 1280))
+		viewport_height = int(css_layout_viewport.get('clientHeight') or layout_viewport.get('clientHeight', 720))
+
+		# For total page dimensions, content size is typically in device pixels, so convert to CSS pixels
+		# by dividing by device pixel ratio
+		raw_page_width = content_size.get('width', viewport_width * device_pixel_ratio)
+		raw_page_height = content_size.get('height', viewport_height * device_pixel_ratio)
+		page_width = int(raw_page_width / device_pixel_ratio)
+		page_height = int(raw_page_height / device_pixel_ratio)
+
+		# For scroll position, use CSS visual viewport if available, otherwise CSS layout viewport
+		# These should already be in CSS pixels
+		scroll_x = int(css_visual_viewport.get('pageX') or css_layout_viewport.get('pageX', 0))
+		scroll_y = int(css_visual_viewport.get('pageY') or css_layout_viewport.get('pageY', 0))
+
+		# Calculate scroll information - pixels that are above/below/left/right of current viewport
+		pixels_above = scroll_y
+		pixels_below = max(0, page_height - viewport_height - scroll_y)
+		pixels_left = scroll_x
+		pixels_right = max(0, page_width - viewport_width - scroll_x)
+
+		page_info = PageInfo(
+			viewport_width=viewport_width,
+			viewport_height=viewport_height,
+			page_width=page_width,
+			page_height=page_height,
+			scroll_x=scroll_x,
+			scroll_y=scroll_y,
+			pixels_above=pixels_above,
+			pixels_below=pixels_below,
+			pixels_left=pixels_left,
+			pixels_right=pixels_right,
+		)
+
+		return page_info
+
+	# ========== Public Helper Methods ==========
+
+	async def get_element_by_index(self, index: int) -> EnhancedDOMTreeNode | None:
+		"""Get DOM element by index from cached selector map.
+
+		Builds DOM if not cached.
+
+		Returns:
+			EnhancedDOMTreeNode or None if index not found
+		"""
+		if not self.selector_map:
+			# Build DOM if not cached
+			await self._build_dom_tree_without_highlights()
+
+		return self.selector_map.get(index) if self.selector_map else None
+
+	def clear_cache(self) -> None:
+		"""Clear cached DOM state to force rebuild on next access."""
+		self.selector_map = None
+		self.current_dom_state = None
+		self.enhanced_dom_tree = None
+		# Keep the DOM service instance to reuse its CDP client connection
+
+	def is_file_input(self, element: EnhancedDOMTreeNode) -> bool:
+		"""Check if element is a file input."""
+		return element.node_name.upper() == 'INPUT' and element.attributes.get('type', '').lower() == 'file'
+
+	@staticmethod
+	def is_element_visible_according_to_all_parents(node: EnhancedDOMTreeNode, html_frames: list[EnhancedDOMTreeNode]) -> bool:
+		"""Check if the element is visible according to all its parent HTML frames.
+
+		Delegates to the DomService static method.
+		"""
+		return DomService.is_element_visible_according_to_all_parents(node, html_frames)
+
+	async def __aexit__(self, exc_type, exc_value, traceback):
+		"""Clean up DOM service on exit."""
+		if self._dom_service:
+			await self._dom_service.__aexit__(exc_type, exc_value, traceback)
+			self._dom_service = None
+
+	def __del__(self):
+		"""Clean up DOM service on deletion."""
+		super().__del__()
+		# DOM service will clean up its own CDP client
+		self._dom_service = None

browser_use/browser/watchdogs/har_recording_watchdog.py

@@ -0,0 +1,779 @@
+"""HAR Recording Watchdog for Browser-Use sessions.
+
+Captures HTTPS network activity via CDP Network domain and writes a HAR 1.2
+file on browser shutdown. Respects `record_har_content` (omit/embed/attach)
+and `record_har_mode` (full/minimal).
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+from dataclasses import dataclass, field
+from importlib import metadata as importlib_metadata
+from pathlib import Path
+from typing import ClassVar
+
+from bubus import BaseEvent
+from cdp_use.cdp.network.events import (
+	DataReceivedEvent,
+	LoadingFailedEvent,
+	LoadingFinishedEvent,
+	RequestWillBeSentEvent,
+	ResponseReceivedEvent,
+)
+from cdp_use.cdp.page.events import FrameNavigatedEvent, LifecycleEventEvent
+
+from browser_use.browser.events import BrowserConnectedEvent, BrowserStopEvent
+from browser_use.browser.watchdog_base import BaseWatchdog
+
+
+@dataclass
+class _HarContent:
+	mime_type: str | None = None
+	text_b64: str | None = None  # for embed
+	file_rel: str | None = None  # for attach
+	size: int | None = None
+
+
+@dataclass
+class _HarEntryBuilder:
+	request_id: str = ''
+	frame_id: str | None = None
+	document_url: str | None = None
+	url: str | None = None
+	method: str | None = None
+	request_headers: dict = field(default_factory=dict)
+	request_body: bytes | None = None
+	post_data: str | None = None  # CDP postData field
+	status: int | None = None
+	status_text: str | None = None
+	response_headers: dict = field(default_factory=dict)
+	mime_type: str | None = None
+	encoded_data: bytearray = field(default_factory=bytearray)
+	failed: bool = False
+	# timing info (CDP timestamps are monotonic seconds); wallTime is epoch seconds
+	ts_request: float | None = None
+	wall_time_request: float | None = None
+	ts_response: float | None = None
+	ts_finished: float | None = None
+	encoded_data_length: int | None = None
+	response_body: bytes | None = None
+	content_length: int | None = None  # From Content-Length header
+	protocol: str | None = None
+	server_ip_address: str | None = None
+	server_port: int | None = None
+	security_details: dict | None = None
+	transfer_size: int | None = None
+
+
+def _is_https(url: str | None) -> bool:
+	return bool(url and url.lower().startswith('https://'))
+
+
+def _origin(url: str) -> str:
+	# Very small origin extractor, assumes https URLs
+	# https://host[:port]/...
+	if not url:
+		return ''
+	try:
+		without_scheme = url.split('://', 1)[1]
+		host_port = without_scheme.split('/', 1)[0]
+		return f'https://{host_port}'
+	except Exception:
+		return ''
+
+
+def _mime_to_extension(mime_type: str | None) -> str:
+	"""Map MIME type to file extension, matching Playwright's behavior."""
+	if not mime_type:
+		return 'bin'
+
+	mime_lower = mime_type.lower().split(';')[0].strip()
+
+	# Common MIME type to extension mapping
+	mime_map = {
+		'text/html': 'html',
+		'text/css': 'css',
+		'text/javascript': 'js',
+		'application/javascript': 'js',
+		'application/x-javascript': 'js',
+		'application/json': 'json',
+		'application/xml': 'xml',
+		'text/xml': 'xml',
+		'text/plain': 'txt',
+		'image/png': 'png',
+		'image/jpeg': 'jpg',
+		'image/jpg': 'jpg',
+		'image/gif': 'gif',
+		'image/webp': 'webp',
+		'image/svg+xml': 'svg',
+		'image/x-icon': 'ico',
+		'font/woff': 'woff',
+		'font/woff2': 'woff2',
+		'application/font-woff': 'woff',
+		'application/font-woff2': 'woff2',
+		'application/x-font-woff': 'woff',
+		'application/x-font-woff2': 'woff2',
+		'font/ttf': 'ttf',
+		'application/x-font-ttf': 'ttf',
+		'font/otf': 'otf',
+		'application/x-font-opentype': 'otf',
+		'application/pdf': 'pdf',
+		'application/zip': 'zip',
+		'application/x-zip-compressed': 'zip',
+		'video/mp4': 'mp4',
+		'video/webm': 'webm',
+		'audio/mpeg': 'mp3',
+		'audio/mp3': 'mp3',
+		'audio/wav': 'wav',
+		'audio/ogg': 'ogg',
+	}
+
+	return mime_map.get(mime_lower, 'bin')
+
+
+def _generate_har_filename(content: bytes, mime_type: str | None) -> str:
+	"""Generate a hash-based filename for HAR attach mode, matching Playwright's format."""
+	content_hash = hashlib.sha1(content).hexdigest()
+	extension = _mime_to_extension(mime_type)
+	return f'{content_hash}.{extension}'
+
+
+class HarRecordingWatchdog(BaseWatchdog):
+	"""Collects HTTPS requests/responses and writes a HAR 1.2 file on stop."""
+
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent]
+	EMITS: ClassVar[list[type[BaseEvent]]] = []
+
+	def __init__(self, *args, **kwargs) -> None:
+		super().__init__(*args, **kwargs)
+		self._enabled: bool = False
+		self._entries: dict[str, _HarEntryBuilder] = {}
+		self._top_level_pages: dict[
+			str, dict
+		] = {}  # frameId -> {url, title, startedDateTime, monotonic_start, onContentLoad, onLoad}
+
+	async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
+		profile = self.browser_session.browser_profile
+		if not profile.record_har_path:
+			return
+
+		# Normalize config
+		self._content_mode = (profile.record_har_content or 'embed').lower()
+		self._mode = (profile.record_har_mode or 'full').lower()
+		self._har_path = Path(str(profile.record_har_path)).expanduser().resolve()
+		self._har_dir = self._har_path.parent
+		self._har_dir.mkdir(parents=True, exist_ok=True)
+
+		try:
+			# Enable Network and Page domains for events
+			cdp_session = await self.browser_session.get_or_create_cdp_session()
+			await cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)
+			await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
+
+			# Query browser version for HAR log.browser
+			try:
+				version_info = await self.browser_session.cdp_client.send.Browser.getVersion()
+				self._browser_name = version_info.get('product') or 'Chromium'
+				self._browser_version = version_info.get('jsVersion') or ''
+			except Exception:
+				self._browser_name = 'Chromium'
+				self._browser_version = ''
+
+			cdp = self.browser_session.cdp_client.register
+			cdp.Network.requestWillBeSent(self._on_request_will_be_sent)
+			cdp.Network.responseReceived(self._on_response_received)
+			cdp.Network.dataReceived(self._on_data_received)
+			cdp.Network.loadingFinished(self._on_loading_finished)
+			cdp.Network.loadingFailed(self._on_loading_failed)
+			cdp.Page.lifecycleEvent(self._on_lifecycle_event)
+			cdp.Page.frameNavigated(self._on_frame_navigated)
+
+			self._enabled = True
+			self.logger.info(f'📊 Starting HAR recording to {self._har_path}')
+		except Exception as e:
+			self.logger.warning(f'Failed to enable HAR recording: {e}')
+			self._enabled = False
+
+	async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
+		if not self._enabled:
+			return
+		try:
+			await self._write_har()
+			self.logger.info(f'📊 HAR file saved: {self._har_path}')
+		except Exception as e:
+			self.logger.warning(f'Failed to write HAR: {e}')
+
+	# =============== CDP Event Handlers (sync) ==================
+	def _on_request_will_be_sent(self, params: RequestWillBeSentEvent, session_id: str | None) -> None:
+		try:
+			req = params.get('request', {}) if hasattr(params, 'get') else getattr(params, 'request', {})
+			url = req.get('url') if isinstance(req, dict) else getattr(req, 'url', None)
+			if not _is_https(url):
+				return  # HTTPS-only requirement (only HTTPS requests are recorded for now)
+
+			request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
+			if not request_id:
+				return
+
+			entry = self._entries.setdefault(request_id, _HarEntryBuilder(request_id=request_id))
+			entry.url = url
+			entry.method = req.get('method') if isinstance(req, dict) else getattr(req, 'method', None)
+			entry.post_data = req.get('postData') if isinstance(req, dict) else getattr(req, 'postData', None)
+
+			# Convert headers to plain dict, handling various formats
+			headers_raw = req.get('headers') if isinstance(req, dict) else getattr(req, 'headers', None)
+			if headers_raw is None:
+				entry.request_headers = {}
+			elif isinstance(headers_raw, dict):
+				entry.request_headers = {k.lower(): str(v) for k, v in headers_raw.items()}
+			elif isinstance(headers_raw, list):
+				entry.request_headers = {
+					h.get('name', '').lower(): str(h.get('value') or '') for h in headers_raw if isinstance(h, dict)
+				}
+			else:
+				# Handle Headers type or other formats - convert to dict
+				try:
+					headers_dict = dict(headers_raw) if hasattr(headers_raw, '__iter__') else {}
+					entry.request_headers = {k.lower(): str(v) for k, v in headers_dict.items()}
+				except Exception:
+					entry.request_headers = {}
+
+			entry.frame_id = params.get('frameId') if hasattr(params, 'get') else getattr(params, 'frameId', None)
+			entry.document_url = (
+				params.get('documentURL')
+				if hasattr(params, 'get')
+				else getattr(params, 'documentURL', None) or entry.document_url
+			)
+
+			# Timing anchors
+			entry.ts_request = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
+			entry.wall_time_request = params.get('wallTime') if hasattr(params, 'get') else getattr(params, 'wallTime', None)
+
+			# Track top-level navigations for page context
+			req_type = params.get('type') if hasattr(params, 'get') else getattr(params, 'type', None)
+			is_same_doc = (
+				params.get('isSameDocument', False) if hasattr(params, 'get') else getattr(params, 'isSameDocument', False)
+			)
+			if req_type == 'Document' and not is_same_doc:
+				# best-effort: consider as navigation
+				if entry.frame_id and url:
+					if entry.frame_id not in self._top_level_pages:
+						self._top_level_pages[entry.frame_id] = {
+							'url': str(url),
+							'title': str(url),  # Default to URL, will be updated from DOM
+							'startedDateTime': entry.wall_time_request,
+							'monotonic_start': entry.ts_request,  # Track monotonic start time for timing calculations
+							'onContentLoad': -1,
+							'onLoad': -1,
+						}
+					else:
+						# Update startedDateTime and monotonic_start if this is earlier
+						page_info = self._top_level_pages[entry.frame_id]
+						if entry.wall_time_request and (
+							page_info['startedDateTime'] is None or entry.wall_time_request < page_info['startedDateTime']
+						):
+							page_info['startedDateTime'] = entry.wall_time_request
+							page_info['monotonic_start'] = entry.ts_request
+		except Exception as e:
+			self.logger.debug(f'requestWillBeSent handling error: {e}')
+
+	def _on_response_received(self, params: ResponseReceivedEvent, session_id: str | None) -> None:
+		try:
+			request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
+			if not request_id or request_id not in self._entries:
+				return
+			response = params.get('response', {}) if hasattr(params, 'get') else getattr(params, 'response', {})
+			entry = self._entries[request_id]
+			entry.status = response.get('status') if isinstance(response, dict) else getattr(response, 'status', None)
+			entry.status_text = (
+				response.get('statusText') if isinstance(response, dict) else getattr(response, 'statusText', None)
+			)
+
+			# Extract Content-Length for compression calculation (before converting headers)
+			headers_raw = response.get('headers') if isinstance(response, dict) else getattr(response, 'headers', None)
+			if headers_raw:
+				if isinstance(headers_raw, dict):
+					cl_str = headers_raw.get('content-length') or headers_raw.get('Content-Length')
+				elif isinstance(headers_raw, list):
+					cl_header = next(
+						(h for h in headers_raw if isinstance(h, dict) and h.get('name', '').lower() == 'content-length'), None
+					)
+					cl_str = cl_header.get('value') if cl_header else None
+				else:
+					cl_str = None
+				if cl_str:
+					try:
+						entry.content_length = int(cl_str)
+					except Exception:
+						pass
+
+			# Convert headers to plain dict, handling various formats
+			if headers_raw is None:
+				entry.response_headers = {}
+			elif isinstance(headers_raw, dict):
+				entry.response_headers = {k.lower(): str(v) for k, v in headers_raw.items()}
+			elif isinstance(headers_raw, list):
+				entry.response_headers = {
+					h.get('name', '').lower(): str(h.get('value') or '') for h in headers_raw if isinstance(h, dict)
+				}
+			else:
+				# Handle Headers type or other formats - convert to dict
+				try:
+					headers_dict = dict(headers_raw) if hasattr(headers_raw, '__iter__') else {}
+					entry.response_headers = {k.lower(): str(v) for k, v in headers_dict.items()}
+				except Exception:
+					entry.response_headers = {}
+
+			entry.mime_type = response.get('mimeType') if isinstance(response, dict) else getattr(response, 'mimeType', None)
+			entry.ts_response = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
+
+			protocol_raw = response.get('protocol') if isinstance(response, dict) else getattr(response, 'protocol', None)
+			if protocol_raw:
+				protocol_lower = str(protocol_raw).lower()
+				if protocol_lower == 'h2' or protocol_lower.startswith('http/2'):
+					entry.protocol = 'HTTP/2.0'
+				elif protocol_lower.startswith('http/1.1'):
+					entry.protocol = 'HTTP/1.1'
+				elif protocol_lower.startswith('http/1.0'):
+					entry.protocol = 'HTTP/1.0'
+				else:
+					entry.protocol = str(protocol_raw).upper()
+
+			entry.server_ip_address = (
+				response.get('remoteIPAddress') if isinstance(response, dict) else getattr(response, 'remoteIPAddress', None)
+			)
+			server_port_raw = response.get('remotePort') if isinstance(response, dict) else getattr(response, 'remotePort', None)
+			if server_port_raw is not None:
+				try:
+					entry.server_port = int(server_port_raw)
+				except (ValueError, TypeError):
+					pass
+
+			# Extract security details (TLS info)
+			security_details_raw = (
+				response.get('securityDetails') if isinstance(response, dict) else getattr(response, 'securityDetails', None)
+			)
+			if security_details_raw:
+				try:
+					entry.security_details = dict(security_details_raw)
+				except Exception:
+					pass
+		except Exception as e:
+			self.logger.debug(f'responseReceived handling error: {e}')
+
+	def _on_data_received(self, params: DataReceivedEvent, session_id: str | None) -> None:
+		try:
+			request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
+			if not request_id or request_id not in self._entries:
+				return
+			data = params.get('data') if hasattr(params, 'get') else getattr(params, 'data', None)
+			if isinstance(data, str):
+				try:
+					self._entries[request_id].encoded_data.extend(data.encode('latin1'))
+				except Exception:
+					pass
+		except Exception as e:
+			self.logger.debug(f'dataReceived handling error: {e}')
+
+	def _on_loading_finished(self, params: LoadingFinishedEvent, session_id: str | None) -> None:
+		try:
+			request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
+			if not request_id or request_id not in self._entries:
+				return
+			entry = self._entries[request_id]
+			entry.ts_finished = params.get('timestamp')
+			# Fetch response body via CDP as dataReceived may be incomplete
+			import asyncio as _asyncio
+
+			async def _fetch_body(self_ref, req_id, sess_id):
+				try:
+					resp = await self_ref.browser_session.cdp_client.send.Network.getResponseBody(
+						params={'requestId': req_id}, session_id=sess_id
+					)
+					data = resp.get('body', b'')
+					if resp.get('base64Encoded'):
+						import base64 as _b64
+
+						data = _b64.b64decode(data)
+					else:
+						# Ensure data is bytes even if CDP returns a string
+						if isinstance(data, str):
+							data = data.encode('utf-8', errors='replace')
+					# Ensure we always have bytes
+					if not isinstance(data, bytes):
+						data = bytes(data) if data else b''
+					entry.response_body = data
+				except Exception:
+					pass
+
+			# Always schedule the response body fetch task
+			_asyncio.create_task(_fetch_body(self, request_id, session_id))
+
+			encoded_length = (
+				params.get('encodedDataLength') if hasattr(params, 'get') else getattr(params, 'encodedDataLength', None)
+			)
+			if encoded_length is not None:
+				try:
+					entry.encoded_data_length = int(encoded_length)
+					entry.transfer_size = entry.encoded_data_length
+				except Exception:
+					entry.encoded_data_length = None
+		except Exception as e:
+			self.logger.debug(f'loadingFinished handling error: {e}')
+
+	def _on_loading_failed(self, params: LoadingFailedEvent, session_id: str | None) -> None:
+		try:
+			request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
+			if request_id and request_id in self._entries:
+				self._entries[request_id].failed = True
+		except Exception as e:
+			self.logger.debug(f'loadingFailed handling error: {e}')
+
+	# ===================== HAR Writing ==========================
+	def _on_lifecycle_event(self, params: LifecycleEventEvent, session_id: str | None) -> None:
+		"""Handle Page.lifecycleEvent for tracking page load timings."""
+		try:
+			frame_id = params.get('frameId') if hasattr(params, 'get') else getattr(params, 'frameId', None)
+			name = params.get('name') if hasattr(params, 'get') else getattr(params, 'name', None)
+			timestamp = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
+
+			if not frame_id or not name or frame_id not in self._top_level_pages:
+				return
+
+			page_info = self._top_level_pages[frame_id]
+			# Use monotonic_start instead of startedDateTime (wall-clock) for timing calculations
+			monotonic_start = page_info.get('monotonic_start')
+
+			if name == 'DOMContentLoaded' and monotonic_start is not None:
+				# Calculate milliseconds since page start using monotonic timestamps
+				try:
+					elapsed_ms = int(round((timestamp - monotonic_start) * 1000))
+					page_info['onContentLoad'] = max(0, elapsed_ms)
+				except Exception:
+					pass
+			elif name == 'load' and monotonic_start is not None:
+				try:
+					elapsed_ms = int(round((timestamp - monotonic_start) * 1000))
+					page_info['onLoad'] = max(0, elapsed_ms)
+				except Exception:
+					pass
+		except Exception as e:
+			self.logger.debug(f'lifecycleEvent handling error: {e}')
+
+	def _on_frame_navigated(self, params: FrameNavigatedEvent, session_id: str | None) -> None:
+		"""Handle Page.frameNavigated to update page title from DOM."""
+		try:
+			frame = params.get('frame') if hasattr(params, 'get') else getattr(params, 'frame', None)
+			if not frame:
+				return
+
+			frame_id = frame.get('id') if isinstance(frame, dict) else getattr(frame, 'id', None)
+			title = (
+				frame.get('name') or frame.get('url')
+				if isinstance(frame, dict)
+				else getattr(frame, 'name', None) or getattr(frame, 'url', None)
+			)
+
+			if frame_id and frame_id in self._top_level_pages:
+				# Try to get actual page title via Runtime.evaluate if possible
+				# For now, use frame name or URL as fallback
+				if title:
+					self._top_level_pages[frame_id]['title'] = str(title)
+		except Exception as e:
+			self.logger.debug(f'frameNavigated handling error: {e}')
+
+	# ===================== HAR Writing ==========================
+	async def _write_har(self) -> None:
+		# Filter by mode and HTTPS already respected at collection time
+		entries = [e for e in self._entries.values() if self._include_entry(e)]
+
+		har_entries = []
+		sidecar_dir: Path | None = None
+		if self._content_mode == 'attach':
+			sidecar_dir = self._har_dir / f'{self._har_path.stem}_har_parts'
+			sidecar_dir.mkdir(parents=True, exist_ok=True)
+
+		for e in entries:
+			content_obj: dict = {'mimeType': e.mime_type or ''}
+
+			# Get body data, preferring response_body over encoded_data
+			if e.response_body is not None:
+				body_data = e.response_body
+			else:
+				body_data = e.encoded_data
+
+			# Defensive conversion: ensure body_data is always bytes
+			if isinstance(body_data, str):
+				body_bytes = body_data.encode('utf-8', errors='replace')
+			elif isinstance(body_data, bytearray):
+				body_bytes = bytes(body_data)
+			elif isinstance(body_data, bytes):
+				body_bytes = body_data
+			else:
+				# Fallback: try to convert to bytes
+				try:
+					body_bytes = bytes(body_data) if body_data else b''
+				except (TypeError, ValueError):
+					body_bytes = b''
+
+			content_size = len(body_bytes)
+
+			# Calculate compression (bytes saved by compression)
+			compression = 0
+			if e.content_length is not None and e.encoded_data_length is not None:
+				compression = max(0, e.content_length - e.encoded_data_length)
+
+			if self._content_mode == 'embed' and content_size > 0:
+				# Prefer plain text; fallback to base64 only if decoding fails
+				try:
+					text_decoded = body_bytes.decode('utf-8')
+					content_obj['text'] = text_decoded
+					content_obj['size'] = content_size
+					content_obj['compression'] = compression
+				except UnicodeDecodeError:
+					content_obj['text'] = base64.b64encode(body_bytes).decode('ascii')
+					content_obj['encoding'] = 'base64'
+					content_obj['size'] = content_size
+					content_obj['compression'] = compression
+			elif self._content_mode == 'attach' and content_size > 0 and sidecar_dir is not None:
+				filename = _generate_har_filename(body_bytes, e.mime_type)
+				(sidecar_dir / filename).write_bytes(body_bytes)
+				content_obj['_file'] = filename
+				content_obj['size'] = content_size
+				content_obj['compression'] = compression
+			else:
+				# omit or empty
+				content_obj['size'] = content_size
+				if content_size > 0:
+					content_obj['compression'] = compression
+
+			started_date_time, total_time_ms, timings = self._compute_timings(e)
+			req_headers_list = [{'name': k, 'value': str(v)} for k, v in (e.request_headers or {}).items()]
+			resp_headers_list = [{'name': k, 'value': str(v)} for k, v in (e.response_headers or {}).items()]
+			request_headers_size = self._calc_headers_size(e.method or 'GET', e.url or '', req_headers_list)
+			response_headers_size = self._calc_headers_size(None, None, resp_headers_list)
+			request_body_size = self._calc_request_body_size(e)
+			request_post_data = None
+			if e.post_data and self._content_mode != 'omit':
+				if self._content_mode == 'embed':
+					request_post_data = {'mimeType': e.request_headers.get('content-type', ''), 'text': e.post_data}
+				elif self._content_mode == 'attach' and sidecar_dir is not None:
+					post_data_bytes = e.post_data.encode('utf-8')
+					req_mime_type = e.request_headers.get('content-type', 'text/plain')
+					req_filename = _generate_har_filename(post_data_bytes, req_mime_type)
+					(sidecar_dir / req_filename).write_bytes(post_data_bytes)
+					request_post_data = {
+						'mimeType': req_mime_type,
+						'_file': req_filename,
+					}
+
+			http_version = e.protocol if e.protocol else 'HTTP/1.1'
+
+			response_body_size = e.transfer_size
+			if response_body_size is None:
+				response_body_size = e.encoded_data_length
+			if response_body_size is None:
+				response_body_size = content_size if content_size > 0 else -1
+
+			entry_dict = {
+				'startedDateTime': started_date_time,
+				'time': total_time_ms,
+				'request': {
+					'method': e.method or 'GET',
+					'url': e.url or '',
+					'httpVersion': http_version,
+					'headers': req_headers_list,
+					'queryString': [],
+					'cookies': [],
+					'headersSize': request_headers_size,
+					'bodySize': request_body_size,
+					'postData': request_post_data,
+				},
+				'response': {
+					'status': e.status or 0,
+					'statusText': e.status_text or '',
+					'httpVersion': http_version,
+					'headers': resp_headers_list,
+					'cookies': [],
+					'content': content_obj,
+					'redirectURL': '',
+					'headersSize': response_headers_size,
+					'bodySize': response_body_size,
+				},
+				'cache': {},
+				'timings': timings,
+				'pageref': self._page_ref_for_entry(e),
+			}
+
+			# Add security/TLS details if available
+			if e.server_ip_address:
+				entry_dict['serverIPAddress'] = e.server_ip_address
+			if e.server_port is not None:
+				entry_dict['_serverPort'] = e.server_port
+			if e.security_details:
+				# Filter to match Playwright's minimal security details set
+				security_filtered = {}
+				if 'protocol' in e.security_details:
+					security_filtered['protocol'] = e.security_details['protocol']
+				if 'subjectName' in e.security_details:
+					security_filtered['subjectName'] = e.security_details['subjectName']
+				if 'issuer' in e.security_details:
+					security_filtered['issuer'] = e.security_details['issuer']
+				if 'validFrom' in e.security_details:
+					security_filtered['validFrom'] = e.security_details['validFrom']
+				if 'validTo' in e.security_details:
+					security_filtered['validTo'] = e.security_details['validTo']
+				if security_filtered:
+					entry_dict['_securityDetails'] = security_filtered
+			if e.transfer_size is not None:
+				entry_dict['response']['_transferSize'] = e.transfer_size
+
+			har_entries.append(entry_dict)
+
+		# Try to include our library version in creator
+		try:
+			bu_version = importlib_metadata.version('browser-use')
+		except Exception:
+			# Fallback when running from source without installed package metadata
+			bu_version = 'dev'
+
+		har_obj = {
+			'log': {
+				'version': '1.2',
+				'creator': {'name': 'browser-use', 'version': bu_version},
+				'browser': {'name': self._browser_name, 'version': self._browser_version},
+				'pages': [
+					{
+						'id': f'page@{pid}',  # Use Playwright format: "page@{frame_id}"
+						'title': page_info.get('title', page_info.get('url', '')),
+						'startedDateTime': self._format_page_started_datetime(page_info.get('startedDateTime')),
+						'pageTimings': (
+							(lambda _ocl, _ol: ({k: v for k, v in (('onContentLoad', _ocl), ('onLoad', _ol)) if v is not None}))(
+								(page_info.get('onContentLoad') if page_info.get('onContentLoad', -1) >= 0 else None),
+								(page_info.get('onLoad') if page_info.get('onLoad', -1) >= 0 else None),
+							)
+						),
+					}
+					for pid, page_info in self._top_level_pages.items()
+				],
+				'entries': har_entries,
+			}
+		}
+
+		tmp_path = self._har_path.with_suffix(self._har_path.suffix + '.tmp')
+		# Write as bytes explicitly to avoid any text/binary mode confusion in different environments
+		tmp_path.write_bytes(json.dumps(har_obj, indent=2, ensure_ascii=False).encode('utf-8'))
+		tmp_path.replace(self._har_path)
+
+	def _format_page_started_datetime(self, timestamp: float | None) -> str:
+		"""Format page startedDateTime from timestamp."""
+		if timestamp is None:
+			return ''
+		try:
+			from datetime import datetime, timezone
+
+			return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat().replace('+00:00', 'Z')
+		except Exception:
+			return ''
+
+	def _page_ref_for_entry(self, e: _HarEntryBuilder) -> str | None:
+		# Use Playwright format: "page@{frame_id}" if frame_id is known
+		if e.frame_id and e.frame_id in self._top_level_pages:
+			return f'page@{e.frame_id}'
+		return None
+
+	def _include_entry(self, e: _HarEntryBuilder) -> bool:
+		if not _is_https(e.url):
+			return False
+		# Filter out favicon requests (matching Playwright behavior)
+		if e.url and '/favicon.ico' in e.url.lower():
+			return False
+		if getattr(self, '_mode', 'full') == 'full':
+			return True
+		# minimal: include main document and same-origin subresources
+		if e.frame_id and e.frame_id in self._top_level_pages:
+			page_info = self._top_level_pages[e.frame_id]
+			page_url = page_info.get('url') if isinstance(page_info, dict) else page_info
+			return _origin(e.url or '') == _origin(page_url or '')
+		return False
+
+	# ===================== Helpers ==============================
+	def _compute_timings(self, e: _HarEntryBuilder) -> tuple[str, int, dict]:
+		# startedDateTime from wall_time_request in ISO8601 Z
+		started = ''
+		try:
+			if e.wall_time_request is not None:
+				from datetime import datetime, timezone
+
+				started = datetime.fromtimestamp(e.wall_time_request, tz=timezone.utc).isoformat().replace('+00:00', 'Z')
+		except Exception:
+			started = ''
+
+		# Calculate timings - CDP doesn't always provide DNS/connect/SSL breakdown
+		# Default to 0 for unavailable timings, calculate what we can from timestamps
+		dns_ms = 0
+		connect_ms = 0
+		ssl_ms = 0
+		send_ms = 0
+		wait_ms = 0
+		receive_ms = 0
+
+		if e.ts_request is not None and e.ts_response is not None:
+			wait_ms = max(0, int(round((e.ts_response - e.ts_request) * 1000)))
+
+		if e.ts_response is not None and e.ts_finished is not None:
+			receive_ms = max(0, int(round((e.ts_finished - e.ts_response) * 1000)))
+
+		# Note: DNS, connect, and SSL timings would require additional CDP events or ResourceTiming API
+		# For now, we structure the timings dict to match Playwright format
+		# but leave DNS/connect/SSL as 0 since CDP doesn't provide this breakdown directly
+
+		total = dns_ms + connect_ms + ssl_ms + send_ms + wait_ms + receive_ms
+		return (
+			started,
+			total,
+			{
+				'dns': dns_ms,
+				'connect': connect_ms,
+				'ssl': ssl_ms,
+				'send': send_ms,
+				'wait': wait_ms,
+				'receive': receive_ms,
+			},
+		)
+
+	def _calc_headers_size(self, method: str | None, url: str | None, headers_list: list[dict]) -> int:
+		try:
+			# Approximate per RFC: sum of header lines + CRLF; include request/status line only for request
+			size = 0
+			if method and url:
+				# Use HTTP/1.1 request line approximation
+				size += len(f'{method} {url} HTTP/1.1\r\n'.encode('latin1'))
+			for h in headers_list:
+				size += len(f'{h.get("name", "")}: {h.get("value", "")}\r\n'.encode('latin1'))
+			size += len(b'\r\n')
+			return size
+		except Exception:
+			return -1
+
+	def _calc_request_body_size(self, e: _HarEntryBuilder) -> int:
+		# Try Content-Length header first; else post_data; else request_body; else 0 for GET/HEAD, -1 if unknown
+		try:
+			cl = None
+			if e.request_headers:
+				cl = e.request_headers.get('content-length') or e.request_headers.get('Content-Length')
+			if cl is not None:
+				return int(cl)
+			if e.post_data:
+				return len(e.post_data.encode('utf-8'))
+			if e.request_body is not None:
+				return len(e.request_body)
+			# GET/HEAD requests typically have no body
+			if e.method and e.method.upper() in ('GET', 'HEAD'):
+				return 0
+		except Exception:
+			pass
+		return -1

browser_use/browser/watchdogs/local_browser_watchdog.py

@@ -0,0 +1,506 @@
+"""Local browser watchdog for managing browser subprocess lifecycle."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar
+
+import psutil
+from bubus import BaseEvent
+from pydantic import PrivateAttr
+
+from browser_use.browser.events import (
+	BrowserKillEvent,
+	BrowserLaunchEvent,
+	BrowserLaunchResult,
+	BrowserStopEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+from browser_use.observability import observe_debug
+
+if TYPE_CHECKING:
+	from browser_use.browser.profile import BrowserChannel
+
+
+class LocalBrowserWatchdog(BaseWatchdog):
+	"""Manages local browser subprocess lifecycle."""
+
+	# Events this watchdog listens to
+	LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [
+		BrowserLaunchEvent,
+		BrowserKillEvent,
+		BrowserStopEvent,
+	]
+
+	# Events this watchdog emits
+	EMITS: ClassVar[list[type[BaseEvent[Any]]]] = []
+
+	# Private state for subprocess management
+	_subprocess: psutil.Process | None = PrivateAttr(default=None)
+	_owns_browser_resources: bool = PrivateAttr(default=True)
+	_temp_dirs_to_cleanup: list[Path] = PrivateAttr(default_factory=list)
+	_original_user_data_dir: str | None = PrivateAttr(default=None)
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='browser_launch_event')
+	async def on_BrowserLaunchEvent(self, event: BrowserLaunchEvent) -> BrowserLaunchResult:
+		"""Launch a local browser process."""
+
+		try:
+			self.logger.debug('[LocalBrowserWatchdog] Received BrowserLaunchEvent, launching local browser...')
+
+			# self.logger.debug('[LocalBrowserWatchdog] Calling _launch_browser...')
+			process, cdp_url = await self._launch_browser()
+			self._subprocess = process
+			# self.logger.debug(f'[LocalBrowserWatchdog] _launch_browser returned: process={process}, cdp_url={cdp_url}')
+
+			return BrowserLaunchResult(cdp_url=cdp_url)
+		except Exception as e:
+			self.logger.error(f'[LocalBrowserWatchdog] Exception in on_BrowserLaunchEvent: {e}', exc_info=True)
+			raise
+
+	async def on_BrowserKillEvent(self, event: BrowserKillEvent) -> None:
+		"""Kill the local browser subprocess."""
+		self.logger.debug('[LocalBrowserWatchdog] Killing local browser process')
+
+		if self._subprocess:
+			await self._cleanup_process(self._subprocess)
+			self._subprocess = None
+
+		# Clean up temp directories if any were created
+		for temp_dir in self._temp_dirs_to_cleanup:
+			self._cleanup_temp_dir(temp_dir)
+		self._temp_dirs_to_cleanup.clear()
+
+		# Restore original user_data_dir if it was modified
+		if self._original_user_data_dir is not None:
+			self.browser_session.browser_profile.user_data_dir = self._original_user_data_dir
+			self._original_user_data_dir = None
+
+		self.logger.debug('[LocalBrowserWatchdog] Browser cleanup completed')
+
+	async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
+		"""Listen for BrowserStopEvent and dispatch BrowserKillEvent without awaiting it."""
+		if self.browser_session.is_local and self._subprocess:
+			self.logger.debug('[LocalBrowserWatchdog] BrowserStopEvent received, dispatching BrowserKillEvent')
+			# Dispatch BrowserKillEvent without awaiting so it gets processed after all BrowserStopEvent handlers
+			self.event_bus.dispatch(BrowserKillEvent())
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='launch_browser_process')
+	async def _launch_browser(self, max_retries: int = 3) -> tuple[psutil.Process, str]:
+		"""Launch browser process and return (process, cdp_url).
+
+		Handles launch errors by falling back to temporary directories if needed.
+
+		Returns:
+			Tuple of (psutil.Process, cdp_url)
+		"""
+		# Keep track of original user_data_dir to restore if needed
+		profile = self.browser_session.browser_profile
+		self._original_user_data_dir = str(profile.user_data_dir) if profile.user_data_dir else None
+		self._temp_dirs_to_cleanup = []
+
+		for attempt in range(max_retries):
+			try:
+				# Get launch args from profile
+				launch_args = profile.get_args()
+
+				# Add debugging port
+				debug_port = self._find_free_port()
+				launch_args.extend(
+					[
+						f'--remote-debugging-port={debug_port}',
+					]
+				)
+				assert '--user-data-dir' in str(launch_args), (
+					'User data dir must be set somewhere in launch args to a non-default path, otherwise Chrome will not let us attach via CDP'
+				)
+
+				# Get browser executable
+				# Priority: custom executable > fallback paths > playwright subprocess
+				if profile.executable_path:
+					browser_path = profile.executable_path
+					self.logger.debug(f'[LocalBrowserWatchdog] 📦 Using custom local browser executable_path= {browser_path}')
+				else:
+					# self.logger.debug('[LocalBrowserWatchdog] 🔍 Looking for local browser binary path...')
+					# Try fallback paths first (Playwright's Chromium preferred by default)
+					browser_path = self._find_installed_browser_path(channel=profile.channel)
+					if not browser_path:
+						self.logger.error(
+							'[LocalBrowserWatchdog] ⚠️ No local browser binary found, installing browser using playwright subprocess...'
+						)
+						browser_path = await self._install_browser_with_playwright()
+
+				self.logger.debug(f'[LocalBrowserWatchdog] 📦 Found local browser installed at executable_path= {browser_path}')
+				if not browser_path:
+					raise RuntimeError('No local Chrome/Chromium install found, and failed to install with playwright')
+
+				# Launch browser subprocess directly
+				self.logger.debug(f'[LocalBrowserWatchdog] 🚀 Launching browser subprocess with {len(launch_args)} args...')
+				self.logger.debug(
+					f'[LocalBrowserWatchdog] 📂 user_data_dir={profile.user_data_dir}, profile_directory={profile.profile_directory}'
+				)
+				subprocess = await asyncio.create_subprocess_exec(
+					browser_path,
+					*launch_args,
+					stdout=asyncio.subprocess.PIPE,
+					stderr=asyncio.subprocess.PIPE,
+				)
+				self.logger.debug(
+					f'[LocalBrowserWatchdog] 🎭 Browser running with browser_pid= {subprocess.pid} 🔗 listening on CDP port :{debug_port}'
+				)
+
+				# Convert to psutil.Process
+				process = psutil.Process(subprocess.pid)
+
+				# Wait for CDP to be ready and get the URL
+				cdp_url = await self._wait_for_cdp_url(debug_port)
+
+				# Success! Clean up only the temp dirs we created but didn't use
+				currently_used_dir = str(profile.user_data_dir)
+				unused_temp_dirs = [tmp_dir for tmp_dir in self._temp_dirs_to_cleanup if str(tmp_dir) != currently_used_dir]
+
+				for tmp_dir in unused_temp_dirs:
+					try:
+						shutil.rmtree(tmp_dir, ignore_errors=True)
+					except Exception:
+						pass
+
+				# Keep only the in-use directory for cleanup during browser kill
+				if currently_used_dir and 'browseruse-tmp-' in currently_used_dir:
+					self._temp_dirs_to_cleanup = [Path(currently_used_dir)]
+				else:
+					self._temp_dirs_to_cleanup = []
+
+				return process, cdp_url
+
+			except Exception as e:
+				error_str = str(e).lower()
+
+				# Check if this is a user_data_dir related error
+				if any(err in error_str for err in ['singletonlock', 'user data directory', 'cannot create', 'already in use']):
+					self.logger.warning(f'Browser launch failed (attempt {attempt + 1}/{max_retries}): {e}')
+
+					if attempt < max_retries - 1:
+						# Create a temporary directory for next attempt
+						tmp_dir = Path(tempfile.mkdtemp(prefix='browseruse-tmp-'))
+						self._temp_dirs_to_cleanup.append(tmp_dir)
+
+						# Update profile to use temp directory
+						profile.user_data_dir = str(tmp_dir)
+						self.logger.debug(f'Retrying with temporary user_data_dir: {tmp_dir}')
+
+						# Small delay before retry
+						await asyncio.sleep(0.5)
+						continue
+
+				# Not a recoverable error or last attempt failed
+				# Restore original user_data_dir before raising
+				if self._original_user_data_dir is not None:
+					profile.user_data_dir = self._original_user_data_dir
+
+				# Clean up any temp dirs we created
+				for tmp_dir in self._temp_dirs_to_cleanup:
+					try:
+						shutil.rmtree(tmp_dir, ignore_errors=True)
+					except Exception:
+						pass
+
+				raise
+
+		# Should not reach here, but just in case
+		if self._original_user_data_dir is not None:
+			profile.user_data_dir = self._original_user_data_dir
+		raise RuntimeError(f'Failed to launch browser after {max_retries} attempts')
+
+	@staticmethod
+	def _find_installed_browser_path(channel: BrowserChannel | None = None) -> str | None:
+		"""Try to find browser executable from common fallback locations.
+
+		If a channel is specified, paths for that browser are searched first.
+		Falls back to all known browser paths if the channel-specific search fails.
+
+		Prioritizes:
+		1. Channel-specific paths (if channel is set to a non-default value)
+		2. Playwright bundled Chromium (when no channel or default channel specified)
+		3. System Chrome stable
+		4. Other system native browsers (Chromium -> Chrome Canary/Dev -> Brave -> Edge)
+		5. Playwright headless-shell fallback
+
+		Returns:
+			Path to browser executable or None if not found
+		"""
+		import glob
+		import platform
+		from pathlib import Path
+
+		from browser_use.browser.profile import BROWSERUSE_DEFAULT_CHANNEL, BrowserChannel
+
+		system = platform.system()
+
+		# Get playwright browsers path from environment variable if set
+		playwright_path = os.environ.get('PLAYWRIGHT_BROWSERS_PATH')
+
+		# Build tagged pattern lists per OS: (browser_group, path)
+		# browser_group is used to match against the requested channel
+		if system == 'Darwin':  # macOS
+			if not playwright_path:
+				playwright_path = '~/Library/Caches/ms-playwright'
+			all_patterns = [
+				('chrome', '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'),
+				('chromium', f'{playwright_path}/chromium-*/chrome-mac*/Chromium.app/Contents/MacOS/Chromium'),
+				('chromium', '/Applications/Chromium.app/Contents/MacOS/Chromium'),
+				('chrome-canary', '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'),
+				('brave', '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'),
+				('msedge', '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'),
+				('chromium', f'{playwright_path}/chromium_headless_shell-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium'),
+			]
+		elif system == 'Linux':
+			if not playwright_path:
+				playwright_path = '~/.cache/ms-playwright'
+			all_patterns = [
+				('chrome', '/usr/bin/google-chrome-stable'),
+				('chrome', '/usr/bin/google-chrome'),
+				('chrome', '/usr/local/bin/google-chrome'),
+				('chromium', f'{playwright_path}/chromium-*/chrome-linux*/chrome'),
+				('chromium', '/usr/bin/chromium'),
+				('chromium', '/usr/bin/chromium-browser'),
+				('chromium', '/usr/local/bin/chromium'),
+				('chromium', '/snap/bin/chromium'),
+				('chrome-beta', '/usr/bin/google-chrome-beta'),
+				('chrome-dev', '/usr/bin/google-chrome-dev'),
+				('brave', '/usr/bin/brave-browser'),
+				('msedge', '/usr/bin/microsoft-edge-stable'),
+				('msedge', '/usr/bin/microsoft-edge'),
+				('chromium', f'{playwright_path}/chromium_headless_shell-*/chrome-linux*/chrome'),
+			]
+		elif system == 'Windows':
+			if not playwright_path:
+				playwright_path = r'%LOCALAPPDATA%\ms-playwright'
+			all_patterns = [
+				('chrome', r'C:\Program Files\Google\Chrome\Application\chrome.exe'),
+				('chrome', r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'),
+				('chrome', r'%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe'),
+				('chrome', r'%PROGRAMFILES%\Google\Chrome\Application\chrome.exe'),
+				('chrome', r'%PROGRAMFILES(X86)%\Google\Chrome\Application\chrome.exe'),
+				('chromium', f'{playwright_path}\\chromium-*\\chrome-win\\chrome.exe'),
+				('chromium', r'C:\Program Files\Chromium\Application\chrome.exe'),
+				('chromium', r'C:\Program Files (x86)\Chromium\Application\chrome.exe'),
+				('chromium', r'%LOCALAPPDATA%\Chromium\Application\chrome.exe'),
+				('brave', r'C:\Program Files\BraveSoftware\Brave-Browser\Application\brave.exe'),
+				('brave', r'C:\Program Files (x86)\BraveSoftware\Brave-Browser\Application\brave.exe'),
+				('msedge', r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe'),
+				('msedge', r'C:\Program Files\Microsoft\Edge\Application\msedge.exe'),
+				('msedge', r'%LOCALAPPDATA%\Microsoft\Edge\Application\msedge.exe'),
+				('chromium', f'{playwright_path}\\chromium_headless_shell-*\\chrome-win\\chrome.exe'),
+			]
+		else:
+			all_patterns = []
+
+		# Map channel enum values to browser group tags
+		_channel_to_group: dict[BrowserChannel, str] = {
+			BrowserChannel.CHROME: 'chrome',
+			BrowserChannel.CHROME_BETA: 'chrome-beta',
+			BrowserChannel.CHROME_DEV: 'chrome-dev',
+			BrowserChannel.CHROME_CANARY: 'chrome-canary',
+			BrowserChannel.CHROMIUM: 'chromium',
+			BrowserChannel.MSEDGE: 'msedge',
+			BrowserChannel.MSEDGE_BETA: 'msedge',
+			BrowserChannel.MSEDGE_DEV: 'msedge',
+			BrowserChannel.MSEDGE_CANARY: 'msedge',
+		}
+
+		# Prioritize the target browser group, then fall back to the rest.
+		if channel and channel != BROWSERUSE_DEFAULT_CHANNEL and channel in _channel_to_group:
+			target_group = _channel_to_group[channel]
+		else:
+			target_group = _channel_to_group[BROWSERUSE_DEFAULT_CHANNEL]
+		prioritized = [p for g, p in all_patterns if g == target_group]
+		rest = [p for g, p in all_patterns if g != target_group]
+		patterns = prioritized + rest
+
+		for pattern in patterns:
+			# Expand user home directory
+			expanded_pattern = Path(pattern).expanduser()
+
+			# Handle Windows environment variables
+			if system == 'Windows':
+				pattern_str = str(expanded_pattern)
+				for env_var in ['%LOCALAPPDATA%', '%PROGRAMFILES%', '%PROGRAMFILES(X86)%']:
+					if env_var in pattern_str:
+						env_key = env_var.strip('%').replace('(X86)', ' (x86)')
+						env_value = os.environ.get(env_key, '')
+						if env_value:
+							pattern_str = pattern_str.replace(env_var, env_value)
+				expanded_pattern = Path(pattern_str)
+
+			# Convert to string for glob
+			pattern_str = str(expanded_pattern)
+
+			# Check if pattern contains wildcards
+			if '*' in pattern_str:
+				# Use glob to expand the pattern
+				matches = glob.glob(pattern_str)
+				if matches:
+					# Sort matches and take the last one (alphanumerically highest version)
+					matches.sort()
+					browser_path = matches[-1]
+					if Path(browser_path).exists() and Path(browser_path).is_file():
+						return browser_path
+			else:
+				# Direct path check
+				if expanded_pattern.exists() and expanded_pattern.is_file():
+					return str(expanded_pattern)
+
+		return None
+
+	async def _install_browser_with_playwright(self) -> str:
+		"""Get browser executable path from playwright in a subprocess to avoid thread issues."""
+		import platform
+
+		# Build command - only use --with-deps on Linux (it fails on Windows/macOS)
+		cmd = ['uvx', 'playwright', 'install', 'chromium']
+		if platform.system() == 'Linux':
+			cmd.append('--with-deps')
+
+		# Run in subprocess with timeout
+		process = await asyncio.create_subprocess_exec(
+			*cmd,
+			stdout=asyncio.subprocess.PIPE,
+			stderr=asyncio.subprocess.PIPE,
+		)
+
+		try:
+			stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=60.0)
+			self.logger.debug(f'[LocalBrowserWatchdog] 📦 Playwright install output: {stdout}')
+			browser_path = self._find_installed_browser_path()
+			if browser_path:
+				return browser_path
+			self.logger.error(f'[LocalBrowserWatchdog] ❌ Playwright local browser installation error: \n{stdout}\n{stderr}')
+			raise RuntimeError('No local browser path found after: uvx playwright install chromium')
+		except TimeoutError:
+			# Kill the subprocess if it times out
+			process.kill()
+			await process.wait()
+			raise RuntimeError('Timeout getting browser path from playwright')
+		except Exception as e:
+			# Make sure subprocess is terminated
+			if process.returncode is None:
+				process.kill()
+				await process.wait()
+			raise RuntimeError(f'Error getting browser path: {e}')
+
+	@staticmethod
+	def _find_free_port() -> int:
+		"""Find a free port for the debugging interface."""
+		import socket
+
+		with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+			s.bind(('127.0.0.1', 0))
+			s.listen(1)
+			port = s.getsockname()[1]
+		return port
+
+	@staticmethod
+	async def _wait_for_cdp_url(port: int, timeout: float = 30) -> str:
+		"""Wait for the browser to start and return the CDP URL."""
+		import aiohttp
+
+		start_time = asyncio.get_event_loop().time()
+
+		while asyncio.get_event_loop().time() - start_time < timeout:
+			try:
+				async with aiohttp.ClientSession() as session:
+					async with session.get(f'http://127.0.0.1:{port}/json/version') as resp:
+						if resp.status == 200:
+							# Chrome is ready
+							return f'http://127.0.0.1:{port}/'
+						else:
+							# Chrome is starting up and returning 502/500 errors
+							await asyncio.sleep(0.1)
+			except Exception:
+				# Connection error - Chrome might not be ready yet
+				await asyncio.sleep(0.1)
+
+		raise TimeoutError(f'Browser did not start within {timeout} seconds')
+
+	@staticmethod
+	async def _cleanup_process(process: psutil.Process) -> None:
+		"""Clean up browser process.
+
+		Args:
+			process: psutil.Process to terminate
+		"""
+		if not process:
+			return
+
+		try:
+			# Try graceful shutdown first
+			process.terminate()
+
+			# Use async wait instead of blocking wait
+			for _ in range(50):  # Wait up to 5 seconds (50 * 0.1)
+				if not process.is_running():
+					return
+				await asyncio.sleep(0.1)
+
+			# If still running after 5 seconds, force kill
+			if process.is_running():
+				process.kill()
+				# Give it a moment to die
+				await asyncio.sleep(0.1)
+
+		except psutil.NoSuchProcess:
+			# Process already gone
+			pass
+		except Exception:
+			# Ignore any other errors during cleanup
+			pass
+
+	def _cleanup_temp_dir(self, temp_dir: Path | str) -> None:
+		"""Clean up temporary directory.
+
+		Args:
+			temp_dir: Path to temporary directory to remove
+		"""
+		if not temp_dir:
+			return
+
+		try:
+			temp_path = Path(temp_dir)
+			# Only remove if it's actually a temp directory we created
+			if 'browseruse-tmp-' in str(temp_path):
+				shutil.rmtree(temp_path, ignore_errors=True)
+		except Exception as e:
+			self.logger.debug(f'Failed to cleanup temp dir {temp_dir}: {e}')
+
+	@property
+	def browser_pid(self) -> int | None:
+		"""Get the browser process ID."""
+		if self._subprocess:
+			return self._subprocess.pid
+		return None
+
+	@staticmethod
+	async def get_browser_pid_via_cdp(browser) -> int | None:
+		"""Get the browser process ID via CDP SystemInfo.getProcessInfo.
+
+		Args:
+			browser: Playwright Browser instance
+
+		Returns:
+			Process ID or None if failed
+		"""
+		try:
+			cdp_session = await browser.new_browser_cdp_session()
+			result = await cdp_session.send('SystemInfo.getProcessInfo')
+			process_info = result.get('processInfo', {})
+			pid = process_info.get('id')
+			await cdp_session.detach()
+			return pid
+		except Exception:
+			# If we can't get PID via CDP, it's not critical
+			return None

browser_use/browser/watchdogs/permissions_watchdog.py

@@ -0,0 +1,43 @@
+"""Permissions watchdog for granting browser permissions on connection."""
+
+from typing import TYPE_CHECKING, ClassVar
+
+from bubus import BaseEvent
+
+from browser_use.browser.events import BrowserConnectedEvent
+from browser_use.browser.watchdog_base import BaseWatchdog
+
+if TYPE_CHECKING:
+	pass
+
+
+class PermissionsWatchdog(BaseWatchdog):
+	"""Grants browser permissions when browser connects."""
+
+	# Event contracts
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+		BrowserConnectedEvent,
+	]
+	EMITS: ClassVar[list[type[BaseEvent]]] = []
+
+	async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
+		"""Grant permissions when browser connects."""
+		permissions = self.browser_session.browser_profile.permissions
+
+		if not permissions:
+			self.logger.debug('No permissions to grant')
+			return
+
+		self.logger.debug(f'🔓 Granting browser permissions: {permissions}')
+
+		try:
+			# Grant permissions using CDP Browser.grantPermissions
+			# origin=None means grant to all origins
+			# Browser domain commands don't use session_id
+			await self.browser_session.cdp_client.send.Browser.grantPermissions(
+				params={'permissions': permissions}  # type: ignore
+			)
+			self.logger.debug(f'✅ Successfully granted permissions: {permissions}')
+		except Exception as e:
+			self.logger.error(f'❌ Failed to grant permissions: {str(e)}')
+			# Don't raise - permissions are not critical to browser operation

browser_use/browser/watchdogs/popups_watchdog.py

@@ -0,0 +1,145 @@
+"""Watchdog for handling JavaScript dialogs (alert, confirm, prompt) automatically."""
+
+import asyncio
+from typing import ClassVar
+
+from bubus import BaseEvent
+from pydantic import PrivateAttr
+
+from browser_use.browser.events import TabCreatedEvent
+from browser_use.browser.watchdog_base import BaseWatchdog
+
+
+class PopupsWatchdog(BaseWatchdog):
+	"""Handles JavaScript dialogs (alert, confirm, prompt) by automatically accepting them immediately."""
+
+	# Events this watchdog listens to and emits
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [TabCreatedEvent]
+	EMITS: ClassVar[list[type[BaseEvent]]] = []
+
+	# Track which targets have dialog handlers registered
+	_dialog_listeners_registered: set[str] = PrivateAttr(default_factory=set)
+
+	def __init__(self, **kwargs):
+		super().__init__(**kwargs)
+		self.logger.debug(f'🚀 PopupsWatchdog initialized with browser_session={self.browser_session}, ID={id(self)}')
+
+	async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
+		"""Set up JavaScript dialog handling when a new tab is created."""
+		target_id = event.target_id
+		self.logger.debug(f'🎯 PopupsWatchdog received TabCreatedEvent for target {target_id}')
+
+		# Skip if we've already registered for this target
+		if target_id in self._dialog_listeners_registered:
+			self.logger.debug(f'Already registered dialog handlers for target {target_id}')
+			return
+
+		self.logger.debug(f'📌 Starting dialog handler setup for target {target_id}')
+		try:
+			# Get all CDP sessions for this target and any child frames
+			cdp_session = await self.browser_session.get_or_create_cdp_session(
+				target_id, focus=False
+			)  # don't auto-focus new tabs! sometimes we need to open tabs in background
+
+			# CRITICAL: Enable Page domain to receive dialog events
+			try:
+				await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
+				self.logger.debug(f'✅ Enabled Page domain for session {cdp_session.session_id[-8:]}')
+			except Exception as e:
+				self.logger.debug(f'Failed to enable Page domain: {e}')
+
+			# Also register for the root CDP client to catch dialogs from any frame
+			if self.browser_session._cdp_client_root:
+				self.logger.debug('📌 Also registering handler on root CDP client')
+				try:
+					# Enable Page domain on root client too
+					await self.browser_session._cdp_client_root.send.Page.enable()
+					self.logger.debug('✅ Enabled Page domain on root CDP client')
+				except Exception as e:
+					self.logger.debug(f'Failed to enable Page domain on root: {e}')
+
+			# Set up async handler for JavaScript dialogs - accept immediately without event dispatch
+			async def handle_dialog(event_data, session_id: str | None = None):
+				"""Handle JavaScript dialog events - accept immediately."""
+				try:
+					dialog_type = event_data.get('type', 'alert')
+					message = event_data.get('message', '')
+
+					# Store the popup message in browser session for inclusion in browser state
+					if message:
+						formatted_message = f'[{dialog_type}] {message}'
+						self.browser_session._closed_popup_messages.append(formatted_message)
+						self.logger.debug(f'📝 Stored popup message: {formatted_message[:100]}')
+
+					# Choose action based on dialog type:
+					# - alert: accept=true (click OK to dismiss)
+					# - confirm: accept=true (click OK to proceed - safer for automation)
+					# - prompt: accept=false (click Cancel since we can't provide input)
+					# - beforeunload: accept=true (allow navigation)
+					should_accept = dialog_type in ('alert', 'confirm', 'beforeunload')
+
+					action_str = 'accepting (OK)' if should_accept else 'dismissing (Cancel)'
+					self.logger.info(f"🔔 JavaScript {dialog_type} dialog: '{message[:100]}' - {action_str}...")
+
+					dismissed = False
+
+					# Approach 1: Use the session that detected the dialog (most reliable)
+					if self.browser_session._cdp_client_root and session_id:
+						try:
+							self.logger.debug(f'🔄 Approach 1: Using detecting session {session_id[-8:]}')
+							await asyncio.wait_for(
+								self.browser_session._cdp_client_root.send.Page.handleJavaScriptDialog(
+									params={'accept': should_accept},
+									session_id=session_id,
+								),
+								timeout=0.5,
+							)
+							dismissed = True
+							self.logger.info('✅ Dialog handled successfully via detecting session')
+						except (TimeoutError, Exception) as e:
+							self.logger.debug(f'Approach 1 failed: {type(e).__name__}')
+
+					# Approach 2: Try with current agent focus session
+					if not dismissed and self.browser_session._cdp_client_root and self.browser_session.agent_focus_target_id:
+						try:
+							# Use public API with focus=False to avoid changing focus during popup dismissal
+							cdp_session = await self.browser_session.get_or_create_cdp_session(
+								self.browser_session.agent_focus_target_id, focus=False
+							)
+							self.logger.debug(f'🔄 Approach 2: Using agent focus session {cdp_session.session_id[-8:]}')
+							await asyncio.wait_for(
+								self.browser_session._cdp_client_root.send.Page.handleJavaScriptDialog(
+									params={'accept': should_accept},
+									session_id=cdp_session.session_id,
+								),
+								timeout=0.5,
+							)
+							dismissed = True
+							self.logger.info('✅ Dialog handled successfully via agent focus session')
+						except (TimeoutError, Exception) as e:
+							self.logger.debug(f'Approach 2 failed: {type(e).__name__}')
+
+				except Exception as e:
+					self.logger.error(f'❌ Critical error in dialog handler: {type(e).__name__}: {e}')
+
+			# Register handler on the specific session
+			cdp_session.cdp_client.register.Page.javascriptDialogOpening(handle_dialog)  # type: ignore[arg-type]
+			self.logger.debug(
+				f'Successfully registered Page.javascriptDialogOpening handler for session {cdp_session.session_id}'
+			)
+
+			# Also register on root CDP client to catch dialogs from any frame
+			if hasattr(self.browser_session._cdp_client_root, 'register'):
+				try:
+					self.browser_session._cdp_client_root.register.Page.javascriptDialogOpening(handle_dialog)  # type: ignore[arg-type]
+					self.logger.debug('Successfully registered dialog handler on root CDP client for all frames')
+				except Exception as root_error:
+					self.logger.warning(f'Failed to register on root CDP client: {root_error}')
+
+			# Mark this target as having dialog handling set up
+			self._dialog_listeners_registered.add(target_id)
+
+			self.logger.debug(f'Set up JavaScript dialog handling for tab {target_id}')
+
+		except Exception as e:
+			self.logger.warning(f'Failed to set up popup handling for tab {target_id}: {e}')

browser_use/browser/watchdogs/recording_watchdog.py

@@ -0,0 +1,223 @@
+"""Recording Watchdog for Browser Use Sessions."""
+
+import asyncio
+from pathlib import Path
+from typing import Any, ClassVar
+
+from bubus import BaseEvent
+from cdp_use.cdp.page.events import ScreencastFrameEvent
+from pydantic import PrivateAttr
+from uuid_extensions import uuid7str
+
+from browser_use.browser.events import AgentFocusChangedEvent, BrowserConnectedEvent, BrowserStopEvent
+from browser_use.browser.profile import ViewportSize
+from browser_use.browser.video_recorder import VideoRecorderService
+from browser_use.browser.watchdog_base import BaseWatchdog
+from browser_use.utils import create_task_with_error_handling
+
+
+class RecordingWatchdog(BaseWatchdog):
+	"""
+	Manages video recording of a browser session using CDP screencasting.
+	"""
+
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent, AgentFocusChangedEvent]
+	EMITS: ClassVar[list[type[BaseEvent]]] = []
+
+	_recorder: VideoRecorderService | None = PrivateAttr(default=None)
+	_current_session_id: str | None = PrivateAttr(default=None)
+	_screencast_params: dict[str, Any] | None = PrivateAttr(default=None)
+
+	async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
+		"""
+		Starts video recording if it is configured in the browser profile.
+		"""
+		profile = self.browser_session.browser_profile
+		if not profile.record_video_dir:
+			return
+
+		video_format = getattr(profile, 'record_video_format', 'mp4').strip('.')
+		output_path = Path(profile.record_video_dir) / f'{uuid7str()}.{video_format}'
+		try:
+			await self.start_recording(output_path, size=profile.record_video_size, framerate=profile.record_video_framerate)
+		except RuntimeError as e:
+			# Preserve prior graceful degradation: a session configured with record_video_dir
+			# should not fail startup when video deps are missing or viewport detection fails.
+			self.logger.warning(f'Skipping video recording: {e}')
+
+	async def start_recording(
+		self,
+		output_path: Path,
+		size: ViewportSize | None = None,
+		framerate: int | None = None,
+	) -> Path:
+		"""
+		Begin recording the current session to `output_path`. Safe to call at any time
+		after the browser has connected.
+
+		Returns the resolved output path. Raises RuntimeError if recording is already active
+		or if the viewport size could not be determined.
+		"""
+		if self._recorder is not None:
+			raise RuntimeError(f'Recording already in progress (output: {self._recorder.output_path})')
+
+		if size is None:
+			self.logger.debug('record size not specified, detecting viewport size...')
+			size = await self._get_current_viewport_size()
+		if not size:
+			raise RuntimeError('Cannot start video recording: viewport size could not be determined.')
+
+		if framerate is None:
+			framerate = self.browser_session.browser_profile.record_video_framerate
+
+		output_path = Path(output_path)
+		self.logger.debug(f'Initializing video recorder → {output_path}')
+		recorder = VideoRecorderService(output_path=output_path, size=size, framerate=framerate)
+		recorder.start()
+		if not recorder._is_active:
+			raise RuntimeError(
+				'Failed to initialize video recorder — ensure optional deps are installed (`pip install "browser-use[video]"`).'
+			)
+
+		self._recorder = recorder
+		self.browser_session.cdp_client.register.Page.screencastFrame(self.on_screencastFrame)
+		self._screencast_params = {
+			'format': 'png',
+			'quality': 90,
+			'maxWidth': size['width'],
+			'maxHeight': size['height'],
+			'everyNthFrame': 1,
+		}
+		await self._start_screencast()
+		return output_path
+
+	async def stop_recording(self) -> Path | None:
+		"""
+		Stop any in-progress recording and finalize the output file.
+
+		Returns the path of the saved video, or None if no recording was active.
+		"""
+		if not self._recorder:
+			return None
+
+		recorder = self._recorder
+		session_id = self._current_session_id
+		self._recorder = None
+		self._current_session_id = None
+		self._screencast_params = None
+
+		if session_id:
+			try:
+				await self.browser_session.cdp_client.send.Page.stopScreencast(session_id=session_id)
+			except Exception as e:
+				self.logger.debug(f'Failed to stop CDP screencast on {session_id}: {e}')
+
+		output_path = recorder.output_path
+		loop = asyncio.get_event_loop()
+		await loop.run_in_executor(None, recorder.stop_and_save)
+		return output_path
+
+	@property
+	def is_recording(self) -> bool:
+		"""Whether a recording is currently in progress."""
+		return self._recorder is not None
+
+	async def on_AgentFocusChangedEvent(self, event: AgentFocusChangedEvent) -> None:
+		"""
+		Switches video recording to the new tab.
+		"""
+		if self._recorder:
+			self.logger.debug(f'Agent focus changed to {event.target_id}, switching screencast...')
+			await self._start_screencast()
+
+	async def _start_screencast(self) -> None:
+		"""Starts screencast on the currently focused tab."""
+		if not self._recorder or not self._screencast_params:
+			return
+
+		try:
+			# Get the current session (for the focused target)
+			cdp_session = await self.browser_session.get_or_create_cdp_session()
+
+			# If we are already recording this session, do nothing
+			if self._current_session_id == cdp_session.session_id:
+				return
+
+			# Stop recording on the previous session
+			if self._current_session_id:
+				try:
+					# Use the root client to stop screencast on the specific session
+					await self.browser_session.cdp_client.send.Page.stopScreencast(session_id=self._current_session_id)
+				except Exception as e:
+					# It's possible the session is already closed
+					self.logger.debug(f'Failed to stop screencast on old session {self._current_session_id}: {e}')
+
+			self._current_session_id = cdp_session.session_id
+
+			# Start recording on the new session
+			await cdp_session.cdp_client.send.Page.startScreencast(
+				params=self._screencast_params,  # type: ignore
+				session_id=cdp_session.session_id,
+			)
+			self.logger.info(f'📹 Started/Switched video recording to target {cdp_session.target_id}')
+		except Exception as e:
+			self.logger.error(f'Failed to switch screencast via CDP: {e}')
+			# If we fail to start on the new tab, we reset current session id
+			self._current_session_id = None
+
+	async def _get_current_viewport_size(self) -> ViewportSize | None:
+		"""Gets the current viewport size directly from the browser via CDP."""
+		try:
+			cdp_session = await self.browser_session.get_or_create_cdp_session()
+			metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
+
+			# Use cssVisualViewport for the most accurate representation of the visible area
+			viewport = metrics.get('cssVisualViewport', {})
+			width = viewport.get('clientWidth')
+			height = viewport.get('clientHeight')
+
+			if width and height:
+				self.logger.debug(f'Detected viewport size: {width}x{height}')
+				return ViewportSize(width=int(width), height=int(height))
+		except Exception as e:
+			self.logger.warning(f'Failed to get viewport size from browser: {e}')
+
+		return None
+
+	def on_screencastFrame(self, event: ScreencastFrameEvent, session_id: str | None) -> None:
+		"""
+		Synchronous handler for incoming screencast frames.
+		"""
+		# Only process frames from the current session we intend to record
+		# This handles race conditions where old session might still send frames before stop completes
+		if self._current_session_id and session_id != self._current_session_id:
+			return
+
+		if not self._recorder:
+			return
+		self._recorder.add_frame(event['data'])
+		create_task_with_error_handling(
+			self._ack_screencast_frame(event, session_id),
+			name='ack_screencast_frame',
+			logger_instance=self.logger,
+			suppress_exceptions=True,
+		)
+
+	async def _ack_screencast_frame(self, event: ScreencastFrameEvent, session_id: str | None) -> None:
+		"""
+		Asynchronously acknowledges a screencast frame.
+		"""
+		try:
+			await self.browser_session.cdp_client.send.Page.screencastFrameAck(
+				params={'sessionId': event['sessionId']}, session_id=session_id
+			)
+		except Exception as e:
+			self.logger.debug(f'Failed to acknowledge screencast frame: {e}')
+
+	async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
+		"""
+		Stops the video recording and finalizes the video file.
+		"""
+		if self._recorder:
+			self.logger.debug('Stopping video recording and saving file...')
+			await self.stop_recording()

browser_use/browser/watchdogs/screenshot_watchdog.py

@@ -0,0 +1,88 @@
+"""Screenshot watchdog for handling screenshot requests using CDP."""
+
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from bubus import BaseEvent
+from cdp_use.cdp.page import CaptureScreenshotParameters
+
+from browser_use.browser.events import ScreenshotEvent
+from browser_use.browser.views import BrowserError
+from browser_use.browser.watchdog_base import BaseWatchdog
+from browser_use.observability import observe_debug
+
+if TYPE_CHECKING:
+	pass
+
+
+class ScreenshotWatchdog(BaseWatchdog):
+	"""Handles screenshot requests using CDP."""
+
+	# Events this watchdog listens to
+	LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [ScreenshotEvent]
+
+	# Events this watchdog emits
+	EMITS: ClassVar[list[type[BaseEvent[Any]]]] = []
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='screenshot_event_handler')
+	async def on_ScreenshotEvent(self, event: ScreenshotEvent) -> str:
+		"""Handle screenshot request using CDP.
+
+		Args:
+			event: ScreenshotEvent with optional full_page and clip parameters
+
+		Returns:
+			Dict with 'screenshot' key containing base64-encoded screenshot or None
+		"""
+		self.logger.debug('[ScreenshotWatchdog] Handler START - on_ScreenshotEvent called')
+		try:
+			# Validate focused target is a top-level page (not iframe/worker)
+			# CDP Page.captureScreenshot only works on page/tab targets
+			focused_target = self.browser_session.get_focused_target()
+
+			if focused_target and focused_target.target_type in ('page', 'tab'):
+				target_id = focused_target.target_id
+			else:
+				# Focused target is iframe/worker/missing - fall back to any page target
+				target_type_str = focused_target.target_type if focused_target else 'None'
+				self.logger.warning(f'[ScreenshotWatchdog] Focused target is {target_type_str}, falling back to page target')
+				page_targets = self.browser_session.get_page_targets()
+				if not page_targets:
+					raise BrowserError('[ScreenshotWatchdog] No page targets available for screenshot')
+				target_id = page_targets[-1].target_id
+
+			cdp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=True)
+
+			# Remove highlights BEFORE taking the screenshot so they don't appear in the image.
+			# Done here (not in finally) so CancelledError is never swallowed — any await in a
+			# finally block can suppress external task cancellation.
+			# remove_highlights() has its own asyncio.timeout(3.0) internally so it won't block.
+			try:
+				await self.browser_session.remove_highlights()
+			except Exception:
+				pass
+
+			# Prepare screenshot parameters
+			params_dict: dict[str, Any] = {'format': 'png', 'captureBeyondViewport': event.full_page}
+			if event.clip:
+				params_dict['clip'] = {
+					'x': event.clip['x'],
+					'y': event.clip['y'],
+					'width': event.clip['width'],
+					'height': event.clip['height'],
+					'scale': 1,
+				}
+			params = CaptureScreenshotParameters(**params_dict)
+
+			# Take screenshot using CDP
+			self.logger.debug(f'[ScreenshotWatchdog] Taking screenshot with params: {params}')
+			result = await cdp_session.cdp_client.send.Page.captureScreenshot(params=params, session_id=cdp_session.session_id)
+
+			# Return base64-encoded screenshot data
+			if result and 'data' in result:
+				self.logger.debug('[ScreenshotWatchdog] Screenshot captured successfully')
+				return result['data']
+
+			raise BrowserError('[ScreenshotWatchdog] Screenshot result missing data')
+		except Exception as e:
+			self.logger.error(f'[ScreenshotWatchdog] Screenshot failed: {e}')
+			raise

browser_use/browser/watchdogs/security_watchdog.py

@@ -0,0 +1,296 @@
+"""Security watchdog for enforcing URL access policies."""
+
+from typing import TYPE_CHECKING, ClassVar
+
+from bubus import BaseEvent
+
+from browser_use.browser.events import (
+	BrowserErrorEvent,
+	NavigateToUrlEvent,
+	NavigationCompleteEvent,
+	TabCreatedEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+
+if TYPE_CHECKING:
+	pass
+
+# Track if we've shown the glob warning
+_GLOB_WARNING_SHOWN = False
+
+
+class SecurityWatchdog(BaseWatchdog):
+	"""Monitors and enforces security policies for URL access."""
+
+	# Event contracts
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+		NavigateToUrlEvent,
+		NavigationCompleteEvent,
+		TabCreatedEvent,
+	]
+	EMITS: ClassVar[list[type[BaseEvent]]] = [
+		BrowserErrorEvent,
+	]
+
+	async def on_NavigateToUrlEvent(self, event: NavigateToUrlEvent) -> None:
+		"""Check if navigation URL is allowed before navigation starts."""
+		# Security check BEFORE navigation
+		if not self._is_url_allowed(event.url):
+			self.logger.warning(f'⛔️ Blocking navigation to disallowed URL: {event.url}')
+			self.event_bus.dispatch(
+				BrowserErrorEvent(
+					error_type='NavigationBlocked',
+					message=f'Navigation blocked to disallowed URL: {event.url}',
+					details={'url': event.url, 'reason': 'not_in_allowed_domains'},
+				)
+			)
+			# Stop event propagation by raising exception
+			raise ValueError(f'Navigation to {event.url} blocked by security policy')
+
+	async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
+		"""Check if navigated URL is allowed (catches redirects to blocked domains)."""
+		# Check if the navigated URL is allowed (in case of redirects)
+		if not self._is_url_allowed(event.url):
+			self.logger.warning(f'⛔️ Navigation to non-allowed URL detected: {event.url}')
+
+			# Dispatch browser error
+			self.event_bus.dispatch(
+				BrowserErrorEvent(
+					error_type='NavigationBlocked',
+					message=f'Navigation blocked to non-allowed URL: {event.url} - redirecting to about:blank',
+					details={'url': event.url, 'target_id': event.target_id},
+				)
+			)
+			# Navigate to about:blank to keep session alive
+			# Agent will see the error and can continue with other tasks
+			try:
+				session = await self.browser_session.get_or_create_cdp_session(target_id=event.target_id)
+				await session.cdp_client.send.Page.navigate(params={'url': 'about:blank'}, session_id=session.session_id)
+				self.logger.info(f'⛔️ Navigated to about:blank after blocked URL: {event.url}')
+			except Exception as e:
+				self.logger.error(f'⛔️ Failed to navigate to about:blank: {type(e).__name__} {e}')
+
+	async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
+		"""Check if new tab URL is allowed."""
+		if not self._is_url_allowed(event.url):
+			self.logger.warning(f'⛔️ New tab created with disallowed URL: {event.url}')
+
+			# Dispatch error and try to close the tab
+			self.event_bus.dispatch(
+				BrowserErrorEvent(
+					error_type='TabCreationBlocked',
+					message=f'Tab created with non-allowed URL: {event.url}',
+					details={'url': event.url, 'target_id': event.target_id},
+				)
+			)
+
+			# Try to close the offending tab
+			try:
+				await self.browser_session._cdp_close_page(event.target_id)
+				self.logger.info(f'⛔️ Closed new tab with non-allowed URL: {event.url}')
+			except Exception as e:
+				self.logger.error(f'⛔️ Failed to close new tab with non-allowed URL: {type(e).__name__} {e}')
+
+	def _is_root_domain(self, domain: str) -> bool:
+		"""Check if a domain is a root domain (no subdomain present).
+
+		Simple heuristic: only add www for domains with exactly 1 dot (domain.tld).
+		For complex cases like country TLDs or subdomains, users should configure explicitly.
+
+		Args:
+			domain: The domain to check
+
+		Returns:
+			True if it's a simple root domain, False otherwise
+		"""
+		# Skip if it contains wildcards or protocol
+		if '*' in domain or '://' in domain:
+			return False
+
+		return domain.count('.') == 1
+
+	def _log_glob_warning(self) -> None:
+		"""Log a warning about glob patterns in allowed_domains."""
+		global _GLOB_WARNING_SHOWN
+		if not _GLOB_WARNING_SHOWN:
+			_GLOB_WARNING_SHOWN = True
+			self.logger.warning(
+				'⚠️ Using glob patterns in allowed_domains. '
+				'Note: Patterns like "*.example.com" will match both subdomains AND the main domain.'
+			)
+
+	def _get_domain_variants(self, host: str) -> tuple[str, str]:
+		"""Get both variants of a domain (with and without www prefix).
+
+		Args:
+			host: The hostname to process
+
+		Returns:
+			Tuple of (original_host, variant_host)
+			- If host starts with www., variant is without www.
+			- Otherwise, variant is with www. prefix
+		"""
+		if host.startswith('www.'):
+			return (host, host[4:])  # ('www.example.com', 'example.com')
+		else:
+			return (host, f'www.{host}')  # ('example.com', 'www.example.com')
+
+	def _is_ip_address(self, host: str) -> bool:
+		"""True iff `host` matches an IPv4 or IPv6 the browser would resolve.
+
+		Mirrors WHATWG host canonicalization so non-standard IPv4 encodings
+		(decimal, hex, octal, short-form, percent-encoded, Unicode digits)
+		can't bypass `block_ip_addresses`. Never raises — unrecognizable
+		hosts return False and fall through to domain-allowlist handling.
+		"""
+		import ipaddress
+		import socket
+		import unicodedata
+		from urllib.parse import unquote
+
+		bare = host.strip('[]')
+		try:
+			bare = unquote(bare)
+		except Exception:
+			pass
+		try:
+			bare = unicodedata.normalize('NFKC', bare)
+		except Exception:
+			pass
+		# IDNA label separators NFKC misses (U+3002, U+FF61 → U+3002).
+		bare = bare.replace('。', '.').replace('｡', '.')
+
+		try:
+			ipaddress.ip_address(bare)
+			return True
+		except Exception:
+			pass
+		# Non-standard IPv4 (decimal, hex, octal, short-form) — `inet_aton`
+		# accepts the same liberal forms the kernel resolver does.
+		try:
+			socket.inet_aton(bare)
+			return True
+		except Exception:
+			return False
+
+	def _is_url_allowed(self, url: str) -> bool:
+		"""Check if a URL is allowed based on the allowed_domains configuration.
+
+		Args:
+			url: The URL to check
+
+		Returns:
+			True if the URL is allowed, False otherwise
+		"""
+
+		# Always allow internal browser targets (before any other checks)
+		if url in ['about:blank', 'chrome://new-tab-page/', 'chrome://new-tab-page', 'chrome://newtab/']:
+			return True
+
+		# Parse the URL to extract components
+		from urllib.parse import urlparse
+
+		try:
+			parsed = urlparse(url)
+		except Exception:
+			# Invalid URL
+			return False
+
+		# Allow data: and blob: URLs (they don't have hostnames)
+		if parsed.scheme in ['data', 'blob']:
+			return True
+
+		# Get the actual host (domain)
+		host = parsed.hostname
+		if not host:
+			return False
+
+		# Check if IP addresses should be blocked (before domain checks)
+		if self.browser_session.browser_profile.block_ip_addresses:
+			if self._is_ip_address(host):
+				return False
+
+		# If no allowed_domains specified, allow all URLs
+		if (
+			not self.browser_session.browser_profile.allowed_domains
+			and not self.browser_session.browser_profile.prohibited_domains
+		):
+			return True
+
+		# Check allowed domains (fast path for sets, slow path for lists with patterns)
+		if self.browser_session.browser_profile.allowed_domains:
+			allowed_domains = self.browser_session.browser_profile.allowed_domains
+
+			if isinstance(allowed_domains, set):
+				# Fast path: O(1) exact hostname match - check both www and non-www variants
+				host_variant, host_alt = self._get_domain_variants(host)
+				return host_variant in allowed_domains or host_alt in allowed_domains
+			else:
+				# Slow path: O(n) pattern matching for lists
+				for pattern in allowed_domains:
+					if self._is_url_match(url, host, parsed.scheme, pattern):
+						return True
+				return False
+
+		# Check prohibited domains (fast path for sets, slow path for lists with patterns)
+		if self.browser_session.browser_profile.prohibited_domains:
+			prohibited_domains = self.browser_session.browser_profile.prohibited_domains
+
+			if isinstance(prohibited_domains, set):
+				# Fast path: O(1) exact hostname match - check both www and non-www variants
+				host_variant, host_alt = self._get_domain_variants(host)
+				return host_variant not in prohibited_domains and host_alt not in prohibited_domains
+			else:
+				# Slow path: O(n) pattern matching for lists
+				for pattern in prohibited_domains:
+					if self._is_url_match(url, host, parsed.scheme, pattern):
+						return False
+				return True
+
+		return True
+
+	def _is_url_match(self, url: str, host: str, scheme: str, pattern: str) -> bool:
+		"""Check if a URL matches a pattern."""
+
+		# Full URL for matching (scheme + host)
+		full_url_pattern = f'{scheme}://{host}'
+
+		# Handle glob patterns
+		if '*' in pattern:
+			self._log_glob_warning()
+			import fnmatch
+
+			# Check if pattern matches the host
+			if pattern.startswith('*.'):
+				# Pattern like *.example.com should match subdomains and main domain
+				domain_part = pattern[2:]  # Remove *.
+				if host == domain_part or host.endswith('.' + domain_part):
+					# Only match http/https URLs for domain-only patterns
+					if scheme in ['http', 'https']:
+						return True
+			elif pattern.endswith('/*'):
+				# Pattern like brave://* or http*://example.com/*
+				if fnmatch.fnmatch(url, pattern):
+					return True
+			else:
+				# Use fnmatch for other glob patterns
+				if fnmatch.fnmatch(
+					full_url_pattern if '://' in pattern else host,
+					pattern,
+				):
+					return True
+		else:
+			# Exact match
+			if '://' in pattern:
+				# Full URL pattern
+				if url.startswith(pattern):
+					return True
+			else:
+				# Domain-only pattern (case-insensitive comparison)
+				if host.lower() == pattern.lower():
+					return True
+				# If pattern is a root domain, also check www subdomain
+				if self._is_root_domain(pattern) and host.lower() == f'www.{pattern.lower()}':
+					return True
+
+		return False

browser_use/browser/watchdogs/storage_state_watchdog.py

@@ -0,0 +1,373 @@
+"""Storage state watchdog for managing browser cookies and storage persistence."""
+
+import asyncio
+import json
+import os
+from pathlib import Path
+from typing import Any, ClassVar
+
+from bubus import BaseEvent
+from cdp_use.cdp.network import Cookie
+from pydantic import Field, PrivateAttr
+
+from browser_use.browser.events import (
+	BrowserConnectedEvent,
+	BrowserStopEvent,
+	LoadStorageStateEvent,
+	SaveStorageStateEvent,
+	StorageStateLoadedEvent,
+	StorageStateSavedEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+from browser_use.utils import create_task_with_error_handling
+
+
+class StorageStateWatchdog(BaseWatchdog):
+	"""Monitors and persists browser storage state including cookies and localStorage."""
+
+	# Event contracts
+	LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+		BrowserConnectedEvent,
+		BrowserStopEvent,
+		SaveStorageStateEvent,
+		LoadStorageStateEvent,
+	]
+	EMITS: ClassVar[list[type[BaseEvent]]] = [
+		StorageStateSavedEvent,
+		StorageStateLoadedEvent,
+	]
+
+	# Configuration
+	auto_save_interval: float = Field(default=30.0)  # Auto-save every 30 seconds
+	save_on_change: bool = Field(default=True)  # Save immediately when cookies change
+
+	# Private state
+	_monitoring_task: asyncio.Task | None = PrivateAttr(default=None)
+	_last_cookie_state: list[dict] = PrivateAttr(default_factory=list)
+	_save_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock)
+
+	async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
+		"""Start monitoring when browser starts."""
+		self.logger.debug('[StorageStateWatchdog] 🍪 Initializing auth/cookies sync <-> with storage_state.json file')
+
+		# Start monitoring
+		await self._start_monitoring()
+
+		# Automatically load storage state after browser start
+		await self.event_bus.dispatch(LoadStorageStateEvent())
+
+	async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
+		"""Stop monitoring when browser stops."""
+		self.logger.debug('[StorageStateWatchdog] Stopping storage_state monitoring')
+		await self._stop_monitoring()
+
+	async def on_SaveStorageStateEvent(self, event: SaveStorageStateEvent) -> None:
+		"""Handle storage state save request."""
+		# Use provided path or fall back to profile default
+		path = event.path
+		if path is None:
+			# Use profile default path if available
+			if self.browser_session.browser_profile.storage_state:
+				path = str(self.browser_session.browser_profile.storage_state)
+			else:
+				path = None  # Skip saving if no path available
+		await self._save_storage_state(path)
+
+	async def on_LoadStorageStateEvent(self, event: LoadStorageStateEvent) -> None:
+		"""Handle storage state load request."""
+		# Use provided path or fall back to profile default
+		path = event.path
+		if path is None:
+			# Use profile default path if available
+			if self.browser_session.browser_profile.storage_state:
+				path = str(self.browser_session.browser_profile.storage_state)
+			else:
+				path = None  # Skip loading if no path available
+		await self._load_storage_state(path)
+
+	async def _start_monitoring(self) -> None:
+		"""Start the monitoring task."""
+		if self._monitoring_task and not self._monitoring_task.done():
+			return
+
+		assert self.browser_session.cdp_client is not None
+
+		self._monitoring_task = create_task_with_error_handling(
+			self._monitor_storage_changes(), name='monitor_storage_changes', logger_instance=self.logger, suppress_exceptions=True
+		)
+		# self.logger'[StorageStateWatchdog] Started storage monitoring task')
+
+	async def _stop_monitoring(self) -> None:
+		"""Stop the monitoring task."""
+		if self._monitoring_task and not self._monitoring_task.done():
+			self._monitoring_task.cancel()
+			try:
+				await self._monitoring_task
+			except asyncio.CancelledError:
+				pass
+			# self.logger.debug('[StorageStateWatchdog] Stopped storage monitoring task')
+
+	async def _check_for_cookie_changes_cdp(self, event: dict) -> None:
+		"""Check if a CDP network event indicates cookie changes.
+
+		This would be called by Network.responseReceivedExtraInfo events
+		if we set up CDP event listeners.
+		"""
+		try:
+			# Check for Set-Cookie headers in the response
+			headers = event.get('headers', {})
+			if 'set-cookie' in headers or 'Set-Cookie' in headers:
+				self.logger.debug('[StorageStateWatchdog] Cookie change detected via CDP')
+
+				# If save on change is enabled, trigger save immediately
+				if self.save_on_change:
+					await self._save_storage_state()
+		except Exception as e:
+			self.logger.warning(f'[StorageStateWatchdog] Error checking for cookie changes: {e}')
+
+	async def _monitor_storage_changes(self) -> None:
+		"""Periodically check for storage changes and auto-save."""
+		while True:
+			try:
+				await asyncio.sleep(self.auto_save_interval)
+
+				# Check if cookies have changed
+				if await self._have_cookies_changed():
+					self.logger.debug('[StorageStateWatchdog] Detected changes to sync with storage_state.json')
+					await self._save_storage_state()
+
+			except asyncio.CancelledError:
+				break
+			except Exception as e:
+				self.logger.error(f'[StorageStateWatchdog] Error in monitoring loop: {e}')
+
+	async def _have_cookies_changed(self) -> bool:
+		"""Check if cookies have changed since last save."""
+		if not self.browser_session.cdp_client:
+			return False
+
+		try:
+			# Get current cookies using CDP
+			current_cookies = await self.browser_session._cdp_get_cookies()
+
+			# Convert to comparable format, using .get() for optional fields
+			current_cookie_set = {
+				(c.get('name', ''), c.get('domain', ''), c.get('path', '')): c.get('value', '') for c in current_cookies
+			}
+
+			last_cookie_set = {
+				(c.get('name', ''), c.get('domain', ''), c.get('path', '')): c.get('value', '') for c in self._last_cookie_state
+			}
+
+			return current_cookie_set != last_cookie_set
+		except Exception as e:
+			self.logger.debug(f'[StorageStateWatchdog] Error comparing cookies: {e}')
+			return False
+
+	async def _save_storage_state(self, path: str | None = None) -> None:
+		"""Save browser storage state to file."""
+		async with self._save_lock:
+			# Check if CDP client is available
+			assert await self.browser_session.get_or_create_cdp_session(target_id=None)
+
+			save_path = path or self.browser_session.browser_profile.storage_state
+			if not save_path:
+				return
+
+			# Skip saving if the storage state is already a dict (indicates it was loaded from memory)
+			# We only save to file if it started as a file path
+			if isinstance(save_path, dict):
+				self.logger.debug('[StorageStateWatchdog] Storage state is already a dict, skipping file save')
+				return
+
+			try:
+				# Get current storage state using CDP
+				storage_state = await self.browser_session._cdp_get_storage_state()
+
+				# Update our last known state
+				self._last_cookie_state = storage_state.get('cookies', []).copy()
+
+				# Convert path to Path object
+				json_path = Path(save_path).expanduser().resolve()
+				json_path.parent.mkdir(parents=True, exist_ok=True)
+
+				# Merge with existing state if file exists
+				merged_state = storage_state
+				if json_path.exists():
+					try:
+						existing_state = json.loads(json_path.read_text())
+						merged_state = self._merge_storage_states(existing_state, dict(storage_state))
+					except Exception as e:
+						self.logger.error(f'[StorageStateWatchdog] Failed to merge with existing state: {e}')
+
+				# Write atomically
+				temp_path = json_path.with_suffix('.json.tmp')
+				temp_path.write_text(json.dumps(merged_state, indent=4, ensure_ascii=False), encoding='utf-8')
+
+				# Backup existing file
+				if json_path.exists():
+					backup_path = json_path.with_suffix('.json.bak')
+					json_path.replace(backup_path)
+
+				# Move temp to final
+				temp_path.replace(json_path)
+
+				# Emit success event
+				self.event_bus.dispatch(
+					StorageStateSavedEvent(
+						path=str(json_path),
+						cookies_count=len(merged_state.get('cookies', [])),
+						origins_count=len(merged_state.get('origins', [])),
+					)
+				)
+
+				self.logger.debug(
+					f'[StorageStateWatchdog] Saved storage state to {json_path} '
+					f'({len(merged_state.get("cookies", []))} cookies, '
+					f'{len(merged_state.get("origins", []))} origins)'
+				)
+
+			except Exception as e:
+				self.logger.error(f'[StorageStateWatchdog] Failed to save storage state: {e}')
+
+	async def _load_storage_state(self, path: str | None = None) -> None:
+		"""Load browser storage state from file."""
+		if not self.browser_session.cdp_client:
+			self.logger.warning('[StorageStateWatchdog] No CDP client available for loading')
+			return
+
+		load_path = path or self.browser_session.browser_profile.storage_state
+		if not load_path or not os.path.exists(str(load_path)):
+			return
+
+		try:
+			# Read the storage state file asynchronously
+			import anyio
+
+			content = await anyio.Path(str(load_path)).read_text()
+			storage = json.loads(content)
+
+			# Apply cookies if present
+			if 'cookies' in storage and storage['cookies']:
+				# Playwright exports session cookies with expires=0/-1. CDP treats expires=0 as expired.
+				# Normalize session cookies by omitting expires
+				normalized_cookies: list[Cookie] = []
+				for cookie in storage['cookies']:
+					if not isinstance(cookie, dict):
+						normalized_cookies.append(cookie)  # type: ignore[arg-type]
+						continue
+					c = dict(cookie)
+					expires = c.get('expires')
+					if expires in (0, 0.0, -1, -1.0):
+						c.pop('expires', None)
+					normalized_cookies.append(Cookie(**c))
+
+				await self.browser_session._cdp_set_cookies(normalized_cookies)
+				self._last_cookie_state = storage['cookies'].copy()
+				self.logger.debug(f'[StorageStateWatchdog] Added {len(storage["cookies"])} cookies from storage state')
+
+			# Apply origins (localStorage/sessionStorage) if present
+			if 'origins' in storage and storage['origins']:
+				for origin in storage['origins']:
+					origin_value = origin.get('origin')
+					if not origin_value:
+						continue
+
+					# Scope storage restoration to its origin to avoid cross-site pollution.
+					if origin.get('localStorage'):
+						lines = []
+						for item in origin['localStorage']:
+							lines.append(f'window.localStorage.setItem({json.dumps(item["name"])}, {json.dumps(item["value"])});')
+						script = (
+							'(function(){\n'
+							f'  if (window.location && window.location.origin !== {json.dumps(origin_value)}) return;\n'
+							'  try {\n'
+							f'    {" ".join(lines)}\n'
+							'  } catch (e) {}\n'
+							'})();'
+						)
+						await self.browser_session._cdp_add_init_script(script)
+
+					if origin.get('sessionStorage'):
+						lines = []
+						for item in origin['sessionStorage']:
+							lines.append(
+								f'window.sessionStorage.setItem({json.dumps(item["name"])}, {json.dumps(item["value"])});'
+							)
+						script = (
+							'(function(){\n'
+							f'  if (window.location && window.location.origin !== {json.dumps(origin_value)}) return;\n'
+							'  try {\n'
+							f'    {" ".join(lines)}\n'
+							'  } catch (e) {}\n'
+							'})();'
+						)
+						await self.browser_session._cdp_add_init_script(script)
+				self.logger.debug(
+					f'[StorageStateWatchdog] Applied localStorage/sessionStorage from {len(storage["origins"])} origins'
+				)
+
+			self.event_bus.dispatch(
+				StorageStateLoadedEvent(
+					path=str(load_path),
+					cookies_count=len(storage.get('cookies', [])),
+					origins_count=len(storage.get('origins', [])),
+				)
+			)
+
+			self.logger.debug(f'[StorageStateWatchdog] Loaded storage state from: {load_path}')
+
+		except Exception as e:
+			self.logger.error(f'[StorageStateWatchdog] Failed to load storage state: {e}')
+
+	@staticmethod
+	def _merge_storage_states(existing: dict[str, Any], new: dict[str, Any]) -> dict[str, Any]:
+		"""Merge two storage states, with new values taking precedence."""
+		merged = existing.copy()
+
+		# Merge cookies
+		existing_cookies = {(c['name'], c['domain'], c['path']): c for c in existing.get('cookies', [])}
+
+		for cookie in new.get('cookies', []):
+			key = (cookie['name'], cookie['domain'], cookie['path'])
+			existing_cookies[key] = cookie
+
+		merged['cookies'] = list(existing_cookies.values())
+
+		# Merge origins
+		existing_origins = {origin['origin']: origin for origin in existing.get('origins', [])}
+
+		for origin in new.get('origins', []):
+			existing_origins[origin['origin']] = origin
+
+		merged['origins'] = list(existing_origins.values())
+
+		return merged
+
+	async def get_current_cookies(self) -> list[dict[str, Any]]:
+		"""Get current cookies using CDP."""
+		if not self.browser_session.cdp_client:
+			return []
+
+		try:
+			cookies = await self.browser_session._cdp_get_cookies()
+			# Cookie is a TypedDict, cast to dict for compatibility
+			return [dict(cookie) for cookie in cookies]
+		except Exception as e:
+			self.logger.error(f'[StorageStateWatchdog] Failed to get cookies: {e}')
+			return []
+
+	async def add_cookies(self, cookies: list[dict[str, Any]]) -> None:
+		"""Add cookies using CDP."""
+		if not self.browser_session.cdp_client:
+			self.logger.warning('[StorageStateWatchdog] No CDP client available for adding cookies')
+			return
+
+		try:
+			# Convert dicts to Cookie objects
+			cookie_objects = [Cookie(**cookie_dict) if isinstance(cookie_dict, dict) else cookie_dict for cookie_dict in cookies]
+			# Set cookies using CDP
+			await self.browser_session._cdp_set_cookies(cookie_objects)
+			self.logger.debug(f'[StorageStateWatchdog] Added {len(cookies)} cookies')
+		except Exception as e:
+			self.logger.error(f'[StorageStateWatchdog] Failed to add cookies: {e}')

browser_use/cli.py

@@ -0,0 +1,464 @@
+"""Browser Use CLI backed by Browser Harness"""
+
+from __future__ import annotations
+
+import sys
+import time
+from contextlib import redirect_stderr, redirect_stdout
+from importlib.metadata import PackageNotFoundError, version
+from io import StringIO
+
+
+def _browser_use_version() -> str:
+	try:
+		return version('browser-use')
+	except PackageNotFoundError:
+		return 'unknown'
+
+
+def _exit_code(result: int | str | None) -> int:
+	if result is None:
+		return 0
+	if isinstance(result, int):
+		return result
+	return 1
+
+
+def _set_harness_client_env() -> None:
+	import os
+
+	os.environ['BH_CLIENT'] = 'browser-use-cli'
+	os.environ['BH_CLIENT_VERSION'] = _browser_use_version()
+
+
+def _capture_via_harness(
+	*,
+	command: str,
+	start_time: float,
+	result: int | str | None = None,
+	error_message: str | None = None,
+) -> None:
+	try:
+		from browser_harness import telemetry as harness_telemetry
+
+		capture_cli_event = getattr(harness_telemetry, 'capture_cli_event', None)
+		if capture_cli_event is None:
+			return
+		_set_harness_client_env()
+		code = _exit_code(result)
+		capture_cli_event(
+			action='error' if code else 'completed',
+			command=command,
+			duration_seconds=time.monotonic() - start_time,
+			exit_code=code,
+			error_message=error_message,
+		)
+	except Exception:
+		pass
+
+
+def _run_mcp_stdio_server(module_name: str) -> None:
+	"""Silence all logging"""
+	import asyncio
+	import importlib
+	import logging
+	import os
+
+	os.environ['BROWSER_USE_LOGGING_LEVEL'] = 'critical'
+	os.environ['BROWSER_USE_SETUP_LOGGING'] = 'false'
+	logging.disable(logging.CRITICAL)
+
+	main = importlib.import_module(module_name).main
+	asyncio.run(main())
+
+
+def _run_mcp_server() -> None:
+	_run_mcp_stdio_server('browser_use.mcp.server')
+
+
+def _run_cli_mcp_server() -> None:
+	_run_mcp_stdio_server('browser_use.mcp.cli_mcp')
+
+
+def _run_install_command(argv: list[str]) -> int:
+	if any(arg in {'-h', '--help'} for arg in argv):
+		print('usage: browser-use install')
+		print()
+		print('Install Chromium browser and system dependencies.')
+		return 0
+
+	import platform
+	import subprocess
+
+	print('Installing Chromium browser + system dependencies...')
+	print('This may take a few minutes...\n')
+
+	cmd = ['uvx', 'playwright', 'install', 'chromium']
+	if platform.system() == 'Linux':
+		cmd.append('--with-deps')
+	cmd.append('--no-shell')
+
+	result = subprocess.run(cmd)
+	if result.returncode == 0:
+		print('\nInstallation complete.')
+		print('Ready to use. Run: uvx browser-use')
+		return 0
+
+	print('\nInstallation failed', file=sys.stderr)
+	return result.returncode or 1
+
+
+def _run_init_command(argv: list[str]) -> int | None:
+	from browser_use.init_cmd import main as init_main
+
+	original_argv = sys.argv
+	try:
+		sys.argv = [original_argv[0], *argv]
+		init_main()
+	except SystemExit as exc:
+		if exc.code is None:
+			return 0
+		if isinstance(exc.code, int):
+			return exc.code
+		print(exc.code, file=sys.stderr)
+		return 1
+	finally:
+		sys.argv = original_argv
+	return 0
+
+
+def _as_browser_use_cli_text(text: str) -> str:
+	return text.replace('Browser Harness', 'Browser Use').replace('browser-harness', 'browser-use')
+
+
+def _normalize_captured_cli_output(func, argv: list[str]) -> int:
+	stdout = StringIO()
+	stderr = StringIO()
+	try:
+		with redirect_stdout(stdout), redirect_stderr(stderr):
+			result = func(argv)
+	except SystemExit as exc:
+		result = exc.code
+
+	out = stdout.getvalue()
+	err = stderr.getvalue()
+	if out:
+		print(_as_browser_use_cli_text(out), end='')
+	if err:
+		print(_as_browser_use_cli_text(err), end='', file=sys.stderr)
+	if result is None:
+		return 0
+	if isinstance(result, int):
+		return result
+	if isinstance(result, str):
+		print(_as_browser_use_cli_text(result), file=sys.stderr)
+		return 1
+	return 1
+
+
+def _patch_browser_harness_cli_text() -> None:
+	from browser_harness import auth, run, telemetry
+
+	run.HELP = _as_browser_use_cli_text(run.HELP)
+	run.USAGE = _as_browser_use_cli_text(run.USAGE)
+
+	original_auth_cli = auth.run_auth_cli
+	original_telemetry_cli = telemetry.run_telemetry_cli
+
+	def run_auth_cli(argv: list[str]) -> int:
+		if any(arg in {'-h', '--help'} for arg in argv):
+			return _normalize_captured_cli_output(original_auth_cli, argv)
+		return original_auth_cli(argv)
+
+	def run_telemetry_cli(argv: list[str]) -> int:
+		if argv and argv != ['status'] and argv != ['enable'] and argv != ['disable']:
+			return _normalize_captured_cli_output(original_telemetry_cli, argv)
+		return original_telemetry_cli(argv)
+
+	auth.run_auth_cli = run_auth_cli
+	telemetry.run_telemetry_cli = run_telemetry_cli
+
+
+_delegated_to_harness = False
+
+
+def _run_browser_harness() -> int | None:
+	from browser_harness import run
+
+	global _delegated_to_harness
+
+	_set_harness_client_env()
+	_patch_browser_harness_cli_text()
+	args = sys.argv[1:]
+	if args and args[0] == 'doctor' and args[1:]:
+		if args[1:] in (['--help'], ['-h']):
+			print('usage: browser-use doctor [--fix-snap]')
+			return 0
+		if args[1:] != ['--fix-snap']:
+			print('usage: browser-use doctor [--fix-snap]', file=sys.stderr)
+			sys.exit(2)
+	_delegated_to_harness = True
+	run.main()
+	return None
+
+
+# Subcommands and flags from the pre-3.0 CLI; maps to hint
+_LEGACY_HINTS: dict[str, str] = {
+	'open': 'new_tab("https://example.com")',
+	'state': 'print(page_info())',
+	'screenshot': 'print(capture_screenshot())',
+	'eval': 'print(js("document.title"))',
+	'cookies': 'print(cdp("Network.getCookies"))',
+	'python': '# the CLI runs Python directly now — pipe it on stdin as shown below',
+	'run': '# write the steps as Python using the pre-imported helpers shown below',
+	'connect': '# connecting is automatic — the default flow attaches to your running Chrome',
+	'close': '# restart the local daemon with `browser-use --reload`; stop cloud browsers with stop_remote_daemon(name)',
+	'sessions': '# named local sessions were removed — one default daemon; use BU_NAME=<name> for cloud daemons',
+	'profile': '# profiles now come from your real Chrome; see the profile-sync interaction skill',
+	'cloud': '# authenticate with `browser-use auth login`, then start_remote_daemon("<name>")',
+	'daemon': '# the daemon starts automatically on every call; restart it with `browser-use --reload`',
+	'record': '# session recording was removed; use capture_screenshot() per step',
+	'mcp': '# MCP server mode is now the --mcp flag: `browser-use --mcp`',
+	'--session': '# use BU_NAME=<name> as an env var for named cloud daemons; local runs use one default daemon',
+	'--headed': '# local control always attaches to your real, visible Chrome — no flag needed',
+	'--cdp-url': '# use the BU_CDP_URL=<url> env var instead of a flag',
+	'--profile': '# use your real Chrome profile; for cloud cookie sync see the profile-sync interaction skill',
+	'--json': '# output is whatever your Python prints — use print(json.dumps(...))',
+	'-c': "# pipe code on stdin instead: echo 'print(page_info())' | browser-use",
+	'--code': "# pipe code on stdin instead: echo 'print(page_info())' | browser-use",
+}
+
+
+def _legacy_command(args: list[str]) -> str | None:
+	if not args:
+		return None
+	first = args[0].split('=', 1)[0]
+	if first in _LEGACY_HINTS:
+		return first
+	return None
+
+
+_CLI3_GUIDE = """Core helpers: new_tab(url), goto_url(url), page_info(), capture_screenshot(),
+  click_at_xy(x, y), type_text(text), fill_input(selector, text), press_key(key),
+  scroll(x, y), js(code), cdp(method, ...), wait_for_load(), wait_for_element(selector),
+  list_tabs(), switch_tab(target), close_tab(target)
+
+Read the full interface now:   browser-use skill show
+Install the CLI 3.0 skill (upgrades the CLI and replaces any pre-existing browser-use skill):
+                               browser-use skill install
+Then load the freshly installed browser-use skill, and follow it to use the latest CLI.
+Skill reference:               https://github.com/browser-use/browser-use/blob/main/browser_use/skills/browser-use/SKILL.md
+Health check:                  browser-use --doctor"""
+
+
+def _legacy_migration_message(command: str) -> str:
+	hint = _LEGACY_HINTS[command]
+	if hint.startswith('#'):
+		replacement = f'  {hint.lstrip("# ")}\n\nExample:\n  browser-use <<\'PY\'\n  new_tab("https://example.com")\n  print(page_info())\n  PY'
+	else:
+		replacement = f"  browser-use <<'PY'\n  {hint}\n  PY"
+	return f"""The browser-use CLI changed in 3.0, and '{command}' was removed.
+
+The old preset subcommands are gone. To use the CLI, you write raw Python and
+pipe it on stdin, and it runs in a persistent browser session. Browser management
+(daemon startup, Chrome/CDP attach, tabs, waiting) is handled for you.
+
+Replacement for '{command}':
+{replacement}
+
+{_CLI3_GUIDE}"""
+
+
+def _unknown_helper_message(name: str) -> str:
+	return f"""'{name}' is not defined in the browser-use CLI.
+
+Example:
+  browser-use <<'PY'
+  new_tab("https://example.com")
+  print(page_info())
+  PY
+
+{_CLI3_GUIDE}"""
+
+
+def _unknown_exec_name(exc: NameError) -> str | None:
+	import re
+
+	name = getattr(exc, 'name', None)
+	if name:
+		return name
+	m = re.search(r"'([A-Za-z_][A-Za-z0-9_]*)'", str(exc))
+	return m.group(1) if m else None
+
+
+def _raised_from_piped_code(exc: BaseException) -> bool:
+	tb = exc.__traceback__
+	last = None
+	while tb is not None:
+		last = tb
+		tb = tb.tb_next
+	return last is not None and last.tb_frame.f_code.co_filename == '<string>'
+
+
+_QUICKSTART = """Welcome to the Browser Use CLI. Allow your coding agent to reliably control a web browser.
+
+The CLI allows your agent to control the browser via Python, and it manages the browser in the background.
+
+  browser-use <<'PY'
+  new_tab("https://news.ycombinator.com")
+  print(page_info())
+  PY
+
+Core helpers: new_tab(url), goto_url(url), page_info(),
+  capture_screenshot(), click_at_xy(x, y), js(code), cdp(method, ...),
+  wait_for_load()
+
+Recommended: install the skill so your coding agent remembers this:
+
+  browser-use skill install
+
+You can also paste this into your agent to get started:
+
+  Install or upgrade browser-use to the latest stable version with uv using
+  Python 3.12, register the skill from `browser-use skill`, and connect it to
+  my browser. Follow https://github.com/browser-use/browser-use if setup or
+  connection fails.
+
+More:
+  browser-use --doctor     check install, daemon, and browser health
+  browser-use --help       full command list
+  docs: https://github.com/browser-use/browser-use/blob/main/browser_use/skills/browser-use/SKILL.md"""
+
+_EMPTY_STDIN_MESSAGE = """browser-use received empty stdin. This CLI executes Python piped on stdin:
+  browser-use <<'PY'
+  print(page_info())
+  PY"""
+
+
+def _command_name(args: list[str]) -> str:
+	if '--cli-mcp' in args:
+		return 'cli-mcp'
+	if '--mcp' in args:
+		return 'mcp'
+	if args and args[0] == 'install':
+		return 'install'
+	if args and args[0] == 'init':
+		return 'init'
+	if '--template' in args or '-t' in args:
+		return 'init'
+	if args and args[0] == 'skill':
+		return 'skill'
+	legacy = _legacy_command(args)
+	if legacy is not None:
+		return f'legacy:{legacy}'
+	return args[0] if args else 'run'
+
+
+def _dispatch(args: list[str]) -> tuple[int | None, str]:
+	if '--cli-mcp' in args:
+		_run_cli_mcp_server()
+		return 0, 'cli-mcp'
+	if '--mcp' in args:
+		_run_mcp_server()
+		return 0, 'mcp'
+	if args and args[0] == 'install':
+		return _run_install_command(args[1:]), 'install'
+	if args and args[0] == 'init':
+		return _run_init_command(args[1:]), 'init'
+	if '--template' in args or '-t' in args:
+		return _run_init_command(args), 'init'
+	if args and args[0] == 'skill':
+		from browser_use.skills.install import handle as handle_skill_command
+
+		return handle_skill_command(args[1:]), 'skill'
+
+	legacy = _legacy_command(args)
+	if legacy is not None:
+		print(_legacy_migration_message(legacy), file=sys.stderr)
+		return 2, f'legacy:{legacy}'
+
+	if not args:
+		if sys.stdin.isatty():
+			print(_QUICKSTART)
+			return 0, 'quickstart'
+		code = sys.stdin.read()
+		if not code.strip():
+			print(_EMPTY_STDIN_MESSAGE, file=sys.stderr)
+			return 1, 'run'
+		sys.stdin = StringIO(code)
+
+	try:
+		return _run_browser_harness(), args[0] if args else 'run'
+	except NameError as exc:
+		name = _unknown_exec_name(exc)
+		if name is None or not _raised_from_piped_code(exc):
+			raise
+		import traceback
+
+		traceback.print_exc()
+		print(_unknown_helper_message(name), file=sys.stderr)
+		return 2, args[0] if args else 'run'
+
+
+class _StderrTail:
+	"""Pass-through stderr wrapper that remembers the tail as error context."""
+
+	def __init__(self, wrapped):
+		self._wrapped = wrapped
+		self.tail = ''
+
+	def write(self, text):
+		self.tail = (self.tail + text)[-500:]
+		return self._wrapped.write(text)
+
+	def __getattr__(self, name):
+		return getattr(self._wrapped, name)
+
+
+def browser_use_tui_main() -> int | None:
+	print('browser-use-tui is deprecated; use browser-use instead.', file=sys.stderr)
+	return main()
+
+
+def main() -> int | None:
+	global _delegated_to_harness
+
+	_delegated_to_harness = False
+	args = sys.argv[1:]
+	start_time = time.monotonic()
+	command = _command_name(args)
+	stderr_tail = _StderrTail(sys.stderr)
+	sys.stderr = stderr_tail
+	try:
+		result, command = _dispatch(args)
+	except SystemExit as exc:
+		result = exc.code
+		if not _delegated_to_harness:
+			_capture_via_harness(
+				command=command,
+				start_time=start_time,
+				result=result,
+				error_message=str(result) if isinstance(result, str) else stderr_tail.tail.strip() or None,
+			)
+		raise
+	except Exception as exc:
+		if not _delegated_to_harness:
+			_capture_via_harness(command=command, start_time=start_time, result=1, error_message=str(exc))
+		raise
+	finally:
+		sys.stderr = stderr_tail._wrapped
+
+	if not _delegated_to_harness:
+		_capture_via_harness(
+			command=command,
+			start_time=start_time,
+			result=result,
+			error_message=(stderr_tail.tail.strip() or None) if _exit_code(result) else None,
+		)
+	return result
+
+
+if __name__ == '__main__':
+	result = main()
+	if result is not None:
+		sys.exit(result)

browser_use/config.py

@@ -0,0 +1,525 @@
+"""Configuration system for browser-use with automatic migration support."""
+
+import json
+import logging
+import os
+from datetime import datetime
+from functools import cache
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import psutil
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+logger = logging.getLogger(__name__)
+
+
+@cache
+def is_running_in_docker() -> bool:
+	"""Detect if we are running in a docker container, for the purpose of optimizing chrome launch flags (dev shm usage, gpu settings, etc.)"""
+	try:
+		if Path('/.dockerenv').exists() or 'docker' in Path('/proc/1/cgroup').read_text().lower():
+			return True
+	except Exception:
+		pass
+
+	try:
+		# if init proc (PID 1) looks like uvicorn/python/uv/etc. then we're in Docker
+		# if init proc (PID 1) looks like bash/systemd/init/etc. then we're probably NOT in Docker
+		init_cmd = ' '.join(psutil.Process(1).cmdline())
+		if ('py' in init_cmd) or ('uv' in init_cmd) or ('app' in init_cmd):
+			return True
+	except Exception:
+		pass
+
+	try:
+		# if less than 10 total running procs, then we're almost certainly in a container
+		if len(psutil.pids()) < 10:
+			return True
+	except Exception:
+		pass
+
+	return False
+
+
+class OldConfig:
+	"""Original lazy-loading configuration class for environment variables."""
+
+	# Cache for directory creation tracking
+	_dirs_created = False
+
+	@property
+	def BROWSER_USE_LOGGING_LEVEL(self) -> str:
+		return os.getenv('BROWSER_USE_LOGGING_LEVEL', 'info').lower()
+
+	@property
+	def ANONYMIZED_TELEMETRY(self) -> bool:
+		return os.getenv('ANONYMIZED_TELEMETRY', 'true').lower()[:1] in 'ty1'
+
+	@property
+	def BROWSER_USE_CLOUD_SYNC(self) -> bool:
+		return os.getenv('BROWSER_USE_CLOUD_SYNC', str(self.ANONYMIZED_TELEMETRY)).lower()[:1] in 'ty1'
+
+	@property
+	def BROWSER_USE_CLOUD_API_URL(self) -> str:
+		url = os.getenv('BROWSER_USE_CLOUD_API_URL', 'https://api.browser-use.com')
+		assert '://' in url, 'BROWSER_USE_CLOUD_API_URL must be a valid URL'
+		return url
+
+	@property
+	def BROWSER_USE_CLOUD_UI_URL(self) -> str:
+		url = os.getenv('BROWSER_USE_CLOUD_UI_URL', '')
+		# Allow empty string as default, only validate if set
+		if url and '://' not in url:
+			raise AssertionError('BROWSER_USE_CLOUD_UI_URL must be a valid URL if set')
+		return url
+
+	@property
+	def BROWSER_USE_MODEL_PRICING_URL(self) -> str:
+		url = os.getenv('BROWSER_USE_MODEL_PRICING_URL', '')
+		if url and '://' not in url:
+			raise AssertionError('BROWSER_USE_MODEL_PRICING_URL must be a valid URL if set')
+		return url
+
+	# Path configuration
+	@property
+	def XDG_CACHE_HOME(self) -> Path:
+		return Path(os.getenv('XDG_CACHE_HOME', '~/.cache')).expanduser().resolve()
+
+	@property
+	def XDG_CONFIG_HOME(self) -> Path:
+		return Path(os.getenv('XDG_CONFIG_HOME', '~/.config')).expanduser().resolve()
+
+	@property
+	def BROWSER_USE_CONFIG_DIR(self) -> Path:
+		path = Path(os.getenv('BROWSER_USE_CONFIG_DIR', str(self.XDG_CONFIG_HOME / 'browseruse'))).expanduser().resolve()
+		self._ensure_dirs()
+		return path
+
+	@property
+	def BROWSER_USE_CONFIG_FILE(self) -> Path:
+		return self.BROWSER_USE_CONFIG_DIR / 'config.json'
+
+	@property
+	def BROWSER_USE_PROFILES_DIR(self) -> Path:
+		path = self.BROWSER_USE_CONFIG_DIR / 'profiles'
+		self._ensure_dirs()
+		return path
+
+	@property
+	def BROWSER_USE_DEFAULT_USER_DATA_DIR(self) -> Path:
+		return self.BROWSER_USE_PROFILES_DIR / 'default'
+
+	@property
+	def BROWSER_USE_EXTENSIONS_DIR(self) -> Path:
+		path = self.BROWSER_USE_CONFIG_DIR / 'extensions'
+		self._ensure_dirs()
+		return path
+
+	def _ensure_dirs(self) -> None:
+		"""Create directories if they don't exist (only once)"""
+		if not self._dirs_created:
+			config_dir = (
+				Path(os.getenv('BROWSER_USE_CONFIG_DIR', str(self.XDG_CONFIG_HOME / 'browseruse'))).expanduser().resolve()
+			)
+			config_dir.mkdir(parents=True, exist_ok=True)
+			(config_dir / 'profiles').mkdir(parents=True, exist_ok=True)
+			(config_dir / 'extensions').mkdir(parents=True, exist_ok=True)
+			self._dirs_created = True
+
+	# LLM API key configuration
+	@property
+	def OPENAI_API_KEY(self) -> str:
+		return os.getenv('OPENAI_API_KEY', '')
+
+	@property
+	def ANTHROPIC_API_KEY(self) -> str:
+		return os.getenv('ANTHROPIC_API_KEY', '')
+
+	@property
+	def GOOGLE_API_KEY(self) -> str:
+		return os.getenv('GOOGLE_API_KEY', '')
+
+	@property
+	def DEEPSEEK_API_KEY(self) -> str:
+		return os.getenv('DEEPSEEK_API_KEY', '')
+
+	@property
+	def GROK_API_KEY(self) -> str:
+		return os.getenv('GROK_API_KEY', '')
+
+	@property
+	def NOVITA_API_KEY(self) -> str:
+		return os.getenv('NOVITA_API_KEY', '')
+
+	@property
+	def AZURE_OPENAI_ENDPOINT(self) -> str:
+		return os.getenv('AZURE_OPENAI_ENDPOINT', '')
+
+	@property
+	def AZURE_OPENAI_KEY(self) -> str:
+		return os.getenv('AZURE_OPENAI_KEY', '')
+
+	@property
+	def SKIP_LLM_API_KEY_VERIFICATION(self) -> bool:
+		return os.getenv('SKIP_LLM_API_KEY_VERIFICATION', 'false').lower()[:1] in 'ty1'
+
+	@property
+	def DEFAULT_LLM(self) -> str:
+		return os.getenv('DEFAULT_LLM', '')
+
+	# Runtime hints
+	@property
+	def IN_DOCKER(self) -> bool:
+		return os.getenv('IN_DOCKER', 'false').lower()[:1] in 'ty1' or is_running_in_docker()
+
+	@property
+	def IS_IN_EVALS(self) -> bool:
+		return os.getenv('IS_IN_EVALS', 'false').lower()[:1] in 'ty1'
+
+	@property
+	def BROWSER_USE_VERSION_CHECK(self) -> bool:
+		return os.getenv('BROWSER_USE_VERSION_CHECK', 'true').lower()[:1] in 'ty1'
+
+	@property
+	def WIN_FONT_DIR(self) -> str:
+		return os.getenv('WIN_FONT_DIR', 'C:\\Windows\\Fonts')
+
+
+class FlatEnvConfig(BaseSettings):
+	"""All environment variables in a flat namespace."""
+
+	model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', case_sensitive=True, extra='allow')
+
+	# Logging and telemetry
+	BROWSER_USE_LOGGING_LEVEL: str = Field(default='info')
+	CDP_LOGGING_LEVEL: str = Field(default='WARNING')
+	BROWSER_USE_DEBUG_LOG_FILE: str | None = Field(default=None)
+	BROWSER_USE_INFO_LOG_FILE: str | None = Field(default=None)
+	ANONYMIZED_TELEMETRY: bool = Field(default=True)
+	BROWSER_USE_CLOUD_SYNC: bool | None = Field(default=None)
+	BROWSER_USE_CLOUD_API_URL: str = Field(default='https://api.browser-use.com')
+	BROWSER_USE_CLOUD_UI_URL: str = Field(default='')
+	BROWSER_USE_MODEL_PRICING_URL: str = Field(default='')
+
+	# Path configuration
+	XDG_CACHE_HOME: str = Field(default='~/.cache')
+	XDG_CONFIG_HOME: str = Field(default='~/.config')
+	BROWSER_USE_CONFIG_DIR: str | None = Field(default=None)
+
+	# LLM API keys
+	OPENAI_API_KEY: str = Field(default='')
+	ANTHROPIC_API_KEY: str = Field(default='')
+	GOOGLE_API_KEY: str = Field(default='')
+	DEEPSEEK_API_KEY: str = Field(default='')
+	GROK_API_KEY: str = Field(default='')
+	NOVITA_API_KEY: str = Field(default='')
+	AZURE_OPENAI_ENDPOINT: str = Field(default='')
+	AZURE_OPENAI_KEY: str = Field(default='')
+	SKIP_LLM_API_KEY_VERIFICATION: bool = Field(default=False)
+	DEFAULT_LLM: str = Field(default='')
+
+	# Runtime hints
+	IN_DOCKER: bool | None = Field(default=None)
+	IS_IN_EVALS: bool = Field(default=False)
+	WIN_FONT_DIR: str = Field(default='C:\\Windows\\Fonts')
+	BROWSER_USE_VERSION_CHECK: bool = Field(default=True)
+
+	# MCP-specific env vars
+	BROWSER_USE_CONFIG_PATH: str | None = Field(default=None)
+	BROWSER_USE_HEADLESS: bool | None = Field(default=None)
+	BROWSER_USE_ALLOWED_DOMAINS: str | None = Field(default=None)
+	BROWSER_USE_LLM_MODEL: str | None = Field(default=None)
+
+	# Proxy env vars
+	BROWSER_USE_PROXY_URL: str | None = Field(default=None)
+	BROWSER_USE_NO_PROXY: str | None = Field(default=None)
+	BROWSER_USE_PROXY_USERNAME: str | None = Field(default=None)
+	BROWSER_USE_PROXY_PASSWORD: str | None = Field(default=None)
+
+	# Extension env vars
+	BROWSER_USE_DISABLE_EXTENSIONS: bool | None = Field(default=None)
+
+
+class DBStyleEntry(BaseModel):
+	"""Database-style entry with UUID and metadata."""
+
+	id: str = Field(default_factory=lambda: str(uuid4()))
+	default: bool = Field(default=False)
+	created_at: str = Field(default_factory=lambda: datetime.utcnow().isoformat())
+
+
+class BrowserProfileEntry(DBStyleEntry):
+	"""Browser profile configuration entry - accepts any BrowserProfile fields."""
+
+	model_config = ConfigDict(extra='allow')
+
+	# Common browser profile fields for reference
+	headless: bool | None = None
+	user_data_dir: str | None = None
+	allowed_domains: list[str] | None = None
+	downloads_path: str | None = None
+
+
+class LLMEntry(DBStyleEntry):
+	"""LLM configuration entry."""
+
+	api_key: str | None = None
+	model: str | None = None
+	temperature: float | None = None
+	max_tokens: int | None = None
+
+
+class AgentEntry(DBStyleEntry):
+	"""Agent configuration entry."""
+
+	max_steps: int | None = None
+	use_vision: bool | None = None
+	system_prompt: str | None = None
+
+
+class DBStyleConfigJSON(BaseModel):
+	"""New database-style configuration format."""
+
+	browser_profile: dict[str, BrowserProfileEntry] = Field(default_factory=dict)
+	llm: dict[str, LLMEntry] = Field(default_factory=dict)
+	agent: dict[str, AgentEntry] = Field(default_factory=dict)
+
+
+def create_default_config() -> DBStyleConfigJSON:
+	"""Create a fresh default configuration."""
+	logger.debug('Creating fresh default config.json')
+
+	new_config = DBStyleConfigJSON()
+
+	# Generate default IDs
+	profile_id = str(uuid4())
+	llm_id = str(uuid4())
+	agent_id = str(uuid4())
+
+	# Create default browser profile entry
+	new_config.browser_profile[profile_id] = BrowserProfileEntry(id=profile_id, default=True, headless=False, user_data_dir=None)
+
+	# Create default LLM entry
+	new_config.llm[llm_id] = LLMEntry(id=llm_id, default=True, model='gpt-4.1-mini', api_key='your-openai-api-key-here')
+
+	# Create default agent entry
+	new_config.agent[agent_id] = AgentEntry(id=agent_id, default=True)
+
+	return new_config
+
+
+def load_and_migrate_config(config_path: Path) -> DBStyleConfigJSON:
+	"""Load config.json or create fresh one if old format detected."""
+	if not config_path.exists():
+		# Create fresh config with defaults
+		config_path.parent.mkdir(parents=True, exist_ok=True)
+		new_config = create_default_config()
+		with open(config_path, 'w') as f:
+			json.dump(new_config.model_dump(), f, indent=2)
+		return new_config
+
+	try:
+		with open(config_path) as f:
+			data = json.load(f)
+
+		# Check if it's already in DB-style format
+		if all(key in data for key in ['browser_profile', 'llm', 'agent']) and all(
+			isinstance(data.get(key, {}), dict) for key in ['browser_profile', 'llm', 'agent']
+		):
+			# Check if the values are DB-style entries (have UUIDs as keys)
+			if data.get('browser_profile') and all(isinstance(v, dict) and 'id' in v for v in data['browser_profile'].values()):
+				# Already in new format
+				return DBStyleConfigJSON(**data)
+
+		# Old format detected - delete it and create fresh config
+		logger.debug(f'Old config format detected at {config_path}, creating fresh config')
+		new_config = create_default_config()
+
+		# Overwrite with new config
+		with open(config_path, 'w') as f:
+			json.dump(new_config.model_dump(), f, indent=2)
+
+		logger.debug(f'Created fresh config.json at {config_path}')
+		return new_config
+
+	except Exception as e:
+		logger.error(f'Failed to load config from {config_path}: {e}, creating fresh config')
+		# On any error, create fresh config
+		new_config = create_default_config()
+		try:
+			with open(config_path, 'w') as f:
+				json.dump(new_config.model_dump(), f, indent=2)
+		except Exception as write_error:
+			logger.error(f'Failed to write fresh config: {write_error}')
+		return new_config
+
+
+class Config:
+	"""Backward-compatible configuration class that merges all config sources.
+
+	Re-reads environment variables on every access to maintain compatibility.
+	"""
+
+	def __init__(self):
+		# Cache for directory creation tracking only
+		self._dirs_created = False
+
+	def __getattr__(self, name: str) -> Any:
+		"""Dynamically proxy all attributes to fresh instances.
+
+		This ensures env vars are re-read on every access.
+		"""
+		# Special handling for internal attributes
+		if name.startswith('_'):
+			raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
+
+		# Create fresh instances on every access
+		old_config = OldConfig()
+
+		# Always use old config for all attributes (it handles env vars with proper transformations)
+		if hasattr(old_config, name):
+			return getattr(old_config, name)
+
+		# For new MCP-specific attributes not in old config
+		env_config = FlatEnvConfig()
+		if hasattr(env_config, name):
+			return getattr(env_config, name)
+
+		# Handle special methods
+		if name == 'get_default_profile':
+			return lambda: self._get_default_profile()
+		elif name == 'get_default_llm':
+			return lambda: self._get_default_llm()
+		elif name == 'get_default_agent':
+			return lambda: self._get_default_agent()
+		elif name == 'load_config':
+			return lambda: self._load_config()
+		elif name == '_ensure_dirs':
+			return lambda: old_config._ensure_dirs()
+
+		raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
+
+	def _get_config_path(self) -> Path:
+		"""Get config path from fresh env config."""
+		env_config = FlatEnvConfig()
+		if env_config.BROWSER_USE_CONFIG_PATH:
+			return Path(env_config.BROWSER_USE_CONFIG_PATH).expanduser()
+		elif env_config.BROWSER_USE_CONFIG_DIR:
+			return Path(env_config.BROWSER_USE_CONFIG_DIR).expanduser() / 'config.json'
+		else:
+			xdg_config = Path(env_config.XDG_CONFIG_HOME).expanduser()
+			return xdg_config / 'browseruse' / 'config.json'
+
+	def _get_db_config(self) -> DBStyleConfigJSON:
+		"""Load and migrate config.json."""
+		config_path = self._get_config_path()
+		return load_and_migrate_config(config_path)
+
+	def _get_default_profile(self) -> dict[str, Any]:
+		"""Get the default browser profile configuration."""
+		db_config = self._get_db_config()
+		for profile in db_config.browser_profile.values():
+			if profile.default:
+				return profile.model_dump(exclude_none=True)
+
+		# Return first profile if no default
+		if db_config.browser_profile:
+			return next(iter(db_config.browser_profile.values())).model_dump(exclude_none=True)
+
+		return {}
+
+	def _get_default_llm(self) -> dict[str, Any]:
+		"""Get the default LLM configuration."""
+		db_config = self._get_db_config()
+		for llm in db_config.llm.values():
+			if llm.default:
+				return llm.model_dump(exclude_none=True)
+
+		# Return first LLM if no default
+		if db_config.llm:
+			return next(iter(db_config.llm.values())).model_dump(exclude_none=True)
+
+		return {}
+
+	def _get_default_agent(self) -> dict[str, Any]:
+		"""Get the default agent configuration."""
+		db_config = self._get_db_config()
+		for agent in db_config.agent.values():
+			if agent.default:
+				return agent.model_dump(exclude_none=True)
+
+		# Return first agent if no default
+		if db_config.agent:
+			return next(iter(db_config.agent.values())).model_dump(exclude_none=True)
+
+		return {}
+
+	def _load_config(self) -> dict[str, Any]:
+		"""Load configuration with env var overrides for MCP components."""
+		config = {
+			'browser_profile': self._get_default_profile(),
+			'llm': self._get_default_llm(),
+			'agent': self._get_default_agent(),
+		}
+
+		# Fresh env config for overrides
+		env_config = FlatEnvConfig()
+
+		# Apply MCP-specific env var overrides
+		if env_config.BROWSER_USE_HEADLESS is not None:
+			config['browser_profile']['headless'] = env_config.BROWSER_USE_HEADLESS
+
+		if env_config.BROWSER_USE_ALLOWED_DOMAINS:
+			domains = [d.strip() for d in env_config.BROWSER_USE_ALLOWED_DOMAINS.split(',') if d.strip()]
+			config['browser_profile']['allowed_domains'] = domains
+
+		# Proxy settings (Chromium) -> consolidated `proxy` dict
+		proxy_dict: dict[str, Any] = {}
+		if env_config.BROWSER_USE_PROXY_URL:
+			proxy_dict['server'] = env_config.BROWSER_USE_PROXY_URL
+		if env_config.BROWSER_USE_NO_PROXY:
+			# store bypass as comma-separated string to match Chrome flag
+			proxy_dict['bypass'] = ','.join([d.strip() for d in env_config.BROWSER_USE_NO_PROXY.split(',') if d.strip()])
+		if env_config.BROWSER_USE_PROXY_USERNAME:
+			proxy_dict['username'] = env_config.BROWSER_USE_PROXY_USERNAME
+		if env_config.BROWSER_USE_PROXY_PASSWORD:
+			proxy_dict['password'] = env_config.BROWSER_USE_PROXY_PASSWORD
+		if proxy_dict:
+			# ensure section exists
+			config.setdefault('browser_profile', {})
+			config['browser_profile']['proxy'] = proxy_dict
+
+		if env_config.OPENAI_API_KEY:
+			config['llm']['api_key'] = env_config.OPENAI_API_KEY
+
+		if env_config.BROWSER_USE_LLM_MODEL:
+			config['llm']['model'] = env_config.BROWSER_USE_LLM_MODEL
+
+		# Extension settings
+		if env_config.BROWSER_USE_DISABLE_EXTENSIONS is not None:
+			config['browser_profile']['enable_default_extensions'] = not env_config.BROWSER_USE_DISABLE_EXTENSIONS
+
+		return config
+
+
+# Create singleton instance
+CONFIG = Config()
+
+
+# Helper functions for MCP components
+def load_browser_use_config() -> dict[str, Any]:
+	"""Load browser-use configuration for MCP components."""
+	return CONFIG.load_config()
+
+
+def get_default_profile(config: dict[str, Any]) -> dict[str, Any]:
+	"""Get default browser profile from config dict."""
+	return config.get('browser_profile', {})
+
+
+def get_default_llm(config: dict[str, Any]) -> dict[str, Any]:
+	"""Get default LLM config from config dict."""
+	return config.get('llm', {})

browser_use/controller/__init__.py

@@ -0,0 +1,3 @@
+from browser_use.tools.service import Controller
+
+__all__ = ['Controller']

browser_use/dom/enhanced_snapshot.py

@@ -0,0 +1,181 @@
+"""
+Enhanced snapshot processing for browser-use DOM tree extraction.
+
+This module provides stateless functions for parsing Chrome DevTools Protocol (CDP) DOMSnapshot data
+to extract visibility, clickability, cursor styles, and other layout information.
+"""
+
+from cdp_use.cdp.domsnapshot.commands import CaptureSnapshotReturns
+from cdp_use.cdp.domsnapshot.types import (
+	LayoutTreeSnapshot,
+	NodeTreeSnapshot,
+)
+
+from browser_use.dom.views import DOMRect, EnhancedSnapshotNode
+
+# Only the ESSENTIAL computed styles for interactivity and visibility detection
+REQUIRED_COMPUTED_STYLES = [
+	# Only styles actually accessed in the codebase (prevents Chrome crashes on heavy sites)
+	'display',  # Used in service.py visibility detection
+	'visibility',  # Used in service.py visibility detection
+	'opacity',  # Used in service.py visibility detection
+	'overflow',  # Used in views.py scrollability detection
+	'overflow-x',  # Used in views.py scrollability detection
+	'overflow-y',  # Used in views.py scrollability detection
+	'cursor',  # Used in enhanced_snapshot.py cursor extraction
+	'pointer-events',  # Used for clickability logic
+	'position',  # Used for visibility logic
+	'background-color',  # Used for visibility logic
+]
+
+
+def _parse_rare_boolean_data(rare_data_set: set[int], index: int) -> bool | None:
+	"""Parse rare boolean data from snapshot - returns True if index is in the rare data set."""
+	return index in rare_data_set
+
+
+def _parse_computed_styles(strings: list[str], style_indices: list[int]) -> dict[str, str]:
+	"""Parse computed styles from layout tree using string indices."""
+	styles = {}
+	for i, style_index in enumerate(style_indices):
+		if i < len(REQUIRED_COMPUTED_STYLES) and 0 <= style_index < len(strings):
+			styles[REQUIRED_COMPUTED_STYLES[i]] = strings[style_index]
+	return styles
+
+
+def build_snapshot_lookup(
+	snapshot: CaptureSnapshotReturns,
+	device_pixel_ratio: float = 1.0,
+) -> dict[int, EnhancedSnapshotNode]:
+	"""Build a lookup table of backend node ID to enhanced snapshot data with everything calculated upfront."""
+	import logging
+
+	logger = logging.getLogger('browser_use.dom.enhanced_snapshot')
+	snapshot_lookup: dict[int, EnhancedSnapshotNode] = {}
+
+	if not snapshot['documents']:
+		return snapshot_lookup
+
+	strings = snapshot['strings']
+	logger.debug(f'🔍 SNAPSHOT: Processing {len(snapshot["documents"])} documents with {len(strings)} strings')
+
+	for doc_idx, document in enumerate(snapshot['documents']):
+		nodes: NodeTreeSnapshot = document['nodes']
+		layout: LayoutTreeSnapshot = document['layout']
+
+		# Build backend node id to snapshot index lookup
+		backend_node_to_snapshot_index = {}
+		if 'backendNodeId' in nodes:
+			for i, backend_node_id in enumerate(nodes['backendNodeId']):
+				backend_node_to_snapshot_index[backend_node_id] = i
+
+		# Log document info
+		doc_url = strings[document.get('documentURL', 0)] if document.get('documentURL', 0) < len(strings) else 'N/A'
+		logger.debug(
+			f'🔍 SNAPSHOT doc[{doc_idx}]: url={doc_url[:80]}... has {len(backend_node_to_snapshot_index)} nodes, '
+			f'layout has {len(layout.get("nodeIndex", []))} entries'
+		)
+
+		# PERFORMANCE: Pre-build layout index map to eliminate O(n²) double lookups
+		# Preserve original behavior: use FIRST occurrence for duplicates
+		layout_index_map = {}
+		if layout and 'nodeIndex' in layout:
+			for layout_idx, node_index in enumerate(layout['nodeIndex']):
+				if node_index not in layout_index_map:  # Only store first occurrence
+					layout_index_map[node_index] = layout_idx
+
+		# Pre-convert rare boolean data from list to set for O(1) lookups.
+		# The raw CDP data uses List[int] which makes `index in list` O(n).
+		# Called once per node, this was O(n²) total — the #1 bottleneck.
+		# At 20k elements: 5,925ms (list) → 2ms (set) = 3,000x speedup.
+		has_clickable_data = 'isClickable' in nodes
+		is_clickable_set: set[int] = set(nodes['isClickable']['index']) if has_clickable_data else set()
+
+		# Build snapshot lookup for each backend node id
+		for backend_node_id, snapshot_index in backend_node_to_snapshot_index.items():
+			is_clickable = None
+			if has_clickable_data:
+				is_clickable = _parse_rare_boolean_data(is_clickable_set, snapshot_index)
+
+			# Find corresponding layout node
+			cursor_style = None
+			is_visible = None
+			bounding_box = None
+			computed_styles = {}
+
+			# Look for layout tree node that corresponds to this snapshot node
+			paint_order = None
+			client_rects = None
+			scroll_rects = None
+			stacking_contexts = None
+			if snapshot_index in layout_index_map:
+				layout_idx = layout_index_map[snapshot_index]
+				if layout_idx < len(layout.get('bounds', [])):
+					# Parse bounding box
+					bounds = layout['bounds'][layout_idx]
+					if len(bounds) >= 4:
+						# IMPORTANT: CDP coordinates are in device pixels, convert to CSS pixels
+						# by dividing by the device pixel ratio
+						raw_x, raw_y, raw_width, raw_height = bounds[0], bounds[1], bounds[2], bounds[3]
+
+						# Apply device pixel ratio scaling to convert device pixels to CSS pixels
+						bounding_box = DOMRect(
+							x=raw_x / device_pixel_ratio,
+							y=raw_y / device_pixel_ratio,
+							width=raw_width / device_pixel_ratio,
+							height=raw_height / device_pixel_ratio,
+						)
+
+					# Parse computed styles for this layout node
+					if layout_idx < len(layout.get('styles', [])):
+						style_indices = layout['styles'][layout_idx]
+						computed_styles = _parse_computed_styles(strings, style_indices)
+						cursor_style = computed_styles.get('cursor')
+
+					# Extract paint order if available
+					if layout_idx < len(layout.get('paintOrders', [])):
+						paint_order = layout.get('paintOrders', [])[layout_idx]
+
+					# Extract client rects if available
+					client_rects_data = layout.get('clientRects', [])
+					if layout_idx < len(client_rects_data):
+						client_rect_data = client_rects_data[layout_idx]
+						if client_rect_data and len(client_rect_data) >= 4:
+							client_rects = DOMRect(
+								x=client_rect_data[0],
+								y=client_rect_data[1],
+								width=client_rect_data[2],
+								height=client_rect_data[3],
+							)
+
+					# Extract scroll rects if available
+					scroll_rects_data = layout.get('scrollRects', [])
+					if layout_idx < len(scroll_rects_data):
+						scroll_rect_data = scroll_rects_data[layout_idx]
+						if scroll_rect_data and len(scroll_rect_data) >= 4:
+							scroll_rects = DOMRect(
+								x=scroll_rect_data[0],
+								y=scroll_rect_data[1],
+								width=scroll_rect_data[2],
+								height=scroll_rect_data[3],
+							)
+
+					# Extract stacking contexts if available
+					if layout_idx < len(layout.get('stackingContexts', [])):
+						stacking_contexts = layout.get('stackingContexts', {}).get('index', [])[layout_idx]
+
+			snapshot_lookup[backend_node_id] = EnhancedSnapshotNode(
+				is_clickable=is_clickable,
+				cursor_style=cursor_style,
+				bounds=bounding_box,
+				clientRects=client_rects,
+				scrollRects=scroll_rects,
+				computed_styles=computed_styles if computed_styles else None,
+				paint_order=paint_order,
+				stacking_contexts=stacking_contexts,
+			)
+
+	# Count how many have bounds (are actually visible/laid out)
+	with_bounds = sum(1 for n in snapshot_lookup.values() if n.bounds)
+	logger.debug(f'🔍 SNAPSHOT: Built lookup with {len(snapshot_lookup)} total entries, {with_bounds} have bounds')
+	return snapshot_lookup

browser_use/dom/markdown_extractor.py

@@ -0,0 +1,547 @@
+"""
+Shared markdown extraction utilities for browser content processing.
+
+This module provides a unified interface for extracting clean markdown from browser content,
+used by both the tools service and page actor.
+"""
+
+import json
+import re
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import TYPE_CHECKING, Any
+
+from browser_use.dom.serializer.html_serializer import HTMLSerializer
+from browser_use.dom.service import DomService
+from browser_use.dom.views import MarkdownChunk
+
+if TYPE_CHECKING:
+	from browser_use.browser.session import BrowserSession
+	from browser_use.browser.watchdogs.dom_watchdog import DOMWatchdog
+
+
+async def extract_clean_markdown(
+	browser_session: 'BrowserSession | None' = None,
+	dom_service: DomService | None = None,
+	target_id: str | None = None,
+	extract_links: bool = False,
+	extract_images: bool = False,
+) -> tuple[str, dict[str, Any]]:
+	"""Extract clean markdown from browser content using enhanced DOM tree.
+
+	This unified function can extract markdown using either a browser session (for tools service)
+	or a DOM service with target ID (for page actor).
+
+	Args:
+	    browser_session: Browser session to extract content from (tools service path)
+	    dom_service: DOM service instance (page actor path)
+	    target_id: Target ID for the page (required when using dom_service)
+	    extract_links: Whether to preserve links in markdown
+	    extract_images: Whether to preserve inline image src URLs in markdown
+
+	Returns:
+	    tuple: (clean_markdown_content, content_statistics)
+
+	Raises:
+	    ValueError: If neither browser_session nor (dom_service + target_id) are provided
+	"""
+	# Validate input parameters
+	if browser_session is not None:
+		if dom_service is not None or target_id is not None:
+			raise ValueError('Cannot specify both browser_session and dom_service/target_id')
+		# Browser session path (tools service)
+		enhanced_dom_tree = await _get_enhanced_dom_tree_from_browser_session(browser_session)
+		current_url = await browser_session.get_current_page_url()
+		method = 'enhanced_dom_tree'
+	elif dom_service is not None and target_id is not None:
+		# DOM service path (page actor)
+		# Lazy fetch all_frames inside get_dom_tree if needed (for cross-origin iframes)
+		enhanced_dom_tree, _ = await dom_service.get_dom_tree(target_id=target_id, all_frames=None)
+		current_url = None  # Not available via DOM service
+		method = 'dom_service'
+	else:
+		raise ValueError('Must provide either browser_session or both dom_service and target_id')
+
+	# Use the HTML serializer with the enhanced DOM tree
+	html_serializer = HTMLSerializer(extract_links=extract_links)
+	page_html = html_serializer.serialize(enhanced_dom_tree)
+
+	original_html_length = len(page_html)
+
+	content, initial_markdown_length, chars_filtered = convert_html_to_markdown(page_html, extract_images=extract_images)
+
+	final_filtered_length = len(content)
+
+	# Content statistics
+	stats = {
+		'method': method,
+		'original_html_chars': original_html_length,
+		'initial_markdown_chars': initial_markdown_length,
+		'filtered_chars_removed': chars_filtered,
+		'final_filtered_chars': final_filtered_length,
+	}
+
+	# Add URL to stats if available
+	if current_url:
+		stats['url'] = current_url
+
+	return content, stats
+
+
+async def _get_enhanced_dom_tree_from_browser_session(browser_session: 'BrowserSession'):
+	"""Get enhanced DOM tree from browser session via DOMWatchdog."""
+	# Get the enhanced DOM tree from DOMWatchdog
+	# This captures the current state of the page including dynamic content, shadow roots, etc.
+	dom_watchdog: DOMWatchdog | None = browser_session._dom_watchdog
+	assert dom_watchdog is not None, 'DOMWatchdog not available'
+
+	# Use cached enhanced DOM tree if available, otherwise build it
+	if dom_watchdog.enhanced_dom_tree is not None:
+		return dom_watchdog.enhanced_dom_tree
+
+	# Build the enhanced DOM tree if not cached
+	await dom_watchdog._build_dom_tree_without_highlights()
+	enhanced_dom_tree = dom_watchdog.enhanced_dom_tree
+	assert enhanced_dom_tree is not None, 'Enhanced DOM tree not available'
+
+	return enhanced_dom_tree
+
+
+# Legacy aliases removed - all code now uses the unified extract_clean_markdown function
+
+
+def convert_html_to_markdown(page_html: str, extract_images: bool = False) -> tuple[str, int, int]:
+	"""Convert serialized page HTML to filtered markdown.
+
+	Returns:
+	    tuple: (filtered_markdown, initial_markdown_chars, chars_filtered)
+	"""
+	from markdownify import markdownify as md
+
+	# 'td', 'th', and headings are the only elements where markdownify sets the _inline context,
+	# which causes img elements to be stripped to just alt text when keep_inline_images_in=[]
+	_keep_inline_images_in = ['td', 'th', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'] if extract_images else []
+	content = md(
+		page_html,
+		heading_style='ATX',  # Use # style headings
+		strip=['script', 'style'],  # Remove these tags
+		bullets='-',  # Use - for unordered lists
+		code_language='',  # Don't add language to code blocks
+		escape_asterisks=False,  # Don't escape asterisks (cleaner output)
+		escape_underscores=False,  # Don't escape underscores (cleaner output)
+		escape_misc=False,  # Don't escape other characters (cleaner output)
+		autolinks=False,  # Don't convert URLs to <> format
+		default_title=False,  # Don't add default title attributes
+		keep_inline_images_in=_keep_inline_images_in,  # Include image src URLs when extract_images=True
+	)
+
+	initial_markdown_length = len(content)
+
+	# Apply light preprocessing to clean up excessive whitespace
+	content, chars_filtered = _preprocess_markdown_content(content)
+
+	return content, initial_markdown_length, chars_filtered
+
+
+def _preprocess_markdown_content(content: str, max_newlines: int = 3) -> tuple[str, int]:
+	"""
+	Light preprocessing of markdown output - minimal cleanup with JSON blob removal.
+
+	Args:
+	    content: Markdown content to lightly filter
+	    max_newlines: Maximum consecutive newlines to allow
+
+	Returns:
+	    tuple: (filtered_content, chars_filtered)
+	"""
+	original_length = len(content)
+
+	# Remove JSON blobs (common in SPAs like LinkedIn, Facebook, etc.)
+	# These are often embedded as `{"key":"value",...}` and can be massive
+	# Match JSON objects/arrays that are at least 100 chars long
+	# This catches SPA state/config data without removing small inline JSON
+	content = re.sub(r'`\{["\w].*?\}`', '', content, flags=re.DOTALL)  # Remove JSON in code blocks
+	content = re.sub(r'\{"\$type":[^}]{100,}\}', '', content)  # Remove JSON with $type fields (common pattern)
+	content = re.sub(r'\{"[^"]{5,}":\{[^}]{100,}\}', '', content)  # Remove nested JSON objects
+
+	# Compress consecutive newlines (4+ newlines become max_newlines)
+	content = re.sub(r'\n{4,}', '\n' * max_newlines, content)
+
+	# Remove lines that are only whitespace
+	lines = content.split('\n')
+	filtered_lines = []
+	for line in lines:
+		stripped = line.strip()
+		# Keep all non-empty lines
+		if stripped:
+			# Skip long lines that actually parse as JSON (SPA state blobs). A prefix
+			# check alone is not enough: markdown links/images also start with '['.
+			if len(stripped) > 100 and stripped[0] in '{[':
+				try:
+					json.loads(stripped)
+					continue
+				except ValueError:
+					pass
+			filtered_lines.append(line)
+
+	content = '\n'.join(filtered_lines)
+	content = content.strip()
+
+	chars_filtered = original_length - len(content)
+	return content, chars_filtered
+
+
+# ---------------------------------------------------------------------------
+# Structure-aware markdown chunking
+# ---------------------------------------------------------------------------
+
+
+class _BlockType(Enum):
+	HEADER = auto()
+	CODE_FENCE = auto()
+	TABLE = auto()
+	LIST_ITEM = auto()
+	PARAGRAPH = auto()
+	BLANK = auto()
+
+
+@dataclass(slots=True)
+class _AtomicBlock:
+	block_type: _BlockType
+	lines: list[str]
+	char_start: int  # offset in original content
+	char_end: int  # offset in original content (exclusive)
+
+
+_TABLE_ROW_RE = re.compile(r'^\s*\|.*\|\s*$')
+_LIST_ITEM_RE = re.compile(r'^(\s*)([-*+]|\d+[.)]) ')
+_LIST_CONTINUATION_RE = re.compile(r'^(\s{2,}|\t)')
+
+
+def _parse_atomic_blocks(content: str) -> list[_AtomicBlock]:
+	"""Phase 1: Walk lines, group into unsplittable blocks."""
+	lines = content.split('\n')
+	blocks: list[_AtomicBlock] = []
+	i = 0
+	offset = 0  # char offset tracking
+
+	while i < len(lines):
+		line = lines[i]
+		line_len = len(line) + 1  # +1 for the newline we split on
+
+		# BLANK
+		if not line.strip():
+			blocks.append(
+				_AtomicBlock(
+					block_type=_BlockType.BLANK,
+					lines=[line],
+					char_start=offset,
+					char_end=offset + line_len,
+				)
+			)
+			offset += line_len
+			i += 1
+			continue
+
+		# CODE FENCE
+		if line.strip().startswith('```'):
+			fence_lines = [line]
+			fence_end = offset + line_len
+			i += 1
+			# Consume until closing fence or EOF
+			while i < len(lines):
+				fence_line = lines[i]
+				fence_line_len = len(fence_line) + 1
+				fence_lines.append(fence_line)
+				fence_end += fence_line_len
+				i += 1
+				if fence_line.strip().startswith('```') and len(fence_lines) > 1:
+					break
+			blocks.append(
+				_AtomicBlock(
+					block_type=_BlockType.CODE_FENCE,
+					lines=fence_lines,
+					char_start=offset,
+					char_end=fence_end,
+				)
+			)
+			offset = fence_end
+			continue
+
+		# HEADER
+		if line.lstrip().startswith('#'):
+			blocks.append(
+				_AtomicBlock(
+					block_type=_BlockType.HEADER,
+					lines=[line],
+					char_start=offset,
+					char_end=offset + line_len,
+				)
+			)
+			offset += line_len
+			i += 1
+			continue
+
+		# TABLE (consecutive |...|  lines)
+		# Header + separator row stay together; each data row is its own block
+		if _TABLE_ROW_RE.match(line):
+			# Collect header line
+			header_lines = [line]
+			header_end = offset + line_len
+			i += 1
+			# Check if next line is separator (contains ---)
+			if i < len(lines) and _TABLE_ROW_RE.match(lines[i]) and '---' in lines[i]:
+				sep = lines[i]
+				sep_len = len(sep) + 1
+				header_lines.append(sep)
+				header_end += sep_len
+				i += 1
+			# Emit header+separator as one atomic block
+			blocks.append(
+				_AtomicBlock(
+					block_type=_BlockType.TABLE,
+					lines=header_lines,
+					char_start=offset,
+					char_end=header_end,
+				)
+			)
+			offset = header_end
+			# Each subsequent table row is its own TABLE block (splittable between rows)
+			while i < len(lines) and _TABLE_ROW_RE.match(lines[i]):
+				row = lines[i]
+				row_len = len(row) + 1
+				blocks.append(
+					_AtomicBlock(
+						block_type=_BlockType.TABLE,
+						lines=[row],
+						char_start=offset,
+						char_end=offset + row_len,
+					)
+				)
+				offset += row_len
+				i += 1
+			continue
+
+		# LIST ITEM (with indented continuations)
+		if _LIST_ITEM_RE.match(line):
+			list_lines = [line]
+			list_end = offset + line_len
+			i += 1
+			# Consume continuation lines (indented or blank between items)
+			while i < len(lines):
+				next_line = lines[i]
+				next_len = len(next_line) + 1
+				# Another list item at same or deeper indent → still part of this block
+				if _LIST_ITEM_RE.match(next_line):
+					list_lines.append(next_line)
+					list_end += next_len
+					i += 1
+					continue
+				# Indented continuation
+				if next_line.strip() and _LIST_CONTINUATION_RE.match(next_line):
+					list_lines.append(next_line)
+					list_end += next_len
+					i += 1
+					continue
+				break
+			blocks.append(
+				_AtomicBlock(
+					block_type=_BlockType.LIST_ITEM,
+					lines=list_lines,
+					char_start=offset,
+					char_end=list_end,
+				)
+			)
+			offset = list_end
+			continue
+
+		# PARAGRAPH (everything else, up to next blank line)
+		para_lines = [line]
+		para_end = offset + line_len
+		i += 1
+		while i < len(lines) and lines[i].strip():
+			# Stop if next line starts a different block type
+			nl = lines[i]
+			if nl.lstrip().startswith('#') or nl.strip().startswith('```') or _TABLE_ROW_RE.match(nl) or _LIST_ITEM_RE.match(nl):
+				break
+			nl_len = len(nl) + 1
+			para_lines.append(nl)
+			para_end += nl_len
+			i += 1
+		blocks.append(
+			_AtomicBlock(
+				block_type=_BlockType.PARAGRAPH,
+				lines=para_lines,
+				char_start=offset,
+				char_end=para_end,
+			)
+		)
+		offset = para_end
+
+	# Fix last block char_end: content may not end with \n
+	if blocks and content and not content.endswith('\n'):
+		blocks[-1] = _AtomicBlock(
+			block_type=blocks[-1].block_type,
+			lines=blocks[-1].lines,
+			char_start=blocks[-1].char_start,
+			char_end=len(content),
+		)
+
+	return blocks
+
+
+def _block_text(block: _AtomicBlock) -> str:
+	return '\n'.join(block.lines)
+
+
+def _get_table_header(block: _AtomicBlock) -> str | None:
+	"""Extract table header + separator rows from a TABLE block."""
+	assert block.block_type == _BlockType.TABLE
+	if len(block.lines) < 2:
+		return None
+	# Header is first line, separator is second line (must contain ---)
+	sep_line = block.lines[1]
+	if '---' in sep_line or '- -' in sep_line:
+		return block.lines[0] + '\n' + block.lines[1]
+	return None
+
+
+def chunk_markdown_by_structure(
+	content: str,
+	max_chunk_chars: int = 100_000,
+	overlap_lines: int = 5,
+	start_from_char: int = 0,
+) -> list[MarkdownChunk]:
+	"""Split markdown into structure-aware chunks.
+
+	Algorithm:
+	  Phase 1 — Parse atomic blocks (headers, code fences, tables, list items, paragraphs).
+	  Phase 2 — Greedy chunk assembly: accumulate blocks until exceeding max_chunk_chars.
+	            A single block exceeding the limit is allowed (soft limit).
+	  Phase 3 — Build overlap prefixes for context carry between chunks.
+
+	Args:
+	    content: Full markdown string.
+	    max_chunk_chars: Target maximum chars per chunk (soft limit for single blocks).
+	    overlap_lines: Number of trailing lines from previous chunk to prepend.
+	    start_from_char: Return chunks starting from the chunk that contains this offset.
+
+	Returns:
+	    List of MarkdownChunk. Empty if start_from_char is past end of content.
+	"""
+	if not content:
+		return [
+			MarkdownChunk(
+				content='',
+				chunk_index=0,
+				total_chunks=1,
+				char_offset_start=0,
+				char_offset_end=0,
+				overlap_prefix='',
+				has_more=False,
+			)
+		]
+
+	if start_from_char >= len(content):
+		return []
+
+	# Phase 1: parse atomic blocks
+	blocks = _parse_atomic_blocks(content)
+	if not blocks:
+		return []
+
+	# Phase 2: greedy chunk assembly with header-preferred splitting
+	raw_chunks: list[list[_AtomicBlock]] = []
+	current_chunk: list[_AtomicBlock] = []
+	current_size = 0
+
+	for block in blocks:
+		block_size = block.char_end - block.char_start
+		# If adding this block would exceed limit AND we already have content, emit chunk
+		if current_size + block_size > max_chunk_chars and current_chunk:
+			# Prefer splitting at a header boundary within the current chunk.
+			# Scan backwards for the last HEADER block; if found and it wouldn't
+			# create a tiny chunk (< 50% of limit), split right before it so the
+			# header starts the next chunk for better semantic coherence.
+			best_split = len(current_chunk)
+			for j in range(len(current_chunk) - 1, 0, -1):
+				if current_chunk[j].block_type == _BlockType.HEADER:
+					prefix_size = sum(b.char_end - b.char_start for b in current_chunk[:j])
+					if prefix_size >= max_chunk_chars * 0.5:
+						best_split = j
+						break
+			raw_chunks.append(current_chunk[:best_split])
+			# Carry remaining blocks (from the header onward) into the next chunk
+			current_chunk = current_chunk[best_split:]
+			current_size = sum(b.char_end - b.char_start for b in current_chunk)
+		current_chunk.append(block)
+		current_size += block_size
+
+	if current_chunk:
+		raw_chunks.append(current_chunk)
+
+	total_chunks = len(raw_chunks)
+
+	# Phase 3: build MarkdownChunk objects with overlap prefixes
+	chunks: list[MarkdownChunk] = []
+	# Track table header from previous chunk for table continuations
+	prev_chunk_last_table_header: str | None = None
+
+	for idx, chunk_blocks in enumerate(raw_chunks):
+		chunk_text = '\n'.join(_block_text(b) for b in chunk_blocks)
+		char_start = chunk_blocks[0].char_start
+		char_end = chunk_blocks[-1].char_end
+
+		# Build overlap prefix
+		overlap = ''
+		if idx > 0:
+			prev_blocks = raw_chunks[idx - 1]
+			prev_text = '\n'.join(_block_text(b) for b in prev_blocks)
+			prev_lines = prev_text.split('\n')
+
+			# Check if current chunk starts with a table continuation
+			first_block = chunk_blocks[0]
+			if first_block.block_type == _BlockType.TABLE and prev_chunk_last_table_header:
+				# Always prepend table header for continuation
+				trailing = prev_lines[-(overlap_lines):] if overlap_lines > 0 else []
+				header_lines = prev_chunk_last_table_header.split('\n')
+				# Deduplicate: don't repeat header lines if they're already in trailing
+				combined = list(header_lines)
+				for tl in trailing:
+					if tl not in combined:
+						combined.append(tl)
+				overlap = '\n'.join(combined)
+			elif overlap_lines > 0:
+				overlap = '\n'.join(prev_lines[-(overlap_lines):])
+
+		# Track table header from this chunk for next iteration.
+		# Only overwrite if this chunk contains a new header+separator block;
+		# otherwise preserve the previous header so tables spanning 3+ chunks
+		# still get the header carried forward.
+		for b in chunk_blocks:
+			if b.block_type == _BlockType.TABLE:
+				hdr = _get_table_header(b)
+				if hdr is not None:
+					prev_chunk_last_table_header = hdr
+
+		has_more = idx < total_chunks - 1
+		chunks.append(
+			MarkdownChunk(
+				content=chunk_text,
+				chunk_index=idx,
+				total_chunks=total_chunks,
+				char_offset_start=char_start,
+				char_offset_end=char_end,
+				overlap_prefix=overlap,
+				has_more=has_more,
+			)
+		)
+
+	# Apply start_from_char filter: return chunks from the one containing that offset
+	if start_from_char > 0:
+		for i, chunk in enumerate(chunks):
+			if chunk.char_offset_end > start_from_char:
+				return chunks[i:]
+		return []  # offset past all chunks
+
+	return chunks

browser_use/dom/playground/extraction.py

@@ -0,0 +1,312 @@
+import asyncio
+import json
+import os
+import time
+
+import anyio
+import pyperclip
+import tiktoken
+
+from browser_use.agent.prompts import AgentMessagePrompt
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser.events import ClickElementEvent, TypeTextEvent
+from browser_use.browser.profile import ViewportSize
+from browser_use.dom.service import DomService
+from browser_use.dom.views import DEFAULT_INCLUDE_ATTRIBUTES
+from browser_use.filesystem.file_system import FileSystem
+
+TIMEOUT = 60
+
+
+async def test_focus_vs_all_elements():
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			# executable_path='/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+			window_size=ViewportSize(width=1100, height=1000),
+			disable_security=False,
+			wait_for_network_idle_page_load_time=1,
+			headless=False,
+			args=['--incognito'],
+			paint_order_filtering=True,
+		),
+	)
+
+	# 10 Sample websites with various interactive elements
+	sample_websites = [
+		'https://browser-use.github.io/stress-tests/challenges/iframe-inception-level2.html',
+		'https://www.google.com/travel/flights',
+		'https://v0-simple-ui-test-site.vercel.app',
+		'https://browser-use.github.io/stress-tests/challenges/iframe-inception-level1.html',
+		'https://browser-use.github.io/stress-tests/challenges/angular-form.html',
+		'https://www.google.com/travel/flights',
+		'https://www.amazon.com/s?k=laptop',
+		'https://github.com/trending',
+		'https://www.reddit.com',
+		'https://www.ycombinator.com/companies',
+		'https://www.kayak.com/flights',
+		'https://www.booking.com',
+		'https://www.airbnb.com',
+		'https://www.linkedin.com/jobs',
+		'https://stackoverflow.com/questions',
+	]
+
+	# 5 Difficult websites with complex elements (iframes, canvas, dropdowns, etc.)
+	difficult_websites = [
+		'https://www.w3schools.com/html/tryit.asp?filename=tryhtml_iframe',  # Nested iframes
+		'https://semantic-ui.com/modules/dropdown.html',  # Complex dropdowns
+		'https://www.dezlearn.com/nested-iframes-example/',  # Cross-origin nested iframes
+		'https://codepen.io/towc/pen/mJzOWJ',  # Canvas elements with interactions
+		'https://jqueryui.com/accordion/',  # Complex accordion/dropdown widgets
+		'https://v0-simple-landing-page-seven-xi.vercel.app/',  # Simple landing page with iframe
+		'https://www.unesco.org/en',
+	]
+
+	# Descriptions for difficult websites
+	difficult_descriptions = {
+		'https://www.w3schools.com/html/tryit.asp?filename=tryhtml_iframe': '🔸 NESTED IFRAMES: Multiple iframe layers',
+		'https://semantic-ui.com/modules/dropdown.html': '🔸 COMPLEX DROPDOWNS: Custom dropdown components',
+		'https://www.dezlearn.com/nested-iframes-example/': '🔸 CROSS-ORIGIN IFRAMES: Different domain iframes',
+		'https://codepen.io/towc/pen/mJzOWJ': '🔸 CANVAS ELEMENTS: Interactive canvas graphics',
+		'https://jqueryui.com/accordion/': '🔸 ACCORDION WIDGETS: Collapsible content sections',
+	}
+
+	websites = sample_websites + difficult_websites
+	current_website_index = 0
+
+	def get_website_list_for_prompt() -> str:
+		"""Get a compact website list for the input prompt."""
+		lines = []
+		lines.append('📋 Websites:')
+
+		# Sample websites (1-10)
+		for i, site in enumerate(sample_websites, 1):
+			current_marker = ' ←' if (i - 1) == current_website_index else ''
+			domain = site.replace('https://', '').split('/')[0]
+			lines.append(f'  {i:2d}.{domain[:15]:<15}{current_marker}')
+
+		# Difficult websites (11-15)
+		for i, site in enumerate(difficult_websites, len(sample_websites) + 1):
+			current_marker = ' ←' if (i - 1) == current_website_index else ''
+			domain = site.replace('https://', '').split('/')[0]
+			desc = difficult_descriptions.get(site, '')
+			challenge = desc.split(': ')[1][:15] if ': ' in desc else ''
+			lines.append(f'  {i:2d}.{domain[:15]:<15} ({challenge}){current_marker}')
+
+		return '\n'.join(lines)
+
+	await browser_session.start()
+
+	# Show startup info
+	print('\n🌐 BROWSER-USE DOM EXTRACTION TESTER')
+	print(f'📊 {len(websites)} websites total: {len(sample_websites)} standard + {len(difficult_websites)} complex')
+	print('🔧 Controls: Type 1-15 to jump | Enter to re-run | "n" next | "q" quit')
+	print('💾 Outputs: tmp/user_message.txt & tmp/element_tree.json\n')
+
+	dom_service = DomService(browser_session)
+
+	while True:
+		# Cycle through websites
+		if current_website_index >= len(websites):
+			current_website_index = 0
+			print('Cycled back to first website!')
+
+		website = websites[current_website_index]
+		# sleep 2
+		await browser_session._cdp_navigate(website)
+		await asyncio.sleep(1)
+
+		last_clicked_index = None  # Track the index for text input
+		while True:
+			try:
+				# 	all_elements_state = await dom_service.get_serialized_dom_tree()
+
+				website_type = 'DIFFICULT' if website in difficult_websites else 'SAMPLE'
+				print(f'\n{"=" * 60}')
+				print(f'[{current_website_index + 1}/{len(websites)}] [{website_type}] Testing: {website}')
+				if website in difficult_descriptions:
+					print(f'{difficult_descriptions[website]}')
+				print(f'{"=" * 60}')
+
+				# Get/refresh the state (includes removing old highlights)
+				print('\nGetting page state...')
+
+				start_time = time.time()
+				all_elements_state = await browser_session.get_browser_state_summary(True)
+				end_time = time.time()
+				get_state_time = end_time - start_time
+				print(f'get_state_summary took {get_state_time:.2f} seconds')
+
+				# Get detailed timing info from DOM service
+				print('\nGetting detailed DOM timing...')
+				serialized_state, _, timing_info = await dom_service.get_serialized_dom_tree()
+
+				# Combine all timing info
+				all_timing = {'get_state_summary_total': get_state_time, **timing_info}
+
+				selector_map = all_elements_state.dom_state.selector_map
+				total_elements = len(selector_map.keys())
+				print(f'Total number of elements: {total_elements}')
+
+				# print(all_elements_state.element_tree.clickable_elements_to_string())
+				prompt = AgentMessagePrompt(
+					browser_state_summary=all_elements_state,
+					file_system=FileSystem(base_dir='./tmp'),
+					include_attributes=DEFAULT_INCLUDE_ATTRIBUTES,
+					step_info=None,
+				)
+				# Write the user message to a file for analysis
+				user_message = prompt.get_user_message(use_vision=False).text
+
+				# clickable_elements_str = all_elements_state.element_tree.clickable_elements_to_string()
+
+				text_to_save = user_message
+
+				os.makedirs('./tmp', exist_ok=True)
+				async with await anyio.open_file('./tmp/user_message.txt', 'w', encoding='utf-8') as f:
+					await f.write(text_to_save)
+
+				# save pure clickable elements to a file
+				if all_elements_state.dom_state._root:
+					async with await anyio.open_file('./tmp/simplified_element_tree.json', 'w', encoding='utf-8') as f:
+						await f.write(json.dumps(all_elements_state.dom_state._root.__json__(), indent=2))
+
+					async with await anyio.open_file('./tmp/original_element_tree.json', 'w', encoding='utf-8') as f:
+						await f.write(json.dumps(all_elements_state.dom_state._root.original_node.__json__(), indent=2))
+
+				# copy the user message to the clipboard
+				# pyperclip.copy(text_to_save)
+
+				encoding = tiktoken.encoding_for_model('gpt-4.1-mini')
+				token_count = len(encoding.encode(text_to_save))
+				print(f'Token count: {token_count}')
+
+				print('User message written to ./tmp/user_message.txt')
+				print('Element tree written to ./tmp/simplified_element_tree.json')
+				print('Original element tree written to ./tmp/original_element_tree.json')
+
+				# Save timing information
+				timing_text = '🔍 DOM EXTRACTION PERFORMANCE ANALYSIS\n'
+				timing_text += f'{"=" * 50}\n\n'
+				timing_text += f'📄 Website: {website}\n'
+				timing_text += f'📊 Total Elements: {total_elements}\n'
+				timing_text += f'🎯 Token Count: {token_count}\n\n'
+
+				timing_text += '⏱️  TIMING BREAKDOWN:\n'
+				timing_text += f'{"─" * 30}\n'
+				for key, value in all_timing.items():
+					timing_text += f'{key:<35}: {value * 1000:>8.2f} ms\n'
+
+				# Calculate percentages
+				total_time = all_timing.get('get_state_summary_total', 0)
+				if total_time > 0 and total_elements > 0:
+					timing_text += '\n📈 PERCENTAGE BREAKDOWN:\n'
+					timing_text += f'{"─" * 30}\n'
+					for key, value in all_timing.items():
+						if key != 'get_state_summary_total':
+							percentage = (value / total_time) * 100
+							timing_text += f'{key:<35}: {percentage:>7.1f}%\n'
+
+				timing_text += '\n🎯 CLICKABLE DETECTION ANALYSIS:\n'
+				timing_text += f'{"─" * 35}\n'
+				clickable_time = all_timing.get('clickable_detection_time', 0)
+				if clickable_time > 0 and total_elements > 0:
+					avg_per_element = (clickable_time / total_elements) * 1000000  # microseconds
+					timing_text += f'Total clickable detection time: {clickable_time * 1000:.2f} ms\n'
+					timing_text += f'Average per element: {avg_per_element:.2f} μs\n'
+					timing_text += f'Clickable detection calls: ~{total_elements} (approx)\n'
+
+				async with await anyio.open_file('./tmp/timing_analysis.txt', 'w', encoding='utf-8') as f:
+					await f.write(timing_text)
+
+				print('Timing analysis written to ./tmp/timing_analysis.txt')
+
+				# also save all_elements_state.element_tree.clickable_elements_to_string() to a file
+				# with open('./tmp/clickable_elements.json', 'w', encoding='utf-8') as f:
+				# 	f.write(json.dumps(all_elements_state.element_tree.__json__(), indent=2))
+				# print('Clickable elements written to ./tmp/clickable_elements.json')
+
+				website_list = get_website_list_for_prompt()
+				answer = input(
+					"🎮 Enter: element index | 'index' click (clickable) | 'index,text' input | 'c,index' copy | Enter re-run | 'n' next | 'q' quit: "
+				)
+
+				if answer.lower() == 'q':
+					return  # Exit completely
+				elif answer.lower() == 'n':
+					print('Moving to next website...')
+					current_website_index += 1
+					break  # Break inner loop to go to next website
+				elif answer.strip() == '':
+					print('Re-running extraction on current page state...')
+					continue  # Continue inner loop to re-extract DOM without reloading page
+				elif answer.strip().isdigit():
+					# Click element format: index
+					try:
+						clicked_index = int(answer)
+						if clicked_index in selector_map:
+							element_node = selector_map[clicked_index]
+							print(f'Clicking element {clicked_index}: {element_node.tag_name}')
+							event = browser_session.event_bus.dispatch(ClickElementEvent(node=element_node))
+							await event
+							print('Click successful.')
+					except ValueError:
+						print(f"Invalid input: '{answer}'. Enter an index, 'index,text', 'c,index', or 'q'.")
+					continue
+
+				try:
+					if answer.lower().startswith('c,'):
+						# Copy element JSON format: c,index
+						parts = answer.split(',', 1)
+						if len(parts) == 2:
+							try:
+								target_index = int(parts[1].strip())
+								if target_index in selector_map:
+									element_node = selector_map[target_index]
+									element_json = json.dumps(element_node.__json__(), indent=2, default=str)
+									pyperclip.copy(element_json)
+									print(f'Copied element {target_index} JSON to clipboard: {element_node.tag_name}')
+								else:
+									print(f'Invalid index: {target_index}')
+							except ValueError:
+								print(f'Invalid index format: {parts[1]}')
+						else:
+							print("Invalid input format. Use 'c,index'.")
+					elif ',' in answer:
+						# Input text format: index,text
+						parts = answer.split(',', 1)
+						if len(parts) == 2:
+							try:
+								target_index = int(parts[0].strip())
+								text_to_input = parts[1]
+								if target_index in selector_map:
+									element_node = selector_map[target_index]
+									print(
+										f"Inputting text '{text_to_input}' into element {target_index}: {element_node.tag_name}"
+									)
+
+									event = await browser_session.event_bus.dispatch(
+										TypeTextEvent(node=element_node, text=text_to_input)
+									)
+
+									print('Input successful.')
+								else:
+									print(f'Invalid index: {target_index}')
+							except ValueError:
+								print(f'Invalid index format: {parts[0]}')
+						else:
+							print("Invalid input format. Use 'index,text'.")
+
+				except Exception as action_e:
+					print(f'Action failed: {action_e}')
+
+			# No explicit highlight removal here, get_state handles it at the start of the loop
+
+			except Exception as e:
+				print(f'Error in loop: {e}')
+				# Optionally add a small delay before retrying
+				await asyncio.sleep(1)
+
+
+if __name__ == '__main__':
+	asyncio.run(test_focus_vs_all_elements())
+	# asyncio.run(test_process_html_file()) # Commented out the other test

browser_use/dom/playground/multi_act.py

@@ -0,0 +1,32 @@
+from browser_use import Agent
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser.profile import ViewportSize
+from browser_use.llm import ChatAzureOpenAI
+
+# Initialize the Azure OpenAI client
+llm = ChatAzureOpenAI(
+	model='gpt-4.1-mini',
+)
+
+
+TASK = """
+Go to https://browser-use.github.io/stress-tests/challenges/react-native-web-form.html and complete the React Native Web form by filling in all required fields and submitting.
+"""
+
+
+async def main():
+	browser = BrowserSession(
+		browser_profile=BrowserProfile(
+			window_size=ViewportSize(width=1100, height=1000),
+		)
+	)
+
+	agent = Agent(task=TASK, llm=llm)
+
+	await agent.run()
+
+
+if __name__ == '__main__':
+	import asyncio
+
+	asyncio.run(main())

browser_use/dom/serializer/clickable_elements.py

@@ -0,0 +1,246 @@
+from browser_use.dom.views import EnhancedDOMTreeNode, NodeType
+
+
+class ClickableElementDetector:
+	@staticmethod
+	def is_interactive(node: EnhancedDOMTreeNode) -> bool:
+		"""Check if this node is clickable/interactive using enhanced scoring."""
+
+		def has_form_control_descendant(element: EnhancedDOMTreeNode, max_depth: int = 2) -> bool:
+			"""Detect nested form controls within limited depth (handles label/span wrappers)."""
+			if max_depth <= 0:
+				return False
+
+			for child in element.children_and_shadow_roots:
+				if child.node_type != NodeType.ELEMENT_NODE:
+					continue
+
+				tag_name = child.tag_name
+				if tag_name in {'input', 'select', 'textarea'}:
+					return True
+
+				if has_form_control_descendant(child, max_depth=max_depth - 1):
+					return True
+
+			return False
+
+		# Skip non-element nodes
+		if node.node_type != NodeType.ELEMENT_NODE:
+			return False
+
+		# # if ax ignored skip
+		# if node.ax_node and node.ax_node.ignored:
+		# 	return False
+
+		# remove html and body nodes
+		if node.tag_name in {'html', 'body'}:
+			return False
+
+		# Check for JavaScript click event listeners detected via CDP (without DOM mutation)
+		# this handles vue.js @click, react onClick, angular (click), etc.
+		if node.has_js_click_listener:
+			return True
+
+		# IFRAME elements should be interactive if they're large enough to potentially need scrolling
+		# Small iframes (< 100px width or height) are unlikely to have scrollable content
+		if node.tag_name and node.tag_name.upper() == 'IFRAME' or node.tag_name.upper() == 'FRAME':
+			if node.snapshot_node and node.snapshot_node.bounds:
+				width = node.snapshot_node.bounds.width
+				height = node.snapshot_node.bounds.height
+				# Only include iframes larger than 100x100px
+				if width > 100 and height > 100:
+					return True
+
+		# RELAXED SIZE CHECK: Allow all elements including size 0 (they might be interactive overlays, etc.)
+		# Note: Size 0 elements can still be interactive (e.g., invisible clickable overlays)
+		# Visibility is determined separately by CSS styles, not just bounding box size
+
+		# Specialized handling for labels used as component wrappers (e.g., Ant Design radio/checkbox)
+		if node.tag_name == 'label':
+			# Skip labels that proxy via "for" to avoid double-activating external inputs
+			if node.attributes and node.attributes.get('for'):
+				return False
+
+			# Detect labels that wrap form controls up to two levels deep (label > span > input)
+			if has_form_control_descendant(node, max_depth=2):
+				return True
+			# Fall through to pointer/role/attribute heuristics for other label cases
+
+		# Span wrappers for UI components (detect clear interactive signals only)
+		if node.tag_name == 'span':
+			if has_form_control_descendant(node, max_depth=2):
+				return True
+			# Allow other heuristics (aria roles, event handlers, pointer) to decide
+
+		# SEARCH ELEMENT DETECTION: Check for search-related classes and attributes
+		if node.attributes:
+			search_indicators = {
+				'search',
+				'magnify',
+				'glass',
+				'lookup',
+				'find',
+				'query',
+				'search-icon',
+				'search-btn',
+				'search-button',
+				'searchbox',
+			}
+
+			# Check class names for search indicators
+			class_list = node.attributes.get('class', '').lower().split()
+			if any(indicator in ' '.join(class_list) for indicator in search_indicators):
+				return True
+
+			# Check id for search indicators
+			element_id = node.attributes.get('id', '').lower()
+			if any(indicator in element_id for indicator in search_indicators):
+				return True
+
+			# Check data attributes for search functionality
+			for attr_name, attr_value in node.attributes.items():
+				if attr_name.startswith('data-') and any(indicator in attr_value.lower() for indicator in search_indicators):
+					return True
+
+		# Enhanced accessibility property checks - direct clear indicators only
+		if node.ax_node and node.ax_node.properties:
+			for prop in node.ax_node.properties:
+				try:
+					# aria disabled
+					if prop.name == 'disabled' and prop.value:
+						return False
+
+					# aria hidden
+					if prop.name == 'hidden' and prop.value:
+						return False
+
+					# Direct interactiveness indicators
+					if prop.name in ['focusable', 'editable', 'settable'] and prop.value:
+						return True
+
+					# Interactive state properties (presence indicates interactive widget)
+					if prop.name in ['checked', 'expanded', 'pressed', 'selected']:
+						# These properties only exist on interactive elements
+						return True
+
+					# Form-related interactiveness
+					if prop.name in ['required', 'autocomplete'] and prop.value:
+						return True
+
+					# Elements with keyboard shortcuts are interactive
+					if prop.name == 'keyshortcuts' and prop.value:
+						return True
+				except (AttributeError, ValueError):
+					# Skip properties we can't process
+					continue
+
+				# ENHANCED TAG CHECK: Include truly interactive elements
+		# Note: 'label' removed - labels are handled by other attribute checks below - other wise labels with "for" attribute can destroy the real clickable element on apartments.com
+		interactive_tags = {
+			'button',
+			'input',
+			'select',
+			'textarea',
+			'a',
+			'details',
+			'summary',
+			'option',
+			'optgroup',
+		}
+		# Check with case-insensitive comparison
+		if node.tag_name and node.tag_name.lower() in interactive_tags:
+			return True
+
+		# SVG elements need special handling - only interactive if they have explicit handlers
+		# svg_tags = {'svg', 'path', 'circle', 'rect', 'polygon', 'ellipse', 'line', 'polyline', 'g'}
+		# if node.tag_name in svg_tags:
+		# 	# Only consider SVG elements interactive if they have:
+		# 	# 1. Explicit event handlers
+		# 	# 2. Interactive role attributes
+		# 	# 3. Cursor pointer style
+		# 	if node.attributes:
+		# 		# Check for event handlers
+		# 		if any(attr.startswith('on') for attr in node.attributes):
+		# 			return True
+		# 		# Check for interactive roles
+		# 		if node.attributes.get('role') in {'button', 'link', 'menuitem'}:
+		# 			return True
+		# 		# Check for cursor pointer (indicating clickability)
+		# 		if node.attributes.get('style') and 'cursor: pointer' in node.attributes.get('style', ''):
+		# 			return True
+		# 	# Otherwise, SVG elements are decorative
+		# 	return False
+
+		# Tertiary check: elements with interactive attributes
+		if node.attributes:
+			# Check for event handlers or interactive attributes
+			interactive_attributes = {'onclick', 'onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', 'tabindex'}
+			if any(attr in node.attributes for attr in interactive_attributes):
+				return True
+
+			# Check for interactive ARIA roles
+			if 'role' in node.attributes:
+				interactive_roles = {
+					'button',
+					'link',
+					'menuitem',
+					'option',
+					'radio',
+					'checkbox',
+					'tab',
+					'textbox',
+					'combobox',
+					'slider',
+					'spinbutton',
+					'search',
+					'searchbox',
+					'row',
+					'cell',
+					'gridcell',
+				}
+				if node.attributes['role'] in interactive_roles:
+					return True
+
+		# Quaternary check: accessibility tree roles
+		if node.ax_node and node.ax_node.role:
+			interactive_ax_roles = {
+				'button',
+				'link',
+				'menuitem',
+				'option',
+				'radio',
+				'checkbox',
+				'tab',
+				'textbox',
+				'combobox',
+				'slider',
+				'spinbutton',
+				'listbox',
+				'search',
+				'searchbox',
+				'row',
+				'cell',
+				'gridcell',
+			}
+			if node.ax_node.role in interactive_ax_roles:
+				return True
+
+		# ICON AND SMALL ELEMENT CHECK: Elements that might be icons
+		if (
+			node.snapshot_node
+			and node.snapshot_node.bounds
+			and 10 <= node.snapshot_node.bounds.width <= 50  # Icon-sized elements
+			and 10 <= node.snapshot_node.bounds.height <= 50
+		):
+			# Check if this small element has interactive properties
+			if node.attributes:
+				# Small elements with these attributes are likely interactive icons
+				icon_attributes = {'class', 'role', 'onclick', 'data-action', 'aria-label'}
+				if any(attr in node.attributes for attr in icon_attributes):
+					return True
+
+		# Final fallback: cursor style indicates interactivity (for cases Chrome missed)
+		if node.snapshot_node and node.snapshot_node.cursor_style and node.snapshot_node.cursor_style == 'pointer':
+			return True
+
+		return False

browser_use/dom/serializer/eval_serializer.py

@@ -0,0 +1,478 @@
+# @file purpose: Concise evaluation serializer for DOM trees - optimized for LLM query writing
+
+
+from browser_use.dom.utils import cap_text_length
+from browser_use.dom.views import (
+	EnhancedDOMTreeNode,
+	NodeType,
+	SimplifiedNode,
+)
+
+# Critical attributes for query writing and form interaction
+# NOTE: Removed 'id' and 'class' to force more robust structural selectors
+EVAL_KEY_ATTRIBUTES = [
+	'id',  # Removed - can have special chars, forces structural selectors
+	'class',  # Removed - can have special chars like +, forces structural selectors
+	'name',
+	'type',
+	'placeholder',
+	'aria-label',
+	'role',
+	'value',
+	# 'href',
+	'data-testid',
+	'alt',  # for images
+	'title',  # useful for tooltips/link context
+	# State attributes (critical for form interaction)
+	'checked',
+	'selected',
+	'disabled',
+	'required',
+	'readonly',
+	# ARIA states
+	'aria-expanded',
+	'aria-pressed',
+	'aria-checked',
+	'aria-selected',
+	'aria-invalid',
+	# Validation attributes (help agents avoid brute force)
+	'pattern',
+	'min',
+	'max',
+	'minlength',
+	'maxlength',
+	'step',
+	'aria-valuemin',
+	'aria-valuemax',
+	'aria-valuenow',
+]
+
+# Semantic elements that should always be shown
+SEMANTIC_ELEMENTS = {
+	'html',  # Always show document root
+	'body',  # Always show body
+	'h1',
+	'h2',
+	'h3',
+	'h4',
+	'h5',
+	'h6',
+	'a',
+	'button',
+	'input',
+	'textarea',
+	'select',
+	'form',
+	'label',
+	'nav',
+	'header',
+	'footer',
+	'main',
+	'article',
+	'section',
+	'table',
+	'thead',
+	'tbody',
+	'tr',
+	'th',
+	'td',
+	'ul',
+	'ol',
+	'li',
+	'img',
+	'iframe',
+	'video',
+	'audio',
+}
+
+# Container elements that can be collapsed if they only wrap one child
+COLLAPSIBLE_CONTAINERS = {'div', 'span', 'section', 'article'}
+
+# SVG child elements to skip (decorative only, no interaction value)
+SVG_ELEMENTS = {
+	'path',
+	'rect',
+	'g',
+	'circle',
+	'ellipse',
+	'line',
+	'polyline',
+	'polygon',
+	'use',
+	'defs',
+	'clipPath',
+	'mask',
+	'pattern',
+	'image',
+	'text',
+	'tspan',
+}
+
+
+class DOMEvalSerializer:
+	"""Ultra-concise DOM serializer for quick LLM query writing."""
+
+	@staticmethod
+	def serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:
+		"""
+		Serialize complete DOM tree structure for LLM understanding.
+
+		Strategy:
+		- Show ALL elements to preserve DOM structure
+		- Non-interactive elements show just tag name
+		- Interactive elements show full attributes + [index]
+		- Self-closing tags only (no closing tags)
+		"""
+		if not node:
+			return ''
+
+		# Skip excluded nodes but process children
+		if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:
+			return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+		# Skip nodes marked as should_display=False
+		if not node.should_display:
+			return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+		formatted_text = []
+		depth_str = depth * '\t'
+
+		if node.original_node.node_type == NodeType.ELEMENT_NODE:
+			tag = node.original_node.tag_name.lower()
+			is_visible = node.original_node.snapshot_node and node.original_node.is_visible
+
+			# Container elements that should be shown even if invisible (might have visible children)
+			container_tags = {'html', 'body', 'div', 'main', 'section', 'article', 'aside', 'header', 'footer', 'nav'}
+
+			# Skip invisible elements UNLESS they're containers or iframes (which might have visible children)
+			if not is_visible and tag not in container_tags and tag not in ['iframe', 'frame']:
+				return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+			# Special handling for iframes - show them with their content
+			if tag in ['iframe', 'frame']:
+				return DOMEvalSerializer._serialize_iframe(node, include_attributes, depth)
+
+			# Skip SVG elements entirely - they're just decorative graphics with no interaction value
+			# Show the <svg> tag itself to indicate graphics, but don't recurse into children
+			if tag == 'svg':
+				line = f'{depth_str}'
+				# Add [i_X] for interactive SVG elements only
+				if node.is_interactive:
+					line += f'[i_{node.original_node.backend_node_id}] '
+				line += '<svg'
+				attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+				if attributes_str:
+					line += f' {attributes_str}'
+				line += ' /> <!-- SVG content collapsed -->'
+				return line
+
+			# Skip SVG child elements entirely (path, rect, g, circle, etc.)
+			if tag in SVG_ELEMENTS:
+				return ''
+
+			# Build compact attributes string
+			attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+
+			# Decide if this element should be shown
+			is_semantic = tag in SEMANTIC_ELEMENTS
+			has_useful_attrs = bool(attributes_str)
+			has_text_content = DOMEvalSerializer._has_direct_text(node)
+			has_children = len(node.children) > 0
+
+			# Build compact element representation
+			line = f'{depth_str}'
+			# Add backend node ID notation - [i_X] for interactive elements only
+			if node.is_interactive:
+				line += f'[i_{node.original_node.backend_node_id}] '
+			# Non-interactive elements don't get an index notation
+			line += f'<{tag}'
+
+			if attributes_str:
+				line += f' {attributes_str}'
+
+			# Add scroll info if element is scrollable
+			if node.original_node.should_show_scroll_info:
+				scroll_text = node.original_node.get_scroll_info_text()
+				if scroll_text:
+					line += f' scroll="{scroll_text}"'
+
+			# Add inline text if present (keep it on same line for compactness)
+			inline_text = DOMEvalSerializer._get_inline_text(node)
+
+			# For containers (html, body, div, etc.), always show children even if there's inline text
+			# For other elements, inline text replaces children (more compact)
+			is_container = tag in container_tags
+
+			if inline_text and not is_container:
+				line += f'>{inline_text}'
+			else:
+				line += ' />'
+
+			formatted_text.append(line)
+
+			# Process children (always for containers, only if no inline_text for others)
+			if has_children and (is_container or not inline_text):
+				children_text = DOMEvalSerializer._serialize_children(node, include_attributes, depth + 1)
+				if children_text:
+					formatted_text.append(children_text)
+
+		elif node.original_node.node_type == NodeType.TEXT_NODE:
+			# Text nodes are handled inline with their parent
+			pass
+
+		elif node.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+			# Shadow DOM - just show children directly with minimal marker
+			if node.children:
+				formatted_text.append(f'{depth_str}#shadow')
+				children_text = DOMEvalSerializer._serialize_children(node, include_attributes, depth + 1)
+				if children_text:
+					formatted_text.append(children_text)
+
+		return '\n'.join(formatted_text)
+
+	@staticmethod
+	def _serialize_children(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+		"""Helper to serialize all children of a node."""
+		children_output = []
+
+		# Check if parent is a list container (ul, ol)
+		is_list_container = node.original_node.node_type == NodeType.ELEMENT_NODE and node.original_node.tag_name.lower() in [
+			'ul',
+			'ol',
+		]
+
+		# Track list items and consecutive links
+		li_count = 0
+		max_list_items = 50
+		consecutive_link_count = 0
+		max_consecutive_links = 50
+		total_links_skipped = 0
+
+		for child in node.children:
+			# Get tag name for this child
+			current_tag = None
+			if child.original_node.node_type == NodeType.ELEMENT_NODE:
+				current_tag = child.original_node.tag_name.lower()
+
+			# If we're in a list container and this child is an li element
+			if is_list_container and current_tag == 'li':
+				li_count += 1
+				# Skip li elements after the 5th one
+				if li_count > max_list_items:
+					continue
+
+			# Track consecutive anchor tags (links)
+			if current_tag == 'a':
+				consecutive_link_count += 1
+				# Skip links after the 5th consecutive one
+				if consecutive_link_count > max_consecutive_links:
+					total_links_skipped += 1
+					continue
+			else:
+				# Reset counter when we hit a non-link element
+				# But first add truncation message if we skipped links
+				if total_links_skipped > 0:
+					depth_str = depth * '\t'
+					children_output.append(f'{depth_str}... ({total_links_skipped} more links in this list)')
+					total_links_skipped = 0
+				consecutive_link_count = 0
+
+			child_text = DOMEvalSerializer.serialize_tree(child, include_attributes, depth)
+			if child_text:
+				children_output.append(child_text)
+
+		# Add truncation message if we skipped items at the end
+		if is_list_container and li_count > max_list_items:
+			depth_str = depth * '\t'
+			children_output.append(
+				f'{depth_str}... ({li_count - max_list_items} more items in this list (truncated) use evaluate to get more.'
+			)
+
+		# Add truncation message for links if we skipped any at the end
+		if total_links_skipped > 0:
+			depth_str = depth * '\t'
+			children_output.append(
+				f'{depth_str}... ({total_links_skipped} more links in this list) (truncated) use evaluate to get more.'
+			)
+
+		return '\n'.join(children_output)
+
+	@staticmethod
+	def _build_compact_attributes(node: EnhancedDOMTreeNode) -> str:
+		"""Build ultra-compact attributes string with only key attributes."""
+		attrs = []
+
+		# Prioritize attributes that help with query writing
+		if node.attributes:
+			for attr in EVAL_KEY_ATTRIBUTES:
+				if attr in node.attributes:
+					value = str(node.attributes[attr]).strip()
+					if not value:
+						continue
+
+					# Special handling for different attributes
+					if attr == 'class':
+						# For class, limit to first 2 classes to save space
+						classes = value.split()[:3]
+						value = ' '.join(classes)
+					elif attr == 'href':
+						# For href, cap at 20 chars to save space
+						value = cap_text_length(value, 80)
+					else:
+						# Cap at 25 chars for other attributes
+						value = cap_text_length(value, 80)
+
+					attrs.append(f'{attr}="{value}"')
+
+		# Note: We intentionally don't add role from ax_node here because:
+		# 1. If role is explicitly set in HTML, it's already captured above via EVAL_KEY_ATTRIBUTES
+		# 2. Inferred roles from AX tree (like link, listitem, LineBreak) are redundant with the tag name
+		# 3. This reduces noise - <a href="..." role="link"> is redundant, we already know <a> is a link
+
+		return ' '.join(attrs)
+
+	@staticmethod
+	def _has_direct_text(node: SimplifiedNode) -> bool:
+		"""Check if node has direct text children (not nested in other elements)."""
+		for child in node.children:
+			if child.original_node.node_type == NodeType.TEXT_NODE:
+				text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+				if len(text) > 1:
+					return True
+		return False
+
+	@staticmethod
+	def _get_inline_text(node: SimplifiedNode) -> str:
+		"""Get text content to display inline (max 40 chars)."""
+		text_parts = []
+		for child in node.children:
+			if child.original_node.node_type == NodeType.TEXT_NODE:
+				text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+				if text and len(text) > 1:
+					text_parts.append(text)
+
+		if not text_parts:
+			return ''
+
+		combined = ' '.join(text_parts)
+		return cap_text_length(combined, 80)
+
+	@staticmethod
+	def _serialize_iframe(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+		"""Handle iframe serialization with content document."""
+		formatted_text = []
+		depth_str = depth * '\t'
+		tag = node.original_node.tag_name.lower()
+
+		# Build minimal iframe marker with key attributes
+		attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+		line = f'{depth_str}<{tag}'
+		if attributes_str:
+			line += f' {attributes_str}'
+
+		# Add scroll info for iframe content
+		if node.original_node.should_show_scroll_info:
+			scroll_text = node.original_node.get_scroll_info_text()
+			if scroll_text:
+				line += f' scroll="{scroll_text}"'
+
+		line += ' />'
+		formatted_text.append(line)
+
+		# If iframe has content document, serialize its content
+		if node.original_node.content_document:
+			# Add marker for iframe content
+			formatted_text.append(f'{depth_str}\t#iframe-content')
+
+			# Process content document children
+			for child_node in node.original_node.content_document.children_nodes or []:
+				# Process html documents
+				if child_node.tag_name.lower() == 'html':
+					# Find and serialize body content only (skip head)
+					for html_child in child_node.children:
+						if html_child.tag_name.lower() == 'body':
+							for body_child in html_child.children:
+								# Recursively process body children (iframe content)
+								DOMEvalSerializer._serialize_document_node(
+									body_child, formatted_text, include_attributes, depth + 2, is_iframe_content=True
+								)
+							break  # Stop after processing body
+				else:
+					# Not an html element - serialize directly
+					DOMEvalSerializer._serialize_document_node(
+						child_node, formatted_text, include_attributes, depth + 1, is_iframe_content=True
+					)
+
+		return '\n'.join(formatted_text)
+
+	@staticmethod
+	def _serialize_document_node(
+		dom_node: EnhancedDOMTreeNode,
+		output: list[str],
+		include_attributes: list[str],
+		depth: int,
+		is_iframe_content: bool = True,
+	) -> None:
+		"""Helper to serialize a document node without SimplifiedNode wrapper.
+
+		Args:
+			is_iframe_content: If True, be more permissive with visibility checks since
+				iframe content might not have snapshot data from parent page.
+		"""
+		depth_str = depth * '\t'
+
+		if dom_node.node_type == NodeType.ELEMENT_NODE:
+			tag = dom_node.tag_name.lower()
+
+			# For iframe content, be permissive - show all semantic elements even without snapshot data
+			# For regular content, skip invisible elements
+			if is_iframe_content:
+				# Only skip if we have snapshot data AND it's explicitly invisible
+				# If no snapshot data, assume visible (cross-origin iframe content)
+				is_visible = (not dom_node.snapshot_node) or dom_node.is_visible
+			else:
+				# Regular strict visibility check
+				is_visible = dom_node.snapshot_node and dom_node.is_visible
+
+			if not is_visible:
+				return
+
+			# Check if semantic or has useful attributes
+			is_semantic = tag in SEMANTIC_ELEMENTS
+			attributes_str = DOMEvalSerializer._build_compact_attributes(dom_node)
+
+			if not is_semantic and not attributes_str:
+				# Skip but process children
+				for child in dom_node.children:
+					DOMEvalSerializer._serialize_document_node(
+						child, output, include_attributes, depth, is_iframe_content=is_iframe_content
+					)
+				return
+
+			# Build element line
+			line = f'{depth_str}<{tag}'
+			if attributes_str:
+				line += f' {attributes_str}'
+
+			# Get direct text content
+			text_parts = []
+			for child in dom_node.children:
+				if child.node_type == NodeType.TEXT_NODE and child.node_value:
+					text = child.node_value.strip()
+					if text and len(text) > 1:
+						text_parts.append(text)
+
+			if text_parts:
+				combined = ' '.join(text_parts)
+				line += f'>{cap_text_length(combined, 100)}'
+			else:
+				line += ' />'
+
+			output.append(line)
+
+			# Process non-text children
+			for child in dom_node.children:
+				if child.node_type != NodeType.TEXT_NODE:
+					DOMEvalSerializer._serialize_document_node(
+						child, output, include_attributes, depth + 1, is_iframe_content=is_iframe_content
+					)

browser_use/dom/serializer/html_serializer.py

@@ -0,0 +1,298 @@
+# @file purpose: Serializes enhanced DOM trees to HTML format including shadow roots
+
+from browser_use.dom.views import EnhancedDOMTreeNode, NodeType
+
+
+class HTMLSerializer:
+	"""Serializes enhanced DOM trees back to HTML format.
+
+	This serializer reconstructs HTML from the enhanced DOM tree, including:
+	- Shadow DOM content (both open and closed)
+	- Iframe content documents
+	- All attributes and text nodes
+	- Proper HTML structure
+
+	Unlike getOuterHTML which only captures light DOM, this captures the full
+	enhanced tree including shadow roots that are crucial for modern SPAs.
+	"""
+
+	def __init__(self, extract_links: bool = False):
+		"""Initialize the HTML serializer.
+
+		Args:
+			extract_links: If True, preserves all links. If False, removes href attributes.
+		"""
+		self.extract_links = extract_links
+
+	def serialize(self, node: EnhancedDOMTreeNode, depth: int = 0) -> str:
+		"""Serialize an enhanced DOM tree node to HTML.
+
+		Args:
+			node: The enhanced DOM tree node to serialize
+			depth: Current depth for indentation (internal use)
+
+		Returns:
+			HTML string representation of the node and its descendants
+		"""
+		if node.node_type == NodeType.DOCUMENT_NODE:
+			# Process document root - serialize all children
+			parts = []
+			for child in node.children_and_shadow_roots:
+				child_html = self.serialize(child, depth)
+				if child_html:
+					parts.append(child_html)
+			return ''.join(parts)
+
+		elif node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+			# Shadow DOM root - wrap in template with shadowrootmode attribute
+			parts = []
+
+			# Add shadow root opening
+			shadow_type = node.shadow_root_type or 'open'
+			parts.append(f'<template shadowroot="{shadow_type.lower()}">')
+
+			# Serialize shadow children
+			for child in node.children:
+				child_html = self.serialize(child, depth + 1)
+				if child_html:
+					parts.append(child_html)
+
+			# Close shadow root
+			parts.append('</template>')
+
+			return ''.join(parts)
+
+		elif node.node_type == NodeType.ELEMENT_NODE:
+			parts = []
+			tag_name = node.tag_name.lower()
+
+			# Skip non-content elements
+			if tag_name in {'style', 'script', 'head', 'meta', 'link', 'title'}:
+				return ''
+
+			# Skip code tags with display:none - these often contain JSON state for SPAs
+			if tag_name == 'code' and node.attributes:
+				style = node.attributes.get('style', '')
+				# Check if element is hidden (display:none) - likely JSON data
+				if 'display:none' in style.replace(' ', '') or 'display: none' in style:
+					return ''
+				# Also check for bpr-guid IDs (LinkedIn's JSON data pattern)
+				element_id = node.attributes.get('id', '')
+				if 'bpr-guid' in element_id or 'data' in element_id or 'state' in element_id:
+					return ''
+
+			# Skip base64 inline images - these are usually placeholders or tracking pixels
+			if tag_name == 'img' and node.attributes:
+				src = node.attributes.get('src', '')
+				if src.startswith('data:image/'):
+					return ''
+
+			# Opening tag
+			parts.append(f'<{tag_name}')
+
+			# Add attributes
+			if node.attributes:
+				attrs = self._serialize_attributes(node.attributes)
+				if attrs:
+					parts.append(' ' + attrs)
+
+			# Handle void elements (self-closing)
+			void_elements = {
+				'area',
+				'base',
+				'br',
+				'col',
+				'embed',
+				'hr',
+				'img',
+				'input',
+				'link',
+				'meta',
+				'param',
+				'source',
+				'track',
+				'wbr',
+			}
+			if tag_name in void_elements:
+				parts.append(' />')
+				return ''.join(parts)
+
+			parts.append('>')
+
+			# Handle table normalization (ensure thead/tbody for markdownify)
+			if tag_name == 'table':
+				# Serialize shadow roots first (same as the general path)
+				if node.shadow_roots:
+					for shadow_root in node.shadow_roots:
+						child_html = self.serialize(shadow_root, depth + 1)
+						if child_html:
+							parts.append(child_html)
+				table_html = self._serialize_table_children(node, depth)
+				parts.append(table_html)
+			# Handle iframe content document
+			elif tag_name in {'iframe', 'frame'} and node.content_document:
+				# Serialize iframe content
+				for child in node.content_document.children_nodes or []:
+					child_html = self.serialize(child, depth + 1)
+					if child_html:
+						parts.append(child_html)
+			else:
+				# Serialize shadow roots FIRST (for declarative shadow DOM)
+				if node.shadow_roots:
+					for shadow_root in node.shadow_roots:
+						child_html = self.serialize(shadow_root, depth + 1)
+						if child_html:
+							parts.append(child_html)
+
+				# Then serialize light DOM children (for slot projection)
+				for child in node.children:
+					child_html = self.serialize(child, depth + 1)
+					if child_html:
+						parts.append(child_html)
+
+			# Closing tag
+			parts.append(f'</{tag_name}>')
+
+			return ''.join(parts)
+
+		elif node.node_type == NodeType.TEXT_NODE:
+			# Return text content with basic HTML escaping
+			if node.node_value:
+				return self._escape_html(node.node_value)
+			return ''
+
+		elif node.node_type == NodeType.COMMENT_NODE:
+			# Skip comments to reduce noise
+			return ''
+
+		else:
+			# Unknown node type - skip
+			return ''
+
+	def _serialize_table_children(self, table_node: EnhancedDOMTreeNode, depth: int) -> str:
+		"""Normalize table structure to ensure thead/tbody for markdownify.
+
+		When a <table> has no <thead> but the first <tr> contains <th> cells,
+		wrap that row in <thead> and remaining rows in <tbody>.
+		"""
+		children = table_node.children
+		if not children:
+			return ''
+
+		# Check if table already has thead
+		child_tags = [c.tag_name for c in children if c.node_type == NodeType.ELEMENT_NODE]
+		has_thead = 'thead' in child_tags
+		has_tbody = 'tbody' in child_tags
+
+		if has_thead or not child_tags:
+			# Already normalized or empty — serialize normally
+			parts = []
+			for child in children:
+				child_html = self.serialize(child, depth + 1)
+				if child_html:
+					parts.append(child_html)
+			return ''.join(parts)
+
+		# Find the first <tr> with <th> cells
+		first_tr = None
+		first_tr_idx = -1
+		for i, child in enumerate(children):
+			if child.node_type == NodeType.ELEMENT_NODE and child.tag_name == 'tr':
+				# Check if this row contains <th> cells
+				has_th = any(c.node_type == NodeType.ELEMENT_NODE and c.tag_name == 'th' for c in child.children)
+				if has_th:
+					first_tr = child
+					first_tr_idx = i
+				break  # Only check the first <tr>
+
+		if first_tr is None:
+			# No header row detected — serialize normally
+			parts = []
+			for child in children:
+				child_html = self.serialize(child, depth + 1)
+				if child_html:
+					parts.append(child_html)
+			return ''.join(parts)
+
+		# Wrap first_tr in <thead>, remaining <tr> in <tbody>
+		parts = []
+
+		# Emit any children before the header row (e.g. colgroup, caption)
+		for child in children[:first_tr_idx]:
+			child_html = self.serialize(child, depth + 1)
+			if child_html:
+				parts.append(child_html)
+
+		# Emit <thead>
+		parts.append('<thead>')
+		parts.append(self.serialize(first_tr, depth + 2))
+		parts.append('</thead>')
+
+		# Collect remaining rows
+		remaining = children[first_tr_idx + 1 :]
+		if remaining and not has_tbody:
+			parts.append('<tbody>')
+			for child in remaining:
+				child_html = self.serialize(child, depth + 2)
+				if child_html:
+					parts.append(child_html)
+			parts.append('</tbody>')
+		else:
+			for child in remaining:
+				child_html = self.serialize(child, depth + 1)
+				if child_html:
+					parts.append(child_html)
+
+		return ''.join(parts)
+
+	def _serialize_attributes(self, attributes: dict[str, str]) -> str:
+		"""Serialize element attributes to HTML attribute string.
+
+		Args:
+			attributes: Dictionary of attribute names to values
+
+		Returns:
+			HTML attribute string (e.g., 'class="foo" id="bar"')
+		"""
+		parts = []
+		for key, value in attributes.items():
+			# Skip href if not extracting links
+			if not self.extract_links and key == 'href':
+				continue
+
+			# Skip data-* attributes as they often contain JSON payloads
+			# These are used by modern SPAs (React, Vue, Angular) for state management
+			if key.startswith('data-'):
+				continue
+
+			# Handle boolean attributes
+			if value == '' or value is None:
+				parts.append(key)
+			else:
+				# Escape attribute value
+				escaped_value = self._escape_attribute(value)
+				parts.append(f'{key}="{escaped_value}"')
+
+		return ' '.join(parts)
+
+	def _escape_html(self, text: str) -> str:
+		"""Escape HTML special characters in text content.
+
+		Args:
+			text: Raw text content
+
+		Returns:
+			HTML-escaped text
+		"""
+		return text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
+
+	def _escape_attribute(self, value: str) -> str:
+		"""Escape HTML special characters in attribute values.
+
+		Args:
+			value: Raw attribute value
+
+		Returns:
+			HTML-escaped attribute value
+		"""
+		return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&#x27;')

browser_use/dom/serializer/paint_order.py

@@ -0,0 +1,212 @@
+from collections import defaultdict
+from dataclasses import dataclass
+
+from browser_use.dom.views import SimplifiedNode
+
+"""
+Helper class for maintaining a union of rectangles (used for order of elements calculation)
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class Rect:
+	"""Closed axis-aligned rectangle with (x1,y1) bottom-left, (x2,y2) top-right."""
+
+	x1: float
+	y1: float
+	x2: float
+	y2: float
+
+	def __post_init__(self):
+		if not (self.x1 <= self.x2 and self.y1 <= self.y2):
+			return False
+
+	# --- fast relations ----------------------------------------------------
+	def area(self) -> float:
+		return (self.x2 - self.x1) * (self.y2 - self.y1)
+
+	def intersects(self, other: 'Rect') -> bool:
+		return not (self.x2 <= other.x1 or other.x2 <= self.x1 or self.y2 <= other.y1 or other.y2 <= self.y1)
+
+	def contains(self, other: 'Rect') -> bool:
+		return self.x1 <= other.x1 and self.y1 <= other.y1 and self.x2 >= other.x2 and self.y2 >= other.y2
+
+
+class RectUnionPure:
+	"""
+	Maintains a *disjoint* set of rectangles.
+	No external dependencies - fine for a few thousand rectangles.
+
+	A safety cap (_MAX_RECTS) prevents exponential explosion on pages with
+	many overlapping translucent layers. Once the cap is hit, contains()
+	conservatively returns False (i.e. nothing is hidden), preserving
+	correctness at the cost of less aggressive paint-order filtering.
+	"""
+
+	__slots__ = ('_rects',)
+
+	# Safety cap: with complex overlapping layers, each add() can fragment
+	# existing rects into up to 4 pieces each. On heavy pages (20k+ elements)
+	# this can cause exponential growth. 5000 is generous enough for normal
+	# pages but prevents runaway memory/CPU.
+	_MAX_RECTS = 5000
+
+	def __init__(self):
+		self._rects: list[Rect] = []
+
+	# -----------------------------------------------------------------
+	def _split_diff(self, a: Rect, b: Rect) -> list[Rect]:
+		r"""
+		Return list of up to 4 rectangles = a \ b.
+		Assumes a intersects b.
+		"""
+		parts = []
+
+		# Bottom slice
+		if a.y1 < b.y1:
+			parts.append(Rect(a.x1, a.y1, a.x2, b.y1))
+		# Top slice
+		if b.y2 < a.y2:
+			parts.append(Rect(a.x1, b.y2, a.x2, a.y2))
+
+		# Middle (vertical) strip: y overlap is [max(a.y1,b.y1), min(a.y2,b.y2)]
+		y_lo = max(a.y1, b.y1)
+		y_hi = min(a.y2, b.y2)
+
+		# Left slice
+		if a.x1 < b.x1:
+			parts.append(Rect(a.x1, y_lo, b.x1, y_hi))
+		# Right slice
+		if b.x2 < a.x2:
+			parts.append(Rect(b.x2, y_lo, a.x2, y_hi))
+
+		return parts
+
+	# -----------------------------------------------------------------
+	def contains(self, r: Rect) -> bool:
+		"""
+		True iff r is fully covered by the current union.
+		"""
+		if not self._rects:
+			return False
+
+		stack = [r]
+		for s in self._rects:
+			new_stack = []
+			for piece in stack:
+				if s.contains(piece):
+					# piece completely gone
+					continue
+				if piece.intersects(s):
+					new_stack.extend(self._split_diff(piece, s))
+				else:
+					new_stack.append(piece)
+			if not new_stack:  # everything eaten – covered
+				return True
+			stack = new_stack
+		return False  # something survived
+
+	# -----------------------------------------------------------------
+	def add(self, r: Rect) -> bool:
+		"""
+		Insert r unless it is already covered.
+		Returns True if the union grew.
+		"""
+		# Safety cap: stop accepting new rects to prevent exponential explosion
+		if len(self._rects) >= self._MAX_RECTS:
+			return False
+
+		if self.contains(r):
+			return False
+
+		pending = [r]
+		i = 0
+		while i < len(self._rects):
+			s = self._rects[i]
+			new_pending = []
+			changed = False
+			for piece in pending:
+				if piece.intersects(s):
+					new_pending.extend(self._split_diff(piece, s))
+					changed = True
+				else:
+					new_pending.append(piece)
+			pending = new_pending
+			if changed:
+				# s unchanged; proceed with next existing rectangle
+				i += 1
+			else:
+				i += 1
+
+		# Any left‑over pieces are new, non‑overlapping areas
+		self._rects.extend(pending)
+		return True
+
+
+class PaintOrderRemover:
+	"""
+	Calculates which elements should be removed based on the paint order parameter.
+	"""
+
+	def __init__(self, root: SimplifiedNode):
+		self.root = root
+
+	def calculate_paint_order(self) -> None:
+		all_simplified_nodes_with_paint_order: list[SimplifiedNode] = []
+
+		def collect_paint_order(node: SimplifiedNode) -> None:
+			if (
+				node.original_node.snapshot_node
+				and node.original_node.snapshot_node.paint_order is not None
+				and node.original_node.snapshot_node.bounds is not None
+			):
+				all_simplified_nodes_with_paint_order.append(node)
+
+			for child in node.children:
+				collect_paint_order(child)
+
+		collect_paint_order(self.root)
+
+		grouped_by_paint_order: defaultdict[int, list[SimplifiedNode]] = defaultdict(list)
+
+		for node in all_simplified_nodes_with_paint_order:
+			if node.original_node.snapshot_node and node.original_node.snapshot_node.paint_order is not None:
+				grouped_by_paint_order[node.original_node.snapshot_node.paint_order].append(node)
+
+		rect_union = RectUnionPure()
+
+		for paint_order, nodes in sorted(grouped_by_paint_order.items(), key=lambda x: -x[0]):
+			rects_to_add = []
+
+			for node in nodes:
+				if not node.original_node.snapshot_node or not node.original_node.snapshot_node.bounds:
+					continue  # shouldn't happen by how we filter them out in the first place
+
+				rect = Rect(
+					x1=node.original_node.snapshot_node.bounds.x,
+					y1=node.original_node.snapshot_node.bounds.y,
+					x2=node.original_node.snapshot_node.bounds.x + node.original_node.snapshot_node.bounds.width,
+					y2=node.original_node.snapshot_node.bounds.y + node.original_node.snapshot_node.bounds.height,
+				)
+
+				if rect_union.contains(rect):
+					node.ignored_by_paint_order = True
+
+				# don't add to the nodes if opacity is less then 0.95 or background-color is transparent
+				if (
+					node.original_node.snapshot_node.computed_styles
+					and node.original_node.snapshot_node.computed_styles.get('background-color', 'rgba(0, 0, 0, 0)')
+					== 'rgba(0, 0, 0, 0)'
+				) or (
+					node.original_node.snapshot_node.computed_styles
+					and float(node.original_node.snapshot_node.computed_styles.get('opacity', '1'))
+					< 0.8  # this is highly vibes based number
+				):
+					continue
+
+				rects_to_add.append(rect)
+
+			for rect in rects_to_add:
+				rect_union.add(rect)
+
+		return None

browser_use/dom/utils.py

@@ -0,0 +1,129 @@
+def cap_text_length(text: str, max_length: int) -> str:
+	"""Cap text length for display."""
+	if len(text) <= max_length:
+		return text
+	return text[:max_length] + '...'
+
+
+def generate_css_selector_for_element(enhanced_node) -> str | None:
+	"""Generate a CSS selector using node properties from version 0.5.0 approach."""
+	import re
+
+	if not enhanced_node or not hasattr(enhanced_node, 'tag_name') or not enhanced_node.tag_name:
+		return None
+
+	# Get base selector from tag name (simplified since we don't have xpath in EnhancedDOMTreeNode)
+	tag_name = enhanced_node.tag_name.lower().strip()
+	if not tag_name or not re.match(r'^[a-zA-Z][a-zA-Z0-9-]*$', tag_name):
+		return None
+
+	css_selector = tag_name
+
+	# Add ID if available (most specific)
+	if enhanced_node.attributes and 'id' in enhanced_node.attributes:
+		element_id = enhanced_node.attributes['id']
+		if element_id and element_id.strip():
+			element_id = element_id.strip()
+			# Validate ID contains only valid characters for # selector
+			if re.match(r'^[a-zA-Z][a-zA-Z0-9_-]*$', element_id):
+				return f'#{element_id}'
+			else:
+				# For IDs with special characters ($, ., :, etc.), use attribute selector
+				# Escape quotes in the ID value
+				escaped_id = element_id.replace('"', '\\"')
+				return f'{tag_name}[id="{escaped_id}"]'
+
+	# Handle class attributes (from version 0.5.0 approach)
+	if enhanced_node.attributes and 'class' in enhanced_node.attributes and enhanced_node.attributes['class']:
+		# Define a regex pattern for valid class names in CSS
+		valid_class_name_pattern = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_-]*$')
+
+		# Iterate through the class attribute values
+		classes = enhanced_node.attributes['class'].split()
+		for class_name in classes:
+			# Skip empty class names
+			if not class_name.strip():
+				continue
+
+			# Check if the class name is valid
+			if valid_class_name_pattern.match(class_name):
+				# Append the valid class name to the CSS selector
+				css_selector += f'.{class_name}'
+
+	# Expanded set of safe attributes that are stable and useful for selection (from v0.5.0)
+	SAFE_ATTRIBUTES = {
+		# Data attributes (if they're stable in your application)
+		'id',
+		# Standard HTML attributes
+		'name',
+		'type',
+		'placeholder',
+		# Accessibility attributes
+		'aria-label',
+		'aria-labelledby',
+		'aria-describedby',
+		'role',
+		# Common form attributes
+		'for',
+		'autocomplete',
+		'required',
+		'readonly',
+		# Media attributes
+		'alt',
+		'title',
+		'src',
+		# Custom stable attributes (add any application-specific ones)
+		'href',
+		'target',
+	}
+
+	# Always include dynamic attributes (include_dynamic_attributes=True equivalent)
+	include_dynamic_attributes = True
+	if include_dynamic_attributes:
+		dynamic_attributes = {
+			'data-id',
+			'data-qa',
+			'data-cy',
+			'data-testid',
+		}
+		SAFE_ATTRIBUTES.update(dynamic_attributes)
+
+	# Handle other attributes (from version 0.5.0 approach)
+	if enhanced_node.attributes:
+		for attribute, value in enhanced_node.attributes.items():
+			if attribute == 'class':
+				continue
+
+			# Skip invalid attribute names
+			if not attribute.strip():
+				continue
+
+			if attribute not in SAFE_ATTRIBUTES:
+				continue
+
+			# Escape special characters in attribute names
+			safe_attribute = attribute.replace(':', r'\:')
+
+			# Handle different value cases
+			if value == '':
+				css_selector += f'[{safe_attribute}]'
+			elif any(char in value for char in '"\'<>`\n\r\t'):
+				# Use contains for values with special characters
+				# For newline-containing text, only use the part before the newline
+				if '\n' in value:
+					value = value.split('\n')[0]
+				# Regex-substitute *any* whitespace with a single space, then strip.
+				collapsed_value = re.sub(r'\s+', ' ', value).strip()
+				# Escape embedded double-quotes.
+				safe_value = collapsed_value.replace('"', '\\"')
+				css_selector += f'[{safe_attribute}*="{safe_value}"]'
+			else:
+				css_selector += f'[{safe_attribute}="{value}"]'
+
+	# Final validation: ensure the selector is safe and doesn't contain problematic characters
+	# Note: quotes are allowed in attribute selectors like [name="value"]
+	if css_selector and not any(char in css_selector for char in ['\n', '\r', '\t']):
+		return css_selector
+
+	# If we get here, the selector was problematic, return just the tag name as fallback
+	return tag_name

browser_use/exceptions.py

@@ -0,0 +1,5 @@
+class LLMException(Exception):
+	def __init__(self, status_code, message):
+		self.status_code = status_code
+		self.message = message
+		super().__init__(f'Error {status_code}: {message}')

browser_use/filesystem/file_system.py

@@ -0,0 +1,941 @@
+import asyncio
+import base64
+import csv
+import io
+import os
+import re
+import shutil
+from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+UNSUPPORTED_BINARY_EXTENSIONS = {
+	'png',
+	'jpg',
+	'jpeg',
+	'gif',
+	'bmp',
+	'svg',
+	'webp',
+	'ico',
+	'mp3',
+	'mp4',
+	'wav',
+	'avi',
+	'mov',
+	'zip',
+	'tar',
+	'gz',
+	'rar',
+	'exe',
+	'bin',
+	'dll',
+	'so',
+}
+
+
+def _build_filename_error_message(file_name: str, supported_extensions: list[str]) -> str:
+	"""Build a specific error message explaining why the filename was rejected and how to fix it."""
+	base = os.path.basename(file_name)
+
+	# Check for binary/image extension
+	if '.' in base:
+		_, ext = base.rsplit('.', 1)
+		ext_lower = ext.lower()
+		if ext_lower in UNSUPPORTED_BINARY_EXTENSIONS:
+			return (
+				f"Error: Cannot write binary/image file '{base}'. "
+				f'The write_file tool only supports text-based files. '
+				f'Supported extensions: {", ".join("." + e for e in supported_extensions)}. '
+				f'For screenshots, the browser automatically captures them - do not try to save screenshots as files.'
+			)
+		if ext_lower not in supported_extensions:
+			return (
+				f"Error: Unsupported file extension '.{ext_lower}' in '{base}'. "
+				f'Supported extensions: {", ".join("." + e for e in supported_extensions)}. '
+				f'Please rename the file to use a supported extension.'
+			)
+
+	# No extension or no dot
+	if '.' not in base:
+		return (
+			f"Error: Filename '{base}' has no extension. "
+			f'Please add a supported extension: {", ".join("." + e for e in supported_extensions)}.'
+		)
+
+	return (
+		f"Error: Invalid filename '{base}'. "
+		f'Filenames must contain only letters, numbers, underscores, hyphens, dots, parentheses, and spaces. '
+		f'Supported extensions: {", ".join("." + e for e in supported_extensions)}.'
+	)
+
+
+DEFAULT_FILE_SYSTEM_PATH = 'browseruse_agent_data'
+
+
+class FileSystemError(Exception):
+	"""Custom exception for file system operations that should be shown to LLM"""
+
+	pass
+
+
+class BaseFile(BaseModel, ABC):
+	"""Base class for all file types"""
+
+	name: str
+	content: str = ''
+
+	# --- Subclass must define this ---
+	@property
+	@abstractmethod
+	def extension(self) -> str:
+		"""File extension (e.g. 'txt', 'md')"""
+		pass
+
+	def write_file_content(self, content: str) -> None:
+		"""Update internal content (formatted)"""
+		self.update_content(content)
+
+	def append_file_content(self, content: str) -> None:
+		"""Append content to internal content"""
+		self.update_content(self.content + content)
+
+	# --- These are shared and implemented here ---
+
+	def update_content(self, content: str) -> None:
+		self.content = content
+
+	def sync_to_disk_sync(self, path: Path) -> None:
+		file_path = path / self.full_name
+		file_path.write_text(self.content)
+
+	async def sync_to_disk(self, path: Path) -> None:
+		file_path = path / self.full_name
+		with ThreadPoolExecutor() as executor:
+			await asyncio.get_event_loop().run_in_executor(executor, lambda: file_path.write_text(self.content))
+
+	async def write(self, content: str, path: Path) -> None:
+		self.write_file_content(content)
+		await self.sync_to_disk(path)
+
+	async def append(self, content: str, path: Path) -> None:
+		self.append_file_content(content)
+		await self.sync_to_disk(path)
+
+	def read(self) -> str:
+		return self.content
+
+	@property
+	def full_name(self) -> str:
+		return f'{self.name}.{self.extension}'
+
+	@property
+	def get_size(self) -> int:
+		return len(self.content)
+
+	@property
+	def get_line_count(self) -> int:
+		return len(self.content.splitlines())
+
+
+class MarkdownFile(BaseFile):
+	"""Markdown file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'md'
+
+
+class TxtFile(BaseFile):
+	"""Plain text file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'txt'
+
+
+class JsonFile(BaseFile):
+	"""JSON file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'json'
+
+
+class CsvFile(BaseFile):
+	"""CSV file implementation with automatic RFC 4180 normalization.
+
+	LLMs frequently produce malformed CSV (missing quotes around fields with commas,
+	inconsistent empty fields, unescaped internal quotes). This class parses the raw
+	content through Python's csv module on every write to guarantee well-formed output.
+	"""
+
+	@property
+	def extension(self) -> str:
+		return 'csv'
+
+	@staticmethod
+	def _normalize_csv(raw: str) -> str:
+		"""Parse and re-serialize CSV content to fix quoting, empty fields, and escaping.
+
+		Handles common LLM mistakes: unquoted fields containing commas,
+		unescaped quotes inside fields, inconsistent empty fields,
+		trailing/leading blank lines, and double-escaped JSON output
+		(literal backslash-n and backslash-quote instead of real newlines/quotes).
+		"""
+		stripped = raw.strip('\n\r')
+		if not stripped:
+			return raw
+
+		# Detect double-escaped LLM tool call output: if the content has no real
+		# newlines but contains literal \n sequences, the entire string is likely
+		# double-escaped JSON. Unescape \" → " first, then \n → newline.
+		if '\n' not in stripped and '\\n' in stripped:
+			stripped = stripped.replace('\\"', '"')
+			stripped = stripped.replace('\\n', '\n')
+
+		reader = csv.reader(io.StringIO(stripped))
+		rows: list[list[str]] = []
+		for row in reader:
+			# Skip completely empty rows (artifacts of blank lines)
+			if row:
+				rows.append(row)
+
+		if not rows:
+			return raw
+
+		out = io.StringIO()
+		writer = csv.writer(out, lineterminator='\n')
+		writer.writerows(rows)
+		# Strip trailing newline so callers (write_file action) control line endings
+		return out.getvalue().rstrip('\n')
+
+	def write_file_content(self, content: str) -> None:
+		"""Normalize CSV content before storing."""
+		self.update_content(self._normalize_csv(content))
+
+	def append_file_content(self, content: str) -> None:
+		"""Normalize the appended CSV rows and merge with existing content."""
+		normalized_new = self._normalize_csv(content)
+		if not normalized_new.strip('\n\r'):
+			return
+		existing = self.content
+		if existing and not existing.endswith('\n'):
+			existing += '\n'
+		combined = existing + normalized_new
+		self.update_content(self._normalize_csv(combined))
+
+
+class JsonlFile(BaseFile):
+	"""JSONL (JSON Lines) file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'jsonl'
+
+
+class PdfFile(BaseFile):
+	"""PDF file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'pdf'
+
+	def sync_to_disk_sync(self, path: Path) -> None:
+		# Lazy import reportlab
+		from reportlab.lib.pagesizes import letter
+		from reportlab.lib.styles import getSampleStyleSheet
+		from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer
+
+		file_path = path / self.full_name
+		try:
+			# Create PDF document
+			doc = SimpleDocTemplate(str(file_path), pagesize=letter)
+			styles = getSampleStyleSheet()
+			story = []
+
+			# Convert markdown content to simple text and add to PDF
+			# For basic implementation, we'll treat content as plain text
+			# This avoids the AGPL license issue while maintaining functionality
+			content_lines = self.content.split('\n')
+
+			for line in content_lines:
+				if line.strip():
+					# Handle basic markdown headers
+					if line.startswith('# '):
+						para = Paragraph(line[2:], styles['Title'])
+					elif line.startswith('## '):
+						para = Paragraph(line[3:], styles['Heading1'])
+					elif line.startswith('### '):
+						para = Paragraph(line[4:], styles['Heading2'])
+					else:
+						para = Paragraph(line, styles['Normal'])
+					story.append(para)
+				else:
+					story.append(Spacer(1, 6))
+
+			doc.build(story)
+		except Exception as e:
+			raise FileSystemError(f"Error: Could not write to file '{self.full_name}'. {str(e)}")
+
+	async def sync_to_disk(self, path: Path) -> None:
+		with ThreadPoolExecutor() as executor:
+			await asyncio.get_event_loop().run_in_executor(executor, lambda: self.sync_to_disk_sync(path))
+
+
+class DocxFile(BaseFile):
+	"""DOCX file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'docx'
+
+	def sync_to_disk_sync(self, path: Path) -> None:
+		file_path = path / self.full_name
+		try:
+			from docx import Document
+
+			doc = Document()
+
+			# Convert content to DOCX paragraphs
+			content_lines = self.content.split('\n')
+
+			for line in content_lines:
+				if line.strip():
+					# Handle basic markdown headers
+					if line.startswith('# '):
+						doc.add_heading(line[2:], level=1)
+					elif line.startswith('## '):
+						doc.add_heading(line[3:], level=2)
+					elif line.startswith('### '):
+						doc.add_heading(line[4:], level=3)
+					else:
+						doc.add_paragraph(line)
+				else:
+					doc.add_paragraph()  # Empty paragraph for spacing
+
+			doc.save(str(file_path))
+		except Exception as e:
+			raise FileSystemError(f"Error: Could not write to file '{self.full_name}'. {str(e)}")
+
+	async def sync_to_disk(self, path: Path) -> None:
+		with ThreadPoolExecutor() as executor:
+			await asyncio.get_event_loop().run_in_executor(executor, lambda: self.sync_to_disk_sync(path))
+
+
+class HtmlFile(BaseFile):
+	"""HTML file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'html'
+
+
+class XmlFile(BaseFile):
+	"""XML file implementation"""
+
+	@property
+	def extension(self) -> str:
+		return 'xml'
+
+
+class FileSystemState(BaseModel):
+	"""Serializable state of the file system"""
+
+	files: dict[str, dict[str, Any]] = Field(default_factory=dict)  # full filename -> file data
+	base_dir: str
+	extracted_content_count: int = 0
+
+
+class FileSystem:
+	"""Enhanced file system with in-memory storage and multiple file type support"""
+
+	def __init__(self, base_dir: str | Path, create_default_files: bool = True):
+		# Handle the Path conversion before calling super().__init__
+		self.base_dir = Path(base_dir) if isinstance(base_dir, str) else base_dir
+		self.base_dir.mkdir(parents=True, exist_ok=True)
+
+		# Create and use a dedicated subfolder for all operations
+		self.data_dir = self.base_dir / DEFAULT_FILE_SYSTEM_PATH
+		if self.data_dir.exists():
+			# clean the data directory
+			shutil.rmtree(self.data_dir)
+		self.data_dir.mkdir(exist_ok=True)
+
+		self._file_types: dict[str, type[BaseFile]] = {
+			'md': MarkdownFile,
+			'txt': TxtFile,
+			'json': JsonFile,
+			'jsonl': JsonlFile,
+			'csv': CsvFile,
+			'pdf': PdfFile,
+			'docx': DocxFile,
+			'html': HtmlFile,
+			'xml': XmlFile,
+		}
+
+		self.files = {}
+		if create_default_files:
+			self.default_files = ['todo.md']
+			self._create_default_files()
+
+		self.extracted_content_count = 0
+
+	def get_allowed_extensions(self) -> list[str]:
+		"""Get allowed extensions"""
+		return list(self._file_types.keys())
+
+	def _get_file_type_class(self, extension: str) -> type[BaseFile] | None:
+		"""Get the appropriate file class for an extension."""
+		return self._file_types.get(extension.lower(), None)
+
+	def _create_default_files(self) -> None:
+		"""Create default results and todo files"""
+		for full_filename in self.default_files:
+			name_without_ext, extension = self._parse_filename(full_filename)
+			file_class = self._get_file_type_class(extension)
+			if not file_class:
+				raise ValueError(f"Error: Invalid file extension '{extension}' for file '{full_filename}'.")
+
+			file_obj = file_class(name=name_without_ext)
+			self.files[full_filename] = file_obj  # Use full filename as key
+			file_obj.sync_to_disk_sync(self.data_dir)
+
+	def _is_valid_filename(self, file_name: str) -> bool:
+		"""Check if filename matches the required pattern: name.extension
+
+		Allows letters, numbers, underscores, hyphens, dots, parentheses, spaces, and Chinese characters
+		in the name part, followed by a dot and a supported extension.
+		"""
+		extensions = '|'.join(self._file_types.keys())
+		# Allow dots, spaces, parens in the name part - match everything up to the last dot
+		pattern = rf'^[a-zA-Z0-9_\-\.\(\) \u4e00-\u9fff]+\.({extensions})$'
+		file_name_base = os.path.basename(file_name)
+		if not re.match(pattern, file_name_base):
+			return False
+		# Ensure the name part (before last dot) is non-empty
+		name_part = file_name_base.rsplit('.', 1)[0]
+		return len(name_part.strip()) > 0
+
+	@staticmethod
+	def sanitize_filename(file_name: str) -> str:
+		"""Sanitize a filename by replacing/removing invalid characters.
+
+		- Replaces spaces with hyphens
+		- Removes characters that are not alphanumeric, underscore, hyphen, dot, parentheses, or Chinese
+		- Preserves the extension
+		- Collapses multiple consecutive hyphens
+		"""
+		base = os.path.basename(file_name)
+		if '.' not in base:
+			return base
+
+		name_part, ext = base.rsplit('.', 1)
+		# Replace spaces with hyphens
+		name_part = name_part.replace(' ', '-')
+		# Remove invalid characters (keep alphanumeric, underscore, hyphen, dot, parens, Chinese)
+		name_part = re.sub(r'[^a-zA-Z0-9_\-\.\(\)\u4e00-\u9fff]', '', name_part)
+		# Collapse multiple hyphens
+		name_part = re.sub(r'-{2,}', '-', name_part)
+		# Strip leading/trailing hyphens and dots
+		name_part = name_part.strip('-.')
+
+		if not name_part:
+			name_part = 'file'
+
+		return f'{name_part}.{ext.lower()}'
+
+	def _resolve_filename(self, file_name: str) -> tuple[str, bool]:
+		"""Resolve a filename, attempting sanitization if the original is invalid.
+
+		Normalizes to basename first to prevent directory traversal (e.g. ../secret.md).
+
+		Returns:
+			(resolved_name, was_changed): The resolved filename and whether it differs from the input.
+			If resolution fails, returns (basename, was_changed).
+		"""
+		base_name = os.path.basename(file_name)
+		was_changed = base_name != file_name
+
+		if self._is_valid_filename(base_name):
+			return base_name, was_changed
+
+		sanitized = self.sanitize_filename(base_name)
+		if sanitized != base_name and self._is_valid_filename(sanitized):
+			return sanitized, True
+
+		return base_name, was_changed
+
+	def _parse_filename(self, filename: str) -> tuple[str, str]:
+		"""Parse filename into name and extension. Always check _is_valid_filename first."""
+		name, extension = filename.rsplit('.', 1)
+		return name, extension.lower()
+
+	def get_dir(self) -> Path:
+		"""Get the file system directory"""
+		return self.data_dir
+
+	def get_file(self, full_filename: str) -> BaseFile | None:
+		"""Get a file object by full filename, trying sanitization if the name is invalid."""
+		resolved, _ = self._resolve_filename(full_filename)
+		if not self._is_valid_filename(resolved):
+			return None
+
+		# Use resolved filename as key
+		return self.files.get(resolved)
+
+	def list_files(self) -> list[str]:
+		"""List all files in the system"""
+		return [file_obj.full_name for file_obj in self.files.values()]
+
+	def display_file(self, full_filename: str) -> str | None:
+		"""Display file content using file-specific display method"""
+		resolved, _ = self._resolve_filename(full_filename)
+		if not self._is_valid_filename(resolved):
+			return None
+
+		file_obj = self.files.get(resolved)
+		if not file_obj:
+			return None
+
+		return file_obj.read()
+
+	async def read_file_structured(self, full_filename: str, external_file: bool = False) -> dict[str, Any]:
+		"""Read file and return structured data including images if applicable.
+
+		Returns:
+			dict with keys:
+				- 'message': str - The message to display
+				- 'images': list[dict] | None - Image data if file is an image: [{"name": str, "data": base64_str}]
+		"""
+		result: dict[str, Any] = {'message': '', 'images': None}
+
+		if external_file:
+			try:
+				try:
+					_, extension = self._parse_filename(full_filename)
+				except Exception:
+					result['message'] = (
+						f'Error: Invalid filename format {full_filename}. Must be alphanumeric with a supported extension.'
+					)
+					return result
+
+				# Text-based extensions: derive from _file_types, excluding those with special readers
+				_special_extensions = {'docx', 'pdf', 'jpg', 'jpeg', 'png'}
+				text_extensions = [ext for ext in self._file_types if ext not in _special_extensions]
+
+				if extension in text_extensions:
+					import anyio
+
+					async with await anyio.open_file(full_filename, 'r') as f:
+						content = await f.read()
+						result['message'] = f'Read from file {full_filename}.\n<content>\n{content}\n</content>'
+						return result
+
+				elif extension == 'docx':
+					from docx import Document
+
+					doc = Document(full_filename)
+					content = '\n'.join([para.text for para in doc.paragraphs])
+					result['message'] = f'Read from file {full_filename}.\n<content>\n{content}\n</content>'
+					return result
+
+				elif extension == 'pdf':
+					import pypdf
+
+					reader = pypdf.PdfReader(full_filename)
+					num_pages = len(reader.pages)
+					MAX_CHARS = 60000  # character-based limit
+
+					# Extract text from all pages with page markers
+					page_texts: list[tuple[int, str]] = []
+					total_chars = 0
+					for i, page in enumerate(reader.pages, 1):
+						text = page.extract_text() or ''
+						page_texts.append((i, text))
+						total_chars += len(text)
+
+					# If small enough, return everything
+					if total_chars <= MAX_CHARS:
+						content_parts = []
+						for page_num, text in page_texts:
+							if text.strip():
+								content_parts.append(f'--- Page {page_num} ---\n{text}')
+						extracted_text = '\n\n'.join(content_parts)
+						result['message'] = (
+							f'Read from file {full_filename} ({num_pages} pages, {total_chars:,} chars).\n'
+							f'<content>\n{extracted_text}\n</content>'
+						)
+						return result
+
+					# Large PDF - use search to prioritize pages with distinctive content
+					import math
+					import re
+
+					# Extract words from each page and count which pages they appear on
+					word_to_pages: dict[str, set[int]] = {}
+					page_words: dict[int, set[str]] = {}
+
+					for page_num, text in page_texts:
+						# Extract words (lowercase, 4+ chars to filter noise)
+						words = set(re.findall(r'\b[a-zA-Z]{4,}\b', text.lower()))
+						page_words[page_num] = words
+						for word in words:
+							if word not in word_to_pages:
+								word_to_pages[word] = set()
+							word_to_pages[word].add(page_num)
+
+					# Score pages using inverse document frequency (IDF)
+					# words appearing on fewer pages get higher weight
+					page_scores: dict[int, float] = {}
+					for page_num, words in page_words.items():
+						score = 0.0
+						for word in words:
+							pages_with_word = len(word_to_pages[word])
+							# IDF: log(total_pages / pages_with_word) - higher for rarer words
+							score += math.log(num_pages / pages_with_word)
+						page_scores[page_num] = score
+
+					# Sort pages by score (highest first), always include page 1
+					sorted_pages = sorted(page_scores.items(), key=lambda x: -x[1])
+					priority_pages = [1]
+					for page_num, _ in sorted_pages:
+						if page_num not in priority_pages:
+							priority_pages.append(page_num)
+
+					# Add remaining pages in order (for pages with no distinctive content)
+					for page_num, _ in page_texts:
+						if page_num not in priority_pages:
+							priority_pages.append(page_num)
+
+					# Build content from prioritized pages, respecting char limit
+					content_parts = []
+					chars_used = 0
+					pages_included = []
+
+					# First pass: add pages in priority order
+					for page_num in priority_pages:
+						text = page_texts[page_num - 1][1]
+						if not text.strip():
+							continue
+						page_header = f'--- Page {page_num} ---\n'
+						truncation_suffix = '\n[...truncated]'
+						remaining = MAX_CHARS - chars_used
+						# Need room for header + suffix + at least some content
+						min_useful = len(page_header) + len(truncation_suffix) + 50
+						if remaining < min_useful:
+							break  # no room left for meaningful content
+						page_content = page_header + text
+						if len(page_content) > remaining:
+							# Truncate page to fit remaining budget exactly
+							page_content = page_content[: remaining - len(truncation_suffix)] + truncation_suffix
+						content_parts.append((page_num, page_content))
+						chars_used += len(page_content)
+						pages_included.append(page_num)
+						if chars_used >= MAX_CHARS:
+							break
+
+					# Sort included pages by page number for readability
+					content_parts.sort(key=lambda x: x[0])
+					extracted_text = '\n\n'.join(part for _, part in content_parts)
+
+					pages_not_shown = num_pages - len(pages_included)
+					if pages_not_shown > 0:
+						skipped = [p for p in range(1, num_pages + 1) if p not in pages_included]
+						truncation_note = (
+							f'\n\n[Showing {len(pages_included)} of {num_pages} pages. '
+							f'Skipped pages: {skipped[:10]}{"..." if len(skipped) > 10 else ""}. '
+							f'Use extract with start_from_char to read further into the file.]'
+						)
+					else:
+						truncation_note = ''
+
+					result['message'] = (
+						f'Read from file {full_filename} ({num_pages} pages, {total_chars:,} chars total).\n'
+						f'<content>\n{extracted_text}{truncation_note}\n</content>'
+					)
+					return result
+
+				elif extension in ['jpg', 'jpeg', 'png']:
+					import anyio
+
+					# Read image file and convert to base64
+					async with await anyio.open_file(full_filename, 'rb') as f:
+						img_data = await f.read()
+
+					base64_str = base64.b64encode(img_data).decode('utf-8')
+
+					result['message'] = f'Read image file {full_filename}.'
+					result['images'] = [{'name': os.path.basename(full_filename), 'data': base64_str}]
+					return result
+
+				else:
+					result['message'] = f'Error: Cannot read file {full_filename} as {extension} extension is not supported.'
+					return result
+
+			except FileNotFoundError:
+				result['message'] = f"Error: File '{full_filename}' not found."
+				return result
+			except PermissionError:
+				result['message'] = f"Error: Permission denied to read file '{full_filename}'."
+				return result
+			except Exception as e:
+				result['message'] = f"Error: Could not read file '{full_filename}'. {str(e)}"
+				return result
+
+		# For internal files, only non-image types are supported
+		resolved, was_sanitized = self._resolve_filename(full_filename)
+		if not self._is_valid_filename(resolved):
+			result['message'] = _build_filename_error_message(full_filename, self.get_allowed_extensions())
+			return result
+
+		file_obj = self.files.get(resolved)
+		if not file_obj:
+			if was_sanitized:
+				result['message'] = f"File '{resolved}' not found. (Filename was auto-corrected from '{full_filename}')"
+			else:
+				result['message'] = f"File '{full_filename}' not found."
+			return result
+
+		try:
+			content = file_obj.read()
+			sanitize_note = f"Note: filename was auto-corrected from '{full_filename}' to '{resolved}'. " if was_sanitized else ''
+			result['message'] = f'{sanitize_note}Read from file {resolved}.\n<content>\n{content}\n</content>'
+			return result
+		except FileSystemError as e:
+			result['message'] = str(e)
+			return result
+		except Exception as e:
+			result['message'] = f"Error: Could not read file '{full_filename}'. {str(e)}"
+			return result
+
+	async def read_file(self, full_filename: str, external_file: bool = False) -> str:
+		"""Read file content using file-specific read method and return appropriate message to LLM.
+
+		Note: For image files, use read_file_structured() to get image data.
+		"""
+		result = await self.read_file_structured(full_filename, external_file)
+		return result['message']
+
+	async def write_file(self, full_filename: str, content: str) -> str:
+		"""Write content to file using file-specific write method"""
+		original_filename = full_filename
+		resolved, was_sanitized = self._resolve_filename(full_filename)
+		if not self._is_valid_filename(resolved):
+			return _build_filename_error_message(full_filename, self.get_allowed_extensions())
+		full_filename = resolved
+
+		try:
+			name_without_ext, extension = self._parse_filename(full_filename)
+			file_class = self._get_file_type_class(extension)
+			if not file_class:
+				raise ValueError(f"Error: Invalid file extension '{extension}' for file '{full_filename}'.")
+
+			# Create or get existing file using full filename as key
+			if full_filename in self.files:
+				file_obj = self.files[full_filename]
+			else:
+				file_obj = file_class(name=name_without_ext)
+				self.files[full_filename] = file_obj  # Use full filename as key
+
+			# Use file-specific write method
+			await file_obj.write(content, self.data_dir)
+			sanitize_note = f" (auto-corrected from '{original_filename}')" if was_sanitized else ''
+			return f'Data written to file {full_filename} successfully.{sanitize_note}'
+		except FileSystemError as e:
+			return str(e)
+		except Exception as e:
+			return f"Error: Could not write to file '{full_filename}'. {str(e)}"
+
+	async def append_file(self, full_filename: str, content: str) -> str:
+		"""Append content to file using file-specific append method"""
+		original_filename = full_filename
+		resolved, was_sanitized = self._resolve_filename(full_filename)
+		if not self._is_valid_filename(resolved):
+			return _build_filename_error_message(full_filename, self.get_allowed_extensions())
+		full_filename = resolved
+
+		file_obj = self.files.get(full_filename)
+		if not file_obj:
+			if was_sanitized:
+				return f"File '{full_filename}' not found. (Filename was auto-corrected from '{original_filename}')"
+			return f"File '{full_filename}' not found."
+
+		try:
+			await file_obj.append(content, self.data_dir)
+			sanitize_note = f" (auto-corrected from '{original_filename}')" if was_sanitized else ''
+			return f'Data appended to file {full_filename} successfully.{sanitize_note}'
+		except FileSystemError as e:
+			return str(e)
+		except Exception as e:
+			return f"Error: Could not append to file '{full_filename}'. {str(e)}"
+
+	async def replace_file_str(self, full_filename: str, old_str: str, new_str: str) -> str:
+		"""Replace old_str with new_str in file_name"""
+		original_filename = full_filename
+		resolved, was_sanitized = self._resolve_filename(full_filename)
+		if not self._is_valid_filename(resolved):
+			return _build_filename_error_message(full_filename, self.get_allowed_extensions())
+		full_filename = resolved
+
+		if not old_str:
+			return 'Error: Cannot replace empty string. Please provide a non-empty string to replace.'
+
+		file_obj = self.files.get(full_filename)
+		if not file_obj:
+			if was_sanitized:
+				return f"File '{full_filename}' not found. (Filename was auto-corrected from '{original_filename}')"
+			return f"File '{full_filename}' not found."
+
+		try:
+			content = file_obj.read()
+			content = content.replace(old_str, new_str)
+			await file_obj.write(content, self.data_dir)
+			sanitize_note = f" (auto-corrected from '{original_filename}')" if was_sanitized else ''
+			return f'Successfully replaced all occurrences of "{old_str}" with "{new_str}" in file {full_filename}{sanitize_note}'
+		except FileSystemError as e:
+			return str(e)
+		except Exception as e:
+			return f"Error: Could not replace string in file '{full_filename}'. {str(e)}"
+
+	async def save_extracted_content(self, content: str) -> str:
+		"""Save extracted content to a numbered file"""
+		initial_filename = f'extracted_content_{self.extracted_content_count}'
+		extracted_filename = f'{initial_filename}.md'
+		file_obj = MarkdownFile(name=initial_filename)
+		await file_obj.write(content, self.data_dir)
+		self.files[extracted_filename] = file_obj
+		self.extracted_content_count += 1
+		return extracted_filename
+
+	def describe(self) -> str:
+		"""List all files with their content information using file-specific display methods"""
+		DISPLAY_CHARS = 400
+		description = ''
+
+		for file_obj in self.files.values():
+			# Skip todo.md from description
+			if file_obj.full_name == 'todo.md':
+				continue
+
+			content = file_obj.read()
+
+			# Handle empty files
+			if not content:
+				description += f'<file>\n{file_obj.full_name} - [empty file]\n</file>\n'
+				continue
+
+			lines = content.splitlines()
+			line_count = len(lines)
+
+			# For small files, display the entire content
+			whole_file_description = (
+				f'<file>\n{file_obj.full_name} - {line_count} lines\n<content>\n{content}\n</content>\n</file>\n'
+			)
+			if len(content) < int(1.5 * DISPLAY_CHARS):
+				description += whole_file_description
+				continue
+
+			# For larger files, display start and end previews
+			half_display_chars = DISPLAY_CHARS // 2
+
+			# Get start preview
+			start_preview = ''
+			start_line_count = 0
+			chars_count = 0
+			for line in lines:
+				if chars_count + len(line) + 1 > half_display_chars:
+					break
+				start_preview += line + '\n'
+				chars_count += len(line) + 1
+				start_line_count += 1
+
+			# Get end preview
+			end_preview = ''
+			end_line_count = 0
+			chars_count = 0
+			for line in reversed(lines):
+				if chars_count + len(line) + 1 > half_display_chars:
+					break
+				end_preview = line + '\n' + end_preview
+				chars_count += len(line) + 1
+				end_line_count += 1
+
+			# Calculate lines in between
+			middle_line_count = line_count - start_line_count - end_line_count
+			if middle_line_count <= 0:
+				description += whole_file_description
+				continue
+
+			start_preview = start_preview.strip('\n').rstrip()
+			end_preview = end_preview.strip('\n').rstrip()
+
+			# Format output
+			if not (start_preview or end_preview):
+				description += f'<file>\n{file_obj.full_name} - {line_count} lines\n<content>\n{middle_line_count} lines...\n</content>\n</file>\n'
+			else:
+				description += f'<file>\n{file_obj.full_name} - {line_count} lines\n<content>\n{start_preview}\n'
+				description += f'... {middle_line_count} more lines ...\n'
+				description += f'{end_preview}\n'
+				description += '</content>\n</file>\n'
+
+		return description.strip('\n')
+
+	def get_todo_contents(self) -> str:
+		"""Get todo file contents"""
+		todo_file = self.get_file('todo.md')
+		return todo_file.read() if todo_file else ''
+
+	def get_state(self) -> FileSystemState:
+		"""Get serializable state of the file system"""
+		files_data = {}
+		for full_filename, file_obj in self.files.items():
+			files_data[full_filename] = {'type': file_obj.__class__.__name__, 'data': file_obj.model_dump()}
+
+		return FileSystemState(
+			files=files_data, base_dir=str(self.base_dir), extracted_content_count=self.extracted_content_count
+		)
+
+	def nuke(self) -> None:
+		"""Delete the file system directory"""
+		shutil.rmtree(self.data_dir)
+
+	@classmethod
+	def from_state(cls, state: FileSystemState) -> 'FileSystem':
+		"""Restore file system from serializable state at the exact same location"""
+		# Create file system without default files
+		fs = cls(base_dir=Path(state.base_dir), create_default_files=False)
+		fs.extracted_content_count = state.extracted_content_count
+
+		# Restore all files
+		for full_filename, file_data in state.files.items():
+			file_type = file_data['type']
+			file_info = file_data['data']
+
+			# Create the appropriate file object based on type
+			file_type_map: dict[str, type[BaseFile]] = {
+				'MarkdownFile': MarkdownFile,
+				'TxtFile': TxtFile,
+				'JsonFile': JsonFile,
+				'JsonlFile': JsonlFile,
+				'CsvFile': CsvFile,
+				'PdfFile': PdfFile,
+				'DocxFile': DocxFile,
+				'HtmlFile': HtmlFile,
+				'XmlFile': XmlFile,
+			}
+
+			file_class = file_type_map.get(file_type)
+			if not file_class:
+				# Skip unknown file types
+				continue
+			file_obj = file_class(**file_info)
+
+			# Add to files dict and sync to disk
+			fs.files[full_filename] = file_obj
+			file_obj.sync_to_disk_sync(fs.data_dir)
+
+		return fs

browser_use/init_cmd.py

@@ -0,0 +1,448 @@
+"""
+Standalone init command for browser-use template generation.
+
+This module provides a minimal command-line interface for generating
+browser-use templates without requiring heavy TUI dependencies.
+"""
+
+import json
+import shutil
+import sys
+from pathlib import Path
+from typing import Any
+from urllib import request
+from urllib.error import URLError
+
+import click
+from InquirerPy import inquirer
+from InquirerPy.base.control import Choice
+from InquirerPy.utils import InquirerPyStyle
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+# Rich console for styled output
+console = Console()
+
+# GitHub template repository URL (for runtime fetching)
+TEMPLATE_REPO_URL = 'https://raw.githubusercontent.com/browser-use/template-library/main'
+
+# Export for backward compatibility with cli.py
+# Templates are fetched at runtime via _get_template_list()
+INIT_TEMPLATES: dict[str, Any] = {}
+
+
+def _fetch_template_list() -> dict[str, Any] | None:
+	"""
+	Fetch template list from GitHub templates.json.
+
+	Returns template dict if successful, None if failed.
+	"""
+	try:
+		url = f'{TEMPLATE_REPO_URL}/templates.json'
+		with request.urlopen(url, timeout=5) as response:
+			data = response.read().decode('utf-8')
+			return json.loads(data)
+	except (URLError, TimeoutError, json.JSONDecodeError, Exception):
+		return None
+
+
+def _get_template_list() -> dict[str, Any]:
+	"""
+	Get template list from GitHub.
+
+	Raises FileNotFoundError if GitHub fetch fails.
+	"""
+	templates = _fetch_template_list()
+	if templates is not None:
+		return templates
+	raise FileNotFoundError('Could not fetch templates from GitHub. Check your internet connection.')
+
+
+def _fetch_from_github(file_path: str) -> str | None:
+	"""
+	Fetch template file from GitHub.
+
+	Returns file content if successful, None if failed.
+	"""
+	try:
+		url = f'{TEMPLATE_REPO_URL}/{file_path}'
+		with request.urlopen(url, timeout=5) as response:
+			return response.read().decode('utf-8')
+	except (URLError, TimeoutError, Exception):
+		return None
+
+
+def _fetch_binary_from_github(file_path: str) -> bytes | None:
+	"""
+	Fetch binary file from GitHub.
+
+	Returns file content if successful, None if failed.
+	"""
+	try:
+		url = f'{TEMPLATE_REPO_URL}/{file_path}'
+		with request.urlopen(url, timeout=5) as response:
+			return response.read()
+	except (URLError, TimeoutError, Exception):
+		return None
+
+
+def _get_template_content(file_path: str) -> str:
+	"""
+	Get template file content from GitHub.
+
+	Raises exception if fetch fails.
+	"""
+	content = _fetch_from_github(file_path)
+
+	if content is not None:
+		return content
+
+	raise FileNotFoundError(f'Could not fetch template from GitHub: {file_path}')
+
+
+# InquirerPy style for template selection (browser-use orange theme)
+inquirer_style = InquirerPyStyle(
+	{
+		'pointer': '#fe750e bold',
+		'highlighted': '#fe750e bold',
+		'question': 'bold',
+		'answer': '#fe750e bold',
+		'questionmark': '#fe750e bold',
+	}
+)
+
+
+def _get_terminal_width() -> int:
+	"""Get current terminal width in columns."""
+	return shutil.get_terminal_size().columns
+
+
+def _format_choice(name: str, metadata: dict[str, Any], width: int, is_default: bool = False) -> str:
+	"""
+	Format a template choice with responsive display based on terminal width.
+
+	Styling:
+	- Featured templates get [FEATURED] prefix
+	- Author name included when width allows (except for default templates)
+	- Everything turns orange when highlighted (InquirerPy's built-in behavior)
+
+	Args:
+		name: Template name
+		metadata: Template metadata (description, featured, author)
+		width: Terminal width in columns
+		is_default: Whether this is a default template (default, advanced, tools)
+
+	Returns:
+		Formatted choice string
+	"""
+	is_featured = metadata.get('featured', False)
+	description = metadata.get('description', '')
+	author_name = metadata.get('author', {}).get('name', '') if isinstance(metadata.get('author'), dict) else ''
+
+	# Build the choice string based on terminal width
+	if width > 100:
+		# Wide: show everything including author (except for default templates)
+		if is_featured:
+			if author_name:
+				return f'[FEATURED] {name} by {author_name} - {description}'
+			else:
+				return f'[FEATURED] {name} - {description}'
+		else:
+			# Non-featured templates
+			if author_name and not is_default:
+				return f'{name} by {author_name} - {description}'
+			else:
+				return f'{name} - {description}'
+
+	elif width > 60:
+		# Medium: show name and description, no author
+		if is_featured:
+			return f'[FEATURED] {name} - {description}'
+		else:
+			return f'{name} - {description}'
+
+	else:
+		# Narrow: show name only
+		return name
+
+
+def _write_init_file(output_path: Path, content: str, force: bool = False) -> bool:
+	"""Write content to a file, with safety checks."""
+	# Check if file already exists
+	if output_path.exists() and not force:
+		console.print(f'[yellow]⚠[/yellow]  File already exists: [cyan]{output_path}[/cyan]')
+		if not click.confirm('Overwrite?', default=False):
+			console.print('[red]✗[/red] Cancelled')
+			return False
+
+	# Ensure parent directory exists
+	output_path.parent.mkdir(parents=True, exist_ok=True)
+
+	# Write file
+	try:
+		output_path.write_text(content, encoding='utf-8')
+		return True
+	except Exception as e:
+		console.print(f'[red]✗[/red] Error writing file: {e}')
+		return False
+
+
+@click.command('browser-use-init')
+@click.option(
+	'--template',
+	'-t',
+	type=str,
+	help='Template to use',
+)
+@click.option(
+	'--output',
+	'-o',
+	type=click.Path(),
+	help='Output file path (default: browser_use_<template>.py)',
+)
+@click.option(
+	'--force',
+	'-f',
+	is_flag=True,
+	help='Overwrite existing files without asking',
+)
+@click.option(
+	'--list',
+	'-l',
+	'list_templates',
+	is_flag=True,
+	help='List available templates',
+)
+def main(
+	template: str | None,
+	output: str | None,
+	force: bool,
+	list_templates: bool,
+):
+	"""
+	Generate a browser-use template file to get started quickly.
+
+	Examples:
+
+	\b
+	# Interactive mode - prompts for template selection
+	uvx browser-use init
+	uvx browser-use init --template
+
+	\b
+	# Generate default template
+	uvx browser-use init --template default
+
+	\b
+	# Generate advanced template with custom filename
+	uvx browser-use init --template advanced --output my_script.py
+
+	\b
+	# List available templates
+	uvx browser-use init --list
+	"""
+
+	# Fetch template list at runtime
+	try:
+		INIT_TEMPLATES = _get_template_list()
+	except FileNotFoundError as e:
+		console.print(f'[red]✗[/red] {e}')
+		sys.exit(1)
+
+	# Handle --list flag
+	if list_templates:
+		console.print('\n[bold]Available templates:[/bold]\n')
+		for name, info in INIT_TEMPLATES.items():
+			console.print(f'  [#fe750e]{name:12}[/#fe750e] - {info["description"]}')
+		console.print()
+		return
+
+	# Interactive template selection if not provided
+	if not template:
+		# Get terminal width for responsive formatting
+		width = _get_terminal_width()
+
+		# Separate default and featured templates
+		default_template_names = ['default', 'advanced', 'tools']
+		featured_templates = [(name, info) for name, info in INIT_TEMPLATES.items() if info.get('featured', False)]
+		other_templates = [
+			(name, info)
+			for name, info in INIT_TEMPLATES.items()
+			if name not in default_template_names and not info.get('featured', False)
+		]
+
+		# Sort by last_modified_date (most recent first)
+		def get_last_modified(item):
+			name, info = item
+			date_str = (
+				info.get('author', {}).get('last_modified_date', '1970-01-01')
+				if isinstance(info.get('author'), dict)
+				else '1970-01-01'
+			)
+			return date_str
+
+		# Sort default templates by last modified
+		default_templates = [(name, INIT_TEMPLATES[name]) for name in default_template_names if name in INIT_TEMPLATES]
+		default_templates.sort(key=get_last_modified, reverse=True)
+
+		# Sort featured and other templates by last modified
+		featured_templates.sort(key=get_last_modified, reverse=True)
+		other_templates.sort(key=get_last_modified, reverse=True)
+
+		# Build choices in order: defaults first, then featured, then others
+		choices = []
+
+		# Add default templates
+		for i, (name, info) in enumerate(default_templates):
+			formatted = _format_choice(name, info, width, is_default=True)
+			choices.append(Choice(name=formatted, value=name))
+
+		# Add featured templates
+		for i, (name, info) in enumerate(featured_templates):
+			formatted = _format_choice(name, info, width, is_default=False)
+			choices.append(Choice(name=formatted, value=name))
+
+		# Add other templates (if any)
+		for name, info in other_templates:
+			formatted = _format_choice(name, info, width, is_default=False)
+			choices.append(Choice(name=formatted, value=name))
+
+		# Use fuzzy prompt for search functionality
+		# Use getattr to avoid static analysis complaining about non-exported names
+		_fuzzy = getattr(inquirer, 'fuzzy')
+		template = _fuzzy(
+			message='Select a template (type to search):',
+			choices=choices,
+			style=inquirer_style,
+			max_height='70%',
+		).execute()
+
+		# Handle user cancellation (Ctrl+C)
+		if template is None:
+			console.print('\n[red]✗[/red] Cancelled')
+			sys.exit(1)
+
+	# Template is guaranteed to be set at this point (either from option or prompt)
+	assert template is not None
+
+	# Create template directory
+	template_dir = Path.cwd() / template
+	if template_dir.exists() and not force:
+		console.print(f'[yellow]⚠[/yellow]  Directory already exists: [cyan]{template_dir}[/cyan]')
+		if not click.confirm('Continue and overwrite files?', default=False):
+			console.print('[red]✗[/red] Cancelled')
+			sys.exit(1)
+
+	# Create directory
+	template_dir.mkdir(parents=True, exist_ok=True)
+
+	# Determine output path
+	if output:
+		output_path = template_dir / Path(output)
+	else:
+		output_path = template_dir / 'main.py'
+
+	# Read template file from GitHub
+	try:
+		template_file = INIT_TEMPLATES[template]['file']
+		content = _get_template_content(template_file)
+	except Exception as e:
+		console.print(f'[red]✗[/red] Error reading template: {e}')
+		sys.exit(1)
+
+	# Write file
+	if _write_init_file(output_path, content, force):
+		console.print(f'\n[green]✓[/green] Created [cyan]{output_path}[/cyan]')
+
+		# Generate additional files if template has a manifest
+		if 'files' in INIT_TEMPLATES[template]:
+			import stat
+
+			for file_spec in INIT_TEMPLATES[template]['files']:
+				source_path = file_spec['source']
+				dest_name = file_spec['dest']
+				dest_path = output_path.parent / dest_name
+				is_binary = file_spec.get('binary', False)
+				is_executable = file_spec.get('executable', False)
+
+				# Skip if we already wrote this file (main.py)
+				if dest_path == output_path:
+					continue
+
+				# Fetch and write file
+				try:
+					if is_binary:
+						file_content = _fetch_binary_from_github(source_path)
+						if file_content:
+							if not dest_path.exists() or force:
+								dest_path.write_bytes(file_content)
+								console.print(f'[green]✓[/green] Created [cyan]{dest_name}[/cyan]')
+						else:
+							console.print(f'[yellow]⚠[/yellow]  Could not fetch [cyan]{dest_name}[/cyan] from GitHub')
+					else:
+						file_content = _get_template_content(source_path)
+						if _write_init_file(dest_path, file_content, force):
+							console.print(f'[green]✓[/green] Created [cyan]{dest_name}[/cyan]')
+							# Make executable if needed
+							if is_executable and sys.platform != 'win32':
+								dest_path.chmod(dest_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+				except Exception as e:
+					console.print(f'[yellow]⚠[/yellow]  Error generating [cyan]{dest_name}[/cyan]: {e}')
+
+		# Create a nice panel for next steps
+		next_steps = Text()
+
+		# Display next steps from manifest if available
+		if 'next_steps' in INIT_TEMPLATES[template]:
+			steps = INIT_TEMPLATES[template]['next_steps']
+			for i, step in enumerate(steps, 1):
+				# Handle footer separately (no numbering)
+				if 'footer' in step:
+					next_steps.append(f'{step["footer"]}\n', style='dim italic')
+					continue
+
+				# Step title
+				next_steps.append(f'\n{i}. {step["title"]}:\n', style='bold')
+
+				# Step commands
+				for cmd in step.get('commands', []):
+					# Replace placeholders
+					cmd = cmd.replace('{template}', template)
+					cmd = cmd.replace('{output}', output_path.name)
+					next_steps.append(f'   {cmd}\n', style='dim')
+
+				# Optional note
+				if 'note' in step:
+					next_steps.append(f'   {step["note"]}\n', style='dim italic')
+
+				next_steps.append('\n')
+		else:
+			# Default workflow for templates without custom next_steps
+			next_steps.append('\n1. Navigate to project directory:\n', style='bold')
+			next_steps.append(f'   cd {template}\n\n', style='dim')
+			next_steps.append('2. Initialize uv project:\n', style='bold')
+			next_steps.append('   uv init\n\n', style='dim')
+			next_steps.append('3. Install browser-use:\n', style='bold')
+			next_steps.append('   uv add browser-use\n\n', style='dim')
+			next_steps.append('4. Set up your API key in .env file or environment:\n', style='bold')
+			next_steps.append('   BROWSER_USE_API_KEY=your-key\n', style='dim')
+			next_steps.append(
+				'   (Get your key at https://cloud.browser-use.com/dashboard/settings?tab=api-keys&new&utm_source=oss&utm_medium=cli)\n\n',
+				style='dim italic',
+			)
+			next_steps.append('5. Run your script:\n', style='bold')
+			next_steps.append(f'   uv run {output_path.name}\n', style='dim')
+
+		console.print(
+			Panel(
+				next_steps,
+				title='[bold]Next steps[/bold]',
+				border_style='#fe750e',
+				padding=(1, 2),
+			)
+		)
+
+
+if __name__ == '__main__':
+	main()

browser_use/integrations/gmail/__init__.py

@@ -0,0 +1,24 @@
+"""
+Gmail Integration for Browser Use
+Provides Gmail API integration for email reading and verification code extraction.
+This integration enables agents to read email content and extract verification codes themselves.
+Usage:
+    from browser_use.integrations.gmail import GmailService, register_gmail_actions
+    # Option 1: Register Gmail actions with file-based authentication
+    tools = Tools()
+    register_gmail_actions(tools)
+    # Option 2: Register Gmail actions with direct access token (recommended for production)
+    tools = Tools()
+    register_gmail_actions(tools, access_token="your_access_token_here")
+    # Option 3: Use the service directly
+    gmail = GmailService(access_token="your_access_token_here")
+    await gmail.authenticate()
+    emails = await gmail.get_recent_emails()
+"""
+
+# @file purpose: Gmail integration for 2FA email authentication and email reading
+
+from .actions import register_gmail_actions
+from .service import GmailService
+
+__all__ = ['GmailService', 'register_gmail_actions']

browser_use/integrations/gmail/actions.py

@@ -0,0 +1,115 @@
+"""
+Gmail Actions for Browser Use
+Defines agent actions for Gmail integration including 2FA code retrieval,
+email reading, and authentication management.
+"""
+
+import logging
+
+from pydantic import BaseModel, Field
+
+from browser_use.agent.views import ActionResult
+from browser_use.tools.service import Tools
+
+from .service import GmailService
+
+logger = logging.getLogger(__name__)
+
+# Global Gmail service instance - initialized when actions are registered
+_gmail_service: GmailService | None = None
+
+
+class GetRecentEmailsParams(BaseModel):
+	"""Parameters for getting recent emails"""
+
+	keyword: str = Field(default='', description='A single keyword for search, e.g. github, airbnb, etc.')
+	max_results: int = Field(default=3, ge=1, le=50, description='Maximum number of emails to retrieve (1-50, default: 3)')
+
+
+def register_gmail_actions(tools: Tools, gmail_service: GmailService | None = None, access_token: str | None = None) -> Tools:
+	"""
+	Register Gmail actions with the provided tools
+	Args:
+	    tools: The browser-use tools to register actions with
+	    gmail_service: Optional pre-configured Gmail service instance
+	    access_token: Optional direct access token (alternative to file-based auth)
+	"""
+	global _gmail_service
+
+	# Use provided service or create a new one with access token if provided
+	if gmail_service:
+		_gmail_service = gmail_service
+	elif access_token:
+		_gmail_service = GmailService(access_token=access_token)
+	else:
+		_gmail_service = GmailService()
+
+	@tools.registry.action(
+		description='Get recent emails from the mailbox with a keyword to retrieve verification codes, OTP, 2FA tokens, magic links, or any recent email content. Keep your query a single keyword.',
+		param_model=GetRecentEmailsParams,
+	)
+	async def get_recent_emails(params: GetRecentEmailsParams) -> ActionResult:
+		"""Get recent emails from the last 5 minutes with full content"""
+		try:
+			if _gmail_service is None:
+				raise RuntimeError('Gmail service not initialized')
+
+			# Ensure authentication
+			if not _gmail_service.is_authenticated():
+				logger.info('📧 Gmail not authenticated, attempting authentication...')
+				authenticated = await _gmail_service.authenticate()
+				if not authenticated:
+					return ActionResult(
+						extracted_content='Failed to authenticate with Gmail. Please ensure Gmail credentials are set up properly.',
+						long_term_memory='Gmail authentication failed',
+					)
+
+			# Use specified max_results (1-50, default 10), last 5 minutes
+			max_results = params.max_results
+			time_filter = '5m'
+
+			# Build query with time filter and optional user query
+			query_parts = [f'newer_than:{time_filter}']
+			if params.keyword.strip():
+				query_parts.append(params.keyword.strip())
+
+			query = ' '.join(query_parts)
+			logger.info(f'🔍 Gmail search query: {query}')
+
+			# Get emails
+			emails = await _gmail_service.get_recent_emails(max_results=max_results, query=query, time_filter=time_filter)
+
+			if not emails:
+				query_info = f" matching '{params.keyword}'" if params.keyword.strip() else ''
+				memory = f'No recent emails found from last {time_filter}{query_info}'
+				return ActionResult(
+					extracted_content=memory,
+					long_term_memory=memory,
+				)
+
+			# Format with full email content for large display
+			content = f'Found {len(emails)} recent email{"s" if len(emails) > 1 else ""} from the last {time_filter}:\n\n'
+
+			for i, email in enumerate(emails, 1):
+				content += f'Email {i}:\n'
+				content += f'From: {email["from"]}\n'
+				content += f'Subject: {email["subject"]}\n'
+				content += f'Date: {email["date"]}\n'
+				content += f'Content:\n{email["body"]}\n'
+				content += '-' * 50 + '\n\n'
+
+			logger.info(f'📧 Retrieved {len(emails)} recent emails')
+			return ActionResult(
+				extracted_content=content,
+				include_extracted_content_only_once=True,
+				long_term_memory=f'Retrieved {len(emails)} recent emails from last {time_filter} for query {query}.',
+			)
+
+		except Exception as e:
+			logger.error(f'Error getting recent emails: {e}')
+			return ActionResult(
+				error=f'Error getting recent emails: {str(e)}',
+				long_term_memory='Failed to get recent emails due to error',
+			)
+
+	return tools

browser_use/integrations/gmail/service.py

@@ -0,0 +1,225 @@
+"""
+Gmail API Service for Browser Use
+Handles Gmail API authentication, email reading, and 2FA code extraction.
+This service provides a clean interface for agents to interact with Gmail.
+"""
+
+import base64
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+import anyio
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from browser_use.config import CONFIG
+
+logger = logging.getLogger(__name__)
+
+
+class GmailService:
+	"""
+	Gmail API service for email reading.
+	Provides functionality to:
+	- Authenticate with Gmail API using OAuth2
+	- Read recent emails with filtering
+	- Return full email content for agent analysis
+	"""
+
+	# Gmail API scopes
+	SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']
+
+	def __init__(
+		self,
+		credentials_file: str | None = None,
+		token_file: str | None = None,
+		config_dir: str | None = None,
+		access_token: str | None = None,
+	):
+		"""
+		Initialize Gmail Service
+		Args:
+		    credentials_file: Path to OAuth credentials JSON from Google Cloud Console
+		    token_file: Path to store/load access tokens
+		    config_dir: Directory to store config files (defaults to browser-use config directory)
+		    access_token: Direct access token (skips file-based auth if provided)
+		"""
+		# Set up configuration directory using browser-use's config system
+		if config_dir is None:
+			self.config_dir = CONFIG.BROWSER_USE_CONFIG_DIR
+		else:
+			self.config_dir = Path(config_dir).expanduser().resolve()
+
+		# Ensure config directory exists (only if not using direct token)
+		if access_token is None:
+			self.config_dir.mkdir(parents=True, exist_ok=True)
+
+		# Set up credential paths
+		self.credentials_file = credentials_file or self.config_dir / 'gmail_credentials.json'
+		self.token_file = token_file or self.config_dir / 'gmail_token.json'
+
+		# Direct access token support
+		self.access_token = access_token
+
+		self.service = None
+		self.creds = None
+		self._authenticated = False
+
+	def is_authenticated(self) -> bool:
+		"""Check if Gmail service is authenticated"""
+		return self._authenticated and self.service is not None
+
+	async def authenticate(self) -> bool:
+		"""
+		Handle OAuth authentication and token management
+		Returns:
+		    bool: True if authentication successful, False otherwise
+		"""
+		try:
+			logger.info('🔐 Authenticating with Gmail API...')
+
+			# Check if using direct access token
+			if self.access_token:
+				logger.info('🔑 Using provided access token')
+				# Create credentials from access token
+				self.creds = Credentials(token=self.access_token, scopes=self.SCOPES)
+				# Test token validity by building service
+				self.service = build('gmail', 'v1', credentials=self.creds)
+				self._authenticated = True
+				logger.info('✅ Gmail API ready with access token!')
+				return True
+
+			# Original file-based authentication flow
+			# Try to load existing tokens
+			if os.path.exists(self.token_file):
+				self.creds = Credentials.from_authorized_user_file(str(self.token_file), self.SCOPES)
+				logger.debug('📁 Loaded existing tokens')
+
+			# If no valid credentials, run OAuth flow
+			if not self.creds or not self.creds.valid:
+				if self.creds and self.creds.expired and self.creds.refresh_token:
+					logger.info('🔄 Refreshing expired tokens...')
+					self.creds.refresh(Request())
+				else:
+					logger.info('🌐 Starting OAuth flow...')
+					if not os.path.exists(self.credentials_file):
+						logger.error(
+							f'❌ Gmail credentials file not found: {self.credentials_file}\n'
+							'Please download it from Google Cloud Console:\n'
+							'1. Go to https://console.cloud.google.com/\n'
+							'2. APIs & Services > Credentials\n'
+							'3. Download OAuth 2.0 Client JSON\n'
+							f"4. Save as 'gmail_credentials.json' in {self.config_dir}/"
+						)
+						return False
+
+					flow = InstalledAppFlow.from_client_secrets_file(str(self.credentials_file), self.SCOPES)
+					# Use specific redirect URI to match OAuth credentials
+					self.creds = flow.run_local_server(port=8080, open_browser=True)
+
+				# Save tokens for next time
+				await anyio.Path(self.token_file).write_text(self.creds.to_json())
+				logger.info(f'💾 Tokens saved to {self.token_file}')
+
+			# Build Gmail service
+			self.service = build('gmail', 'v1', credentials=self.creds)
+			self._authenticated = True
+			logger.info('✅ Gmail API ready!')
+			return True
+
+		except Exception as e:
+			logger.error(f'❌ Gmail authentication failed: {e}')
+			return False
+
+	async def get_recent_emails(self, max_results: int = 10, query: str = '', time_filter: str = '1h') -> list[dict[str, Any]]:
+		"""
+		Get recent emails with optional query filter
+		Args:
+		    max_results: Maximum number of emails to fetch
+		    query: Gmail search query (e.g., 'from:noreply@example.com')
+		    time_filter: Time filter (e.g., '5m', '1h', '1d')
+		Returns:
+		    List of email dictionaries with parsed content
+		"""
+		if not self.is_authenticated():
+			logger.error('❌ Gmail service not authenticated. Call authenticate() first.')
+			return []
+
+		try:
+			# Add time filter to query if provided
+			if time_filter and 'newer_than:' not in query:
+				query = f'newer_than:{time_filter} {query}'.strip()
+
+			logger.info(f'📧 Fetching {max_results} recent emails...')
+			if query:
+				logger.debug(f'🔍 Query: {query}')
+
+			# Get message list
+			assert self.service is not None
+			results = self.service.users().messages().list(userId='me', maxResults=max_results, q=query).execute()
+
+			messages = results.get('messages', [])
+			if not messages:
+				logger.info('📭 No messages found')
+				return []
+
+			logger.info(f'📨 Found {len(messages)} messages, fetching details...')
+
+			# Get full message details
+			emails = []
+			for i, message in enumerate(messages, 1):
+				logger.debug(f'📖 Reading email {i}/{len(messages)}...')
+
+				full_message = self.service.users().messages().get(userId='me', id=message['id'], format='full').execute()
+
+				email_data = self._parse_email(full_message)
+				emails.append(email_data)
+
+			return emails
+
+		except HttpError as error:
+			logger.error(f'❌ Gmail API error: {error}')
+			return []
+		except Exception as e:
+			logger.error(f'❌ Unexpected error fetching emails: {e}')
+			return []
+
+	def _parse_email(self, message: dict[str, Any]) -> dict[str, Any]:
+		"""Parse Gmail message into readable format"""
+		headers = {h['name']: h['value'] for h in message['payload']['headers']}
+
+		return {
+			'id': message['id'],
+			'thread_id': message['threadId'],
+			'subject': headers.get('Subject', ''),
+			'from': headers.get('From', ''),
+			'to': headers.get('To', ''),
+			'date': headers.get('Date', ''),
+			'timestamp': int(message['internalDate']),
+			'body': self._extract_body(message['payload']),
+			'raw_message': message,
+		}
+
+	def _extract_body(self, payload: dict[str, Any]) -> str:
+		"""Extract email body from payload"""
+		body = ''
+
+		if payload.get('body', {}).get('data'):
+			# Simple email body
+			body = base64.urlsafe_b64decode(payload['body']['data']).decode('utf-8')
+		elif payload.get('parts'):
+			# Multi-part email
+			for part in payload['parts']:
+				if part['mimeType'] == 'text/plain' and part.get('body', {}).get('data'):
+					part_body = base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
+					body += part_body
+				elif part['mimeType'] == 'text/html' and not body and part.get('body', {}).get('data'):
+					# Fallback to HTML if no plain text
+					body = base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
+
+		return body

browser_use/llm/README.md

@@ -0,0 +1,25 @@
+# Browser Use LLMs
+
+We officially support the following LLMs:
+
+- OpenAI
+- Anthropic
+- Google
+- Groq
+- Ollama
+- DeepSeek
+
+- Mistral
+
+## Mistral specifics
+
+Use `ChatMistral` with `MISTRAL_API_KEY` (and optional `MISTRAL_BASE_URL`). Structured outputs automatically strip unsupported JSON schema keywords (`minLength`, `maxLength`, `pattern`, `format`), and generation uses `max_tokens` plus the optional `safe_prompt` flag.
+
+- Cerebras
+
+
+## Migrating from LangChain
+
+Because of how we implemented the LLMs, we can technically support anything. If you want to use a LangChain model, you can use the `ChatLangchain` (NOT OFFICIALLY SUPPORTED) class.
+
+You can find all the details in the [LangChain example](/examples/models/langchain/example.py). We suggest you grab that code and use it as a reference.

browser_use/llm/__init__.py

@@ -0,0 +1,161 @@
+"""
+We have switched all of our code from langchain to openai.types.chat.chat_completion_message_param.
+
+For easier transition we have
+"""
+
+from typing import TYPE_CHECKING
+
+# Lightweight imports that are commonly used
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	SystemMessage,
+	UserMessage,
+)
+from browser_use.llm.messages import (
+	ContentPartImageParam as ContentImage,
+)
+from browser_use.llm.messages import (
+	ContentPartRefusalParam as ContentRefusal,
+)
+from browser_use.llm.messages import (
+	ContentPartTextParam as ContentText,
+)
+
+# Type stubs for lazy imports
+if TYPE_CHECKING:
+	from browser_use.llm.anthropic.chat import ChatAnthropic
+	from browser_use.llm.aws.chat_anthropic import ChatAnthropicBedrock
+	from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+	from browser_use.llm.azure.chat import ChatAzureOpenAI
+	from browser_use.llm.browser_use.chat import ChatBrowserUse
+	from browser_use.llm.cerebras.chat import ChatCerebras
+	from browser_use.llm.deepseek.chat import ChatDeepSeek
+	from browser_use.llm.google.chat import ChatGoogle
+	from browser_use.llm.groq.chat import ChatGroq
+	from browser_use.llm.mistral.chat import ChatMistral
+	from browser_use.llm.oci_raw.chat import ChatOCIRaw
+	from browser_use.llm.ollama.chat import ChatOllama
+	from browser_use.llm.openai.chat import ChatOpenAI
+	from browser_use.llm.openrouter.chat import ChatOpenRouter
+	from browser_use.llm.vercel.chat import ChatVercel
+
+	# Type stubs for model instances - enables IDE autocomplete
+	openai_gpt_4o: ChatOpenAI
+	openai_gpt_4o_mini: ChatOpenAI
+	openai_gpt_4_1_mini: ChatOpenAI
+	openai_o1: ChatOpenAI
+	openai_o1_mini: ChatOpenAI
+	openai_o1_pro: ChatOpenAI
+	openai_o3: ChatOpenAI
+	openai_o3_mini: ChatOpenAI
+	openai_o3_pro: ChatOpenAI
+	openai_o4_mini: ChatOpenAI
+	openai_gpt_5: ChatOpenAI
+	openai_gpt_5_mini: ChatOpenAI
+	openai_gpt_5_nano: ChatOpenAI
+
+	azure_gpt_4o: ChatAzureOpenAI
+	azure_gpt_4o_mini: ChatAzureOpenAI
+	azure_gpt_4_1_mini: ChatAzureOpenAI
+	azure_o1: ChatAzureOpenAI
+	azure_o1_mini: ChatAzureOpenAI
+	azure_o1_pro: ChatAzureOpenAI
+	azure_o3: ChatAzureOpenAI
+	azure_o3_mini: ChatAzureOpenAI
+	azure_o3_pro: ChatAzureOpenAI
+	azure_gpt_5: ChatAzureOpenAI
+	azure_gpt_5_mini: ChatAzureOpenAI
+
+	google_gemini_2_0_flash: ChatGoogle
+	google_gemini_2_0_pro: ChatGoogle
+	google_gemini_2_5_pro: ChatGoogle
+	google_gemini_2_5_flash: ChatGoogle
+	google_gemini_2_5_flash_lite: ChatGoogle
+
+# Models are imported on-demand via __getattr__
+
+# Lazy imports mapping for heavy chat models
+_LAZY_IMPORTS = {
+	'ChatAnthropic': ('browser_use.llm.anthropic.chat', 'ChatAnthropic'),
+	'ChatAnthropicBedrock': ('browser_use.llm.aws.chat_anthropic', 'ChatAnthropicBedrock'),
+	'ChatAWSBedrock': ('browser_use.llm.aws.chat_bedrock', 'ChatAWSBedrock'),
+	'ChatAzureOpenAI': ('browser_use.llm.azure.chat', 'ChatAzureOpenAI'),
+	'ChatBrowserUse': ('browser_use.llm.browser_use.chat', 'ChatBrowserUse'),
+	'ChatCerebras': ('browser_use.llm.cerebras.chat', 'ChatCerebras'),
+	'ChatDeepSeek': ('browser_use.llm.deepseek.chat', 'ChatDeepSeek'),
+	'ChatGoogle': ('browser_use.llm.google.chat', 'ChatGoogle'),
+	'ChatGroq': ('browser_use.llm.groq.chat', 'ChatGroq'),
+	'ChatMistral': ('browser_use.llm.mistral.chat', 'ChatMistral'),
+	'ChatOCIRaw': ('browser_use.llm.oci_raw.chat', 'ChatOCIRaw'),
+	'ChatOllama': ('browser_use.llm.ollama.chat', 'ChatOllama'),
+	'ChatOpenAI': ('browser_use.llm.openai.chat', 'ChatOpenAI'),
+	'ChatOpenRouter': ('browser_use.llm.openrouter.chat', 'ChatOpenRouter'),
+	'ChatVercel': ('browser_use.llm.vercel.chat', 'ChatVercel'),
+}
+
+# Cache for model instances - only created when accessed
+_model_cache: dict[str, 'BaseChatModel'] = {}
+
+
+def __getattr__(name: str):
+	"""Lazy import mechanism for heavy chat model imports and model instances."""
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		try:
+			from importlib import import_module
+
+			module = import_module(module_path)
+			attr = getattr(module, attr_name)
+			return attr
+		except ImportError as e:
+			raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
+
+	# Check cache first for model instances
+	if name in _model_cache:
+		return _model_cache[name]
+
+	# Try to get model instances from models module on-demand
+	try:
+		from browser_use.llm.models import __getattr__ as models_getattr
+
+		attr = models_getattr(name)
+		# Cache in our clean cache dict
+		_model_cache[name] = attr
+		return attr
+	except (AttributeError, ImportError):
+		pass
+
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	# Message types -> for easier transition from langchain
+	'BaseMessage',
+	'UserMessage',
+	'SystemMessage',
+	'AssistantMessage',
+	# Content parts with better names
+	'ContentText',
+	'ContentRefusal',
+	'ContentImage',
+	# Chat models
+	'BaseChatModel',
+	'ChatOpenAI',
+	'ChatBrowserUse',
+	'ChatDeepSeek',
+	'ChatGoogle',
+	'ChatAnthropic',
+	'ChatAnthropicBedrock',
+	'ChatAWSBedrock',
+	'ChatGroq',
+	'ChatMistral',
+	'ChatAzureOpenAI',
+	'ChatOCIRaw',
+	'ChatOllama',
+	'ChatOpenRouter',
+	'ChatVercel',
+	'ChatCerebras',
+]

browser_use/llm/anthropic/chat.py

@@ -0,0 +1,453 @@
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from anthropic import (
+	APIConnectionError,
+	APIStatusError,
+	AsyncAnthropic,
+	NotGiven,
+	RateLimitError,
+	omit,
+)
+from anthropic.types import CacheControlEphemeralParam, Message, ToolParam
+from anthropic.types.model_param import ModelParam
+from anthropic.types.text_block import TextBlock
+from anthropic.types.tool_choice_tool_param import ToolChoiceToolParam
+from httpx import Timeout
+from pydantic import BaseModel
+
+from browser_use.llm.anthropic.serializer import AnthropicMessageSerializer
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelOutputTruncatedError, ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatAnthropic(BaseChatModel):
+	"""
+	A wrapper around Anthropic's chat model.
+	"""
+
+	# Model configuration
+	model: str | ModelParam
+	max_tokens: int = 8192
+	temperature: float | None = None
+	top_p: float | None = None
+	seed: int | None = None
+	output_config: dict[str, Any] | None = None
+	thinking: dict[str, Any] | None = None
+	betas: list[str] | None = None
+	fallbacks: list[dict[str, Any]] | None = None
+	inference_geo: str | None = None
+
+	# Client initialization parameters
+	api_key: str | None = None
+	auth_token: str | None = None
+	base_url: str | httpx.URL | None = None
+	timeout: float | Timeout | None | NotGiven = NotGiven()
+	max_retries: int = 10
+	default_headers: Mapping[str, str] | None = None
+	default_query: Mapping[str, object] | None = None
+	http_client: httpx.AsyncClient | None = None
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'anthropic'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary."""
+		# Define base client params
+		base_params = {
+			'api_key': self.api_key,
+			'auth_token': self.auth_token,
+			'base_url': self.base_url,
+			'timeout': self.timeout,
+			'max_retries': self.max_retries,
+			'default_headers': self.default_headers,
+			'default_query': self.default_query,
+			'http_client': self.http_client,
+		}
+
+		# Create client_params dict with non-None values and non-NotGiven values
+		client_params = {}
+		for k, v in base_params.items():
+			if v is not None and v is not NotGiven():
+				client_params[k] = v
+
+		return client_params
+
+	def _is_adaptive_thinking_only_model(self) -> bool:
+		model = self.name.lower()
+		return 'claude-fable-5' in model or 'claude-mythos-5' in model
+
+	def _requires_auto_tool_choice(self) -> bool:
+		model = self.name.lower()
+		if 'claude-fable-5' in model or 'claude-mythos-5' in model:
+			return True
+		if self.thinking is None:
+			return False
+		return self.thinking.get('type') != 'disabled'
+
+	def _validate_thinking_config(self) -> None:
+		if not self.thinking or not self._is_adaptive_thinking_only_model():
+			return
+
+		thinking_type = self.thinking.get('type')
+		if thinking_type in {'enabled', 'disabled'} or 'budget_tokens' in self.thinking:
+			raise ValueError(
+				f'{self.model} only supports adaptive thinking. Omit thinking or use adaptive display options such as '
+				'{"type": "adaptive", "display": "summarized"}.'
+			)
+
+	def _get_betas_for_invoke(self) -> list[str] | None:
+		betas = self.betas
+
+		if self.fallbacks is None:
+			return betas
+
+		betas = list(betas or [])
+		if not any(beta.startswith('server-side-fallback-') for beta in betas):
+			betas.append('server-side-fallback-2026-06-01')
+		return betas
+
+	def _get_extra_body_for_invoke(self) -> dict[str, Any] | None:
+		extra_body: dict[str, Any] = {}
+
+		if self.output_config is not None:
+			extra_body['output_config'] = self.output_config
+
+		if self.fallbacks is not None:
+			extra_body['fallbacks'] = self.fallbacks
+
+		if self.inference_geo is not None:
+			extra_body['inference_geo'] = self.inference_geo
+
+		return extra_body or None
+
+	def _get_client_params_for_invoke(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary for invoke."""
+		self._validate_thinking_config()
+
+		client_params = {}
+
+		if self.temperature is not None:
+			client_params['temperature'] = self.temperature
+
+		if self.max_tokens is not None:
+			client_params['max_tokens'] = self.max_tokens
+
+		if self.top_p is not None:
+			client_params['top_p'] = self.top_p
+
+		if self.seed is not None:
+			client_params['seed'] = self.seed
+
+		if self.thinking is not None:
+			client_params['thinking'] = self.thinking
+
+		betas = self._get_betas_for_invoke()
+		if betas is not None:
+			client_params['betas'] = betas
+
+		extra_body = self._get_extra_body_for_invoke()
+		if extra_body is not None:
+			client_params['extra_body'] = extra_body
+
+		return client_params
+
+	def get_client(self) -> AsyncAnthropic:
+		"""
+		Returns an AsyncAnthropic client.
+
+		Returns:
+			AsyncAnthropic: An instance of the AsyncAnthropic client.
+		"""
+		client_params = self._get_client_params()
+		return AsyncAnthropic(**client_params)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	async def _create_message(self, **params: Any) -> Any:
+		betas = params.pop('betas', None)
+		client = self.get_client()
+		if betas is not None:
+			return await client.beta.messages.create(**params, betas=betas)
+		return await client.messages.create(**params)
+
+	def _is_message_like_response(self, response: Any) -> bool:
+		return all(hasattr(response, attr) for attr in ('content', 'usage', 'stop_reason'))
+
+	def _get_cache_creation_tokens(self, response: Any) -> tuple[int | None, int | None]:
+		cache_creation = getattr(response.usage, 'cache_creation', None)
+		if cache_creation is None:
+			return None, None
+		return (
+			getattr(cache_creation, 'ephemeral_5m_input_tokens', None),
+			getattr(cache_creation, 'ephemeral_1h_input_tokens', None),
+		)
+
+	def _get_pricing_multiplier(self) -> float | None:
+		if self.inference_geo == 'us':
+			return 1.1
+		return None
+
+	def _get_usage(self, response: Any) -> ChatInvokeUsage | None:
+		cache_creation_5m_tokens, cache_creation_1h_tokens = self._get_cache_creation_tokens(response)
+		usage = ChatInvokeUsage(
+			prompt_tokens=response.usage.input_tokens
+			+ (
+				response.usage.cache_read_input_tokens or 0
+			),  # Total tokens in Anthropic are a bit fucked, you have to add cached tokens to the prompt tokens
+			completion_tokens=response.usage.output_tokens,
+			total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+			prompt_cached_tokens=response.usage.cache_read_input_tokens,
+			prompt_cache_creation_tokens=response.usage.cache_creation_input_tokens,
+			prompt_cache_creation_5m_tokens=cache_creation_5m_tokens,
+			prompt_cache_creation_1h_tokens=cache_creation_1h_tokens,
+			prompt_image_tokens=None,
+			pricing_multiplier=self._get_pricing_multiplier(),
+		)
+		return usage
+
+	def _get_stop_details(self, response: Any) -> dict[str, Any] | None:
+		stop_details = getattr(response, 'stop_details', None)
+		if stop_details is None:
+			return None
+		if hasattr(stop_details, 'model_dump'):
+			return stop_details.model_dump()
+		if isinstance(stop_details, dict):
+			return stop_details
+		return {key: getattr(stop_details, key) for key in ('type', 'category', 'explanation') if hasattr(stop_details, key)}
+
+	def _extract_content_blocks(self, response: Any) -> tuple[str, str | None, str | None]:
+		text_parts: list[str] = []
+		thinking_parts: list[str] = []
+		redacted_thinking_parts: list[str] = []
+
+		for content_block in response.content:
+			block_type = getattr(content_block, 'type', None)
+			if isinstance(content_block, TextBlock) or block_type == 'text':
+				text = getattr(content_block, 'text', None)
+				if text:
+					text_parts.append(text)
+			elif block_type == 'thinking':
+				thinking_text = getattr(content_block, 'thinking', None)
+				if thinking_text:
+					thinking_parts.append(thinking_text)
+			elif block_type == 'redacted_thinking':
+				redacted_text = getattr(content_block, 'data', None) or getattr(content_block, 'redacted_thinking', None)
+				if redacted_text:
+					redacted_thinking_parts.append(str(redacted_text))
+
+		if text_parts:
+			completion = ''.join(text_parts)
+		elif response.content:
+			completion = str(response.content[0])
+		else:
+			completion = ''
+
+		thinking = '\n'.join(thinking_parts) if thinking_parts else None
+		redacted_thinking = '\n'.join(redacted_thinking_parts) if redacted_thinking_parts else None
+		return completion, thinking, redacted_thinking
+
+	def _json_candidates_from_text(self, text: str) -> list[str]:
+		candidates: list[str] = []
+		stripped = text.strip()
+		if stripped:
+			candidates.append(stripped)
+
+		if stripped.startswith('```') and stripped.endswith('```'):
+			lines = stripped.splitlines()
+			if len(lines) >= 3:
+				candidates.append('\n'.join(lines[1:-1]).strip())
+
+		for start_char, end_char in (('{', '}'), ('[', ']')):
+			start = stripped.find(start_char)
+			end = stripped.rfind(end_char)
+			if start != -1 and end > start:
+				candidates.append(stripped[start : end + 1])
+
+		return list(dict.fromkeys(candidate for candidate in candidates if candidate))
+
+	def _completion_from_text_response(
+		self, response: Any, output_format: type[T], usage: ChatInvokeUsage | None
+	) -> ChatInvokeCompletion[T] | None:
+		response_text, thinking, redacted_thinking = self._extract_content_blocks(response)
+		for candidate in self._json_candidates_from_text(response_text):
+			try:
+				completion = output_format.model_validate_json(candidate)
+			except Exception:
+				try:
+					completion = output_format.model_validate(json.loads(candidate))
+				except Exception:
+					continue
+			return ChatInvokeCompletion(
+				completion=completion,
+				thinking=thinking,
+				redacted_thinking=redacted_thinking,
+				usage=usage,
+				stop_reason=response.stop_reason,
+				stop_details=self._get_stop_details(response),
+			)
+		return None
+
+	@overload
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T], **kwargs: Any) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		anthropic_messages, system_prompt = AnthropicMessageSerializer.serialize_messages(messages)
+
+		try:
+			if output_format is None:
+				# Normal completion without structured output
+				response = await self._create_message(
+					model=self.model,
+					messages=anthropic_messages,
+					system=system_prompt or omit,
+					**self._get_client_params_for_invoke(),
+				)
+
+				# Ensure we have a valid Message object before accessing attributes
+				if not isinstance(response, Message) and not self._is_message_like_response(response):
+					raise ModelProviderError(
+						message=f'Unexpected response type from Anthropic API: {type(response).__name__}. Response: {str(response)[:200]}',
+						status_code=502,
+						model=self.name,
+					)
+
+				usage = self._get_usage(response)
+
+				response_text, thinking, redacted_thinking = self._extract_content_blocks(response)
+
+				return ChatInvokeCompletion(
+					completion=response_text,
+					thinking=thinking,
+					redacted_thinking=redacted_thinking,
+					usage=usage,
+					stop_reason=response.stop_reason,
+					stop_details=self._get_stop_details(response),
+				)
+
+			else:
+				# Use tool calling for structured output
+				# Create a tool that represents the output format
+				tool_name = output_format.__name__
+				schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+
+				# Remove title from schema if present (Anthropic doesn't like it in parameters)
+				if 'title' in schema:
+					del schema['title']
+
+				tool = ToolParam(
+					name=tool_name,
+					description=f'Extract information in the format of {tool_name}',
+					input_schema=schema,
+					cache_control=CacheControlEphemeralParam(type='ephemeral'),
+				)
+
+				if self._requires_auto_tool_choice():
+					tool_choice = {'type': 'auto'}
+				else:
+					# Force the model to use this tool
+					tool_choice = ToolChoiceToolParam(type='tool', name=tool_name)
+
+				response = await self._create_message(
+					model=self.model,
+					messages=anthropic_messages,
+					tools=[tool],
+					system=system_prompt or omit,
+					tool_choice=tool_choice,
+					**self._get_client_params_for_invoke(),
+				)
+
+				# Ensure we have a valid Message object before accessing attributes
+				if not isinstance(response, Message) and not self._is_message_like_response(response):
+					raise ModelProviderError(
+						message=f'Unexpected response type from Anthropic API: {type(response).__name__}. Response: {str(response)[:200]}',
+						status_code=502,
+						model=self.name,
+					)
+
+				usage = self._get_usage(response)
+
+				if response.stop_reason == 'max_tokens':
+					raise ModelOutputTruncatedError(
+						message=(
+							f'Model output was truncated at max_tokens={self.max_tokens}; the structured'
+							' output is incomplete. Increase max_tokens or request shorter output.'
+						),
+						model=self.name,
+					)
+
+				# Extract the tool use block
+				for content_block in response.content:
+					if hasattr(content_block, 'type') and content_block.type == 'tool_use':
+						# Parse the tool input as the structured output
+						try:
+							return ChatInvokeCompletion(
+								completion=output_format.model_validate(content_block.input),
+								usage=usage,
+								stop_reason=response.stop_reason,
+								stop_details=self._get_stop_details(response),
+							)
+						except Exception as e:
+							# If validation fails, try to fix common model output issues
+							_input = content_block.input
+							if isinstance(_input, str):
+								_input = json.loads(_input)
+							elif isinstance(_input, dict):
+								# Model sometimes double-serializes fields
+								for key, value in _input.items():
+									if isinstance(value, str) and value.startswith(('[', '{')):
+										try:
+											_input[key] = json.loads(value)
+										except json.JSONDecodeError:
+											cleaned = value.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
+											try:
+												_input[key] = json.loads(cleaned)
+											except json.JSONDecodeError:
+												pass
+							else:
+								raise
+							return ChatInvokeCompletion(
+								completion=output_format.model_validate(_input),
+								usage=usage,
+								stop_reason=response.stop_reason,
+								stop_details=self._get_stop_details(response),
+							)
+
+				if self._requires_auto_tool_choice():
+					text_completion = self._completion_from_text_response(response, output_format, usage)
+					if text_completion is not None:
+						return text_completion
+
+				# If no tool use block found, raise an error
+				raise ValueError('Expected tool use in response but none found')
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=e.message, model=self.name) from e
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.message, model=self.name) from e
+		except APIStatusError as e:
+			raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+		except ModelProviderError:
+			raise  # don't re-wrap with the generic 502
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

browser_use/llm/anthropic/serializer.py

@@ -0,0 +1,328 @@
+import json
+from typing import overload
+
+from anthropic.types import (
+	Base64ImageSourceParam,
+	CacheControlEphemeralParam,
+	ImageBlockParam,
+	MessageParam,
+	TextBlockParam,
+	ToolUseBlockParam,
+	URLImageSourceParam,
+)
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SupportedImageMediaType,
+	SystemMessage,
+	UserMessage,
+)
+
+NonSystemMessage = UserMessage | AssistantMessage
+
+
+class AnthropicMessageSerializer:
+	"""Serializer for converting between custom message types and Anthropic message param types."""
+
+	@staticmethod
+	def _is_base64_image(url: str) -> bool:
+		"""Check if the URL is a base64 encoded image."""
+		return url.startswith('data:image/')
+
+	@staticmethod
+	def _parse_base64_url(url: str) -> tuple[SupportedImageMediaType, str]:
+		"""Parse a base64 data URL to extract media type and data."""
+		# Format: data:image/jpeg;base64,<data>
+		if not url.startswith('data:'):
+			raise ValueError(f'Invalid base64 URL: {url}')
+
+		header, data = url.split(',', 1)
+		media_type = header.split(';')[0].replace('data:', '')
+
+		# Ensure it's a supported media type
+		supported_types = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']
+		if media_type not in supported_types:
+			# Default to jpeg if not recognized
+			media_type = 'image/jpeg'
+
+		return media_type, data  # type: ignore
+
+	@staticmethod
+	def _serialize_cache_control(use_cache: bool) -> CacheControlEphemeralParam | None:
+		"""Serialize cache control."""
+		if use_cache:
+			return CacheControlEphemeralParam(type='ephemeral')
+		return None
+
+	@staticmethod
+	def _serialize_content_part_text(part: ContentPartTextParam, use_cache: bool) -> TextBlockParam:
+		"""Convert a text content part to Anthropic's TextBlockParam."""
+		return TextBlockParam(
+			text=part.text, type='text', cache_control=AnthropicMessageSerializer._serialize_cache_control(use_cache)
+		)
+
+	@staticmethod
+	def _serialize_content_part_image(part: ContentPartImageParam) -> ImageBlockParam:
+		"""Convert an image content part to Anthropic's ImageBlockParam."""
+		url = part.image_url.url
+
+		if AnthropicMessageSerializer._is_base64_image(url):
+			# Handle base64 encoded images
+			media_type, data = AnthropicMessageSerializer._parse_base64_url(url)
+			return ImageBlockParam(
+				source=Base64ImageSourceParam(
+					data=data,
+					media_type=media_type,
+					type='base64',
+				),
+				type='image',
+			)
+		else:
+			# Handle URL images
+			return ImageBlockParam(source=URLImageSourceParam(url=url, type='url'), type='image')
+
+	@staticmethod
+	def _serialize_content_to_str(
+		content: str | list[ContentPartTextParam], use_cache: bool = False
+	) -> list[TextBlockParam] | str:
+		"""Serialize content to a string."""
+		cache_control = AnthropicMessageSerializer._serialize_cache_control(use_cache)
+
+		if isinstance(content, str):
+			if cache_control:
+				return [TextBlockParam(text=content, type='text', cache_control=cache_control)]
+			else:
+				return content
+
+		serialized_blocks: list[TextBlockParam] = []
+		for i, part in enumerate(content):
+			is_last = i == len(content) - 1
+			if part.type == 'text':
+				serialized_blocks.append(
+					AnthropicMessageSerializer._serialize_content_part_text(part, use_cache=use_cache and is_last)
+				)
+
+		return serialized_blocks
+
+	@staticmethod
+	def _serialize_content(
+		content: str | list[ContentPartTextParam | ContentPartImageParam],
+		use_cache: bool = False,
+	) -> str | list[TextBlockParam | ImageBlockParam]:
+		"""Serialize content to Anthropic format."""
+		if isinstance(content, str):
+			if use_cache:
+				return [TextBlockParam(text=content, type='text', cache_control=CacheControlEphemeralParam(type='ephemeral'))]
+			else:
+				return content
+
+		serialized_blocks: list[TextBlockParam | ImageBlockParam] = []
+		for i, part in enumerate(content):
+			is_last = i == len(content) - 1
+			if part.type == 'text':
+				serialized_blocks.append(
+					AnthropicMessageSerializer._serialize_content_part_text(part, use_cache=use_cache and is_last)
+				)
+			elif part.type == 'image_url':
+				serialized_blocks.append(AnthropicMessageSerializer._serialize_content_part_image(part))
+
+		return serialized_blocks
+
+	@staticmethod
+	def _serialize_tool_calls_to_content(tool_calls, use_cache: bool = False) -> list[ToolUseBlockParam]:
+		"""Convert tool calls to Anthropic's ToolUseBlockParam format."""
+		blocks: list[ToolUseBlockParam] = []
+		for i, tool_call in enumerate(tool_calls):
+			# Parse the arguments JSON string to object
+
+			try:
+				input_obj = json.loads(tool_call.function.arguments)
+			except json.JSONDecodeError:
+				# If arguments aren't valid JSON, use as string
+				input_obj = {'arguments': tool_call.function.arguments}
+
+			is_last = i == len(tool_calls) - 1
+			blocks.append(
+				ToolUseBlockParam(
+					id=tool_call.id,
+					input=input_obj,
+					name=tool_call.function.name,
+					type='tool_use',
+					cache_control=AnthropicMessageSerializer._serialize_cache_control(use_cache and is_last),
+				)
+			)
+		return blocks
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> MessageParam: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> SystemMessage: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> MessageParam: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> MessageParam | SystemMessage:
+		"""Serialize a custom message to an Anthropic MessageParam.
+
+		Note: Anthropic doesn't have a 'system' role. System messages should be
+		handled separately as the system parameter in the API call, not as a message.
+		If a SystemMessage is passed here, it will be converted to a user message.
+		"""
+		if isinstance(message, UserMessage):
+			content = AnthropicMessageSerializer._serialize_content(message.content, use_cache=message.cache)
+			return MessageParam(role='user', content=content)
+
+		elif isinstance(message, SystemMessage):
+			# Anthropic doesn't have system messages in the messages array
+			# System prompts are passed separately. Convert to user message.
+			return message
+
+		elif isinstance(message, AssistantMessage):
+			# Handle content and tool calls
+			blocks: list[TextBlockParam | ToolUseBlockParam] = []
+
+			# Add content blocks if present
+			if message.content is not None:
+				if isinstance(message.content, str):
+					# String content: only cache if it's the only/last block (no tool calls)
+					blocks.append(
+						TextBlockParam(
+							text=message.content,
+							type='text',
+							cache_control=AnthropicMessageSerializer._serialize_cache_control(
+								message.cache and not message.tool_calls
+							),
+						)
+					)
+				else:
+					# Process content parts (text and refusal)
+					for i, part in enumerate(message.content):
+						# Only last content block gets cache if there are no tool calls
+						is_last_content = (i == len(message.content) - 1) and not message.tool_calls
+						if part.type == 'text':
+							blocks.append(
+								AnthropicMessageSerializer._serialize_content_part_text(
+									part, use_cache=message.cache and is_last_content
+								)
+							)
+							# # Note: Anthropic doesn't have a specific refusal block type,
+							# # so we convert refusals to text blocks
+							# elif part.type == 'refusal':
+							# 	blocks.append(TextBlockParam(text=f'[Refusal] {part.refusal}', type='text'))
+
+			# Add tool use blocks if present
+			if message.tool_calls:
+				tool_blocks = AnthropicMessageSerializer._serialize_tool_calls_to_content(
+					message.tool_calls, use_cache=message.cache
+				)
+				blocks.extend(tool_blocks)
+
+			# If no content or tool calls, add empty text block
+			# (Anthropic requires at least one content block)
+			if not blocks:
+				blocks.append(
+					TextBlockParam(
+						text='', type='text', cache_control=AnthropicMessageSerializer._serialize_cache_control(message.cache)
+					)
+				)
+
+			# If caching is enabled or we have multiple blocks, return blocks as-is
+			# Otherwise, simplify single text blocks to plain string
+			if message.cache or len(blocks) > 1:
+				content = blocks
+			else:
+				# Only simplify when no caching and single block
+				single_block = blocks[0]
+				if single_block['type'] == 'text' and not single_block.get('cache_control'):
+					content = single_block['text']
+				else:
+					content = blocks
+
+			return MessageParam(
+				role='assistant',
+				content=content,
+			)
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def _clean_cache_messages(messages: list[NonSystemMessage]) -> list[NonSystemMessage]:
+		"""Clean cache settings so only the last cache=True message remains cached.
+
+		Because of how Claude caching works, only the last cache message matters.
+		This method automatically removes cache=True from all messages except the last one.
+
+		Args:
+			messages: List of non-system messages to clean
+
+		Returns:
+			List of messages with cleaned cache settings
+		"""
+		if not messages:
+			return messages
+
+		# Create a copy to avoid modifying the original
+		cleaned_messages = [msg.model_copy(deep=True) for msg in messages]
+
+		# Find the last message with cache=True
+		last_cache_index = -1
+		for i in range(len(cleaned_messages) - 1, -1, -1):
+			if cleaned_messages[i].cache:
+				last_cache_index = i
+				break
+
+		# If we found a cached message, disable cache for all others
+		if last_cache_index != -1:
+			for i, msg in enumerate(cleaned_messages):
+				if i != last_cache_index and msg.cache:
+					# Set cache to False for all messages except the last cached one
+					msg.cache = False
+
+		return cleaned_messages
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> tuple[list[MessageParam], list[TextBlockParam] | str | None]:
+		"""Serialize a list of messages, extracting any system message.
+
+		Returns:
+		    A tuple of (messages, system_message) where system_message is extracted
+		    from any SystemMessage in the list.
+		"""
+		messages = [m.model_copy(deep=True) for m in messages]
+
+		# Separate system messages from normal messages
+		normal_messages: list[NonSystemMessage] = []
+		system_message: SystemMessage | None = None
+
+		for message in messages:
+			if isinstance(message, SystemMessage):
+				system_message = message
+			else:
+				normal_messages.append(message)
+
+		# Clean cache messages so only the last cache=True message remains cached
+		normal_messages = AnthropicMessageSerializer._clean_cache_messages(normal_messages)
+
+		# Serialize normal messages
+		serialized_messages: list[MessageParam] = []
+		for message in normal_messages:
+			serialized_messages.append(AnthropicMessageSerializer.serialize(message))
+
+		# Serialize system message
+		serialized_system_message: list[TextBlockParam] | str | None = None
+		if system_message:
+			serialized_system_message = AnthropicMessageSerializer._serialize_content_to_str(
+				system_message.content, use_cache=system_message.cache
+			)
+
+		return serialized_messages, serialized_system_message

browser_use/llm/aws/__init__.py

@@ -0,0 +1,36 @@
+from typing import TYPE_CHECKING
+
+# Type stubs for lazy imports
+if TYPE_CHECKING:
+	from browser_use.llm.aws.chat_anthropic import ChatAnthropicBedrock
+	from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+
+# Lazy imports mapping for AWS chat models
+_LAZY_IMPORTS = {
+	'ChatAnthropicBedrock': ('browser_use.llm.aws.chat_anthropic', 'ChatAnthropicBedrock'),
+	'ChatAWSBedrock': ('browser_use.llm.aws.chat_bedrock', 'ChatAWSBedrock'),
+}
+
+
+def __getattr__(name: str):
+	"""Lazy import mechanism for AWS chat models."""
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		try:
+			from importlib import import_module
+
+			module = import_module(module_path)
+			attr = getattr(module, attr_name)
+			# Cache the imported attribute in the module's globals
+			globals()[name] = attr
+			return attr
+		except ImportError as e:
+			raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
+
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'ChatAWSBedrock',
+	'ChatAnthropicBedrock',
+]

browser_use/llm/aws/chat_anthropic.py

@@ -0,0 +1,270 @@
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypeVar, overload
+
+from anthropic import (
+	APIConnectionError,
+	APIStatusError,
+	AsyncAnthropicBedrock,
+	RateLimitError,
+	omit,
+)
+from anthropic.types import CacheControlEphemeralParam, Message, ToolParam
+from anthropic.types.text_block import TextBlock
+from anthropic.types.tool_choice_tool_param import ToolChoiceToolParam
+from pydantic import BaseModel
+
+from browser_use.llm.anthropic.serializer import AnthropicMessageSerializer
+from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+if TYPE_CHECKING:
+	from boto3.session import Session  # pyright: ignore
+
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatAnthropicBedrock(ChatAWSBedrock):
+	"""
+	AWS Bedrock Anthropic Claude chat model.
+
+	This is a convenience class that provides Claude-specific defaults
+	for the AWS Bedrock service. It inherits all functionality from
+	ChatAWSBedrock but sets Anthropic Claude as the default model.
+	"""
+
+	# Anthropic Claude specific defaults
+	model: str = 'anthropic.claude-3-5-sonnet-20240620-v1:0'
+	max_tokens: int = 8192
+	temperature: float | None = None
+	top_p: float | None = None
+	top_k: int | None = None
+	stop_sequences: list[str] | None = None
+	seed: int | None = None
+
+	# AWS credentials and configuration
+	aws_access_key: str | None = None
+	aws_secret_key: str | None = None
+	aws_session_token: str | None = None
+	aws_region: str | None = None
+	session: 'Session | None' = None
+
+	# Client initialization parameters
+	max_retries: int = 10
+	default_headers: Mapping[str, str] | None = None
+	default_query: Mapping[str, object] | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'anthropic_bedrock'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary for Bedrock."""
+		client_params: dict[str, Any] = {}
+
+		if self.session:
+			credentials = self.session.get_credentials()
+			client_params.update(
+				{
+					'aws_access_key': credentials.access_key,
+					'aws_secret_key': credentials.secret_key,
+					'aws_session_token': credentials.token,
+					'aws_region': self.session.region_name,
+				}
+			)
+		else:
+			# Use individual credentials
+			if self.aws_access_key:
+				client_params['aws_access_key'] = self.aws_access_key
+			if self.aws_secret_key:
+				client_params['aws_secret_key'] = self.aws_secret_key
+			if self.aws_region:
+				client_params['aws_region'] = self.aws_region
+			if self.aws_session_token:
+				client_params['aws_session_token'] = self.aws_session_token
+
+		# Add optional parameters
+		if self.max_retries:
+			client_params['max_retries'] = self.max_retries
+		if self.default_headers:
+			client_params['default_headers'] = self.default_headers
+		if self.default_query:
+			client_params['default_query'] = self.default_query
+
+		return client_params
+
+	def _get_client_params_for_invoke(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary for invoke."""
+		client_params = {}
+
+		if self.temperature is not None:
+			client_params['temperature'] = self.temperature
+		if self.max_tokens is not None:
+			client_params['max_tokens'] = self.max_tokens
+		if self.top_p is not None:
+			client_params['top_p'] = self.top_p
+		if self.top_k is not None:
+			client_params['top_k'] = self.top_k
+		if self.seed is not None:
+			client_params['seed'] = self.seed
+		if self.stop_sequences is not None:
+			client_params['stop_sequences'] = self.stop_sequences
+
+		return client_params
+
+	def get_client(self) -> AsyncAnthropicBedrock:
+		"""
+		Returns an AsyncAnthropicBedrock client.
+
+		Returns:
+			AsyncAnthropicBedrock: An instance of the AsyncAnthropicBedrock client.
+		"""
+		client_params = self._get_client_params()
+		return AsyncAnthropicBedrock(**client_params)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_cache_creation_tokens(self, response: Message) -> tuple[int | None, int | None]:
+		cache_creation = getattr(response.usage, 'cache_creation', None)
+		if cache_creation is None:
+			return None, None
+		return (
+			getattr(cache_creation, 'ephemeral_5m_input_tokens', None),
+			getattr(cache_creation, 'ephemeral_1h_input_tokens', None),
+		)
+
+	def _get_usage(self, response: Message) -> ChatInvokeUsage | None:
+		"""Extract usage information from the response."""
+		cache_creation_5m_tokens, cache_creation_1h_tokens = self._get_cache_creation_tokens(response)
+		usage = ChatInvokeUsage(
+			prompt_tokens=response.usage.input_tokens
+			+ (
+				response.usage.cache_read_input_tokens or 0
+			),  # Total tokens in Anthropic are a bit fucked, you have to add cached tokens to the prompt tokens
+			completion_tokens=response.usage.output_tokens,
+			total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+			prompt_cached_tokens=response.usage.cache_read_input_tokens,
+			prompt_cache_creation_tokens=response.usage.cache_creation_input_tokens,
+			prompt_cache_creation_5m_tokens=cache_creation_5m_tokens,
+			prompt_cache_creation_1h_tokens=cache_creation_1h_tokens,
+			prompt_image_tokens=None,
+		)
+		return usage
+
+	@overload
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T], **kwargs: Any) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		anthropic_messages, system_prompt = AnthropicMessageSerializer.serialize_messages(messages)
+
+		try:
+			if output_format is None:
+				# Normal completion without structured output
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					system=system_prompt or omit,
+					**self._get_client_params_for_invoke(),
+				)
+
+				usage = self._get_usage(response)
+
+				# Extract text from the first content block
+				first_content = response.content[0]
+				if isinstance(first_content, TextBlock):
+					response_text = first_content.text
+				else:
+					# If it's not a text block, convert to string
+					response_text = str(first_content)
+
+				return ChatInvokeCompletion(
+					completion=response_text,
+					usage=usage,
+				)
+
+			else:
+				# Use tool calling for structured output
+				# Create a tool that represents the output format
+				tool_name = output_format.__name__
+				schema = output_format.model_json_schema()
+
+				# Remove title from schema if present (Anthropic doesn't like it in parameters)
+				if 'title' in schema:
+					del schema['title']
+
+				tool = ToolParam(
+					name=tool_name,
+					description=f'Extract information in the format of {tool_name}',
+					input_schema=schema,
+					cache_control=CacheControlEphemeralParam(type='ephemeral'),
+				)
+
+				# Force the model to use this tool
+				tool_choice = ToolChoiceToolParam(type='tool', name=tool_name)
+
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					tools=[tool],
+					system=system_prompt or omit,
+					tool_choice=tool_choice,
+					**self._get_client_params_for_invoke(),
+				)
+
+				usage = self._get_usage(response)
+
+				# Extract the tool use block
+				for content_block in response.content:
+					if hasattr(content_block, 'type') and content_block.type == 'tool_use':
+						# Parse the tool input as the structured output
+						try:
+							return ChatInvokeCompletion(completion=output_format.model_validate(content_block.input), usage=usage)
+						except Exception as e:
+							# If validation fails, try to fix common model output issues
+							_input = content_block.input
+							if isinstance(_input, str):
+								_input = json.loads(_input)
+							elif isinstance(_input, dict):
+								# Model sometimes double-serializes fields
+								for key, value in _input.items():
+									if isinstance(value, str) and value.startswith(('[', '{')):
+										try:
+											_input[key] = json.loads(value)
+										except json.JSONDecodeError:
+											cleaned = value.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
+											try:
+												_input[key] = json.loads(cleaned)
+											except json.JSONDecodeError:
+												pass
+							else:
+								raise
+							return ChatInvokeCompletion(
+								completion=output_format.model_validate(_input),
+								usage=usage,
+							)
+
+				# If no tool use block found, raise an error
+				raise ValueError('Expected tool use in response but none found')
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=e.message, model=self.name) from e
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.message, model=self.name) from e
+		except APIStatusError as e:
+			raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

browser_use/llm/aws/chat_bedrock.py

@@ -0,0 +1,279 @@
+import json
+from dataclasses import dataclass
+from os import getenv
+from typing import TYPE_CHECKING, Any, TypeVar, overload
+
+from pydantic import BaseModel
+
+from browser_use.llm.aws.serializer import AWSBedrockMessageSerializer
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+if TYPE_CHECKING:
+	from boto3 import client as AwsClient  # type: ignore
+	from boto3.session import Session  # type: ignore
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatAWSBedrock(BaseChatModel):
+	"""
+	AWS Bedrock chat model supporting multiple providers (Anthropic, Meta, etc.).
+
+	This class provides access to various models via AWS Bedrock,
+	supporting both text generation and structured output via tool calling.
+
+	To use this model, you need to either:
+	1. Set the following environment variables:
+	   - AWS_ACCESS_KEY_ID
+	   - AWS_SECRET_ACCESS_KEY
+	   - AWS_SESSION_TOKEN (only required when using temporary credentials)
+	   - AWS_REGION
+	2. Or provide a boto3 Session object
+	3. Or use AWS SSO authentication
+	"""
+
+	# Model configuration
+	model: str = 'anthropic.claude-3-5-sonnet-20240620-v1:0'
+	max_tokens: int | None = 4096
+	temperature: float | None = None
+	top_p: float | None = None
+	seed: int | None = None
+	stop_sequences: list[str] | None = None
+
+	# AWS credentials and configuration
+	aws_access_key_id: str | None = None
+	aws_secret_access_key: str | None = None
+	aws_session_token: str | None = None
+	aws_region: str | None = None
+	aws_sso_auth: bool = False
+	session: 'Session | None' = None
+
+	# Request parameters
+	request_params: dict[str, Any] | None = None
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'aws_bedrock'
+
+	def _get_client(self) -> 'AwsClient':  # type: ignore
+		"""Get the AWS Bedrock client."""
+		try:
+			from boto3 import client as AwsClient  # type: ignore
+		except ImportError:
+			raise ImportError(
+				'`boto3` not installed. Please install using `pip install browser-use[aws] or pip install browser-use[all]`'
+			)
+
+		if self.session:
+			return self.session.client('bedrock-runtime')
+
+		# Get credentials from environment or instance parameters
+		access_key = self.aws_access_key_id or getenv('AWS_ACCESS_KEY_ID')
+		secret_key = self.aws_secret_access_key or getenv('AWS_SECRET_ACCESS_KEY')
+		session_token = self.aws_session_token or getenv('AWS_SESSION_TOKEN')
+		region = self.aws_region or getenv('AWS_REGION') or getenv('AWS_DEFAULT_REGION')
+
+		if self.aws_sso_auth:
+			return AwsClient(service_name='bedrock-runtime', region_name=region)
+		else:
+			if not access_key or not secret_key:
+				raise ModelProviderError(
+					message='AWS credentials not found. Please set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables (and AWS_SESSION_TOKEN if using temporary credentials) or provide a boto3 session.',
+					model=self.name,
+				)
+
+			return AwsClient(
+				service_name='bedrock-runtime',
+				region_name=region,
+				aws_access_key_id=access_key,
+				aws_secret_access_key=secret_key,
+				aws_session_token=session_token,
+			)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_inference_config(self) -> dict[str, Any]:
+		"""Get the inference configuration for the request."""
+		config = {}
+		if self.max_tokens is not None:
+			config['maxTokens'] = self.max_tokens
+		if self.temperature is not None:
+			config['temperature'] = self.temperature
+		if self.top_p is not None:
+			config['topP'] = self.top_p
+		if self.stop_sequences is not None:
+			config['stopSequences'] = self.stop_sequences
+		if self.seed is not None:
+			config['seed'] = self.seed
+		return config
+
+	def _format_tools_for_request(self, output_format: type[BaseModel]) -> list[dict[str, Any]]:
+		"""Format a Pydantic model as a tool for structured output."""
+		schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+
+		return [
+			{
+				'toolSpec': {
+					'name': f'extract_{output_format.__name__.lower()}',
+					'description': f'Extract information in the format of {output_format.__name__}',
+					'inputSchema': {'json': schema},
+				}
+			}
+		]
+
+	def _get_usage(self, response: dict[str, Any]) -> ChatInvokeUsage | None:
+		"""Extract usage information from the response."""
+		if 'usage' not in response:
+			return None
+
+		usage_data = response['usage']
+		return ChatInvokeUsage(
+			prompt_tokens=usage_data.get('inputTokens', 0),
+			completion_tokens=usage_data.get('outputTokens', 0),
+			total_tokens=usage_data.get('totalTokens', 0),
+			prompt_cached_tokens=None,  # Bedrock doesn't provide this
+			prompt_cache_creation_tokens=None,
+			prompt_image_tokens=None,
+		)
+
+	@overload
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T], **kwargs: Any) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Invoke the AWS Bedrock model with the given messages.
+
+		Args:
+			messages: List of chat messages
+			output_format: Optional Pydantic model class for structured output
+
+		Returns:
+			Either a string response or an instance of output_format
+		"""
+		try:
+			from botocore.exceptions import ClientError  # type: ignore
+		except ImportError:
+			raise ImportError(
+				'`boto3` not installed. Please install using `pip install browser-use[aws] or pip install browser-use[all]`'
+			)
+
+		bedrock_messages, system_message = AWSBedrockMessageSerializer.serialize_messages(messages)
+
+		try:
+			# Prepare the request body
+			body: dict[str, Any] = {}
+
+			if system_message:
+				body['system'] = system_message
+
+			inference_config = self._get_inference_config()
+			if inference_config:
+				body['inferenceConfig'] = inference_config
+
+			# Handle structured output via tool calling
+			if output_format is not None:
+				tools = self._format_tools_for_request(output_format)
+				body['toolConfig'] = {'tools': tools}
+
+			# Add any additional request parameters
+			if self.request_params:
+				body.update(self.request_params)
+
+			# Filter out None values
+			body = {k: v for k, v in body.items() if v is not None}
+
+			# Make the API call
+			client = self._get_client()
+			response = client.converse(modelId=self.model, messages=bedrock_messages, **body)
+
+			usage = self._get_usage(response)
+
+			# Extract the response content
+			if 'output' in response and 'message' in response['output']:
+				message = response['output']['message']
+				content = message.get('content', [])
+
+				if output_format is None:
+					# Return text response
+					text_content = []
+					for item in content:
+						if 'text' in item:
+							text_content.append(item['text'])
+
+					response_text = '\n'.join(text_content) if text_content else ''
+					return ChatInvokeCompletion(
+						completion=response_text,
+						usage=usage,
+					)
+				else:
+					# Handle structured output from tool calls
+					for item in content:
+						if 'toolUse' in item:
+							tool_use = item['toolUse']
+							tool_input = tool_use.get('input', {})
+
+							try:
+								# Validate and return the structured output
+								return ChatInvokeCompletion(
+									completion=output_format.model_validate(tool_input),
+									usage=usage,
+								)
+							except Exception as e:
+								# If validation fails, try to parse as JSON first
+								if isinstance(tool_input, str):
+									try:
+										data = json.loads(tool_input)
+										return ChatInvokeCompletion(
+											completion=output_format.model_validate(data),
+											usage=usage,
+										)
+									except json.JSONDecodeError:
+										pass
+								raise ModelProviderError(
+									message=f'Failed to validate structured output: {str(e)}',
+									model=self.name,
+								) from e
+
+					# If no tool use found but output_format was requested
+					raise ModelProviderError(
+						message='Expected structured output but no tool use found in response',
+						model=self.name,
+					)
+
+			# If no valid content found
+			if output_format is None:
+				return ChatInvokeCompletion(
+					completion='',
+					usage=usage,
+				)
+			else:
+				raise ModelProviderError(
+					message='No valid content found in response',
+					model=self.name,
+				)
+
+		except ClientError as e:
+			error_code = e.response.get('Error', {}).get('Code', 'Unknown')
+			error_message = e.response.get('Error', {}).get('Message', str(e))
+
+			if error_code in ['ThrottlingException', 'TooManyRequestsException']:
+				raise ModelRateLimitError(message=error_message, model=self.name) from e
+			else:
+				raise ModelProviderError(message=error_message, model=self.name) from e
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

browser_use/llm/aws/serializer.py

@@ -0,0 +1,257 @@
+import base64
+import json
+import re
+from typing import Any, overload
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartRefusalParam,
+	ContentPartTextParam,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+
+class AWSBedrockMessageSerializer:
+	"""Serializer for converting between custom message types and AWS Bedrock message format."""
+
+	@staticmethod
+	def _is_base64_image(url: str) -> bool:
+		"""Check if the URL is a base64 encoded image."""
+		return url.startswith('data:image/')
+
+	@staticmethod
+	def _is_url_image(url: str) -> bool:
+		"""Check if the URL is a regular HTTP/HTTPS image URL."""
+		return url.startswith(('http://', 'https://')) and any(
+			url.lower().endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp']
+		)
+
+	@staticmethod
+	def _parse_base64_url(url: str) -> tuple[str, bytes]:
+		"""Parse a base64 data URL to extract format and raw bytes."""
+		# Format: data:image/jpeg;base64,<data>
+		if not url.startswith('data:'):
+			raise ValueError(f'Invalid base64 URL: {url}')
+
+		header, data = url.split(',', 1)
+
+		# Extract format from mime type
+		mime_match = re.search(r'image/(\w+)', header)
+		if mime_match:
+			format_name = mime_match.group(1).lower()
+			# Map common formats
+			format_mapping = {'jpg': 'jpeg', 'jpeg': 'jpeg', 'png': 'png', 'gif': 'gif', 'webp': 'webp'}
+			image_format = format_mapping.get(format_name, 'jpeg')
+		else:
+			image_format = 'jpeg'  # Default format
+
+		# Decode base64 data
+		try:
+			image_bytes = base64.b64decode(data)
+		except Exception as e:
+			raise ValueError(f'Failed to decode base64 image data: {e}')
+
+		return image_format, image_bytes
+
+	@staticmethod
+	def _download_and_convert_image(url: str) -> tuple[str, bytes]:
+		"""Download an image from URL and convert to base64 bytes."""
+		try:
+			import httpx
+		except ImportError:
+			raise ImportError('httpx not available. Please install it to use URL images with AWS Bedrock.')
+
+		try:
+			response = httpx.get(url, timeout=30)
+			response.raise_for_status()
+
+			# Detect format from content type or URL
+			content_type = response.headers.get('content-type', '').lower()
+			if 'jpeg' in content_type or url.lower().endswith(('.jpg', '.jpeg')):
+				image_format = 'jpeg'
+			elif 'png' in content_type or url.lower().endswith('.png'):
+				image_format = 'png'
+			elif 'gif' in content_type or url.lower().endswith('.gif'):
+				image_format = 'gif'
+			elif 'webp' in content_type or url.lower().endswith('.webp'):
+				image_format = 'webp'
+			else:
+				image_format = 'jpeg'  # Default format
+
+			return image_format, response.content
+
+		except Exception as e:
+			raise ValueError(f'Failed to download image from {url}: {e}')
+
+	@staticmethod
+	def _serialize_content_part_text(part: ContentPartTextParam) -> dict[str, Any]:
+		"""Convert a text content part to AWS Bedrock format."""
+		return {'text': part.text}
+
+	@staticmethod
+	def _serialize_content_part_image(part: ContentPartImageParam) -> dict[str, Any]:
+		"""Convert an image content part to AWS Bedrock format."""
+		url = part.image_url.url
+
+		if AWSBedrockMessageSerializer._is_base64_image(url):
+			# Handle base64 encoded images
+			image_format, image_bytes = AWSBedrockMessageSerializer._parse_base64_url(url)
+		elif AWSBedrockMessageSerializer._is_url_image(url):
+			# Download and convert URL images
+			image_format, image_bytes = AWSBedrockMessageSerializer._download_and_convert_image(url)
+		else:
+			raise ValueError(f'Unsupported image URL format: {url}')
+
+		return {
+			'image': {
+				'format': image_format,
+				'source': {
+					'bytes': image_bytes,
+				},
+			}
+		}
+
+	@staticmethod
+	def _serialize_user_content(
+		content: str | list[ContentPartTextParam | ContentPartImageParam],
+	) -> list[dict[str, Any]]:
+		"""Serialize content for user messages."""
+		if isinstance(content, str):
+			return [{'text': content}]
+
+		content_blocks: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_text(part))
+			elif part.type == 'image_url':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_image(part))
+
+		return content_blocks
+
+	@staticmethod
+	def _serialize_system_content(
+		content: str | list[ContentPartTextParam],
+	) -> list[dict[str, Any]]:
+		"""Serialize content for system messages."""
+		if isinstance(content, str):
+			return [{'text': content}]
+
+		content_blocks: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_text(part))
+
+		return content_blocks
+
+	@staticmethod
+	def _serialize_assistant_content(
+		content: str | list[ContentPartTextParam | ContentPartRefusalParam] | None,
+	) -> list[dict[str, Any]]:
+		"""Serialize content for assistant messages."""
+		if content is None:
+			return []
+		if isinstance(content, str):
+			return [{'text': content}]
+
+		content_blocks: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_text(part))
+			# Skip refusal content parts - AWS Bedrock doesn't need them
+
+		return content_blocks
+
+	@staticmethod
+	def _serialize_tool_call(tool_call: ToolCall) -> dict[str, Any]:
+		"""Convert a tool call to AWS Bedrock format."""
+		try:
+			arguments = json.loads(tool_call.function.arguments)
+		except json.JSONDecodeError:
+			# If arguments aren't valid JSON, wrap them
+			arguments = {'arguments': tool_call.function.arguments}
+
+		return {
+			'toolUse': {
+				'toolUseId': tool_call.id,
+				'name': tool_call.function.name,
+				'input': arguments,
+			}
+		}
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> dict[str, Any]: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> SystemMessage: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> dict[str, Any]: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> dict[str, Any] | SystemMessage:
+		"""Serialize a custom message to AWS Bedrock format."""
+
+		if isinstance(message, UserMessage):
+			return {
+				'role': 'user',
+				'content': AWSBedrockMessageSerializer._serialize_user_content(message.content),
+			}
+
+		elif isinstance(message, SystemMessage):
+			# System messages are handled separately in AWS Bedrock
+			return message
+
+		elif isinstance(message, AssistantMessage):
+			content_blocks: list[dict[str, Any]] = []
+
+			# Add content blocks if present
+			if message.content is not None:
+				content_blocks.extend(AWSBedrockMessageSerializer._serialize_assistant_content(message.content))
+
+			# Add tool use blocks if present
+			if message.tool_calls:
+				for tool_call in message.tool_calls:
+					content_blocks.append(AWSBedrockMessageSerializer._serialize_tool_call(tool_call))
+
+			# AWS Bedrock requires at least one content block
+			if not content_blocks:
+				content_blocks = [{'text': ''}]
+
+			return {
+				'role': 'assistant',
+				'content': content_blocks,
+			}
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None]:
+		"""
+		Serialize a list of messages, extracting any system message.
+
+		Returns:
+			Tuple of (bedrock_messages, system_message) where system_message is extracted
+			from any SystemMessage in the list.
+		"""
+		bedrock_messages: list[dict[str, Any]] = []
+		system_message: list[dict[str, Any]] | None = None
+
+		for message in messages:
+			if isinstance(message, SystemMessage):
+				# Extract system message content
+				system_message = AWSBedrockMessageSerializer._serialize_system_content(message.content)
+			else:
+				# Serialize and add to regular messages
+				serialized = AWSBedrockMessageSerializer.serialize(message)
+				bedrock_messages.append(serialized)
+
+		return bedrock_messages, system_message

browser_use/llm/azure/chat.py

@@ -0,0 +1,287 @@
+import os
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from openai import APIConnectionError, APIStatusError, RateLimitError
+from openai import AsyncAzureOpenAI as AsyncAzureOpenAIClient
+from openai.types.responses import Response
+from openai.types.shared import ChatModel
+from pydantic import BaseModel
+
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.openai.like import ChatOpenAILike
+from browser_use.llm.openai.responses_serializer import ResponsesAPIMessageSerializer
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar('T', bound=BaseModel)
+
+
+# List of models that only support the Responses API
+RESPONSES_API_ONLY_MODELS: list[str] = [
+	'gpt-5.1-codex',
+	'gpt-5.1-codex-mini',
+	'gpt-5.1-codex-max',
+	'gpt-5-codex',
+	'codex-mini-latest',
+	'computer-use-preview',
+]
+
+
+@dataclass
+class ChatAzureOpenAI(ChatOpenAILike):
+	"""
+	A class for to interact with any provider using the OpenAI API schema.
+
+	Args:
+	    model (str): The name of the OpenAI model to use. Defaults to "not-provided".
+	    api_key (Optional[str]): The API key to use. Defaults to "not-provided".
+	    use_responses_api (bool): If True, use the Responses API instead of Chat Completions API.
+	        This is required for certain models like gpt-5.1-codex-mini on Azure OpenAI with
+	        api_version >= 2025-03-01-preview. Set to 'auto' to automatically detect based on model.
+	"""
+
+	# Model configuration
+	model: str | ChatModel
+
+	# Client initialization parameters
+	api_key: str | None = None
+	api_version: str | None = '2024-12-01-preview'
+	azure_endpoint: str | None = None
+	azure_deployment: str | None = None
+	base_url: str | None = None
+	azure_ad_token: str | None = None
+	azure_ad_token_provider: Any | None = None
+
+	default_headers: dict[str, str] | None = None
+	default_query: dict[str, Any] | None = None
+
+	# Responses API support
+	use_responses_api: bool | str = 'auto'  # True, False, or 'auto'
+
+	client: AsyncAzureOpenAIClient | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'azure'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		_client_params: dict[str, Any] = {}
+
+		self.api_key = self.api_key or os.getenv('AZURE_OPENAI_KEY') or os.getenv('AZURE_OPENAI_API_KEY')
+		self.azure_endpoint = self.azure_endpoint or os.getenv('AZURE_OPENAI_ENDPOINT')
+		self.azure_deployment = self.azure_deployment or os.getenv('AZURE_OPENAI_DEPLOYMENT')
+		params_mapping = {
+			'api_key': self.api_key,
+			'api_version': self.api_version,
+			'organization': self.organization,
+			'azure_endpoint': self.azure_endpoint,
+			'azure_deployment': self.azure_deployment,
+			'base_url': self.base_url,
+			'azure_ad_token': self.azure_ad_token,
+			'azure_ad_token_provider': self.azure_ad_token_provider,
+			'http_client': self.http_client,
+		}
+		if self.default_headers is not None:
+			_client_params['default_headers'] = self.default_headers
+		if self.default_query is not None:
+			_client_params['default_query'] = self.default_query
+
+		_client_params.update({k: v for k, v in params_mapping.items() if v is not None})
+
+		return _client_params
+
+	def get_client(self) -> AsyncAzureOpenAIClient:
+		"""
+		Returns an asynchronous OpenAI client.
+
+		Returns:
+			AsyncAzureOpenAIClient: An instance of the asynchronous OpenAI client.
+		"""
+		if self.client:
+			return self.client
+
+		_client_params: dict[str, Any] = self._get_client_params()
+
+		if self.http_client:
+			_client_params['http_client'] = self.http_client
+		else:
+			# Create a new async HTTP client with custom limits
+			_client_params['http_client'] = httpx.AsyncClient(
+				limits=httpx.Limits(max_connections=20, max_keepalive_connections=6)
+			)
+
+		self.client = AsyncAzureOpenAIClient(**_client_params)
+
+		return self.client
+
+	def _should_use_responses_api(self) -> bool:
+		"""Determine if the Responses API should be used based on model and settings."""
+		if isinstance(self.use_responses_api, bool):
+			return self.use_responses_api
+
+		# Auto-detect: use Responses API for models that require it
+		model_lower = str(self.model).lower()
+		for responses_only_model in RESPONSES_API_ONLY_MODELS:
+			if responses_only_model.lower() in model_lower:
+				return True
+		return False
+
+	def _get_usage_from_responses(self, response: Response) -> ChatInvokeUsage | None:
+		"""Extract usage information from a Responses API response."""
+		if response.usage is None:
+			return None
+
+		# Get cached tokens from input_tokens_details if available
+		cached_tokens = None
+		if response.usage.input_tokens_details is not None:
+			cached_tokens = getattr(response.usage.input_tokens_details, 'cached_tokens', None)
+
+		return ChatInvokeUsage(
+			prompt_tokens=response.usage.input_tokens,
+			prompt_cached_tokens=cached_tokens,
+			prompt_cache_creation_tokens=None,
+			prompt_image_tokens=None,
+			completion_tokens=response.usage.output_tokens,
+			total_tokens=response.usage.total_tokens,
+		)
+
+	async def _ainvoke_responses_api(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Invoke the model using the Responses API.
+
+		This is used for models that require the Responses API (e.g., gpt-5.1-codex-mini)
+		or when use_responses_api is explicitly set to True.
+		"""
+		# Serialize messages to Responses API input format
+		input_messages = ResponsesAPIMessageSerializer.serialize_messages(messages)
+
+		try:
+			model_params: dict[str, Any] = {
+				'model': self.model,
+				'input': input_messages,
+			}
+
+			if self.temperature is not None:
+				model_params['temperature'] = self.temperature
+
+			if self.max_completion_tokens is not None:
+				model_params['max_output_tokens'] = self.max_completion_tokens
+
+			if self.top_p is not None:
+				model_params['top_p'] = self.top_p
+
+			if self.service_tier is not None:
+				model_params['service_tier'] = self.service_tier
+
+			# Handle reasoning models
+			if self.reasoning_models and any(str(m).lower() in str(self.model).lower() for m in self.reasoning_models):
+				# For reasoning models, use reasoning parameter instead of reasoning_effort
+				model_params['reasoning'] = {'effort': self.reasoning_effort}
+				model_params.pop('temperature', None)
+
+			if output_format is None:
+				# Return string response
+				response = await self.get_client().responses.create(**model_params)
+
+				usage = self._get_usage_from_responses(response)
+				return ChatInvokeCompletion(
+					completion=response.output_text or '',
+					usage=usage,
+					stop_reason=response.status if response.status else None,
+				)
+
+			else:
+				# For structured output, use the text.format parameter
+				json_schema = SchemaOptimizer.create_optimized_json_schema(
+					output_format,
+					remove_min_items=self.remove_min_items_from_schema,
+					remove_defaults=self.remove_defaults_from_schema,
+				)
+
+				model_params['text'] = {
+					'format': {
+						'type': 'json_schema',
+						'name': 'agent_output',
+						'strict': True,
+						'schema': json_schema,
+					}
+				}
+
+				# Add JSON schema to system prompt if requested
+				if self.add_schema_to_system_prompt and input_messages and input_messages[0].get('role') == 'system':
+					schema_text = f'\n<json_schema>\n{json_schema}\n</json_schema>'
+					content = input_messages[0].get('content', '')
+					if isinstance(content, str):
+						input_messages[0]['content'] = content + schema_text
+					elif isinstance(content, list):
+						input_messages[0]['content'] = list(content) + [{'type': 'input_text', 'text': schema_text}]
+					model_params['input'] = input_messages
+
+				if self.dont_force_structured_output:
+					# Remove the text format parameter if not forcing structured output
+					model_params.pop('text', None)
+
+				response = await self.get_client().responses.create(**model_params)
+
+				if not response.output_text:
+					raise ModelProviderError(
+						message='Failed to parse structured output from model response',
+						status_code=500,
+						model=self.name,
+					)
+
+				usage = self._get_usage_from_responses(response)
+				parsed = output_format.model_validate_json(response.output_text)
+
+				return ChatInvokeCompletion(
+					completion=parsed,
+					usage=usage,
+					stop_reason=response.status if response.status else None,
+				)
+
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.message, model=self.name) from e
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e
+
+		except APIStatusError as e:
+			raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e
+
+	@overload
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T], **kwargs: Any) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Invoke the model with the given messages.
+
+		This method routes to either the Responses API or the Chat Completions API
+		based on the model and settings.
+
+		Args:
+			messages: List of chat messages
+			output_format: Optional Pydantic model class for structured output
+
+		Returns:
+			Either a string response or an instance of output_format
+		"""
+		if self._should_use_responses_api():
+			return await self._ainvoke_responses_api(messages, output_format, **kwargs)
+		else:
+			# Use the parent class implementation (Chat Completions API)
+			return await super().ainvoke(messages, output_format, **kwargs)

browser_use/llm/base.py

@@ -0,0 +1,59 @@
+"""
+We have switched all of our code from langchain to openai.types.chat.chat_completion_message_param.
+
+For easier transition we have
+"""
+
+from typing import Any, Protocol, TypeVar, overload, runtime_checkable
+
+from pydantic import BaseModel
+
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@runtime_checkable
+class BaseChatModel(Protocol):
+	_verified_api_keys: bool = False
+
+	model: str
+
+	@property
+	def provider(self) -> str: ...
+
+	@property
+	def name(self) -> str: ...
+
+	@property
+	def model_name(self) -> str:
+		# for legacy support
+		return self.model
+
+	@overload
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T], **kwargs: Any) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]: ...
+
+	@classmethod
+	def __get_pydantic_core_schema__(
+		cls,
+		source_type: type,
+		handler: Any,
+	) -> Any:
+		"""
+		Allow this Protocol to be used in Pydantic models -> very useful to typesafe the agent settings for example.
+		Returns a schema that allows any object (since this is a Protocol).
+		"""
+		from pydantic_core import core_schema
+
+		# Return a schema that accepts any object for Protocol types
+		return core_schema.any_schema()

browser_use/llm/browser_use/__init__.py

@@ -0,0 +1,3 @@
+from browser_use.llm.browser_use.chat import ChatBrowserUse
+
+__all__ = ['ChatBrowserUse']

browser_use/llm/browser_use/chat.py

@@ -0,0 +1,310 @@
+"""
+ChatBrowserUse - Client for browser-use cloud API
+
+This wraps the BaseChatModel protocol and sends requests to the browser-use cloud API
+for optimized browser automation LLM inference.
+"""
+
+import asyncio
+import logging
+import os
+import random
+from typing import Any, TypeVar, overload
+
+import httpx
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion
+from browser_use.observability import observe
+
+T = TypeVar('T', bound=BaseModel)
+
+logger = logging.getLogger(__name__)
+
+# HTTP status codes that should trigger a retry
+RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
+
+
+class ChatBrowserUse(BaseChatModel):
+	"""
+	Client for browser-use cloud API.
+
+	This sends requests to the browser-use cloud API which uses optimized models
+	and prompts for browser automation tasks.
+
+	Usage:
+		agent = Agent(
+			task="Find the number of stars of the browser-use repo",
+			llm=ChatBrowserUse(model='openai/gpt-5.5'),
+		)
+	"""
+
+	def __init__(
+		self,
+		model: str = 'bu-2-0',
+		api_key: str | None = None,
+		base_url: str | None = None,
+		timeout: float = 120.0,
+		max_retries: int = 5,
+		retry_base_delay: float = 1.0,
+		retry_max_delay: float = 60.0,
+		**kwargs,
+	):
+		"""
+		Initialize ChatBrowserUse client.
+
+		Args:
+			model: Model name to use. Options:
+				- 'bu-2-0' or 'bu-latest': Default model (latest premium)
+				- 'bu-1-0': Previous generation model
+				- 'browser-use/bu-30b-a3b-preview': Browser Use Open Source Model
+				- Provider-prefixed ids resolved by the gateway, e.g. 'anthropic/claude-sonnet-4-6',
+				  'openai/gpt-5.5', 'google/gemini-3-pro'.
+			api_key: API key for browser-use cloud. Defaults to BROWSER_USE_API_KEY env var.
+			base_url: Base URL for the API. Defaults to BROWSER_USE_LLM_URL env var or production URL.
+			timeout: Request timeout in seconds.
+			max_retries: Maximum number of retries for transient errors (default: 5).
+			retry_base_delay: Base delay in seconds for exponential backoff (default: 1.0).
+			retry_max_delay: Maximum delay in seconds between retries (default: 60.0).
+		"""
+		# Accept 'bu-*' aliases and any provider-prefixed id; the gateway resolves the
+		# latter (anthropic/*, openai/*, google/*, browser-use/*), so we don't enumerate them.
+		bu_aliases = ['bu-latest', 'bu-1-0', 'bu-2-0']
+		is_valid = model in bu_aliases or '/' in model
+		if not is_valid:
+			raise ValueError(
+				f"Invalid model: '{model}'. Use a 'bu-*' alias ({', '.join(bu_aliases)}) "
+				"or a provider-prefixed id like 'anthropic/claude-sonnet-4-6', "
+				"'openai/gpt-5.5', or 'google/gemini-3-pro'."
+			)
+
+		# Normalize bu-latest to the current latest model
+		if model == 'bu-latest':
+			self.model = 'bu-2-0'
+		else:
+			self.model = model
+
+		self.fast = False
+		self.api_key = api_key or os.getenv('BROWSER_USE_API_KEY')
+		self.base_url = base_url or os.getenv('BROWSER_USE_LLM_URL', 'https://llm.api.browser-use.com')
+		self.timeout = timeout
+		self.max_retries = max_retries
+		self.retry_base_delay = retry_base_delay
+		self.retry_max_delay = retry_max_delay
+
+		if not self.api_key:
+			raise ValueError(
+				'BROWSER_USE_API_KEY is not set. To use ChatBrowserUse, get a key at:\n'
+				'https://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=chat_browser_use'
+			)
+
+	@property
+	def provider(self) -> str:
+		return 'browser-use'
+
+	@property
+	def name(self) -> str:
+		return self.model
+
+	@overload
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: None = None, request_type: str = 'browser_agent', **kwargs: Any
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T], request_type: str = 'browser_agent', **kwargs: Any
+	) -> ChatInvokeCompletion[T]: ...
+
+	@observe(name='chat_browser_use_ainvoke')
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: type[T] | None = None,
+		request_type: str = 'browser_agent',
+		**kwargs: Any,
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Send request to browser-use cloud API.
+
+		Args:
+			messages: List of messages to send
+			output_format: Expected output format (Pydantic model)
+			request_type: Type of request - 'browser_agent' or 'judge'
+			**kwargs: Additional arguments, including:
+				- session_id: Session ID for sticky routing (same session → same container)
+
+		Returns:
+			ChatInvokeCompletion with structured response and usage info
+		"""
+		# Get ANONYMIZED_TELEMETRY setting from config
+		from browser_use.config import CONFIG
+
+		anonymized_telemetry = CONFIG.ANONYMIZED_TELEMETRY
+
+		# Extract session_id from kwargs for sticky routing
+		session_id = kwargs.get('session_id')
+
+		# Prepare request payload
+		payload: dict[str, Any] = {
+			'model': self.model,
+			'messages': [self._serialize_message(msg) for msg in messages],
+			'fast': self.fast,
+			'request_type': request_type,
+			'anonymized_telemetry': anonymized_telemetry,
+		}
+
+		# Add session_id for sticky routing if provided
+		if session_id:
+			payload['session_id'] = session_id
+
+		# Add output format schema if provided
+		if output_format is not None:
+			payload['output_format'] = output_format.model_json_schema()
+
+		last_error: Exception | None = None
+
+		# Retry loop with exponential backoff
+		for attempt in range(self.max_retries):
+			try:
+				result = await self._make_request(payload)
+				break
+			except httpx.HTTPStatusError as e:
+				last_error = e
+				status_code = e.response.status_code
+
+				# Check if this is a retryable error
+				if status_code in RETRYABLE_STATUS_CODES and attempt < self.max_retries - 1:
+					delay = min(self.retry_base_delay * (2**attempt), self.retry_max_delay)
+					jitter = random.uniform(0, delay * 0.1)
+					total_delay = delay + jitter
+					logger.warning(
+						f'⚠️ Got {status_code} error, retrying in {total_delay:.1f}s... (attempt {attempt + 1}/{self.max_retries})'
+					)
+					await asyncio.sleep(total_delay)
+					continue
+
+				# Non-retryable HTTP error or exhausted retries
+				self._raise_http_error(e)
+
+			except (httpx.TimeoutException, httpx.ConnectError) as e:
+				last_error = e
+				# Network errors are retryable
+				if attempt < self.max_retries - 1:
+					delay = min(self.retry_base_delay * (2**attempt), self.retry_max_delay)
+					jitter = random.uniform(0, delay * 0.1)
+					total_delay = delay + jitter
+					error_type = 'timeout' if isinstance(e, httpx.TimeoutException) else 'connection error'
+					logger.warning(
+						f'⚠️ Got {error_type}, retrying in {total_delay:.1f}s... (attempt {attempt + 1}/{self.max_retries})'
+					)
+					await asyncio.sleep(total_delay)
+					continue
+
+				# Exhausted retries
+				if isinstance(e, httpx.TimeoutException):
+					raise ValueError(f'Request timed out after {self.timeout}s (retried {self.max_retries} times)')
+				raise ValueError(f'Failed to connect to browser-use API after {self.max_retries} attempts: {e}')
+
+			except Exception as e:
+				raise ValueError(f'Failed to connect to browser-use API: {e}')
+		else:
+			# Loop completed without break (all retries exhausted)
+			if last_error is not None:
+				if isinstance(last_error, httpx.HTTPStatusError):
+					self._raise_http_error(last_error)
+				raise ValueError(f'Request failed after {self.max_retries} attempts: {last_error}')
+			raise RuntimeError('Retry loop completed without return or exception')
+
+		# Parse response - server returns structured data as dict
+		if output_format is not None:
+			# Server returns structured data as a dict, validate it
+			completion_data = result['completion']
+			logger.debug(
+				f'📥 Got structured data from service: {list(completion_data.keys()) if isinstance(completion_data, dict) else type(completion_data)}'
+			)
+
+			# Convert action dicts to ActionModel instances if needed
+			# llm-use returns dicts to avoid validation with empty ActionModel
+			if isinstance(completion_data, dict) and 'action' in completion_data:
+				actions = completion_data['action']
+				if actions and isinstance(actions[0], dict):
+					from typing import get_args
+
+					# Get ActionModel type from output_format
+					action_model_type = get_args(output_format.model_fields['action'].annotation)[0]
+
+					# Convert dicts to ActionModel instances
+					completion_data['action'] = [action_model_type.model_validate(action_dict) for action_dict in actions]
+
+			completion = output_format.model_validate(completion_data)
+		else:
+			completion = result['completion']
+
+		# Parse usage info
+		usage = None
+		if 'usage' in result and result['usage'] is not None:
+			from browser_use.llm.views import ChatInvokeUsage
+
+			usage = ChatInvokeUsage(**result['usage'])
+
+		return ChatInvokeCompletion(
+			completion=completion,
+			usage=usage,
+		)
+
+	async def _make_request(self, payload: dict) -> dict:
+		"""Make a single API request."""
+		async with httpx.AsyncClient(timeout=self.timeout) as client:
+			response = await client.post(
+				f'{self.base_url}/v1/chat/completions',
+				json=payload,
+				headers={
+					'Authorization': f'Bearer {self.api_key}',
+					'Content-Type': 'application/json',
+				},
+			)
+			response.raise_for_status()
+			return response.json()
+
+	def _raise_http_error(self, e: httpx.HTTPStatusError) -> None:
+		"""Raise appropriate ModelProviderError for HTTP errors."""
+		error_detail = ''
+		try:
+			error_data = e.response.json()
+			error_detail = error_data.get('detail', str(e))
+		except Exception:
+			error_detail = str(e)
+
+		status_code = e.response.status_code
+
+		if status_code == 401:
+			raise ModelProviderError(
+				message=f'BROWSER_USE_API_KEY is invalid. Get a new key at:\nhttps://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=chat_browser_use\n{error_detail}',
+				status_code=401,
+				model=self.name,
+			)
+		elif status_code == 402:
+			raise ModelProviderError(
+				message=f'Browser Use credits exhausted. Add more at:\nhttps://cloud.browser-use.com/billing?utm_source=oss&utm_medium=chat_browser_use\n{error_detail}',
+				status_code=402,
+				model=self.name,
+			)
+		elif status_code == 429:
+			raise ModelRateLimitError(message=f'Rate limit exceeded. {error_detail}', status_code=429, model=self.name)
+		elif status_code in {500, 502, 503, 504}:
+			raise ModelProviderError(message=f'Server error. {error_detail}', status_code=status_code, model=self.name)
+		else:
+			raise ModelProviderError(message=f'API request failed: {error_detail}', status_code=status_code, model=self.name)
+
+	def _serialize_message(self, message: BaseMessage) -> dict:
+		"""Serialize a message to JSON format."""
+		# Handle Union types by checking the actual message type
+		msg_dict = message.model_dump()
+		return {
+			'role': msg_dict['role'],
+			'content': msg_dict['content'],
+		}

browser_use/llm/cerebras/chat.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from openai import (
+	APIConnectionError,
+	APIError,
+	APIStatusError,
+	APITimeoutError,
+	AsyncOpenAI,
+	RateLimitError,
+)
+from openai.types.chat import ChatCompletion
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.cerebras.serializer import CerebrasMessageSerializer
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatCerebras(BaseChatModel):
+	"""Cerebras inference wrapper (OpenAI-compatible)."""
+
+	model: str = 'llama3.1-8b'
+
+	# Generation parameters
+	max_tokens: int | None = 4096
+	temperature: float | None = 0.2
+	top_p: float | None = None
+	seed: int | None = None
+
+	# Connection parameters
+	api_key: str | None = None
+	base_url: str | httpx.URL | None = 'https://api.cerebras.ai/v1'
+	timeout: float | httpx.Timeout | None = None
+	client_params: dict[str, Any] | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'cerebras'
+
+	def _client(self) -> AsyncOpenAI:
+		return AsyncOpenAI(
+			api_key=self.api_key,
+			base_url=self.base_url,
+			timeout=self.timeout,
+			**(self.client_params or {}),
+		)
+
+	@property
+	def name(self) -> str:
+		return self.model
+
+	def _get_usage(self, response: ChatCompletion) -> ChatInvokeUsage | None:
+		if response.usage is not None:
+			usage = ChatInvokeUsage(
+				prompt_tokens=response.usage.prompt_tokens,
+				prompt_cached_tokens=None,
+				prompt_cache_creation_tokens=None,
+				prompt_image_tokens=None,
+				completion_tokens=response.usage.completion_tokens,
+				total_tokens=response.usage.total_tokens,
+			)
+		else:
+			usage = None
+		return usage
+
+	@overload
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: None = None,
+		**kwargs: Any,
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: type[T],
+		**kwargs: Any,
+	) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: type[T] | None = None,
+		**kwargs: Any,
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Cerebras ainvoke supports:
+		1. Regular text/multi-turn conversation
+		2. JSON Output (response_format)
+		"""
+		client = self._client()
+		cerebras_messages = CerebrasMessageSerializer.serialize_messages(messages)
+		common: dict[str, Any] = {}
+
+		if self.temperature is not None:
+			common['temperature'] = self.temperature
+		if self.max_tokens is not None:
+			common['max_tokens'] = self.max_tokens
+		if self.top_p is not None:
+			common['top_p'] = self.top_p
+		if self.seed is not None:
+			common['seed'] = self.seed
+
+		# ① Regular multi-turn conversation/text output
+		if output_format is None:
+			try:
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=cerebras_messages,  # type: ignore
+					**common,
+				)
+				usage = self._get_usage(resp)
+				return ChatInvokeCompletion(
+					completion=resp.choices[0].message.content or '',
+					usage=usage,
+				)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		# ② JSON Output path (response_format)
+		if output_format is not None and hasattr(output_format, 'model_json_schema'):
+			try:
+				# For Cerebras, we'll use a simpler approach without response_format
+				# Instead, we'll ask the model to return JSON and parse it
+				import json
+
+				# Get the schema to guide the model
+				schema = output_format.model_json_schema()
+				schema_str = json.dumps(schema, indent=2)
+
+				# Create a prompt that asks for the specific JSON structure
+				json_prompt = f"""
+Please respond with a JSON object that follows this exact schema:
+{schema_str}
+
+Your response must be valid JSON only, no other text.
+"""
+
+				# Add or modify the last user message to include the JSON prompt
+				if cerebras_messages and cerebras_messages[-1]['role'] == 'user':
+					if isinstance(cerebras_messages[-1]['content'], str):
+						cerebras_messages[-1]['content'] += json_prompt
+					elif isinstance(cerebras_messages[-1]['content'], list):
+						cerebras_messages[-1]['content'].append({'type': 'text', 'text': json_prompt})
+				else:
+					# Add as a new user message
+					cerebras_messages.append({'role': 'user', 'content': json_prompt})
+
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=cerebras_messages,  # type: ignore
+					**common,
+				)
+				content = resp.choices[0].message.content
+				if not content:
+					raise ModelProviderError('Empty JSON content in Cerebras response', model=self.name)
+
+				usage = self._get_usage(resp)
+
+				# Try to extract JSON from the response
+				import re
+
+				json_match = re.search(r'\{.*\}', content, re.DOTALL)
+				if json_match:
+					json_str = json_match.group(0)
+				else:
+					json_str = content
+
+				parsed = output_format.model_validate_json(json_str)
+				return ChatInvokeCompletion(
+					completion=parsed,
+					usage=usage,
+				)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		raise ModelProviderError('No valid ainvoke execution path for Cerebras LLM', model=self.name)

browser_use/llm/cerebras/serializer.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import json
+from typing import Any, overload
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+MessageDict = dict[str, Any]
+
+
+class CerebrasMessageSerializer:
+	"""Serializer for converting browser-use messages to Cerebras messages."""
+
+	# -------- content 处理 --------------------------------------------------
+	@staticmethod
+	def _serialize_text_part(part: ContentPartTextParam) -> str:
+		return part.text
+
+	@staticmethod
+	def _serialize_image_part(part: ContentPartImageParam) -> dict[str, Any]:
+		url = part.image_url.url
+		if url.startswith('data:'):
+			return {'type': 'image_url', 'image_url': {'url': url}}
+		return {'type': 'image_url', 'image_url': {'url': url}}
+
+	@staticmethod
+	def _serialize_content(content: Any) -> str | list[dict[str, Any]]:
+		if content is None:
+			return ''
+		if isinstance(content, str):
+			return content
+		serialized: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				serialized.append({'type': 'text', 'text': CerebrasMessageSerializer._serialize_text_part(part)})
+			elif part.type == 'image_url':
+				serialized.append(CerebrasMessageSerializer._serialize_image_part(part))
+			elif part.type == 'refusal':
+				serialized.append({'type': 'text', 'text': f'[Refusal] {part.refusal}'})
+		return serialized
+
+	# -------- Tool-call 处理 -------------------------------------------------
+	@staticmethod
+	def _serialize_tool_calls(tool_calls: list[ToolCall]) -> list[dict[str, Any]]:
+		cerebras_tool_calls: list[dict[str, Any]] = []
+		for tc in tool_calls:
+			try:
+				arguments = json.loads(tc.function.arguments)
+			except json.JSONDecodeError:
+				arguments = {'arguments': tc.function.arguments}
+			cerebras_tool_calls.append(
+				{
+					'id': tc.id,
+					'type': 'function',
+					'function': {
+						'name': tc.function.name,
+						'arguments': arguments,
+					},
+				}
+			)
+		return cerebras_tool_calls
+
+	# -------- 单条消息序列化 -------------------------------------------------
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> MessageDict: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> MessageDict: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> MessageDict: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> MessageDict:
+		if isinstance(message, UserMessage):
+			return {
+				'role': 'user',
+				'content': CerebrasMessageSerializer._serialize_content(message.content),
+			}
+		if isinstance(message, SystemMessage):
+			return {
+				'role': 'system',
+				'content': CerebrasMessageSerializer._serialize_content(message.content),
+			}
+		if isinstance(message, AssistantMessage):
+			msg: MessageDict = {
+				'role': 'assistant',
+				'content': CerebrasMessageSerializer._serialize_content(message.content),
+			}
+			if message.tool_calls:
+				msg['tool_calls'] = CerebrasMessageSerializer._serialize_tool_calls(message.tool_calls)
+			return msg
+		raise ValueError(f'Unknown message type: {type(message)}')
+
+	# -------- 列表序列化 -----------------------------------------------------
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> list[MessageDict]:
+		return [CerebrasMessageSerializer.serialize(m) for m in messages]

browser_use/llm/deepseek/chat.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from openai import (
+	APIConnectionError,
+	APIError,
+	APIStatusError,
+	APITimeoutError,
+	AsyncOpenAI,
+	RateLimitError,
+)
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.deepseek.serializer import DeepSeekMessageSerializer
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatDeepSeek(BaseChatModel):
+	"""DeepSeek /chat/completions wrapper (OpenAI-compatible)."""
+
+	model: str = 'deepseek-chat'
+
+	# Generation parameters
+	max_tokens: int | None = None
+	temperature: float | None = None
+	top_p: float | None = None
+	seed: int | None = None
+
+	# Connection parameters
+	api_key: str | None = None
+	base_url: str | httpx.URL | None = 'https://api.deepseek.com/v1'
+	timeout: float | httpx.Timeout | None = None
+	client_params: dict[str, Any] | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'deepseek'
+
+	def _client(self) -> AsyncOpenAI:
+		return AsyncOpenAI(
+			api_key=self.api_key,
+			base_url=self.base_url,
+			timeout=self.timeout,
+			**(self.client_params or {}),
+		)
+
+	@property
+	def name(self) -> str:
+		return self.model
+
+	@overload
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: None = None,
+		tools: list[dict[str, Any]] | None = None,
+		stop: list[str] | None = None,
+		**kwargs: Any,
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: type[T],
+		tools: list[dict[str, Any]] | None = None,
+		stop: list[str] | None = None,
+		**kwargs: Any,
+	) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: type[T] | None = None,
+		tools: list[dict[str, Any]] | None = None,
+		stop: list[str] | None = None,
+		**kwargs: Any,
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		DeepSeek ainvoke supports:
+		1. Regular text/multi-turn conversation
+		2. Function Calling
+		3. JSON Output (response_format)
+		4. Conversation prefix continuation (beta, prefix, stop)
+		"""
+		client = self._client()
+		ds_messages = DeepSeekMessageSerializer.serialize_messages(messages)
+		common: dict[str, Any] = {}
+
+		if self.temperature is not None:
+			common['temperature'] = self.temperature
+		if self.max_tokens is not None:
+			common['max_tokens'] = self.max_tokens
+		if self.top_p is not None:
+			common['top_p'] = self.top_p
+		if self.seed is not None:
+			common['seed'] = self.seed
+
+		# Beta conversation prefix continuation (see official documentation)
+		if self.base_url and str(self.base_url).endswith('/beta'):
+			# The last assistant message must have prefix
+			if ds_messages and isinstance(ds_messages[-1], dict) and ds_messages[-1].get('role') == 'assistant':
+				ds_messages[-1]['prefix'] = True
+			if stop:
+				common['stop'] = stop
+
+		# ① Regular multi-turn conversation/text output
+		if output_format is None and not tools:
+			try:
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=ds_messages,  # type: ignore
+					**common,
+				)
+				return ChatInvokeCompletion(
+					completion=resp.choices[0].message.content or '',
+					usage=None,
+				)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		# ② Function Calling path (with tools or output_format)
+		if tools or (output_format is not None and hasattr(output_format, 'model_json_schema')):
+			try:
+				call_tools = tools
+				tool_choice = None
+				if output_format is not None and hasattr(output_format, 'model_json_schema'):
+					tool_name = output_format.__name__
+					schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+					schema.pop('title', None)
+					call_tools = [
+						{
+							'type': 'function',
+							'function': {
+								'name': tool_name,
+								'description': f'Return a JSON object of type {tool_name}',
+								'parameters': schema,
+							},
+						}
+					]
+					tool_choice = {'type': 'function', 'function': {'name': tool_name}}
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=ds_messages,  # type: ignore
+					tools=call_tools,  # type: ignore
+					tool_choice=tool_choice,  # type: ignore
+					**common,
+				)
+				msg = resp.choices[0].message
+				if not msg.tool_calls:
+					raise ValueError('Expected tool_calls in response but got none')
+				raw_args = msg.tool_calls[0].function.arguments
+				if isinstance(raw_args, str):
+					parsed = json.loads(raw_args)
+				else:
+					parsed = raw_args
+				# --------- Fix: only use model_validate when output_format is not None ----------
+				if output_format is not None:
+					return ChatInvokeCompletion(
+						completion=output_format.model_validate(parsed),
+						usage=None,
+					)
+				else:
+					# If no output_format, return dict directly
+					return ChatInvokeCompletion(
+						completion=parsed,
+						usage=None,
+					)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		# ③ JSON Output path (official response_format)
+		if output_format is not None and hasattr(output_format, 'model_json_schema'):
+			try:
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=ds_messages,  # type: ignore
+					response_format={'type': 'json_object'},
+					**common,
+				)
+				content = resp.choices[0].message.content
+				if not content:
+					raise ModelProviderError('Empty JSON content in DeepSeek response', model=self.name)
+				parsed = output_format.model_validate_json(content)
+				return ChatInvokeCompletion(
+					completion=parsed,
+					usage=None,
+				)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		raise ModelProviderError('No valid ainvoke execution path for DeepSeek LLM', model=self.name)

browser_use/llm/deepseek/serializer.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import json
+from typing import Any, overload
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+MessageDict = dict[str, Any]
+
+
+class DeepSeekMessageSerializer:
+	"""Serializer for converting browser-use messages to DeepSeek messages."""
+
+	# -------- content 处理 --------------------------------------------------
+	@staticmethod
+	def _serialize_text_part(part: ContentPartTextParam) -> str:
+		return part.text
+
+	@staticmethod
+	def _serialize_image_part(part: ContentPartImageParam) -> dict[str, Any]:
+		url = part.image_url.url
+		if url.startswith('data:'):
+			return {'type': 'image_url', 'image_url': {'url': url}}
+		return {'type': 'image_url', 'image_url': {'url': url}}
+
+	@staticmethod
+	def _serialize_content(content: Any) -> str | list[dict[str, Any]]:
+		if content is None:
+			return ''
+		if isinstance(content, str):
+			return content
+		serialized: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				serialized.append({'type': 'text', 'text': DeepSeekMessageSerializer._serialize_text_part(part)})
+			elif part.type == 'image_url':
+				serialized.append(DeepSeekMessageSerializer._serialize_image_part(part))
+			elif part.type == 'refusal':
+				serialized.append({'type': 'text', 'text': f'[Refusal] {part.refusal}'})
+		return serialized
+
+	# -------- Tool-call 处理 -------------------------------------------------
+	@staticmethod
+	def _serialize_tool_calls(tool_calls: list[ToolCall]) -> list[dict[str, Any]]:
+		deepseek_tool_calls: list[dict[str, Any]] = []
+		for tc in tool_calls:
+			try:
+				arguments = json.loads(tc.function.arguments)
+			except json.JSONDecodeError:
+				arguments = {'arguments': tc.function.arguments}
+			deepseek_tool_calls.append(
+				{
+					'id': tc.id,
+					'type': 'function',
+					'function': {
+						'name': tc.function.name,
+						'arguments': arguments,
+					},
+				}
+			)
+		return deepseek_tool_calls
+
+	# -------- 单条消息序列化 -------------------------------------------------
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> MessageDict: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> MessageDict: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> MessageDict: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> MessageDict:
+		if isinstance(message, UserMessage):
+			return {
+				'role': 'user',
+				'content': DeepSeekMessageSerializer._serialize_content(message.content),
+			}
+		if isinstance(message, SystemMessage):
+			return {
+				'role': 'system',
+				'content': DeepSeekMessageSerializer._serialize_content(message.content),
+			}
+		if isinstance(message, AssistantMessage):
+			msg: MessageDict = {
+				'role': 'assistant',
+				'content': DeepSeekMessageSerializer._serialize_content(message.content),
+			}
+			if message.tool_calls:
+				msg['tool_calls'] = DeepSeekMessageSerializer._serialize_tool_calls(message.tool_calls)
+			return msg
+		raise ValueError(f'Unknown message type: {type(message)}')
+
+	# -------- 列表序列化 -----------------------------------------------------
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> list[MessageDict]:
+		return [DeepSeekMessageSerializer.serialize(m) for m in messages]