chore: import upstream snapshot with attribution

browser_use/agent/__init__.py

@@ -0,0 +1,32 @@
+"""Agent package exports."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+	from browser_use.agent.service import Agent
+	from browser_use.beta.service import BetaAgentError
+
+_LAZY_IMPORTS = {
+	'Agent': ('browser_use.agent.service', 'Agent'),
+	'BetaAgentError': ('browser_use.beta.service', 'BetaAgentError'),
+}
+
+
+def __getattr__(name: str):
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		from importlib import import_module
+
+		module = import_module(module_path)
+		attr = getattr(module, attr_name)
+		globals()[name] = attr
+		return attr
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'Agent',
+	'BetaAgentError',
+]

browser_use/agent/cloud_events.py

@@ -0,0 +1,284 @@
+import base64
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+import anyio
+from bubus import BaseEvent
+from pydantic import Field, field_validator
+from uuid_extensions import uuid7str
+
+MAX_STRING_LENGTH = 500000  # 100K chars ~ 25k tokens should be enough
+MAX_URL_LENGTH = 100000
+MAX_TASK_LENGTH = 100000
+MAX_COMMENT_LENGTH = 2000
+MAX_FILE_CONTENT_SIZE = 50 * 1024 * 1024  # 50MB
+
+
+class UpdateAgentTaskEvent(BaseEvent):
+	# Required fields for identification
+	id: str  # The task ID to update
+	user_id: str = Field(max_length=255)  # For authorization
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+
+	# Optional fields that can be updated
+	stopped: bool | None = None
+	paused: bool | None = None
+	done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
+	finished_at: datetime | None = None
+	agent_state: dict | None = None
+	user_feedback_type: str | None = Field(None, max_length=10)  # UserFeedbackType enum value as string
+	user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
+	gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
+
+	@classmethod
+	def from_agent(cls, agent) -> 'UpdateAgentTaskEvent':
+		"""Create an UpdateAgentTaskEvent from an Agent instance"""
+		if not hasattr(agent, '_task_start_time'):
+			raise ValueError('Agent must have _task_start_time attribute')
+
+		done_output = agent.history.final_result() if agent.history else None
+		if done_output and len(done_output) > MAX_STRING_LENGTH:
+			done_output = done_output[:MAX_STRING_LENGTH]
+		return cls(
+			id=str(agent.task_id),
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			stopped=agent.state.stopped if hasattr(agent.state, 'stopped') else False,
+			paused=agent.state.paused if hasattr(agent.state, 'paused') else False,
+			done_output=done_output,
+			finished_at=datetime.now(timezone.utc) if agent.history and agent.history.is_done() else None,
+			agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
+			user_feedback_type=None,
+			user_comment=None,
+			gif_url=None,
+			# user_feedback_type and user_comment would be set by the API/frontend
+			# gif_url would be set after GIF generation if needed
+		)
+
+
+class CreateAgentOutputFileEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	task_id: str
+	file_name: str = Field(max_length=255)
+	file_content: str | None = None  # Base64 encoded file content
+	content_type: str | None = Field(None, max_length=100)  # MIME type for file uploads
+	created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+	@field_validator('file_content')
+	@classmethod
+	def validate_file_size(cls, v: str | None) -> str | None:
+		"""Validate base64 file content size."""
+		if v is None:
+			return v
+		# Remove data URL prefix if present
+		if ',' in v:
+			v = v.split(',')[1]
+		# Estimate decoded size (base64 is ~33% larger)
+		estimated_size = len(v) * 3 / 4
+		if estimated_size > MAX_FILE_CONTENT_SIZE:
+			raise ValueError(f'File content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
+		return v
+
+	@classmethod
+	async def from_agent_and_file(cls, agent, output_path: str) -> 'CreateAgentOutputFileEvent':
+		"""Create a CreateAgentOutputFileEvent from a file path"""
+
+		gif_path = Path(output_path)
+		if not gif_path.exists():
+			raise FileNotFoundError(f'File not found: {output_path}')
+
+		gif_size = os.path.getsize(gif_path)
+
+		# Read GIF content for base64 encoding if needed
+		gif_content = None
+		if gif_size < 50 * 1024 * 1024:  # Only read if < 50MB
+			async with await anyio.open_file(gif_path, 'rb') as f:
+				gif_bytes = await f.read()
+				gif_content = base64.b64encode(gif_bytes).decode('utf-8')
+
+		return cls(
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			task_id=str(agent.task_id),
+			file_name=gif_path.name,
+			file_content=gif_content,  # Base64 encoded
+			content_type='image/gif',
+		)
+
+
+class CreateAgentStepEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)  # Added for authorization checks
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	agent_task_id: str
+	step: int
+	evaluation_previous_goal: str = Field(max_length=MAX_STRING_LENGTH)
+	memory: str = Field(max_length=MAX_STRING_LENGTH)
+	next_goal: str = Field(max_length=MAX_STRING_LENGTH)
+	actions: list[dict]
+	screenshot_url: str | None = Field(None, max_length=MAX_FILE_CONTENT_SIZE)  # ~50MB for base64 images
+	url: str = Field(default='', max_length=MAX_URL_LENGTH)
+
+	@field_validator('screenshot_url')
+	@classmethod
+	def validate_screenshot_size(cls, v: str | None) -> str | None:
+		"""Validate screenshot URL or base64 content size."""
+		if v is None or not v.startswith('data:'):
+			return v
+		# It's base64 data, check size
+		if ',' in v:
+			base64_part = v.split(',')[1]
+			estimated_size = len(base64_part) * 3 / 4
+			if estimated_size > MAX_FILE_CONTENT_SIZE:
+				raise ValueError(f'Screenshot content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
+		return v
+
+	@classmethod
+	def from_agent_step(
+		cls, agent, model_output, result: list, actions_data: list[dict], browser_state_summary
+	) -> 'CreateAgentStepEvent':
+		"""Create a CreateAgentStepEvent from agent step data"""
+		# Get first action details if available
+		first_action = model_output.action[0] if model_output.action else None
+
+		# Extract current state from model output
+		current_state = model_output.current_state if hasattr(model_output, 'current_state') else None
+
+		# Capture screenshot as base64 data URL if available
+		screenshot_url = None
+		if browser_state_summary.screenshot:
+			screenshot_url = f'data:image/png;base64,{browser_state_summary.screenshot}'
+			import logging
+
+			logger = logging.getLogger(__name__)
+			logger.debug(f'📸 Including screenshot in CreateAgentStepEvent, length: {len(browser_state_summary.screenshot)}')
+		else:
+			import logging
+
+			logger = logging.getLogger(__name__)
+			logger.debug('📸 No screenshot in browser_state_summary for CreateAgentStepEvent')
+
+		return cls(
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			agent_task_id=str(agent.task_id),
+			step=agent.state.n_steps,
+			evaluation_previous_goal=current_state.evaluation_previous_goal if current_state else '',
+			memory=current_state.memory if current_state else '',
+			next_goal=current_state.next_goal if current_state else '',
+			actions=actions_data,  # List of action dicts
+			url=browser_state_summary.url,
+			screenshot_url=screenshot_url,
+		)
+
+
+class CreateAgentTaskEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)  # Added for authorization checks
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	agent_session_id: str
+	llm_model: str = Field(max_length=200)  # LLMModel enum value as string
+	stopped: bool = False
+	paused: bool = False
+	task: str = Field(max_length=MAX_TASK_LENGTH)
+	done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
+	scheduled_task_id: str | None = None
+	started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	finished_at: datetime | None = None
+	agent_state: dict = Field(default_factory=dict)
+	user_feedback_type: str | None = Field(None, max_length=10)  # UserFeedbackType enum value as string
+	user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
+	gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
+
+	@classmethod
+	def from_agent(cls, agent) -> 'CreateAgentTaskEvent':
+		"""Create a CreateAgentTaskEvent from an Agent instance"""
+		return cls(
+			id=str(agent.task_id),
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			agent_session_id=str(agent.session_id),
+			task=agent.task,
+			llm_model=agent.llm.model_name,
+			agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
+			stopped=False,
+			paused=False,
+			done_output=None,
+			started_at=datetime.fromtimestamp(agent._task_start_time, tz=timezone.utc),
+			finished_at=None,
+			user_feedback_type=None,
+			user_comment=None,
+			gif_url=None,
+		)
+
+
+class CreateAgentSessionEvent(BaseEvent):
+	# Model fields
+	id: str = Field(default_factory=uuid7str)
+	user_id: str = Field(max_length=255)
+	device_id: str | None = Field(None, max_length=255)  # Device ID for auth lookup
+	browser_session_id: str = Field(max_length=255)
+	browser_session_live_url: str = Field(max_length=MAX_URL_LENGTH)
+	browser_session_cdp_url: str = Field(max_length=MAX_URL_LENGTH)
+	browser_session_stopped: bool = False
+	browser_session_stopped_at: datetime | None = None
+	is_source_api: bool | None = None
+	browser_state: dict = Field(default_factory=dict)
+	browser_session_data: dict | None = None
+
+	@classmethod
+	def from_agent(cls, agent) -> 'CreateAgentSessionEvent':
+		"""Create a CreateAgentSessionEvent from an Agent instance"""
+		return cls(
+			id=str(agent.session_id),
+			user_id='',  # To be filled by cloud handler
+			device_id=agent.cloud_sync.auth_client.device_id
+			if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
+			else None,
+			browser_session_id=agent.browser_session.id,
+			browser_session_live_url='',  # To be filled by cloud handler
+			browser_session_cdp_url='',  # To be filled by cloud handler
+			browser_state={
+				'viewport': agent.browser_profile.viewport if agent.browser_profile else {'width': 1280, 'height': 720},
+				'user_agent': agent.browser_profile.user_agent if agent.browser_profile else None,
+				'headless': agent.browser_profile.headless if agent.browser_profile else True,
+				'initial_url': None,  # Will be updated during execution
+				'final_url': None,  # Will be updated during execution
+				'total_pages_visited': 0,  # Will be updated during execution
+				'session_duration_seconds': 0,  # Will be updated during execution
+			},
+			browser_session_data={
+				'cookies': [],
+				'secrets': {},
+				# TODO: send secrets safely so tasks can be replayed on cloud seamlessly
+				# 'secrets': dict(agent.sensitive_data) if agent.sensitive_data else {},
+				'allowed_domains': agent.browser_profile.allowed_domains if agent.browser_profile else [],
+			},
+		)
+
+
+class UpdateAgentSessionEvent(BaseEvent):
+	"""Event to update an existing agent session"""
+
+	# Model fields
+	id: str  # Session ID to update
+	user_id: str = Field(max_length=255)
+	device_id: str | None = Field(None, max_length=255)
+	browser_session_stopped: bool | None = None
+	browser_session_stopped_at: datetime | None = None
+	end_reason: str | None = Field(None, max_length=100)  # Why the session ended

browser_use/agent/gif.py

@@ -0,0 +1,419 @@
+from __future__ import annotations
+
+import base64
+import io
+import logging
+import os
+import platform
+from typing import TYPE_CHECKING
+
+from browser_use.agent.views import AgentHistoryList
+from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
+from browser_use.config import CONFIG
+
+if TYPE_CHECKING:
+	from PIL import Image, ImageFont
+
+logger = logging.getLogger(__name__)
+
+
+def decode_unicode_escapes_to_utf8(text: str) -> str:
+	"""Handle decoding any unicode escape sequences embedded in a string (needed to render non-ASCII languages like chinese or arabic in the GIF overlay text)"""
+
+	if r'\u' not in text:
+		# doesn't have any escape sequences that need to be decoded
+		return text
+
+	try:
+		# Try to decode Unicode escape sequences
+		return text.encode('latin1').decode('unicode_escape')
+	except (UnicodeEncodeError, UnicodeDecodeError):
+		# logger.debug(f"Failed to decode unicode escape sequences while generating gif text: {text}")
+		return text
+
+
+def create_history_gif(
+	task: str,
+	history: AgentHistoryList,
+	#
+	output_path: str = 'agent_history.gif',
+	duration: int = 3000,
+	show_goals: bool = True,
+	show_task: bool = True,
+	show_logo: bool = False,
+	font_size: int = 40,
+	title_font_size: int = 56,
+	goal_font_size: int = 44,
+	margin: int = 40,
+	line_spacing: float = 1.5,
+) -> None:
+	"""Create a GIF from the agent's history with overlaid task and goal text."""
+	if not history.history:
+		logger.warning('No history to create GIF from')
+		return
+
+	from PIL import Image, ImageFont
+
+	images = []
+
+	# if history is empty, we can't create a gif
+	if not history.history:
+		logger.warning('No history to create GIF from')
+		return
+
+	# Get all screenshots from history (including None placeholders)
+	screenshots = history.screenshots(return_none_if_not_screenshot=True)
+
+	if not screenshots:
+		logger.warning('No screenshots found in history')
+		return
+
+	# Find the first non-placeholder screenshot
+	# A screenshot is considered a placeholder if:
+	# 1. It's the exact 4px placeholder for about:blank pages, OR
+	# 2. It comes from a new tab page (chrome://newtab/, about:blank, etc.)
+	first_real_screenshot = None
+	for screenshot in screenshots:
+		if screenshot and screenshot != PLACEHOLDER_4PX_SCREENSHOT:
+			first_real_screenshot = screenshot
+			break
+
+	if not first_real_screenshot:
+		logger.warning('No valid screenshots found (all are placeholders or from new tab pages)')
+		return
+
+	# Try to load nicer fonts
+	try:
+		# Try different font options in order of preference
+		# ArialUni is a font that comes with Office and can render most non-alphabet characters
+		font_options = [
+			'PingFang',
+			'STHeiti Medium',
+			'Microsoft YaHei',  # 微软雅黑
+			'SimHei',  # 黑体
+			'SimSun',  # 宋体
+			'Noto Sans CJK SC',  # 思源黑体
+			'WenQuanYi Micro Hei',  # 文泉驿微米黑
+			'Helvetica',
+			'Arial',
+			'DejaVuSans',
+			'Verdana',
+		]
+		font_loaded = False
+
+		for font_name in font_options:
+			try:
+				if platform.system() == 'Windows':
+					# Need to specify the abs font path on Windows
+					font_name = os.path.join(CONFIG.WIN_FONT_DIR, font_name + '.ttf')
+				regular_font = ImageFont.truetype(font_name, font_size)
+				title_font = ImageFont.truetype(font_name, title_font_size)
+				font_loaded = True
+				break
+			except OSError:
+				continue
+
+		if not font_loaded:
+			raise OSError('No preferred fonts found')
+
+	except OSError:
+		regular_font = ImageFont.load_default()
+		title_font = ImageFont.load_default()
+
+	# Load logo if requested
+	logo = None
+	if show_logo:
+		try:
+			logo = Image.open('./static/browser-use.png')
+			# Resize logo to be small (e.g., 40px height)
+			logo_height = 150
+			aspect_ratio = logo.width / logo.height
+			logo_width = int(logo_height * aspect_ratio)
+			logo = logo.resize((logo_width, logo_height), Image.Resampling.LANCZOS)
+		except Exception as e:
+			logger.warning(f'Could not load logo: {e}')
+
+	# Create task frame if requested
+	if show_task and task:
+		# Find the first non-placeholder screenshot for the task frame
+		first_real_screenshot = None
+		for item in history.history:
+			screenshot_b64 = item.state.get_screenshot()
+			if screenshot_b64 and screenshot_b64 != PLACEHOLDER_4PX_SCREENSHOT:
+				first_real_screenshot = screenshot_b64
+				break
+
+		if first_real_screenshot:
+			task_frame = _create_task_frame(
+				task,
+				first_real_screenshot,
+				title_font,  # type: ignore
+				regular_font,  # type: ignore
+				logo,
+				line_spacing,
+			)
+			images.append(task_frame)
+		else:
+			logger.warning('No real screenshots found for task frame, skipping task frame')
+
+	# Process each history item with its corresponding screenshot
+	for i, (item, screenshot) in enumerate(zip(history.history, screenshots), 1):
+		if not screenshot:
+			continue
+
+		# Skip placeholder screenshots from about:blank pages
+		# These are 4x4 white PNGs encoded as a specific base64 string
+		if screenshot == PLACEHOLDER_4PX_SCREENSHOT:
+			logger.debug(f'Skipping placeholder screenshot from about:blank page at step {i}')
+			continue
+
+		# Skip screenshots from new tab pages
+		from browser_use.utils import is_new_tab_page
+
+		if is_new_tab_page(item.state.url):
+			logger.debug(f'Skipping screenshot from new tab page ({item.state.url}) at step {i}')
+			continue
+
+		# Convert base64 screenshot to PIL Image
+		img_data = base64.b64decode(screenshot)
+		image = Image.open(io.BytesIO(img_data))
+
+		if show_goals and item.model_output:
+			image = _add_overlay_to_image(
+				image=image,
+				step_number=i,
+				goal_text=item.model_output.current_state.next_goal,
+				regular_font=regular_font,  # type: ignore
+				title_font=title_font,  # type: ignore
+				margin=margin,
+				logo=logo,
+			)
+
+		images.append(image)
+
+	if images:
+		# Save the GIF
+		images[0].save(
+			output_path,
+			save_all=True,
+			append_images=images[1:],
+			duration=duration,
+			loop=0,
+			optimize=False,
+		)
+		logger.info(f'Created GIF at {output_path}')
+	else:
+		logger.warning('No images found in history to create GIF')
+
+
+def _create_task_frame(
+	task: str,
+	first_screenshot: str,
+	title_font: ImageFont.FreeTypeFont,
+	regular_font: ImageFont.FreeTypeFont,
+	logo: Image.Image | None = None,
+	line_spacing: float = 1.5,
+) -> Image.Image:
+	"""Create initial frame showing the task."""
+	from PIL import Image, ImageDraw, ImageFont
+
+	img_data = base64.b64decode(first_screenshot)
+	template = Image.open(io.BytesIO(img_data))
+	image = Image.new('RGB', template.size, (0, 0, 0))
+	draw = ImageDraw.Draw(image)
+
+	# Calculate vertical center of image
+	center_y = image.height // 2
+
+	# Draw task text with dynamic font size based on task length
+	margin = 140  # Increased margin
+	max_width = image.width - (2 * margin)
+
+	# Dynamic font size calculation based on task length
+	# Start with base font size (regular + 16)
+	base_font_size = regular_font.size + 16
+	min_font_size = max(regular_font.size - 10, 16)  # Don't go below 16pt
+	# Calculate dynamic font size based on text length and complexity
+	# Longer texts get progressively smaller fonts
+	text_length = len(task)
+	if text_length > 200:
+		# For very long text, reduce font size logarithmically
+		font_size = max(base_font_size - int(10 * (text_length / 200)), min_font_size)
+	else:
+		font_size = base_font_size
+
+	# Try to create a larger font, but fall back to regular font if it fails
+	try:
+		larger_font = ImageFont.truetype(regular_font.path, font_size)  # type: ignore
+	except (OSError, AttributeError):
+		# Fall back to regular font if .path is not available or font loading fails
+		larger_font = regular_font
+
+	# Generate wrapped text with the calculated font size
+	wrapped_text = _wrap_text(task, larger_font, max_width)
+
+	# Calculate line height with spacing
+	line_height = larger_font.size * line_spacing
+
+	# Split text into lines and draw with custom spacing
+	lines = wrapped_text.split('\n')
+	total_height = line_height * len(lines)
+
+	# Start position for first line
+	text_y = center_y - (total_height / 2) + 50  # Shifted down slightly
+
+	for line in lines:
+		# Get line width for centering
+		line_bbox = draw.textbbox((0, 0), line, font=larger_font)
+		text_x = (image.width - (line_bbox[2] - line_bbox[0])) // 2
+
+		draw.text(
+			(text_x, text_y),
+			line,
+			font=larger_font,
+			fill=(255, 255, 255),
+		)
+		text_y += line_height
+
+	# Add logo if provided (top right corner)
+	if logo:
+		logo_margin = 20
+		logo_x = image.width - logo.width - logo_margin
+		image.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
+
+	return image
+
+
+def _add_overlay_to_image(
+	image: Image.Image,
+	step_number: int,
+	goal_text: str,
+	regular_font: ImageFont.FreeTypeFont,
+	title_font: ImageFont.FreeTypeFont,
+	margin: int,
+	logo: Image.Image | None = None,
+	display_step: bool = True,
+	text_color: tuple[int, int, int, int] = (255, 255, 255, 255),
+	text_box_color: tuple[int, int, int, int] = (0, 0, 0, 255),
+) -> Image.Image:
+	"""Add step number and goal overlay to an image."""
+
+	from PIL import Image, ImageDraw
+
+	goal_text = decode_unicode_escapes_to_utf8(goal_text)
+	image = image.convert('RGBA')
+	txt_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
+	draw = ImageDraw.Draw(txt_layer)
+	if display_step:
+		# Add step number (bottom left)
+		step_text = str(step_number)
+		step_bbox = draw.textbbox((0, 0), step_text, font=title_font)
+		step_width = step_bbox[2] - step_bbox[0]
+		step_height = step_bbox[3] - step_bbox[1]
+
+		# Position step number in bottom left
+		x_step = margin + 10  # Slight additional offset from edge
+		y_step = image.height - margin - step_height - 10  # Slight offset from bottom
+
+		# Draw rounded rectangle background for step number
+		padding = 20  # Increased padding
+		step_bg_bbox = (
+			x_step - padding,
+			y_step - padding,
+			x_step + step_width + padding,
+			y_step + step_height + padding,
+		)
+		draw.rounded_rectangle(
+			step_bg_bbox,
+			radius=15,  # Add rounded corners
+			fill=text_box_color,
+		)
+
+		# Draw step number
+		draw.text(
+			(x_step, y_step),
+			step_text,
+			font=title_font,
+			fill=text_color,
+		)
+
+	# Draw goal text (centered, bottom)
+	max_width = image.width - (4 * margin)
+	wrapped_goal = _wrap_text(goal_text, title_font, max_width)
+	goal_bbox = draw.multiline_textbbox((0, 0), wrapped_goal, font=title_font)
+	goal_width = goal_bbox[2] - goal_bbox[0]
+	goal_height = goal_bbox[3] - goal_bbox[1]
+
+	# Center goal text horizontally, place above step number
+	x_goal = (image.width - goal_width) // 2
+	y_goal = y_step - goal_height - padding * 4  # More space between step and goal
+
+	# Draw rounded rectangle background for goal
+	padding_goal = 25  # Increased padding for goal
+	goal_bg_bbox = (
+		x_goal - padding_goal,  # Remove extra space for logo
+		y_goal - padding_goal,
+		x_goal + goal_width + padding_goal,
+		y_goal + goal_height + padding_goal,
+	)
+	draw.rounded_rectangle(
+		goal_bg_bbox,
+		radius=15,  # Add rounded corners
+		fill=text_box_color,
+	)
+
+	# Draw goal text
+	draw.multiline_text(
+		(x_goal, y_goal),
+		wrapped_goal,
+		font=title_font,
+		fill=text_color,
+		align='center',
+	)
+
+	# Add logo if provided (top right corner)
+	if logo:
+		logo_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
+		logo_margin = 20
+		logo_x = image.width - logo.width - logo_margin
+		logo_layer.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
+		txt_layer = Image.alpha_composite(logo_layer, txt_layer)
+
+	# Composite and convert
+	result = Image.alpha_composite(image, txt_layer)
+	return result.convert('RGB')
+
+
+def _wrap_text(text: str, font: ImageFont.FreeTypeFont, max_width: int) -> str:
+	"""
+	Wrap text to fit within a given width.
+
+	Args:
+	    text: Text to wrap
+	    font: Font to use for text
+	    max_width: Maximum width in pixels
+
+	Returns:
+	    Wrapped text with newlines
+	"""
+	text = decode_unicode_escapes_to_utf8(text)
+	words = text.split()
+	lines = []
+	current_line = []
+
+	for word in words:
+		current_line.append(word)
+		line = ' '.join(current_line)
+		bbox = font.getbbox(line)
+		if bbox[2] > max_width:
+			if len(current_line) == 1:
+				lines.append(current_line.pop())
+			else:
+				current_line.pop()
+				lines.append(' '.join(current_line))
+				current_line = [word]
+
+	if current_line:
+		lines.append(' '.join(current_line))
+
+	return '\n'.join(lines)

browser_use/agent/judge.py

@@ -0,0 +1,225 @@
+"""Judge system for evaluating browser-use agent execution traces."""
+
+import base64
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal
+
+from browser_use.llm.messages import (
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	ImageURL,
+	SystemMessage,
+	UserMessage,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _encode_image(image_path: str) -> str | None:
+	"""Encode image to base64 string."""
+	try:
+		path = Path(image_path)
+		if not path.exists():
+			return None
+		with open(path, 'rb') as f:
+			return base64.b64encode(f.read()).decode('utf-8')
+	except Exception as e:
+		logger.warning(f'Failed to encode image {image_path}: {e}')
+		return None
+
+
+def _truncate_text(text: str, max_length: int, from_beginning: bool = False) -> str:
+	"""Truncate text to maximum length with eval system indicator."""
+	if len(text) <= max_length:
+		return text
+	if from_beginning:
+		return '...[text truncated]' + text[-max_length + 23 :]
+	else:
+		return text[: max_length - 23] + '...[text truncated]...'
+
+
+def construct_judge_messages(
+	task: str,
+	final_result: str,
+	agent_steps: list[str],
+	screenshot_paths: list[str],
+	max_images: int = 10,
+	ground_truth: str | None = None,
+	use_vision: bool | Literal['auto'] = True,
+) -> list[BaseMessage]:
+	"""
+	Construct messages for judge evaluation of agent trace.
+
+	Args:
+		task: The original task description
+		final_result: The final result returned to the user
+		agent_steps: List of formatted agent step descriptions
+		screenshot_paths: List of screenshot file paths
+		max_images: Maximum number of screenshots to include
+		ground_truth: Optional ground truth answer or criteria that must be satisfied for success
+
+	Returns:
+		List of messages for LLM judge evaluation
+	"""
+	task_truncated = _truncate_text(task, 40000)
+	final_result_truncated = _truncate_text(final_result, 40000)
+	steps_text = '\n'.join(agent_steps)
+	steps_text_truncated = _truncate_text(steps_text, 40000)
+
+	# Only include screenshots if use_vision is not False
+	encoded_images: list[ContentPartImageParam] = []
+	if use_vision is not False:
+		# Select last N screenshots
+		selected_screenshots = screenshot_paths[-max_images:] if len(screenshot_paths) > max_images else screenshot_paths
+
+		# Encode screenshots
+		for img_path in selected_screenshots:
+			encoded = _encode_image(img_path)
+			if encoded:
+				encoded_images.append(
+					ContentPartImageParam(
+						image_url=ImageURL(
+							url=f'data:image/png;base64,{encoded}',
+							media_type='image/png',
+						)
+					)
+				)
+
+	current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
+
+	# System prompt for judge - conditionally add ground truth section
+	ground_truth_section = ''
+	if ground_truth:
+		ground_truth_section = """
+**GROUND TRUTH VALIDATION (HIGHEST PRIORITY):**
+The <ground_truth> section contains verified correct information for this task. This can be:
+- **Evaluation criteria**: Specific conditions that must be met (e.g., "The success popup should show up", "Must extract exactly 5 items")
+- **Factual answers**: The correct answer to a question or information retrieval task (e.g. "10/11/24", "Paris")
+- **Expected outcomes**: What should happen after task completion (e.g., "Google Doc must be created", "File should be downloaded")
+
+The ground truth takes ABSOLUTE precedence over all other evaluation criteria. If the ground truth is not satisfied by the agent's execution and final response, the verdict MUST be false.
+"""
+
+	system_prompt = f"""You are an expert judge evaluating browser automation agent performance.
+
+<evaluation_framework>
+{ground_truth_section}
+**PRIMARY EVALUATION CRITERIA (in order of importance):**
+1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome.
+2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested?
+3. **Tool Effectiveness**: Did the browser interactions work as expected? Were tools used appropriately? How many % of the tools failed? 
+4. **Agent Reasoning**: Quality of decision-making, planning, and problem-solving throughout the trajectory. 
+5. **Browser Handling**: Navigation stability, error recovery, and technical execution. If the browser crashes, does not load or a captcha blocks the task, the score must be very low.
+
+**VERDICT GUIDELINES:**
+- true: Task completed as requested, human-like execution, all of the users criteria were met and the agent did not make up any information.
+- false: Task not completed, or only partially completed.
+
+**Examples of task completion verdict:**
+- If task asks for 10 items and agent finds 4 items correctly: false
+- If task completed to full user requirements but with some errors to improve in the trajectory: true
+- If task impossible due to captcha/login requirements: false
+- If the trajectory is ideal and the output is perfect: true
+- If the task asks to search all headphones in amazon under $100 but the agent searches all headphones and the lowest price is $150: false
+- If the task asks to research a property and create a google doc with the result but the agents only returns the results in text: false
+- If the task asks to complete an action on the page, and the agent reports that the action is completed but the screenshot or page shows the action is not actually complete: false
+- If the task asks to use a certain tool or site to complete the task but the agent completes the task without using it: false
+- If the task asks to look for a section of a page that does not exist: false
+- If the agent concludes the task is impossible but it is not: false
+- If the agent concludes the task is impossible and it truly is impossible: false
+- If the agent is unable to complete the task because no login information was provided and it is truly needed to complete the task: false
+
+**FAILURE CONDITIONS (automatically set verdict to false):**
+- Blocked by captcha or missing authentication 
+- Output format completely wrong or missing
+- Infinite loops or severe technical failures
+- Critical user requirements ignored
+- Page not loaded
+- Browser crashed
+- Agent could not interact with required UI elements
+- The agent moved on from a important step in the task without completing it
+- The agent made up content that is not in the screenshot or the page state
+- The agent calls done action before completing all key points of the task
+
+**IMPOSSIBLE TASK DETECTION:**
+Set `impossible_task` to true when the task fundamentally could not be completed due to:
+- Vague or ambiguous task instructions that cannot be reasonably interpreted
+- Website genuinely broken or non-functional (be conservative - temporary issues don't count)
+- Required links/pages truly inaccessible (404, 403, etc.)
+- Task requires authentication/login but no credentials were provided
+- Task asks for functionality that doesn't exist on the target site
+- Other insurmountable external obstacles beyond the agent's control
+
+Do NOT mark as impossible if:
+- Agent made poor decisions but task was achievable
+- Temporary page loading issues that could be retried
+- Agent didn't try the right approach
+- Website works but agent struggled with it
+
+**CAPTCHA DETECTION:**
+Set `reached_captcha` to true if:
+- Screenshots show captcha challenges (reCAPTCHA, hCaptcha, etc.)
+- Agent reports being blocked by bot detection
+- Error messages indicate captcha/verification requirements
+- Any evidence the agent encountered anti-bot measures during execution
+
+**IMPORTANT EVALUATION NOTES:**
+- **evaluate for action** - For each key step of the trace, double check whether the action that the agent tried to performed actually happened. If the required action did not actually occur, the verdict should be false.
+- **screenshot is not entire content** - The agent has the entire DOM content, but the screenshot is only part of the content. If the agent extracts information from the page, but you do not see it in the screenshot, you can assume this information is there.
+- **Penalize poor tool usage** - Wrong tools, inefficient approaches, ignoring available information.
+- **current date/time is {current_date}** - content with recent dates is real, not fabricated.
+- **IMPORTANT**: be very picky about the user's request - Have very high standard for the agent completing the task exactly to the user's request. 
+- **IMPORTANT**: be initially doubtful of the agent's self reported success, be sure to verify that its methods are valid and fulfill the user's desires to a tee.
+
+</evaluation_framework>
+
+<response_format>
+Respond with EXACTLY this JSON structure (no additional text before or after):
+
+{{
+	"reasoning": "Breakdown of user task into key points. Detailed analysis covering: what went well, what didn't work, trajectory quality assessment, tool usage evaluation, output quality review, and overall user satisfaction prediction.",
+	"verdict": true or false,
+	"failure_reason": "Max 5 sentences explanation of why the task was not completed successfully in case of failure. If verdict is true, use an empty string.",
+	"impossible_task": true or false,
+	"reached_captcha": true or false
+}}
+</response_format>
+"""
+
+	# Build user prompt with conditional ground truth section
+	ground_truth_prompt = ''
+	if ground_truth:
+		ground_truth_prompt = f"""
+<ground_truth>
+{ground_truth}
+</ground_truth>
+"""
+
+	user_prompt = f"""
+<task>
+{task_truncated or 'No task provided'}
+</task>
+{ground_truth_prompt}
+<agent_trajectory>
+{steps_text_truncated or 'No agent trajectory provided'}
+</agent_trajectory>
+
+<final_result>
+{final_result_truncated or 'No final result provided'}
+</final_result>
+
+{len(encoded_images)} screenshots from execution are attached.
+
+Evaluate this agent execution given the criteria and respond with the exact JSON structure requested."""
+
+	# Build messages with screenshots
+	content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=user_prompt)]
+	content_parts.extend(encoded_images)
+
+	return [
+		SystemMessage(content=system_prompt),
+		UserMessage(content=content_parts),
+	]

browser_use/agent/message_manager/service.py

@@ -0,0 +1,597 @@
+from __future__ import annotations
+
+import logging
+from typing import Literal
+
+from browser_use.agent.message_manager.views import (
+	HistoryItem,
+)
+from browser_use.agent.prompts import AgentMessagePrompt
+from browser_use.agent.views import (
+	ActionResult,
+	AgentOutput,
+	AgentStepInfo,
+	MessageCompactionSettings,
+	MessageManagerState,
+)
+from browser_use.browser.views import BrowserStateSummary
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.messages import (
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SystemMessage,
+	UserMessage,
+)
+from browser_use.observability import observe_debug
+from browser_use.utils import (
+	collect_sensitive_data_values,
+	match_url_with_domain_pattern,
+	redact_sensitive_string,
+	time_execution_sync,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ========== Logging Helper Functions ==========
+# These functions are used ONLY for formatting debug log output.
+# They do NOT affect the actual message content sent to the LLM.
+# All logging functions start with _log_ for easy identification.
+
+
+def _log_get_message_emoji(message: BaseMessage) -> str:
+	"""Get emoji for a message type - used only for logging display"""
+	emoji_map = {
+		'UserMessage': '💬',
+		'SystemMessage': '🧠',
+		'AssistantMessage': '🔨',
+	}
+	return emoji_map.get(message.__class__.__name__, '🎮')
+
+
+def _log_format_message_line(message: BaseMessage, content: str, is_last_message: bool, terminal_width: int) -> list[str]:
+	"""Format a single message for logging display"""
+	try:
+		lines = []
+
+		# Get emoji and token info
+		emoji = _log_get_message_emoji(message)
+		# token_str = str(message.metadata.tokens).rjust(4)
+		# TODO: fix the token count
+		token_str = '??? (TODO)'
+		prefix = f'{emoji}[{token_str}]: '
+
+		# Calculate available width (emoji=2 visual cols + [token]: =8 chars)
+		content_width = terminal_width - 10
+
+		# Handle last message wrapping
+		if is_last_message and len(content) > content_width:
+			# Find a good break point
+			break_point = content.rfind(' ', 0, content_width)
+			if break_point > content_width * 0.7:  # Keep at least 70% of line
+				first_line = content[:break_point]
+				rest = content[break_point + 1 :]
+			else:
+				# No good break point, just truncate
+				first_line = content[:content_width]
+				rest = content[content_width:]
+
+			lines.append(prefix + first_line)
+
+			# Second line with 10-space indent
+			if rest:
+				if len(rest) > terminal_width - 10:
+					rest = rest[: terminal_width - 10]
+				lines.append(' ' * 10 + rest)
+		else:
+			# Single line - truncate if needed
+			if len(content) > content_width:
+				content = content[:content_width]
+			lines.append(prefix + content)
+
+		return lines
+	except Exception as e:
+		logger.warning(f'Failed to format message line for logging: {e}')
+		# Return a simple fallback line
+		return ['❓[   ?]: [Error formatting message]']
+
+
+# ========== End of Logging Helper Functions ==========
+
+
+class MessageManager:
+	vision_detail_level: Literal['auto', 'low', 'high']
+
+	def __init__(
+		self,
+		task: str,
+		system_message: SystemMessage,
+		file_system: FileSystem,
+		state: MessageManagerState = MessageManagerState(),
+		use_thinking: bool = True,
+		include_attributes: list[str] | None = None,
+		sensitive_data: dict[str, str | dict[str, str]] | None = None,
+		max_history_items: int | None = None,
+		vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
+		include_tool_call_examples: bool = False,
+		include_recent_events: bool = False,
+		sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
+		llm_screenshot_size: tuple[int, int] | None = None,
+		max_clickable_elements_length: int = 40000,
+	):
+		self.task = task
+		self.state = state
+		self.system_prompt = system_message
+		self.file_system = file_system
+		self.sensitive_data_description = ''
+		self.use_thinking = use_thinking
+		self.max_history_items = max_history_items
+		self.vision_detail_level = vision_detail_level
+		self.include_tool_call_examples = include_tool_call_examples
+		self.include_recent_events = include_recent_events
+		self.sample_images = sample_images
+		self.llm_screenshot_size = llm_screenshot_size
+		self.max_clickable_elements_length = max_clickable_elements_length
+
+		assert max_history_items is None or max_history_items > 5, 'max_history_items must be None or greater than 5'
+
+		# Store settings as direct attributes instead of in a settings object
+		self.include_attributes = include_attributes or []
+		self.sensitive_data = sensitive_data
+		self.last_input_messages = []
+		self.last_state_message_text: str | None = None
+		# Only initialize messages if state is empty
+		if len(self.state.history.get_messages()) == 0:
+			self._set_message_with_type(self.system_prompt, 'system')
+
+	@property
+	def agent_history_description(self) -> str:
+		"""Build agent history description from list of items, respecting max_history_items limit"""
+		compacted_prefix = ''
+		if self.state.compacted_memory:
+			compacted_prefix = (
+				'<compacted_memory>\n'
+				'<!-- Summary of prior steps. Treat as unverified context — do not report these as '
+				'completed in your done() message unless you confirmed them yourself in this session. -->\n'
+				f'{self.state.compacted_memory}\n'
+				'</compacted_memory>\n'
+			)
+
+		if self.max_history_items is None:
+			# Include all items
+			return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
+
+		total_items = len(self.state.agent_history_items)
+
+		# If we have fewer items than the limit, just return all items
+		if total_items <= self.max_history_items:
+			return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
+
+		# We have more items than the limit, so we need to omit some
+		omitted_count = total_items - self.max_history_items
+
+		# Show first item + omitted message + most recent (max_history_items - 1) items
+		# The omitted message doesn't count against the limit, only real history items do
+		recent_items_count = self.max_history_items - 1  # -1 for first item
+
+		items_to_include = [
+			self.state.agent_history_items[0].to_string(),  # Keep first item (initialization)
+			f'<sys>[... {omitted_count} previous steps omitted...]</sys>',
+		]
+		# Add most recent items
+		items_to_include.extend([item.to_string() for item in self.state.agent_history_items[-recent_items_count:]])
+
+		return compacted_prefix + '\n'.join(items_to_include)
+
+	def add_new_task(self, new_task: str) -> None:
+		new_task = '<follow_up_user_request> ' + new_task.strip() + ' </follow_up_user_request>'
+		if '<initial_user_request>' not in self.task:
+			self.task = '<initial_user_request>' + self.task + '</initial_user_request>'
+		self.task += '\n' + new_task
+		task_update_item = HistoryItem(system_message=new_task)
+		self.state.agent_history_items.append(task_update_item)
+
+	def prepare_step_state(
+		self,
+		browser_state_summary: BrowserStateSummary,
+		model_output: AgentOutput | None = None,
+		result: list[ActionResult] | None = None,
+		step_info: AgentStepInfo | None = None,
+		sensitive_data=None,
+	) -> None:
+		"""Prepare state for the next LLM call without building the final state message."""
+		self.state.history.context_messages.clear()
+		self._update_agent_history_description(model_output, result, step_info)
+
+		effective_sensitive_data = sensitive_data if sensitive_data is not None else self.sensitive_data
+		if effective_sensitive_data is not None:
+			self.sensitive_data = effective_sensitive_data
+			self.sensitive_data_description = self._get_sensitive_data_description(browser_state_summary.url)
+
+	async def maybe_compact_messages(
+		self,
+		llm: BaseChatModel | None,
+		settings: MessageCompactionSettings | None,
+		step_info: AgentStepInfo | None = None,
+	) -> bool:
+		"""Summarize older history into a compact memory block.
+
+		Step interval is the primary trigger; char count is a minimum floor.
+		"""
+		if not settings or not settings.enabled:
+			return False
+		if llm is None:
+			return False
+		if step_info is None:
+			return False
+
+		# Step cadence gate
+		steps_since = step_info.step_number - (self.state.last_compaction_step or 0)
+		if steps_since < settings.compact_every_n_steps:
+			return False
+
+		# Char floor gate
+		history_items = self.state.agent_history_items
+		full_history_text = '\n'.join(item.to_string() for item in history_items).strip()
+		trigger_char_count = settings.trigger_char_count or 40000
+		if len(full_history_text) < trigger_char_count:
+			return False
+
+		logger.debug(f'Compacting message history (items={len(history_items)}, chars={len(full_history_text)})')
+
+		# Build compaction input
+		compaction_sections = []
+		if self.state.compacted_memory:
+			compaction_sections.append(
+				f'<previous_compacted_memory>\n{self.state.compacted_memory}\n</previous_compacted_memory>'
+			)
+		compaction_sections.append(f'<agent_history>\n{full_history_text}\n</agent_history>')
+		if settings.include_read_state and self.state.read_state_description:
+			compaction_sections.append(f'<read_state>\n{self.state.read_state_description}\n</read_state>')
+		compaction_input = '\n\n'.join(compaction_sections)
+
+		if self.sensitive_data:
+			filtered = self._filter_sensitive_data(UserMessage(content=compaction_input))
+			compaction_input = filtered.text
+
+		system_prompt = (
+			'You are summarizing an agent run for prompt compaction.\n'
+			'Capture task requirements, key facts, decisions, partial progress, errors, and next steps.\n'
+			'Preserve important entities, values, URLs, and file paths.\n'
+			'CRITICAL: Only mark a step as completed if you see explicit success confirmation in the history. '
+			'If a step was started but not explicitly confirmed complete, mark it as "IN-PROGRESS". '
+			'Never infer completion from context — only report what was confirmed.\n'
+			'Return plain text only. Do not include tool calls or JSON.'
+		)
+		if settings.summary_max_chars:
+			system_prompt += f' Keep under {settings.summary_max_chars} characters if possible.'
+
+		messages = [SystemMessage(content=system_prompt), UserMessage(content=compaction_input)]
+		try:
+			response = await llm.ainvoke(messages)
+			summary = (response.completion or '').strip()
+		except Exception as e:
+			logger.warning(f'Failed to compact messages: {e}')
+			return False
+
+		if not summary:
+			return False
+
+		if settings.summary_max_chars and len(summary) > settings.summary_max_chars:
+			summary = summary[: settings.summary_max_chars].rstrip() + '…'
+
+		self.state.compacted_memory = summary
+		self.state.compaction_count += 1
+		self.state.last_compaction_step = step_info.step_number
+
+		# Keep first item + most recent items
+		keep_last = max(0, settings.keep_last_items)
+		if len(history_items) > keep_last + 1:
+			if keep_last == 0:
+				self.state.agent_history_items = [history_items[0]]
+			else:
+				self.state.agent_history_items = [history_items[0]] + history_items[-keep_last:]
+
+		logger.debug(f'Compaction complete (summary_chars={len(summary)}, history_items={len(self.state.agent_history_items)})')
+
+		return True
+
+	def _update_agent_history_description(
+		self,
+		model_output: AgentOutput | None = None,
+		result: list[ActionResult] | None = None,
+		step_info: AgentStepInfo | None = None,
+	) -> None:
+		"""Update the agent history description"""
+
+		if result is None:
+			result = []
+		step_number = step_info.step_number if step_info else None
+
+		self.state.read_state_description = ''
+		self.state.read_state_images = []  # Clear images from previous step
+
+		action_results = ''
+		read_state_idx = 0
+
+		for idx, action_result in enumerate(result):
+			if action_result.include_extracted_content_only_once and action_result.extracted_content:
+				self.state.read_state_description += (
+					f'<read_state_{read_state_idx}>\n{action_result.extracted_content}\n</read_state_{read_state_idx}>\n'
+				)
+				read_state_idx += 1
+				logger.debug(f'Added extracted_content to read_state_description: {action_result.extracted_content}')
+
+			# Store images for one-time inclusion in the next message
+			if action_result.images:
+				self.state.read_state_images.extend(action_result.images)
+				logger.debug(f'Added {len(action_result.images)} image(s) to read_state_images')
+
+			if action_result.long_term_memory:
+				action_results += f'{action_result.long_term_memory}\n'
+				logger.debug(f'Added long_term_memory to action_results: {action_result.long_term_memory}')
+			elif action_result.extracted_content and not action_result.include_extracted_content_only_once:
+				action_results += f'{action_result.extracted_content}\n'
+				logger.debug(f'Added extracted_content to action_results: {action_result.extracted_content}')
+
+			if action_result.error:
+				if len(action_result.error) > 200:
+					error_text = action_result.error[:100] + '......' + action_result.error[-100:]
+				else:
+					error_text = action_result.error
+				action_results += f'{error_text}\n'
+				logger.debug(f'Added error to action_results: {error_text}')
+
+		# Simple 60k character limit for read_state_description
+		MAX_CONTENT_SIZE = 60000
+		if len(self.state.read_state_description) > MAX_CONTENT_SIZE:
+			self.state.read_state_description = (
+				self.state.read_state_description[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
+			)
+			logger.debug(f'Truncated read_state_description to {MAX_CONTENT_SIZE} characters')
+
+		self.state.read_state_description = self.state.read_state_description.strip('\n')
+
+		if action_results:
+			action_results = f'Result\n{action_results}'
+		action_results = action_results.strip('\n') if action_results else None
+
+		# Simple 60k character limit for action_results
+		if action_results and len(action_results) > MAX_CONTENT_SIZE:
+			action_results = action_results[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
+			logger.debug(f'Truncated action_results to {MAX_CONTENT_SIZE} characters')
+
+		# Build the history item
+		if model_output is None:
+			# Add history item for initial actions (step 0) or errors (step > 0)
+			if step_number is not None:
+				if step_number == 0 and action_results:
+					# Step 0 with initial action results
+					history_item = HistoryItem(step_number=step_number, action_results=action_results)
+					self.state.agent_history_items.append(history_item)
+				elif step_number > 0:
+					# Error case for steps > 0
+					history_item = HistoryItem(step_number=step_number, error='Agent failed to output in the right format.')
+					self.state.agent_history_items.append(history_item)
+		else:
+			history_item = HistoryItem(
+				step_number=step_number,
+				evaluation_previous_goal=model_output.current_state.evaluation_previous_goal,
+				memory=model_output.current_state.memory,
+				next_goal=model_output.current_state.next_goal,
+				action_results=action_results,
+			)
+			self.state.agent_history_items.append(history_item)
+
+	def _get_sensitive_data_description(self, current_page_url) -> str:
+		sensitive_data = self.sensitive_data
+		if not sensitive_data:
+			return ''
+
+		# Collect placeholders for sensitive data
+		placeholders: set[str] = set()
+
+		for key, value in sensitive_data.items():
+			if isinstance(value, dict):
+				# New format: {domain: {key: value}}
+				if current_page_url and match_url_with_domain_pattern(current_page_url, key, True):
+					placeholders.update(value.keys())
+			else:
+				# Old format: {key: value}
+				placeholders.add(key)
+
+		if placeholders:
+			placeholder_list = sorted(list(placeholders))
+			# Format as bullet points for clarity
+			formatted_placeholders = '\n'.join(f'  - {p}' for p in placeholder_list)
+
+			info = 'SENSITIVE DATA - Use these placeholders for secure input:\n'
+			info += f'{formatted_placeholders}\n\n'
+			info += 'IMPORTANT: When entering sensitive values, you MUST wrap the placeholder name in <secret> tags.\n'
+			info += f'Example: To enter the value for "{placeholder_list[0]}", use: <secret>{placeholder_list[0]}</secret>\n'
+			info += 'The system will automatically replace these tags with the actual secret values.'
+			return info
+
+		return ''
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='create_state_messages')
+	@time_execution_sync('--create_state_messages')
+	def create_state_messages(
+		self,
+		browser_state_summary: BrowserStateSummary,
+		model_output: AgentOutput | None = None,
+		result: list[ActionResult] | None = None,
+		step_info: AgentStepInfo | None = None,
+		use_vision: bool | Literal['auto'] = True,
+		page_filtered_actions: str | None = None,
+		sensitive_data=None,
+		available_file_paths: list[str] | None = None,  # Always pass current available_file_paths
+		unavailable_skills_info: str | None = None,  # Information about skills that cannot be used yet
+		plan_description: str | None = None,  # Rendered plan for injection into agent state
+		skip_state_update: bool = False,
+	) -> None:
+		"""Create single state message with all content"""
+
+		if not skip_state_update:
+			self.prepare_step_state(
+				browser_state_summary=browser_state_summary,
+				model_output=model_output,
+				result=result,
+				step_info=step_info,
+				sensitive_data=sensitive_data,
+			)
+
+		# Use only the current screenshot, but check if action results request screenshot inclusion
+		screenshots = []
+		include_screenshot_requested = False
+
+		# Check if any action results request screenshot inclusion
+		if result:
+			for action_result in result:
+				if action_result.metadata and action_result.metadata.get('include_screenshot'):
+					include_screenshot_requested = True
+					logger.debug('Screenshot inclusion requested by action result')
+					break
+
+		# Handle different use_vision modes:
+		# - "auto": Only include screenshot if explicitly requested by action (e.g., screenshot)
+		# - True: Always include screenshot
+		# - False: Never include screenshot
+		include_screenshot = False
+		if use_vision is True:
+			# Always include screenshot when use_vision=True
+			include_screenshot = True
+		elif use_vision == 'auto':
+			# Only include screenshot if explicitly requested by action when use_vision="auto"
+			include_screenshot = include_screenshot_requested
+		# else: use_vision is False, never include screenshot (include_screenshot stays False)
+
+		if include_screenshot and browser_state_summary.screenshot:
+			screenshots.append(browser_state_summary.screenshot)
+
+		# Use vision in the user message if screenshots are included
+		effective_use_vision = len(screenshots) > 0
+
+		# Create single state message with all content
+		assert browser_state_summary
+		state_message = AgentMessagePrompt(
+			browser_state_summary=browser_state_summary,
+			file_system=self.file_system,
+			agent_history_description=self.agent_history_description,
+			read_state_description=self.state.read_state_description,
+			task=self.task,
+			include_attributes=self.include_attributes,
+			step_info=step_info,
+			page_filtered_actions=page_filtered_actions,
+			max_clickable_elements_length=self.max_clickable_elements_length,
+			sensitive_data=self.sensitive_data_description,
+			available_file_paths=available_file_paths,
+			screenshots=screenshots,
+			vision_detail_level=self.vision_detail_level,
+			include_recent_events=self.include_recent_events,
+			sample_images=self.sample_images,
+			read_state_images=self.state.read_state_images,
+			llm_screenshot_size=self.llm_screenshot_size,
+			unavailable_skills_info=unavailable_skills_info,
+			plan_description=plan_description,
+		).get_user_message(effective_use_vision)
+
+		# Store state message text for history
+		self.last_state_message_text = state_message.text
+
+		# Set the state message with caching enabled
+		self._set_message_with_type(state_message, 'state')
+
+	def _log_history_lines(self) -> str:
+		"""Generate a formatted log string of message history for debugging / printing to terminal"""
+		# TODO: fix logging
+
+		# try:
+		# 	total_input_tokens = 0
+		# 	message_lines = []
+		# 	terminal_width = shutil.get_terminal_size((80, 20)).columns
+
+		# 	for i, m in enumerate(self.state.history.messages):
+		# 		try:
+		# 			total_input_tokens += m.metadata.tokens
+		# 			is_last_message = i == len(self.state.history.messages) - 1
+
+		# 			# Extract content for logging
+		# 			content = _log_extract_message_content(m.message, is_last_message, m.metadata)
+
+		# 			# Format the message line(s)
+		# 			lines = _log_format_message_line(m, content, is_last_message, terminal_width)
+		# 			message_lines.extend(lines)
+		# 		except Exception as e:
+		# 			logger.warning(f'Failed to format message {i} for logging: {e}')
+		# 			# Add a fallback line for this message
+		# 			message_lines.append('❓[   ?]: [Error formatting this message]')
+
+		# 	# Build final log message
+		# 	return (
+		# 		f'📜 LLM Message history ({len(self.state.history.messages)} messages, {total_input_tokens} tokens):\n'
+		# 		+ '\n'.join(message_lines)
+		# 	)
+		# except Exception as e:
+		# 	logger.warning(f'Failed to generate history log: {e}')
+		# 	# Return a minimal fallback message
+		# 	return f'📜 LLM Message history (error generating log: {e})'
+
+		return ''
+
+	@time_execution_sync('--get_messages')
+	def get_messages(self) -> list[BaseMessage]:
+		"""Get current message list, potentially trimmed to max tokens"""
+
+		# Log message history for debugging
+		logger.debug(self._log_history_lines())
+		self.last_input_messages = self.state.history.get_messages()
+		return self.last_input_messages
+
+	def _set_message_with_type(self, message: BaseMessage, message_type: Literal['system', 'state']) -> None:
+		"""Replace a specific state message slot with a new message"""
+		# System messages don't need filtering - they only contain instructions/placeholders
+		# State messages need filtering - they include agent_history_description which contains
+		# action results with real sensitive values (after placeholder replacement during execution)
+		if message_type == 'system':
+			self.state.history.system_message = message
+		elif message_type == 'state':
+			if self.sensitive_data:
+				message = self._filter_sensitive_data(message)
+			self.state.history.state_message = message
+		else:
+			raise ValueError(f'Invalid state message type: {message_type}')
+
+	def _add_context_message(self, message: BaseMessage) -> None:
+		"""Add a contextual message specific to this step (e.g., validation errors, retry instructions, timeout warnings)"""
+		# Context messages typically contain error messages and validation info, not action results
+		# with sensitive data, so filtering is not needed here
+		self.state.history.context_messages.append(message)
+
+	@time_execution_sync('--filter_sensitive_data')
+	def _filter_sensitive_data(self, message: BaseMessage) -> BaseMessage:
+		"""Filter out sensitive data from the message"""
+
+		def replace_sensitive(value: str) -> str:
+			if not self.sensitive_data:
+				return value
+
+			sensitive_values = collect_sensitive_data_values(self.sensitive_data)
+
+			# If there are no valid sensitive data entries, just return the original value
+			if not sensitive_values:
+				logger.warning('No valid entries found in sensitive_data dictionary')
+				return value
+
+			return redact_sensitive_string(value, sensitive_values)
+
+		if isinstance(message.content, str):
+			message.content = replace_sensitive(message.content)
+		elif isinstance(message.content, list):
+			for i, item in enumerate(message.content):
+				if isinstance(item, ContentPartTextParam):
+					item.text = replace_sensitive(item.text)
+					message.content[i] = item
+		return message

browser_use/agent/message_manager/utils.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+import anyio
+
+from browser_use.llm.messages import BaseMessage
+
+logger = logging.getLogger(__name__)
+
+
+async def save_conversation(
+	input_messages: list[BaseMessage],
+	response: Any,
+	target: str | Path,
+	encoding: str | None = None,
+) -> None:
+	"""Save conversation history to file asynchronously."""
+	target_path = Path(target)
+	# create folders if not exists
+	if target_path.parent:
+		await anyio.Path(target_path.parent).mkdir(parents=True, exist_ok=True)
+
+	await anyio.Path(target_path).write_text(
+		await _format_conversation(input_messages, response),
+		encoding=encoding or 'utf-8',
+	)
+
+
+async def _format_conversation(messages: list[BaseMessage], response: Any) -> str:
+	"""Format the conversation including messages and response."""
+	lines = []
+
+	# Format messages
+	for message in messages:
+		lines.append(f' {message.role} ')
+
+		lines.append(message.text)
+		lines.append('')  # Empty line after each message
+
+	# Format response
+	lines.append(json.dumps(json.loads(response.model_dump_json(exclude_unset=True)), indent=2, ensure_ascii=False))
+
+	return '\n'.join(lines)
+
+
+# Note: _write_messages_to_file and _write_response_to_file have been merged into _format_conversation
+# This is more efficient for async operations and reduces file I/O

browser_use/agent/message_manager/views.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from browser_use.llm.messages import (
+	BaseMessage,
+)
+
+if TYPE_CHECKING:
+	pass
+
+
+class HistoryItem(BaseModel):
+	"""Represents a single agent history item with its data and string representation"""
+
+	step_number: int | None = None
+	evaluation_previous_goal: str | None = None
+	memory: str | None = None
+	next_goal: str | None = None
+	action_results: str | None = None
+	error: str | None = None
+	system_message: str | None = None
+
+	model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
+
+	def model_post_init(self, __context) -> None:
+		"""Validate that error and system_message are not both provided"""
+		if self.error is not None and self.system_message is not None:
+			raise ValueError('Cannot have both error and system_message at the same time')
+
+	def to_string(self) -> str:
+		"""Get string representation of the history item"""
+		step_str = 'step' if self.step_number is not None else 'step_unknown'
+
+		if self.error:
+			return f"""<{step_str}>
+{self.error}"""
+		elif self.system_message:
+			return self.system_message
+		else:
+			content_parts = []
+
+			# Only include evaluation_previous_goal if it's not None/empty
+			if self.evaluation_previous_goal:
+				content_parts.append(f'{self.evaluation_previous_goal}')
+
+			# Always include memory
+			if self.memory:
+				content_parts.append(f'{self.memory}')
+
+			# Only include next_goal if it's not None/empty
+			if self.next_goal:
+				content_parts.append(f'{self.next_goal}')
+
+			if self.action_results:
+				content_parts.append(self.action_results)
+
+			content = '\n'.join(content_parts)
+
+			return f"""<{step_str}>
+{content}"""
+
+
+class MessageHistory(BaseModel):
+	"""History of messages"""
+
+	system_message: BaseMessage | None = None
+	state_message: BaseMessage | None = None
+	context_messages: list[BaseMessage] = Field(default_factory=list)
+	model_config = ConfigDict(arbitrary_types_allowed=True)
+
+	def get_messages(self) -> list[BaseMessage]:
+		"""Get all messages in the correct order: system -> state -> contextual"""
+		messages = []
+		if self.system_message:
+			messages.append(self.system_message)
+		if self.state_message:
+			messages.append(self.state_message)
+		messages.extend(self.context_messages)
+
+		return messages
+
+
+class MessageManagerState(BaseModel):
+	"""Holds the state for MessageManager"""
+
+	history: MessageHistory = Field(default_factory=MessageHistory)
+	tool_id: int = 1
+	agent_history_items: list[HistoryItem] = Field(
+		default_factory=lambda: [HistoryItem(step_number=0, system_message='Agent initialized')]
+	)
+	read_state_description: str = ''
+	# Images to include in the next state message (cleared after each step)
+	read_state_images: list[dict[str, Any]] = Field(default_factory=list)
+	compacted_memory: str | None = None
+	compaction_count: int = 0
+	last_compaction_step: int | None = None
+
+	model_config = ConfigDict(arbitrary_types_allowed=True)

browser_use/agent/prompts.py

@@ -0,0 +1,588 @@
+import importlib.resources
+from datetime import datetime
+from typing import TYPE_CHECKING, Literal, Optional
+
+from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
+from browser_use.dom.views import NodeType, SimplifiedNode
+from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage
+from browser_use.observability import observe_debug
+from browser_use.utils import is_new_tab_page, sanitize_surrogates
+
+if TYPE_CHECKING:
+	from browser_use.agent.views import AgentStepInfo
+	from browser_use.browser.views import BrowserStateSummary
+	from browser_use.filesystem.file_system import FileSystem
+
+
+def _is_anthropic_4_5_model(model_name: str | None) -> bool:
+	"""Check if the model is Claude Opus 4.5 or Haiku 4.5 (requires 4096+ token prompts for caching)."""
+	if not model_name:
+		return False
+	model_lower = model_name.lower()
+	# Check for Opus 4.5 or Haiku 4.5 variants
+	is_opus_4_5 = 'opus' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
+	is_haiku_4_5 = 'haiku' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
+	return is_opus_4_5 or is_haiku_4_5
+
+
+class SystemPrompt:
+	def __init__(
+		self,
+		max_actions_per_step: int = 3,
+		override_system_message: str | None = None,
+		extend_system_message: str | None = None,
+		use_thinking: bool = True,
+		flash_mode: bool = False,
+		is_anthropic: bool = False,
+		is_browser_use_model: bool = False,
+		model_name: str | None = None,
+	):
+		self.max_actions_per_step = max_actions_per_step
+		self.use_thinking = use_thinking
+		self.flash_mode = flash_mode
+		self.is_anthropic = is_anthropic
+		self.is_browser_use_model = is_browser_use_model
+		self.model_name = model_name
+		# Check if this is an Anthropic 4.5 model that needs longer prompts for caching
+		self.is_anthropic_4_5 = _is_anthropic_4_5_model(model_name)
+		prompt = ''
+		if override_system_message is not None:
+			prompt = override_system_message
+		else:
+			self._load_prompt_template()
+			prompt = self.prompt_template.format(max_actions=self.max_actions_per_step)
+
+		if extend_system_message:
+			prompt += f'\n{extend_system_message}'
+
+		self.system_message = SystemMessage(content=prompt, cache=True)
+
+	def _load_prompt_template(self) -> None:
+		"""Load the prompt template from the markdown file."""
+		try:
+			# Choose the appropriate template based on model type and mode
+			# Browser-use models use simplified prompts optimized for fine-tuned models
+			if self.is_browser_use_model:
+				if self.flash_mode:
+					template_filename = 'system_prompt_browser_use_flash.md'
+				elif self.use_thinking:
+					template_filename = 'system_prompt_browser_use.md'
+				else:
+					template_filename = 'system_prompt_browser_use_no_thinking.md'
+			# Anthropic 4.5 models (Opus 4.5, Haiku 4.5) need 4096+ token prompts for caching
+			elif self.is_anthropic_4_5 and self.flash_mode:
+				template_filename = 'system_prompt_anthropic_flash.md'
+			elif self.flash_mode and self.is_anthropic:
+				template_filename = 'system_prompt_flash_anthropic.md'
+			elif self.flash_mode:
+				template_filename = 'system_prompt_flash.md'
+			elif self.use_thinking:
+				template_filename = 'system_prompt.md'
+			else:
+				template_filename = 'system_prompt_no_thinking.md'
+
+			# This works both in development and when installed as a package
+			with (
+				importlib.resources.files('browser_use.agent.system_prompts')
+				.joinpath(template_filename)
+				.open('r', encoding='utf-8') as f
+			):
+				self.prompt_template = f.read()
+		except Exception as e:
+			raise RuntimeError(f'Failed to load system prompt template: {e}')
+
+	def get_system_message(self) -> SystemMessage:
+		"""
+		Get the system prompt for the agent.
+
+		Returns:
+		    SystemMessage: Formatted system prompt
+		"""
+		return self.system_message
+
+
+class AgentMessagePrompt:
+	vision_detail_level: Literal['auto', 'low', 'high']
+
+	def __init__(
+		self,
+		browser_state_summary: 'BrowserStateSummary',
+		file_system: 'FileSystem',
+		agent_history_description: str | None = None,
+		read_state_description: str | None = None,
+		task: str | None = None,
+		include_attributes: list[str] | None = None,
+		step_info: Optional['AgentStepInfo'] = None,
+		page_filtered_actions: str | None = None,
+		max_clickable_elements_length: int = 40000,
+		sensitive_data: str | None = None,
+		available_file_paths: list[str] | None = None,
+		screenshots: list[str] | None = None,
+		vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
+		include_recent_events: bool = False,
+		sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
+		read_state_images: list[dict] | None = None,
+		llm_screenshot_size: tuple[int, int] | None = None,
+		unavailable_skills_info: str | None = None,
+		plan_description: str | None = None,
+	):
+		self.browser_state: 'BrowserStateSummary' = browser_state_summary
+		self.file_system: 'FileSystem | None' = file_system
+		self.agent_history_description: str | None = agent_history_description
+		self.read_state_description: str | None = read_state_description
+		self.task: str | None = task
+		self.include_attributes = include_attributes
+		self.step_info = step_info
+		self.page_filtered_actions: str | None = page_filtered_actions
+		self.max_clickable_elements_length: int = max_clickable_elements_length
+		self.sensitive_data: str | None = sensitive_data
+		self.available_file_paths: list[str] | None = available_file_paths
+		self.screenshots = screenshots or []
+		self.vision_detail_level = vision_detail_level
+		self.include_recent_events = include_recent_events
+		self.sample_images = sample_images or []
+		self.read_state_images = read_state_images or []
+		self.unavailable_skills_info: str | None = unavailable_skills_info
+		self.plan_description: str | None = plan_description
+		self.llm_screenshot_size = llm_screenshot_size
+		assert self.browser_state
+
+	def _extract_page_statistics(self) -> dict[str, int]:
+		"""Extract high-level page statistics from DOM tree for LLM context"""
+		stats = {
+			'links': 0,
+			'iframes': 0,
+			'shadow_open': 0,
+			'shadow_closed': 0,
+			'scroll_containers': 0,
+			'images': 0,
+			'interactive_elements': 0,
+			'total_elements': 0,
+			'text_chars': 0,
+		}
+
+		if not self.browser_state.dom_state or not self.browser_state.dom_state._root:
+			return stats
+
+		def traverse_node(node: SimplifiedNode) -> None:
+			"""Recursively traverse simplified DOM tree to count elements"""
+			if not node or not node.original_node:
+				return
+
+			original = node.original_node
+			stats['total_elements'] += 1
+
+			# Count by node type and tag
+			if original.node_type == NodeType.ELEMENT_NODE:
+				tag = original.tag_name.lower() if original.tag_name else ''
+
+				if tag == 'a':
+					stats['links'] += 1
+				elif tag in ('iframe', 'frame'):
+					stats['iframes'] += 1
+				elif tag == 'img':
+					stats['images'] += 1
+
+				# Check if scrollable
+				if original.is_actually_scrollable:
+					stats['scroll_containers'] += 1
+
+				# Check if interactive
+				if node.is_interactive:
+					stats['interactive_elements'] += 1
+
+				# Check if this element hosts shadow DOM
+				if node.is_shadow_host:
+					# Check if any shadow children are closed
+					has_closed_shadow = any(
+						child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE
+						and child.original_node.shadow_root_type
+						and child.original_node.shadow_root_type.lower() == 'closed'
+						for child in node.children
+					)
+					if has_closed_shadow:
+						stats['shadow_closed'] += 1
+					else:
+						stats['shadow_open'] += 1
+
+			elif original.node_type == NodeType.TEXT_NODE:
+				stats['text_chars'] += len(original.node_value.strip())
+
+			elif original.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+				# Shadow DOM fragment - these are the actual shadow roots
+				# But don't double-count since we count them at the host level above
+				pass
+
+			# Traverse children
+			for child in node.children:
+				traverse_node(child)
+
+		traverse_node(self.browser_state.dom_state._root)
+		return stats
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')
+	def _get_browser_state_description(self) -> str:
+		# Extract page statistics first
+		page_stats = self._extract_page_statistics()
+
+		# Format statistics
+		stats_text = '<page_stats>'
+		if page_stats['total_elements'] < 10:
+			stats_text += 'Page appears empty (SPA not loaded?) - '
+		# Skeleton screen: many elements but almost no text = loading placeholders
+		elif page_stats['total_elements'] > 20 and page_stats['text_chars'] < page_stats['total_elements'] * 5:
+			stats_text += 'Page appears to show skeleton/placeholder content (still loading?) - '
+		stats_text += f'{page_stats["links"]} links, {page_stats["interactive_elements"]} interactive, '
+		stats_text += f'{page_stats["iframes"]} iframes'
+		if page_stats['shadow_open'] > 0 or page_stats['shadow_closed'] > 0:
+			stats_text += f', {page_stats["shadow_open"]} shadow(open), {page_stats["shadow_closed"]} shadow(closed)'
+		if page_stats['images'] > 0:
+			stats_text += f', {page_stats["images"]} images'
+		stats_text += f', {page_stats["total_elements"]} total elements'
+		stats_text += '</page_stats>\n'
+
+		elements_text = self.browser_state.dom_state.llm_representation(include_attributes=self.include_attributes)
+
+		if len(elements_text) > self.max_clickable_elements_length:
+			elements_text = elements_text[: self.max_clickable_elements_length]
+			truncated_text = f' (truncated to {self.max_clickable_elements_length} characters)'
+		else:
+			truncated_text = ''
+
+		has_content_above = False
+		has_content_below = False
+		# Enhanced page information for the model
+		page_info_text = ''
+		if self.browser_state.page_info:
+			pi = self.browser_state.page_info
+			# Compute page statistics dynamically
+			pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
+			pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
+			has_content_above = pages_above > 0
+			has_content_below = pages_below > 0
+			page_info_text = '<page_info>'
+			page_info_text += f'{pages_above:.1f} pages above, {pages_below:.1f} pages below'
+			if pages_below > 0.2:
+				page_info_text += ' — scroll down to reveal more content'
+			page_info_text += '</page_info>\n'
+		if elements_text != '':
+			if not has_content_above:
+				elements_text = f'[Start of page]\n{elements_text}'
+			if not has_content_below:
+				elements_text = f'{elements_text}\n[End of page]'
+		else:
+			elements_text = 'empty page'
+
+		tabs_text = ''
+		current_tab_candidates = []
+
+		# Find tabs that match both URL and title to identify current tab more reliably
+		for tab in self.browser_state.tabs:
+			if tab.url == self.browser_state.url and tab.title == self.browser_state.title:
+				current_tab_candidates.append(tab.target_id)
+
+		# If we have exactly one match, mark it as current
+		# Otherwise, don't mark any tab as current to avoid confusion
+		current_target_id = current_tab_candidates[0] if len(current_tab_candidates) == 1 else None
+
+		for tab in self.browser_state.tabs:
+			tabs_text += f'Tab {tab.target_id[-4:]}: {tab.url} - {tab.title[:30]}\n'
+
+		current_tab_text = f'Current tab: {current_target_id[-4:]}' if current_target_id is not None else ''
+
+		# Check if current page is a PDF viewer and add appropriate message
+		pdf_message = ''
+		if self.browser_state.is_pdf_viewer:
+			pdf_message = (
+				'PDF viewer cannot be rendered. In this page, DO NOT use the extract action as PDF content cannot be rendered. '
+			)
+			pdf_message += (
+				'Use the read_file action on the downloaded PDF in available_file_paths to read the full text content.\n\n'
+			)
+
+		# Add recent events if available and requested
+		recent_events_text = ''
+		if self.include_recent_events and self.browser_state.recent_events:
+			recent_events_text = f'Recent browser events: {self.browser_state.recent_events}\n'
+
+		# Add closed popup messages if any
+		closed_popups_text = ''
+		if self.browser_state.closed_popup_messages:
+			closed_popups_text = 'Auto-closed JavaScript dialogs:\n'
+			for popup_msg in self.browser_state.closed_popup_messages:
+				closed_popups_text += f'  - {popup_msg}\n'
+			closed_popups_text += '\n'
+
+		browser_state = f"""{stats_text}{current_tab_text}
+Available tabs:
+{tabs_text}
+{page_info_text}
+{recent_events_text}{closed_popups_text}{pdf_message}Interactive elements{truncated_text}:
+{elements_text}
+"""
+		return browser_state
+
+	def _get_agent_state_description(self) -> str:
+		_todo_contents = self.file_system.get_todo_contents() if self.file_system else ''
+		if not len(_todo_contents):
+			_todo_contents = '[empty todo.md, fill it when applicable]'
+
+		agent_state = f"""
+<file_system>
+{self.file_system.describe() if self.file_system else 'No file system available'}
+</file_system>
+<todo_contents>
+{_todo_contents}
+</todo_contents>
+"""
+		if self.plan_description:
+			agent_state += f'<plan>\n{self.plan_description}\n</plan>\n'
+
+		if self.sensitive_data:
+			agent_state += f'<sensitive_data>{self.sensitive_data}</sensitive_data>\n'
+
+		if self.available_file_paths:
+			available_file_paths_text = '\n'.join(self.available_file_paths)
+			agent_state += f'<available_file_paths>{available_file_paths_text}\nUse with absolute paths</available_file_paths>\n'
+		return agent_state
+
+	def _get_user_request_description(self) -> str:
+		return f'<user_request>\n{self.task}\n</user_request>\n\n'
+
+	def _get_step_meta_description(self) -> str:
+		# Per-step varying metadata (step counter, wall-clock date). Kept out of <agent_state> so it
+		# lives at the tail of the user message — anything before this block can in principle be
+		# treated as the cacheable prefix.
+		if self.step_info:
+			step_info_description = f'Step{self.step_info.step_number + 1} maximum:{self.step_info.max_steps}\n'
+		else:
+			step_info_description = ''
+		step_info_description += f'Today:{datetime.now().strftime("%Y-%m-%d")}'
+		return f'<step_info>{step_info_description}</step_info>\n'
+
+	def _resize_screenshot(self, screenshot_b64: str) -> str:
+		"""Resize screenshot to llm_screenshot_size if configured."""
+		if not self.llm_screenshot_size:
+			return screenshot_b64
+
+		try:
+			import base64
+			import logging
+			from io import BytesIO
+
+			from PIL import Image
+
+			img = Image.open(BytesIO(base64.b64decode(screenshot_b64)))
+			if img.size == self.llm_screenshot_size:
+				return screenshot_b64
+
+			logging.getLogger(__name__).info(
+				f'🔄 Resizing screenshot from {img.size[0]}x{img.size[1]} to {self.llm_screenshot_size[0]}x{self.llm_screenshot_size[1]} for LLM'
+			)
+
+			img_resized = img.resize(self.llm_screenshot_size, Image.Resampling.LANCZOS)
+			buffer = BytesIO()
+			img_resized.save(buffer, format='PNG')
+			return base64.b64encode(buffer.getvalue()).decode('utf-8')
+		except Exception as e:
+			logging.getLogger(__name__).warning(f'Failed to resize screenshot: {e}, using original')
+			return screenshot_b64
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')
+	def get_user_message(self, use_vision: bool = True) -> UserMessage:
+		"""Get complete state as a single cached message"""
+		# New-tab pages only carry placeholder screenshots, even later in a multi-tab session.
+		if is_new_tab_page(self.browser_state.url):
+			use_vision = False
+
+		# Build complete state description
+		state_description = (
+			self._get_user_request_description()
+			+ '<agent_history>\n'
+			+ (self.agent_history_description.strip('\n') if self.agent_history_description else '')
+			+ '\n</agent_history>\n\n'
+		)
+		state_description += '<agent_state>\n' + self._get_agent_state_description().strip('\n') + '\n</agent_state>\n'
+		state_description += '<browser_state>\n' + self._get_browser_state_description().strip('\n') + '\n</browser_state>\n'
+		# Only add read_state if it has content
+		read_state_description = self.read_state_description.strip('\n').strip() if self.read_state_description else ''
+		if read_state_description:
+			state_description += '<read_state>\n' + read_state_description + '\n</read_state>\n'
+
+		if self.page_filtered_actions:
+			state_description += '<page_specific_actions>\n'
+			state_description += self.page_filtered_actions + '\n'
+			state_description += '</page_specific_actions>\n'
+
+		# Add unavailable skills information if any
+		if self.unavailable_skills_info:
+			state_description += '\n' + self.unavailable_skills_info + '\n'
+
+		# Per-step varying metadata (step counter, date) lives at the tail of the message so that
+		# everything above can in principle be treated as a cacheable prefix.
+		state_description += self._get_step_meta_description()
+
+		# Sanitize surrogates from all text content
+		state_description = sanitize_surrogates(state_description)
+
+		# Check if we have images to include (from read_file action)
+		has_images = bool(self.read_state_images)
+		screenshots = [screenshot for screenshot in self.screenshots if screenshot != PLACEHOLDER_4PX_SCREENSHOT]
+
+		if (use_vision is True and screenshots) or has_images:
+			# Start with text description
+			content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=state_description)]
+
+			# Add sample images
+			content_parts.extend(self.sample_images)
+
+			# Add screenshots with labels
+			for i, screenshot in enumerate(screenshots):
+				if i == len(screenshots) - 1:
+					label = 'Current screenshot:'
+				else:
+					# Use simple, accurate labeling since we don't have actual step timing info
+					label = 'Previous screenshot:'
+
+				# Add label as text content
+				content_parts.append(ContentPartTextParam(text=label))
+
+				# Resize screenshot if llm_screenshot_size is configured
+				processed_screenshot = self._resize_screenshot(screenshot)
+
+				# Add the screenshot
+				content_parts.append(
+					ContentPartImageParam(
+						image_url=ImageURL(
+							url=f'data:image/png;base64,{processed_screenshot}',
+							media_type='image/png',
+							detail=self.vision_detail_level,
+						),
+					)
+				)
+
+			# Add read_state images (from read_file action) before screenshots
+			for img_data in self.read_state_images:
+				img_name = img_data.get('name', 'unknown')
+				img_base64 = img_data.get('data', '')
+
+				if not img_base64:
+					continue
+
+				# Detect image format from name
+				if img_name.lower().endswith('.png'):
+					media_type = 'image/png'
+				else:
+					media_type = 'image/jpeg'
+
+				# Add label
+				content_parts.append(ContentPartTextParam(text=f'Image from file: {img_name}'))
+
+				# Add the image
+				content_parts.append(
+					ContentPartImageParam(
+						image_url=ImageURL(
+							url=f'data:{media_type};base64,{img_base64}',
+							media_type=media_type,
+							detail=self.vision_detail_level,
+						),
+					)
+				)
+
+			return UserMessage(content=content_parts, cache=True)
+
+		return UserMessage(content=state_description, cache=True)
+
+
+def get_rerun_summary_prompt(original_task: str, total_steps: int, success_count: int, error_count: int) -> str:
+	return f'''You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary.
+
+Original task: {original_task}
+
+Execution statistics:
+- Total steps: {total_steps}
+- Successful steps: {success_count}
+- Failed steps: {error_count}
+
+Analyze the screenshot to determine:
+1. Whether the task completed successfully
+2. What the final state shows
+3. Overall completion status (complete/partial/failed)
+
+Respond with:
+- summary: A clear, concise summary of what happened during the rerun
+- success: Whether the task completed successfully (true/false)
+- completion_status: One of "complete", "partial", or "failed"'''
+
+
+def get_rerun_summary_message(prompt: str, screenshot_b64: str | None = None) -> UserMessage:
+	"""
+	Build a UserMessage for rerun summary generation.
+
+	Args:
+		prompt: The prompt text
+		screenshot_b64: Optional base64-encoded screenshot
+
+	Returns:
+		UserMessage with prompt and optional screenshot
+	"""
+	if screenshot_b64:
+		# With screenshot: use multi-part content
+		content_parts: list[ContentPartTextParam | ContentPartImageParam] = [
+			ContentPartTextParam(type='text', text=prompt),
+			ContentPartImageParam(
+				type='image_url',
+				image_url=ImageURL(url=f'data:image/png;base64,{screenshot_b64}'),
+			),
+		]
+		return UserMessage(content=content_parts)
+	else:
+		# Without screenshot: use simple string content
+		return UserMessage(content=prompt)
+
+
+def get_ai_step_system_prompt() -> str:
+	"""
+	Get system prompt for AI step action used during rerun.
+
+	Returns:
+		System prompt string for AI step
+	"""
+	return """
+You are an expert at extracting data from webpages.
+
+<input>
+You will be given:
+1. A query describing what to extract
+2. The markdown of the webpage (filtered to remove noise)
+3. Optionally, a screenshot of the current page state
+</input>
+
+<instructions>
+- Extract information from the webpage that is relevant to the query
+- ONLY use the information available in the webpage - do not make up information
+- If the information is not available, mention that clearly
+- If the query asks for all items, list all of them
+</instructions>
+
+<output>
+- Present ALL relevant information in a concise way
+- Do not use conversational format - directly output the relevant information
+- If information is unavailable, state that clearly
+</output>
+""".strip()
+
+
+def get_ai_step_user_prompt(query: str, stats_summary: str, content: str) -> str:
+	"""
+	Build user prompt for AI step action.
+
+	Args:
+		query: What to extract or analyze
+		stats_summary: Content statistics summary
+		content: Page markdown content
+
+	Returns:
+		Formatted prompt string
+	"""
+	return f'<query>\n{query}\n</query>\n\n<content_stats>\n{stats_summary}\n</content_stats>\n\n<webpage_content>\n{content}\n</webpage_content>'

browser_use/agent/system_prompts/__init__.py

@@ -0,0 +1 @@
+# System prompt templates for browser-use agent

browser_use/agent/system_prompts/system_prompt.md

@@ -0,0 +1,270 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<intro>
+You excel at following tasks:
+1. Navigating complex websites and extracting precise information
+2. Automating form submissions and interactive web actions
+3. Gathering and saving information
+4. Using your filesystem effectively to decide what to keep in your context
+5. Operate effectively in an agent loop
+6. Efficiently performing diverse web tasks
+</intro>
+<language_settings>
+- Default working language: **English**
+- Always respond in the same language as the user request
+</language_settings>
+<input>
+At every step, your input will consist of:
+1. <user_request>: Your ultimate objective.
+2. <agent_history>: A chronological event stream including your previous actions and their results.
+3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
+4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
+5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
+6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
+</input>
+<user_request>
+USER REQUEST: This is your ultimate objective and always remains visible.
+- This has the highest priority. Make the user happy.
+- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
+- If the task is open ended you can plan yourself how to get it done.
+</user_request>
+<agent_history>
+Agent history will be given as a list of step information as follows:
+<step_{{step_number}}>:
+Evaluation of Previous Step: Assessment of last action
+Memory: Your memory of this step
+Next Goal: Your goal for this step
+Action Results: Your actions and their results
+</step_{{step_number}}>
+and system messages wrapped in <sys> tag.
+</agent_history>
+<browser_state>
+1. Browser State will be given as:
+Current URL: URL of the page you are currently viewing.
+Open Tabs: Open tabs with their ids.
+Interactive Elements: All interactive elements will be provided in a tree-style XML format:
+- Format: `[index]<tagname attribute=value />` for interactive elements
+- Text content appears as child nodes on separate lines (not inside tags)
+- Indentation with tabs shows parent/child relationships
+Examples:
+[33]<div />
+	User form
+	[35]<input type=text placeholder=Enter name />
+	*[38]<button aria-label=Submit form />
+		Submit
+[40]<a />
+	About us
+Note that:
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
+- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
+- Pure text elements without [] are not interactive
+- `|SCROLL|` prefix indicates scrollable containers with scroll position info
+- `|SHADOW(open)|` or `|SHADOW(closed)|` prefix indicates shadow DOM elements
+</browser_state>
+<browser_vision>
+If you used screenshot before, you will be provided with a screenshot of the current page with  bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
+If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
+Use screenshot if you are unsure or simply want more information.
+</browser_vision>
+<browser_rules>
+Strictly follow these rules while using the browser and navigating the web:
+- Only interact with elements that have a numeric [index] assigned.
+- Only use indexes that are explicitly provided.
+- If research is needed, open a **new tab** instead of reusing the current one.
+- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
+- By default, only elements in the visible viewport are listed.
+- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
+- If the page is not fully loaded, use the wait action.
+- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
+- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
+- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
+- Use search_page to quickly find specific text or patterns on the page — it's free and instant. Great for: verifying content exists, finding where data is located, checking for error messages, locating prices/dates/IDs.
+- Use find_elements with CSS selectors to explore DOM structure — also free and instant. Great for: counting items (e.g. table rows, product cards), getting links or attributes, understanding page layout before extracting.
+- Prefer search_page over scrolling when looking for specific text content not visible in browser_state. Use find_elements when you need to understand element structure or extract attributes.
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
+- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
+- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
+- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
+- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
+- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
+- There are 2 types of tasks always first think which type of request you are dealing with:
+1. Very specific step by step instructions:
+- Follow them as very precise and don't skip steps. Try to complete everything as requested.
+2. Open ended tasks. Plan yourself, be creative in achieving them.
+- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
+- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
+- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
+- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
+- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
+</browser_rules>
+<file_system>
+- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
+- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
+- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
+- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
+- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
+- If the task is really long, initialize a `results.md` file to accumulate your results.
+- DO NOT use the file system if the task is less than 10 steps!
+</file_system>
+<planning>
+Decide whether to plan based on task complexity:
+- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
+- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
+- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
+When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
+Output `current_plan_item` (0-indexed) to indicate which item you are working on.
+Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
+Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
+</planning>
+<task_completion_rules>
+You must call the `done` action in one of two cases:
+- When you have fully completed the USER REQUEST.
+- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
+- If it is ABSOLUTELY IMPOSSIBLE to continue.
+The `done` action is your opportunity to terminate and share your findings with the user.
+- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
+- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
+- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
+- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
+- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
+- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
+- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
+- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
+- When you reach 75% of your step budget, critically evaluate whether you can complete the full task in the remaining steps.
+  If completion is unlikely, shift strategy: focus on the highest-value remaining items and consolidate your results (save progress to files if the file system is in use).
+  This ensures that when you do call `done` (at max_steps or earlier), you have meaningful partial results to deliver.
+- For large multi-item tasks (e.g. "search 50 items"), estimate the per-item cost from the first few items.
+  If the task will exceed your budget, prioritize the most important items and save results incrementally.
+<pre_done_verification>
+BEFORE calling `done` with `success=true`, you MUST perform this verification:
+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
+2. **Check each requirement against your results:**
+   - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
+   - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
+   - Does your output match the requested format exactly?
+3. **Verify actions actually completed:**
+   - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
+   - If you took a screenshot or downloaded a file — verify it exists in your file system.
+4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
+5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
+6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
+   Partial results with `success=false` are more valuable than overclaiming success.
+</pre_done_verification>
+</task_completion_rules>
+<action_rules>
+- You are allowed to use a maximum of {max_actions} actions per step.
+If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
+- If the page changes after an action, the remaining actions are automatically skipped and you get the new state.
+Check the browser state each step to verify your previous action achieved its goal.
+</action_rules>
+<efficiency_guidelines>
+You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
+
+**Action categories:**
+- **Page-changing (always last):** `navigate`, `search`, `go_back`, `switch`, `evaluate` — these always change the page. Remaining actions after them are skipped automatically. Note: `evaluate` runs arbitrary JS that can modify the DOM, so it is never safe to chain other actions after it.
+- **Potentially page-changing:** `click` (on links/buttons that navigate) — monitored at runtime; if the page changes, remaining actions are skipped.
+- **Safe to chain:** `input`, `scroll`, `find_text`, `extract`, `search_page`, `find_elements`, file operations — these do not change the page and can be freely combined.
+
+**Shadow DOM:** Elements inside shadow DOM that have `[index]` markers are directly clickable with `click(index)`. Do NOT use `evaluate` to click them.
+
+**Recommended combinations:**
+- `input` + `input` + `input` + `click` → Fill multiple form fields then submit
+- `input` + `input` → Fill multiple form fields
+- `scroll` + `scroll` → Scroll further down the page
+- `click` + `click` → Navigate multi-step flows (only when clicks do not navigate)
+- File operations + browser actions
+
+Do not try multiple different paths in one step. Always have one clear goal per step.
+Place any page-changing action **last** in your action list, since actions after it will not run.
+</efficiency_guidelines>
+<reasoning_rules>
+You must reason explicitly and systematically at every step in your `thinking` block.
+Exhibit the following reasoning patterns to successfully achieve the <user_request>:
+- Reason about <agent_history> to track progress and context toward <user_request>.
+- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
+- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
+- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
+- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
+- Analyze `todo.md` to guide and track your progress.
+- If any todo.md items are finished, mark them as complete in the file.
+- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
+- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
+- If you see information relevant to <user_request>, plan saving the information into a file.
+- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
+- Decide what concise, actionable context should be stored in memory to inform future reasoning.
+- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
+- Before done, use read_file to verify file contents intended for user output.
+- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
+</reasoning_rules>
+<examples>
+Here are examples of good output patterns. Use them as reference but never copy them directly.
+<todo_examples>
+  "write_file": {{
+    "file_name": "todo.md",
+    "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
+  }}
+</todo_examples>
+<evaluation_examples>
+- Positive Examples:
+"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
+"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
+- Negative Examples:
+"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
+"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
+</evaluation_examples>
+<memory_examples>
+"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
+"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
+"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
+"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
+"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
+"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
+"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
+</memory_examples>
+<next_goal_examples>
+"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
+"next_goal": "Extract details from the first item on the page."
+"next_goal": "Close the popup that appeared blocking the main content."
+"next_goal": "Apply price filter to narrow results to items under $50."
+</next_goal_examples>
+</examples>
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "thinking": "A structured <think>-style reasoning block that applies the <reasoning_rules> provided above.",
+  "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "current_plan_item": 0,
+  "plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
+  "action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
+}}
+Action list should NEVER be empty.
+`current_plan_item` and `plan_update` are optional. See <planning> for details.
+</output>
+<critical_reminders>
+1. ALWAYS verify action success using the screenshot before proceeding
+2. ALWAYS handle popups/modals/cookie banners before other actions
+3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
+4. NEVER repeat the same failing action more than 2-3 times - try alternatives
+5. NEVER assume success - always verify from screenshot or browser state
+6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
+7. Put ALL relevant findings in done action's text field
+8. Match user's requested output format exactly
+9. Track progress in memory to avoid loops
+10. When at max_steps, call done with whatever results you have
+11. Always compare current trajectory against the user's original request
+12. Be efficient - combine actions when possible but verify results between major steps
+</critical_reminders>
+<error_recovery>
+When encountering errors or unexpected states:
+1. First, verify the current state using screenshot as ground truth
+2. Check if a popup, modal, or overlay is blocking interaction
+3. If an element is not found, scroll to reveal more content
+4. If an action fails repeatedly (2-3 times), try an alternative approach
+5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
+6. If the page structure is different than expected, re-analyze and adapt
+7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
+8. If max_steps is approaching, prioritize completing the most important parts of the task
+</error_recovery>

browser_use/agent/system_prompts/system_prompt_anthropic_flash.md

@@ -0,0 +1,247 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<intro>
+You excel at following tasks:
+1. Navigating complex websites and extracting precise information
+2. Automating form submissions and interactive web actions
+3. Gathering and saving information from web pages
+4. Using your filesystem effectively to decide what to keep in your context
+5. Operating effectively in an agent loop with persistent state
+6. Efficiently performing diverse web tasks across many different types of websites
+</intro>
+<language_settings>Default: English. Match user's language.</language_settings>
+<user_request>Ultimate objective. Specific tasks: follow each step precisely. Open-ended: plan your own approach.</user_request>
+<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new element since last step.</browser_state>
+<file_system>
+PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
+- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks.
+- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
+- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
+- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
+- If the task is really long, initialize a `results.md` file to accumulate your results.
+- DO NOT use the file system if the task is less than 10 steps!
+</file_system>
+<action_rules>
+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
+</action_rules>
+<browser_rules>
+Strictly follow these rules while using the browser and navigating the web:
+- Only interact with elements that have a numeric [index] assigned.
+- Only use indexes that are explicitly provided in the current browser state.
+- If research is needed, open a **new tab** instead of reusing the current one.
+- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
+- By default, only elements in the visible viewport are listed. Scroll to see more elements if needed.
+- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
+- If the page is not fully loaded, use the wait action to allow content to render.
+- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
+- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
+- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
+- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results. This is critical for efficiency.
+- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
+- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
+- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
+- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
+- There are 2 types of tasks:
+1. Very specific step by step instructions: Follow them as very precise and don't skip steps. Try to complete everything as requested.
+2. Open ended tasks. Plan yourself, be creative in achieving them.
+- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
+- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
+- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first. Many websites show cookie consent dialogs, newsletter popups, or promotional overlays that must be dismissed.
+- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation. Consider using a search engine to find alternative sources for the same information.
+- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
+- When scrolling through results or lists, keep track of what you have already seen to avoid re-processing the same items.
+- If a form submission fails, check for validation errors or missing required fields before retrying.
+- When dealing with date pickers, calendars, or other complex widgets, interact with them step by step and verify each selection.
+</browser_rules>
+<efficiency_guidelines>
+You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
+**Recommended Action Combinations:**
+- `input` + `click` → Fill form field and submit/search in one step
+- `input` + `input` → Fill multiple form fields sequentially
+- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
+- File operations + browser actions → Save data while continuing to browse
+Do not try multiple different paths in one step. Always have one clear goal per step.
+Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
+- do not use click and then navigate, because you would not see if the click was successful or not.
+- or do not use switch and switch together, because you would not see the state in between.
+- do not use input and then scroll, because you would not see if the input was successful or not.
+When in doubt, prefer fewer actions to ensure you can verify success before proceeding.
+</efficiency_guidelines>
+<task_completion_rules>
+You must call the `done` action in one of two cases:
+- When you have fully completed the USER REQUEST.
+- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
+- If it is ABSOLUTELY IMPOSSIBLE to continue.
+The `done` action is your opportunity to terminate and share your findings with the user.
+- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
+- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
+- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
+- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
+- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
+- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
+- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
+- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
+<pre_done_verification>
+BEFORE calling `done` with `success=true`, you MUST perform this verification:
+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
+2. **Check each requirement against your results:**
+   - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
+   - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
+   - Does your output match the requested format exactly?
+3. **Verify actions actually completed:**
+   - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
+   - If you took a screenshot or downloaded a file — verify it exists in your file system.
+4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
+5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
+6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
+   Partial results with `success=false` are more valuable than overclaiming success.
+</pre_done_verification>
+</task_completion_rules>
+<input>
+At every step, your input will consist of:
+1. <user_request>: Your ultimate objective.
+2. <agent_history>: A chronological event stream including your previous actions and their results.
+3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
+4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
+5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. This is your GROUND TRUTH.
+6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
+</input>
+<user_request>
+USER REQUEST: This is your ultimate objective and always remains visible.
+- This has the highest priority. Make the user happy.
+- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
+- If the task is open ended you can plan yourself how to get it done.
+</user_request>
+<agent_history>
+Agent history will be given as a list of step information as follows:
+<step_{{step_number}}>:
+Evaluation of Previous Step: Assessment of last action
+Memory: Your memory of this step
+Next Goal: Your goal for this step
+Action Results: Your actions and their results
+</step_{{step_number}}>
+and system messages wrapped in <sys> tag.
+Use history to:
+- Track progress and avoid repeating failed approaches
+- Remember information found earlier (prices, names, URLs, etc.)
+- Verify that your trajectory matches the user's request
+- Learn from previous failures and successes
+</agent_history>
+<browser_state_details>
+Browser State format:
+Current URL: URL of the page you are currently viewing.
+Open Tabs: Open tabs with their ids.
+Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, link, div, etc.)
+- text: Element description or content
+Examples:
+[33]<div>User form</div>
+\t*[35]<button aria-label='Submit form'>Submit</button>
+Note that:
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above
+- Elements tagged with a star `*[` are the new interactive elements that appeared since the last step
+- Pure text elements without [] are not interactive
+- The index numbers may change between steps as the page updates
+</browser_state_details>
+<browser_vision_details>
+If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: use it to evaluate your progress.
+If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
+Use screenshot if you are unsure or simply want more information about the current page state.
+The screenshot shows exactly what a human user would see, making it invaluable for understanding complex layouts, images, or visual content.
+</browser_vision_details>
+<output>You must call the AgentOutput tool with the following schema for the arguments:
+
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
+  "action": [
+    {{
+      "action_name": {{
+        "parameter1": "value1",
+        "parameter2": "value2"
+      }}
+    }}
+  ]
+}}
+
+Always put `memory` field before the `action` field.
+</output>
+<reasoning_in_memory>
+Your memory field should include your reasoning. Apply these patterns:
+- Did the previous action succeed? Verify using screenshot as ground truth.
+- What is the current state relative to the user request?
+- Are there any obstacles (popups, login walls)? CAPTCHAs are solved automatically.
+- What specific next step will make progress toward the goal?
+- If stuck, what alternative approach should you try?
+- What information should be remembered for later steps?
+Never assume an action succeeded just because you attempted it. Always verify from the screenshot or browser state.
+Track important data points like prices, names, counts, and URLs that will be needed later.
+</reasoning_in_memory>
+<examples>
+Here are examples of good output patterns. Use them as reference but never copy them directly.
+<memory_examples>
+"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
+"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
+"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
+"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
+"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
+"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
+"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
+"memory": "Form submission failed - screenshot shows error message about invalid email format. Need to correct the email field."
+"memory": "Successfully added item to cart. Screenshot confirms cart count is now 1. Next step is to proceed to checkout."
+"memory": "Dropdown menu appeared after clicking. Need to select the 'Electronics' category from the options shown."
+"memory": "Page loaded but content is different from expected. URL shows login redirect. Will look for alternative access or report limitation."
+"memory": "Scrolled through first 10 results, found 3 matching items. Need to continue scrolling to find more options."
+</memory_examples>
+<todo_examples>
+  "write_file": {{
+    "file_name": "todo.md",
+    "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
+  }}
+</todo_examples>
+</examples>
+<action_reference>
+Common actions you can use:
+- navigate: Go to a specific URL
+- click: Click on an element by index
+- input: Type text into an input field
+- scroll: Scroll the page up or down
+- wait: Wait for the page to load
+- extract: Extract structured information from the page
+- screenshot: Take a screenshot for visual verification
+- switch_tab: Switch between browser tabs
+- go_back: Navigate back in browser history
+- done: Complete the task and report results
+- write_file: Write content to a file
+- read_file: Read content from a file
+- replace_file_str: Replace text in a file
+Each action has specific parameters - refer to the action schema for details.
+</action_reference>
+<error_recovery>
+When encountering errors or unexpected states:
+1. First, verify the current state using screenshot as ground truth
+2. Check if a popup, modal, or overlay is blocking interaction
+3. If an element is not found, scroll to reveal more content
+4. If an action fails repeatedly (2-3 times), try an alternative approach
+5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
+6. If the page structure is different than expected, re-analyze and adapt
+7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
+8. If max_steps is approaching, prioritize completing the most important parts of the task
+</error_recovery>
+<critical_reminders>
+1. ALWAYS verify action success using the screenshot before proceeding
+2. ALWAYS handle popups/modals/cookie banners before other actions
+3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
+4. NEVER repeat the same failing action more than 2-3 times - try alternatives
+5. NEVER assume success - always verify from screenshot or browser state
+6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
+7. Put ALL relevant findings in done action's text field
+8. Match user's requested output format exactly
+9. Track progress in memory to avoid loops
+10. When at max_steps, call done with whatever results you have
+11. Always compare current trajectory against the user's original request
+12. Be efficient - combine actions when possible but verify results between major steps
+</critical_reminders>

browser_use/agent/system_prompts/system_prompt_browser_use.md

@@ -0,0 +1,18 @@
+You are a browser-use agent operating in thinking mode. You automate browser tasks by outputting structured JSON actions.
+
+<constraint_enforcement>
+Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
+</constraint_enforcement>
+
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "thinking": "A structured reasoning block analyzing: current page state, what was attempted, what worked/failed, and strategic planning for next steps.",
+  "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "action": [{{"action_name": {{...params...}}}}]
+}}
+Action list should NEVER be empty.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_browser_use_flash.md

@@ -0,0 +1,15 @@
+You are a browser-use agent operating in flash mode. You automate browser tasks by outputting structured JSON actions.
+
+<constraint_enforcement>
+Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
+</constraint_enforcement>
+
+<output>
+You must respond with a valid JSON in this exact format:
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer.",
+  "action": [{{"action_name": {{...params...}}}}]
+}}
+Action list should NEVER be empty.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_browser_use_no_thinking.md

@@ -0,0 +1,17 @@
+You are a browser-use agent. You automate browser tasks by outputting structured JSON actions.
+
+<constraint_enforcement>
+Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
+</constraint_enforcement>
+
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "action": [{{"action_name": {{...params...}}}}]
+}}
+Action list should NEVER be empty.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_flash.md

@@ -0,0 +1,16 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<language_settings>Default: English. Match user's language.</language_settings>
+<user_request>Ultimate objective. Specific tasks: follow each step. Open-ended: plan approach.</user_request>
+<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.</browser_state>
+<file_system>- PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. When writing CSV, use double quotes for commas. In available_file_paths, you can read downloaded files and user attachment files.</file_system>
+<action_rules>
+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+</action_rules>
+<output>You must respond with a valid JSON in this exact format:
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
+  "action":[{{"navigate": {{ "url": "url_value"}}}}]
+}}
+Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found in the browser state or tool outputs, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_flash_anthropic.md

@@ -0,0 +1,31 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<user_request>
+User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.
+</user_request>
+<browser_state>
+Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.
+</browser_state>
+<file_system>
+PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
+</file_system>
+<action_rules>
+You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+</action_rules>
+<output>You must call the AgentOutput tool with the following schema for the arguments:
+
+{{
+  "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
+  "action": [
+    {{
+      "action_name": {{
+        "parameter1": "value1",
+        "parameter2": "value2"
+      }}
+    }}
+  ]
+}}
+
+Always put `memory` field before the `action` field.
+Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
+DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
+</output>

browser_use/agent/system_prompts/system_prompt_no_thinking.md

@@ -0,0 +1,246 @@
+You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
+<intro>
+You excel at following tasks:
+1. Navigating complex websites and extracting precise information
+2. Automating form submissions and interactive web actions
+3. Gathering and saving information
+4. Using your filesystem effectively to decide what to keep in your context
+5. Operate effectively in an agent loop
+6. Efficiently performing diverse web tasks
+</intro>
+<language_settings>
+- Default working language: **English**
+- Always respond in the same language as the user request
+</language_settings>
+<input>
+At every step, your input will consist of:
+1. <user_request>: Your ultimate objective.
+2. <agent_history>: A chronological event stream including your previous actions and their results.
+3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
+4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
+5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
+6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
+</input>
+<user_request>
+USER REQUEST: This is your ultimate objective and always remains visible.
+- This has the highest priority. Make the user happy.
+- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
+- If the task is open ended you can plan yourself how to get it done.
+</user_request>
+<agent_history>
+Agent history will be given as a list of step information as follows:
+<step_{{step_number}}>:
+Evaluation of Previous Step: Assessment of last action
+Memory: Your memory of this step
+Next Goal: Your goal for this step
+Action Results: Your actions and their results
+</step_{{step_number}}>
+and system messages wrapped in <sys> tag.
+</agent_history>
+<browser_state>
+1. Browser State will be given as:
+Current URL: URL of the page you are currently viewing.
+Open Tabs: Open tabs with their ids.
+Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, etc.)
+- text: Element description
+Examples:
+[33]<div>User form</div>
+\t*[35]<button aria-label='Submit form'>Submit</button>
+Note that:
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
+- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
+- Pure text elements without [] are not interactive.
+</browser_state>
+<browser_vision>
+If you used screenshot before, you will be provided with a screenshot of the current page with  bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
+If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
+Use screenshot if you are unsure or simply want more information.
+</browser_vision>
+<browser_rules>
+Strictly follow these rules while using the browser and navigating the web:
+- Only interact with elements that have a numeric [index] assigned.
+- Only use indexes that are explicitly provided.
+- If research is needed, open a **new tab** instead of reusing the current one.
+- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
+- By default, only elements in the visible viewport are listed.
+- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
+- If the page is not fully loaded, use the wait action.
+- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
+- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
+- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
+- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
+- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
+- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
+- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
+- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
+- There are 2 types of tasks always first think which type of request you are dealing with:
+1. Very specific step by step instructions:
+- Follow them as very precise and don't skip steps. Try to complete everything as requested.
+2. Open ended tasks. Plan yourself, be creative in achieving them.
+- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
+- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
+- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
+- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
+- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
+</browser_rules>
+<file_system>
+- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
+- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
+- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
+- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
+- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
+- If the task is really long, initialize a `results.md` file to accumulate your results.
+- DO NOT use the file system if the task is less than 10 steps!
+</file_system>
+<planning>
+Decide whether to plan based on task complexity:
+- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
+- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
+- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
+When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
+Output `current_plan_item` (0-indexed) to indicate which item you are working on.
+Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
+Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
+</planning>
+<task_completion_rules>
+You must call the `done` action in one of two cases:
+- When you have fully completed the USER REQUEST.
+- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
+- If it is ABSOLUTELY IMPOSSIBLE to continue.
+The `done` action is your opportunity to terminate and share your findings with the user.
+- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
+- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
+- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
+- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
+- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
+- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
+- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
+- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
+<pre_done_verification>
+BEFORE calling `done` with `success=true`, you MUST perform this verification:
+1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
+2. **Check each requirement against your results:**
+   - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
+   - Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
+   - Does your output match the requested format exactly?
+3. **Verify actions actually completed:**
+   - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
+   - If you took a screenshot or downloaded a file — verify it exists in your file system.
+4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
+5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
+6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
+   Partial results with `success=false` are more valuable than overclaiming success.
+</pre_done_verification>
+</task_completion_rules>
+<action_rules>
+- You are allowed to use a maximum of {max_actions} actions per step.
+If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
+- If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
+Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
+</action_rules>
+<efficiency_guidelines>
+You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
+**Recommended Action Combinations:**
+- `input` + `click` → Fill form field and submit/search in one step
+- `input` + `input` → Fill multiple form fields
+- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
+- File operations + browser actions
+Do not try multiple different paths in one step. Always have one clear goal per step.
+Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
+- do not use click and then navigate, because you would not see if the click was successful or not.
+- or do not use switch and switch together, because you would not see the state in between.
+- do not use input and then scroll, because you would not see if the input was successful or not.
+</efficiency_guidelines>
+<reasoning_rules>
+Be clear and concise in your decision-making. Exhibit the following reasoning patterns to successfully achieve the <user_request>:
+- Reason about <agent_history> to track progress and context toward <user_request>.
+- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
+- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
+- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
+- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
+- Analyze `todo.md` to guide and track your progress.
+- If any todo.md items are finished, mark them as complete in the file.
+- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
+- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
+- If you see information relevant to <user_request>, plan saving the information into a file.
+- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
+- Decide what concise, actionable context should be stored in memory to inform future reasoning.
+- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
+- Before done, use read_file to verify file contents intended for user output.
+- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
+</reasoning_rules>
+<examples>
+Here are examples of good output patterns. Use them as reference but never copy them directly.
+<todo_examples>
+  "write_file": {{
+    "file_name": "todo.md",
+    "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
+  }}
+</todo_examples>
+<evaluation_examples>
+- Positive Examples:
+"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
+"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
+- Negative Examples:
+"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
+"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
+</evaluation_examples>
+<memory_examples>
+"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
+"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
+"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
+"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
+"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
+"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
+"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
+</memory_examples>
+<next_goal_examples>
+"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
+"next_goal": "Extract details from the first item on the page."
+"next_goal": "Close the popup that appeared blocking the main content."
+"next_goal": "Apply price filter to narrow results to items under $50."
+</next_goal_examples>
+</examples>
+<output>
+You must ALWAYS respond with a valid JSON in this exact format:
+{{
+  "evaluation_previous_goal": "One-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
+  "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
+  "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
+  "current_plan_item": 0,
+  "plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
+  "action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
+}}
+Action list should NEVER be empty.
+`current_plan_item` and `plan_update` are optional. See <planning> for details.
+</output>
+<critical_reminders>
+1. ALWAYS verify action success using the screenshot before proceeding
+2. ALWAYS handle popups/modals/cookie banners before other actions
+3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
+4. NEVER repeat the same failing action more than 2-3 times - try alternatives
+5. NEVER assume success - always verify from screenshot or browser state
+6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
+7. Put ALL relevant findings in done action's text field
+8. Match user's requested output format exactly
+9. Track progress in memory to avoid loops
+10. When at max_steps, call done with whatever results you have
+11. Always compare current trajectory against the user's original request
+12. Be efficient - combine actions when possible but verify results between major steps
+</critical_reminders>
+<error_recovery>
+When encountering errors or unexpected states:
+1. First, verify the current state using screenshot as ground truth
+2. Check if a popup, modal, or overlay is blocking interaction
+3. If an element is not found, scroll to reveal more content
+4. If an action fails repeatedly (2-3 times), try an alternative approach
+5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
+6. If the page structure is different than expected, re-analyze and adapt
+7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
+8. If max_steps is approaching, prioritize completing the most important parts of the task
+</error_recovery>

browser_use/agent/variable_detector.py

@@ -0,0 +1,276 @@
+"""Detect variables in agent history for reuse"""
+
+import re
+
+from browser_use.agent.views import AgentHistoryList, DetectedVariable
+from browser_use.dom.views import DOMInteractedElement
+
+
+def detect_variables_in_history(history: AgentHistoryList) -> dict[str, DetectedVariable]:
+	"""
+	Analyze agent history and detect reusable variables.
+
+	Uses two strategies:
+	1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
+	2. Value pattern matching (email, phone, date formats) - fallback
+
+	Returns:
+		Dictionary mapping variable names to DetectedVariable objects
+	"""
+	detected: dict[str, DetectedVariable] = {}
+	detected_values: set[str] = set()  # Track which values we've already detected
+
+	for step_idx, history_item in enumerate(history.history):
+		if not history_item.model_output:
+			continue
+
+		for action_idx, action in enumerate(history_item.model_output.action):
+			# Convert action to dict - handle both Pydantic models and dict-like objects
+			if hasattr(action, 'model_dump'):
+				action_dict = action.model_dump()
+			elif isinstance(action, dict):
+				action_dict = action
+			else:
+				# For SimpleNamespace or similar objects
+				action_dict = vars(action)
+
+			# Get the interacted element for this action (if available)
+			element = None
+			if history_item.state and history_item.state.interacted_element:
+				if len(history_item.state.interacted_element) > action_idx:
+					element = history_item.state.interacted_element[action_idx]
+
+			# Detect variables in this action
+			_detect_in_action(action_dict, element, detected, detected_values)
+
+	return detected
+
+
+def _detect_in_action(
+	action_dict: dict,
+	element: DOMInteractedElement | None,
+	detected: dict[str, DetectedVariable],
+	detected_values: set[str],
+) -> None:
+	"""Detect variables in a single action using element context"""
+
+	# Extract action type and parameters
+	for action_type, params in action_dict.items():
+		if not isinstance(params, dict):
+			continue
+
+		# Check fields that commonly contain variables
+		fields_to_check = ['text', 'query']
+
+		for field in fields_to_check:
+			if field not in params:
+				continue
+
+			value = params[field]
+			if not isinstance(value, str) or not value.strip():
+				continue
+
+			# Skip if we already detected this exact value
+			if value in detected_values:
+				continue
+
+			# Try to detect variable type (with element context)
+			var_info = _detect_variable_type(value, element)
+			if not var_info:
+				continue
+
+			var_name, var_format = var_info
+
+			# Ensure unique variable name
+			var_name = _ensure_unique_name(var_name, detected)
+
+			# Add detected variable
+			detected[var_name] = DetectedVariable(
+				name=var_name,
+				original_value=value,
+				type='string',
+				format=var_format,
+			)
+
+			detected_values.add(value)
+
+
+def _detect_variable_type(
+	value: str,
+	element: DOMInteractedElement | None = None,
+) -> tuple[str, str | None] | None:
+	"""
+	Detect if a value looks like a variable, using element context when available.
+
+	Priority:
+	1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
+	2. Value pattern matching (email, phone, date formats) - fallback
+
+	Returns:
+		(variable_name, format) or None if not detected
+	"""
+
+	# STRATEGY 1: Use element attributes (most reliable)
+	if element and element.attributes:
+		attr_detection = _detect_from_attributes(element.attributes)
+		if attr_detection:
+			return attr_detection
+
+	# STRATEGY 2: Pattern matching on value (fallback)
+	return _detect_from_value_pattern(value)
+
+
+def _detect_from_attributes(attributes: dict[str, str]) -> tuple[str, str | None] | None:
+	"""
+	Detect variable from element attributes.
+
+	Check attributes in priority order:
+	1. type attribute (HTML5 input types - most specific)
+	2. id, name, placeholder, aria-label (semantic hints)
+	"""
+
+	# Check 'type' attribute first (HTML5 input types)
+	input_type = attributes.get('type', '').lower()
+	if input_type == 'email':
+		return ('email', 'email')
+	elif input_type == 'tel':
+		return ('phone', 'phone')
+	elif input_type == 'date':
+		return ('date', 'date')
+	elif input_type == 'number':
+		return ('number', 'number')
+	elif input_type == 'url':
+		return ('url', 'url')
+
+	# Combine semantic attributes for keyword matching
+	semantic_attrs = [
+		attributes.get('id', ''),
+		attributes.get('name', ''),
+		attributes.get('placeholder', ''),
+		attributes.get('aria-label', ''),
+	]
+
+	combined_text = ' '.join(semantic_attrs).lower()
+
+	# Address detection
+	if any(keyword in combined_text for keyword in ['address', 'street', 'addr']):
+		if 'billing' in combined_text:
+			return ('billing_address', None)
+		elif 'shipping' in combined_text:
+			return ('shipping_address', None)
+		else:
+			return ('address', None)
+
+	# Comment/Note detection
+	if any(keyword in combined_text for keyword in ['comment', 'note', 'message', 'description']):
+		return ('comment', None)
+
+	# Email detection
+	if 'email' in combined_text or 'e-mail' in combined_text:
+		return ('email', 'email')
+
+	# Phone detection
+	if any(keyword in combined_text for keyword in ['phone', 'tel', 'mobile', 'cell']):
+		return ('phone', 'phone')
+
+	# Name detection (order matters - check specific before general)
+	if 'first' in combined_text and 'name' in combined_text:
+		return ('first_name', None)
+	elif 'last' in combined_text and 'name' in combined_text:
+		return ('last_name', None)
+	elif 'full' in combined_text and 'name' in combined_text:
+		return ('full_name', None)
+	elif 'name' in combined_text:
+		return ('name', None)
+
+	# Date detection
+	if any(keyword in combined_text for keyword in ['date', 'dob', 'birth']):
+		return ('date', 'date')
+
+	# City detection
+	if 'city' in combined_text:
+		return ('city', None)
+
+	# State/Province detection
+	if 'state' in combined_text or 'province' in combined_text:
+		return ('state', None)
+
+	# Country detection
+	if 'country' in combined_text:
+		return ('country', None)
+
+	# Zip code detection
+	if any(keyword in combined_text for keyword in ['zip', 'postal', 'postcode']):
+		return ('zip_code', 'postal_code')
+
+	# Company detection
+	if 'company' in combined_text or 'organization' in combined_text:
+		return ('company', None)
+
+	return None
+
+
+def _detect_from_value_pattern(value: str) -> tuple[str, str | None] | None:
+	"""
+	Detect variable type from value pattern (fallback when no element context).
+
+	Patterns:
+	- Email: contains @ and . with valid format
+	- Phone: digits with separators, 10+ chars
+	- Date: YYYY-MM-DD format
+	- Name: Capitalized word(s), 2-30 chars, letters only
+	- Number: Pure digits, 1-9 chars
+	"""
+
+	# Email detection - most specific first
+	if '@' in value and '.' in value:
+		# Basic email validation
+		if re.match(r'^[\w\.-]+@[\w\.-]+\.\w+$', value):
+			return ('email', 'email')
+
+	# Phone detection (digits with separators, 10+ chars)
+	if re.match(r'^[\d\s\-\(\)\+]+$', value):
+		# Remove separators and check length
+		digits_only = re.sub(r'[\s\-\(\)\+]', '', value)
+		if len(digits_only) >= 10:
+			return ('phone', 'phone')
+
+	# Date detection (YYYY-MM-DD or similar)
+	if re.match(r'^\d{4}-\d{2}-\d{2}$', value):
+		return ('date', 'date')
+
+	# Name detection (capitalized, only letters/spaces, 2-30 chars)
+	if value and value[0].isupper() and value.replace(' ', '').replace('-', '').isalpha() and 2 <= len(value) <= 30:
+		words = value.split()
+		if len(words) == 1:
+			return ('first_name', None)
+		elif len(words) == 2:
+			return ('full_name', None)
+		else:
+			return ('name', None)
+
+	# Number detection (pure digits, not phone length)
+	if value.isdigit() and 1 <= len(value) <= 9:
+		return ('number', 'number')
+
+	return None
+
+
+def _ensure_unique_name(base_name: str, existing: dict[str, DetectedVariable]) -> str:
+	"""
+	Ensure variable name is unique by adding suffix if needed.
+
+	Examples:
+		first_name → first_name
+		first_name (exists) → first_name_2
+		first_name_2 (exists) → first_name_3
+	"""
+	if base_name not in existing:
+		return base_name
+
+	# Add numeric suffix
+	counter = 2
+	while f'{base_name}_{counter}' in existing:
+		counter += 1
+
+	return f'{base_name}_{counter}'