chore: import upstream snapshot with attribution

browser_use/tools/extraction/__init__.py

@@ -0,0 +1,4 @@
+from browser_use.tools.extraction.schema_utils import schema_dict_to_pydantic_model
+from browser_use.tools.extraction.views import ExtractionResult
+
+__all__ = ['schema_dict_to_pydantic_model', 'ExtractionResult']

browser_use/tools/extraction/schema_utils.py

@@ -0,0 +1,168 @@
+"""Converts a JSON Schema dict to a runtime Pydantic model for structured extraction."""
+
+import logging
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, create_model
+
+logger = logging.getLogger(__name__)
+
+# Keywords that indicate composition/reference patterns we don't support
+_UNSUPPORTED_KEYWORDS = frozenset(
+	{
+		'$ref',
+		'allOf',
+		'anyOf',
+		'oneOf',
+		'not',
+		'$defs',
+		'definitions',
+		'if',
+		'then',
+		'else',
+		'dependentSchemas',
+		'dependentRequired',
+	}
+)
+
+# Primitive JSON Schema type → Python type
+_PRIMITIVE_MAP: dict[str, type] = {
+	'string': str,
+	'number': float,
+	'integer': int,
+	'boolean': bool,
+	'null': type(None),
+}
+
+
+class _StrictBase(BaseModel):
+	model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True)
+
+
+def _check_unsupported(schema: dict) -> None:
+	"""Raise ValueError if the schema uses unsupported composition keywords."""
+	for kw in _UNSUPPORTED_KEYWORDS:
+		if kw in schema:
+			raise ValueError(f'Unsupported JSON Schema keyword: {kw}')
+
+
+def _resolve_type(schema: dict, name: str) -> Any:
+	"""Recursively resolve a JSON Schema node to a Python type.
+
+	Returns a Python type suitable for use as a field type in pydantic.create_model.
+	"""
+	_check_unsupported(schema)
+
+	json_type = schema.get('type', 'string')
+
+	# Enums — constrain to str (Literal would be stricter but LLMs are flaky)
+	if 'enum' in schema:
+		return str
+
+	# Object with properties → nested pydantic model
+	if json_type == 'object':
+		properties = schema.get('properties', {})
+		if properties:
+			return _build_model(schema, name)
+		return dict
+
+	# Array
+	if json_type == 'array':
+		items_schema = schema.get('items')
+		if items_schema:
+			item_type = _resolve_type(items_schema, f'{name}_item')
+			return list[item_type]
+		return list
+
+	# Primitive
+	base = _PRIMITIVE_MAP.get(json_type, str)
+
+	# Nullable
+	if schema.get('nullable', False):
+		return base | None
+
+	return base
+
+
+_PRIMITIVE_DEFAULTS: dict[str, Any] = {
+	'string': '',
+	'number': 0.0,
+	'integer': 0,
+	'boolean': False,
+}
+
+
+def _build_model(schema: dict, name: str) -> type[BaseModel]:
+	"""Build a pydantic model from an object-type JSON Schema node."""
+	_check_unsupported(schema)
+
+	properties = schema.get('properties', {})
+	required_fields = set(schema.get('required', []))
+	fields: dict[str, Any] = {}
+
+	for prop_name, prop_schema in properties.items():
+		prop_type = _resolve_type(prop_schema, f'{name}_{prop_name}')
+
+		if prop_name in required_fields:
+			default = ...
+		elif 'default' in prop_schema:
+			default = prop_schema['default']
+		elif prop_schema.get('nullable', False):
+			# _resolve_type already made the type include None
+			default = None
+		else:
+			# Non-required, non-nullable, no explicit default.
+			# Use a type-appropriate zero value for primitives/arrays;
+			# fall back to None (with | None) for enums and nested objects
+			# where no in-set or constructible default exists.
+			json_type = prop_schema.get('type', 'string')
+			if 'enum' in prop_schema:
+				# Can't pick an arbitrary enum member as default — use None
+				# so absent fields serialize as null, not an out-of-set value.
+				prop_type = prop_type | None
+				default = None
+			elif json_type in _PRIMITIVE_DEFAULTS:
+				default = _PRIMITIVE_DEFAULTS[json_type]
+			elif json_type == 'array':
+				default = []
+			else:
+				# Nested object or unknown — must allow None as sentinel
+				prop_type = prop_type | None
+				default = None
+
+		field_kwargs: dict[str, Any] = {}
+		if 'description' in prop_schema:
+			field_kwargs['description'] = prop_schema['description']
+
+		if isinstance(default, list) and not default:
+			fields[prop_name] = (prop_type, Field(default_factory=list, **field_kwargs))
+		else:
+			fields[prop_name] = (prop_type, Field(default, **field_kwargs))
+
+	return create_model(name, __base__=_StrictBase, **fields)
+
+
+def schema_dict_to_pydantic_model(schema: dict) -> type[BaseModel]:
+	"""Convert a JSON Schema dict to a runtime Pydantic model.
+
+	The schema must be ``{"type": "object", "properties": {...}, ...}``.
+	Unsupported keywords ($ref, allOf, anyOf, oneOf, etc.) raise ValueError.
+
+	Returns:
+		A dynamically-created Pydantic BaseModel subclass.
+
+	Raises:
+		ValueError: If the schema is invalid or uses unsupported features.
+	"""
+	_check_unsupported(schema)
+
+	top_type = schema.get('type')
+	if top_type != 'object':
+		raise ValueError(f'Top-level schema must have type "object", got {top_type!r}')
+
+	properties = schema.get('properties')
+	if not properties:
+		raise ValueError('Top-level schema must have at least one property')
+
+	model_name = schema.get('title', 'DynamicExtractionModel')
+	return _build_model(schema, model_name)

browser_use/tools/extraction/views.py

@@ -0,0 +1,17 @@
+"""Pydantic models for the extraction subsystem."""
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ExtractionResult(BaseModel):
+	"""Metadata about a structured extraction, stored in ActionResult.metadata."""
+
+	model_config = ConfigDict(extra='forbid')
+
+	data: dict[str, Any] = Field(description='The validated extraction payload')
+	schema_used: dict[str, Any] = Field(description='The JSON Schema that was enforced')
+	is_partial: bool = Field(default=False, description='True if content was truncated before extraction')
+	source_url: str | None = Field(default=None, description='URL the content was extracted from')
+	content_stats: dict[str, Any] = Field(default_factory=dict, description='Content processing statistics')

browser_use/tools/registry/service.py

@@ -0,0 +1,611 @@
+import asyncio
+import functools
+import inspect
+import logging
+import re
+from collections.abc import Callable
+from inspect import Parameter, iscoroutinefunction, signature
+from types import UnionType
+from typing import Any, Generic, Optional, TypeVar, Union, get_args, get_origin
+
+import pyotp
+from pydantic import BaseModel, Field, RootModel, create_model
+
+from browser_use.browser import BrowserSession
+from browser_use.browser.views import BrowserError
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.base import BaseChatModel
+from browser_use.observability import observe_debug
+from browser_use.telemetry.service import ProductTelemetry
+from browser_use.tools.registry.views import (
+	ActionModel,
+	ActionRegistry,
+	RegisteredAction,
+	SpecialActionParameters,
+)
+from browser_use.utils import is_new_tab_page, match_url_with_domain_pattern, time_execution_async
+
+Context = TypeVar('Context')
+
+logger = logging.getLogger(__name__)
+
+
+class Registry(Generic[Context]):
+	"""Service for registering and managing actions"""
+
+	def __init__(self, exclude_actions: list[str] | None = None):
+		self.registry = ActionRegistry()
+		self.telemetry = ProductTelemetry()
+		# Create a new list to avoid mutable default argument issues
+		self.exclude_actions = list(exclude_actions) if exclude_actions is not None else []
+
+	def exclude_action(self, action_name: str) -> None:
+		"""Exclude an action from the registry after initialization.
+
+		If the action is already registered, it will be removed from the registry.
+		The action is also added to the exclude_actions list to prevent re-registration.
+		"""
+		# Add to exclude list to prevent future registration
+		if action_name not in self.exclude_actions:
+			self.exclude_actions.append(action_name)
+
+		# Remove from registry if already registered
+		if action_name in self.registry.actions:
+			del self.registry.actions[action_name]
+			logger.debug(f'Excluded action "{action_name}" from registry')
+
+	def _get_special_param_types(self) -> dict[str, type | UnionType | None]:
+		"""Get the expected types for special parameters from SpecialActionParameters"""
+		# Manually define the expected types to avoid issues with Optional handling.
+		# we should try to reduce this list to 0 if possible, give as few standardized objects to all the actions
+		# but each driver should decide what is relevant to expose the action methods,
+		# e.g. CDP client, 2fa code getters, sensitive_data wrappers, other context, etc.
+		return {
+			'context': None,  # Context is a TypeVar, so we can't validate type
+			'browser_session': BrowserSession,
+			'page_url': str,
+			'cdp_client': None,  # CDPClient type from cdp_use, but we don't import it here
+			'page_extraction_llm': BaseChatModel,
+			'available_file_paths': list,
+			'has_sensitive_data': bool,
+			'file_system': FileSystem,
+			'extraction_schema': None,  # dict | None, skip type validation
+		}
+
+	def _normalize_action_function_signature(
+		self,
+		func: Callable,
+		description: str,
+		param_model: type[BaseModel] | None = None,
+	) -> tuple[Callable, type[BaseModel]]:
+		"""
+		Normalize action function to accept only kwargs.
+
+		Returns:
+			- Normalized function that accepts (*_, params: ParamModel, **special_params)
+			- The param model to use for registration
+		"""
+		sig = signature(func)
+		parameters = list(sig.parameters.values())
+		special_param_types = self._get_special_param_types()
+		special_param_names = set(special_param_types.keys())
+
+		# Step 1: Validate no **kwargs in original function signature
+		# if it needs default values it must use a dedicated param_model: BaseModel instead
+		for param in parameters:
+			if param.kind == Parameter.VAR_KEYWORD:
+				raise ValueError(
+					f"Action '{func.__name__}' has **{param.name} which is not allowed. "
+					f'Actions must have explicit positional parameters only.'
+				)
+
+		# Step 2: Separate special and action parameters
+		action_params = []
+		special_params = []
+		param_model_provided = param_model is not None
+
+		for i, param in enumerate(parameters):
+			# Check if this is a Type 1 pattern (first param is BaseModel)
+			if i == 0 and param_model_provided and param.name not in special_param_names:
+				# This is Type 1 pattern - skip the params argument
+				continue
+
+			if param.name in special_param_names:
+				# Validate special parameter type
+				expected_type = special_param_types.get(param.name)
+				if param.annotation != Parameter.empty and expected_type is not None:
+					# Handle Optional types - normalize both sides
+					param_type = param.annotation
+					origin = get_origin(param_type)
+					if origin is Union:
+						args = get_args(param_type)
+						# Find non-None type
+						param_type = next((arg for arg in args if arg is not type(None)), param_type)
+
+					# Check if types are compatible (exact match, subclass, or generic list)
+					types_compatible = (
+						param_type == expected_type
+						or (
+							inspect.isclass(param_type)
+							and inspect.isclass(expected_type)
+							and issubclass(param_type, expected_type)
+						)
+						or
+						# Handle list[T] vs list comparison
+						(expected_type is list and (param_type is list or get_origin(param_type) is list))
+					)
+
+					if not types_compatible:
+						expected_type_name = getattr(expected_type, '__name__', str(expected_type))
+						param_type_name = getattr(param_type, '__name__', str(param_type))
+						raise ValueError(
+							f"Action '{func.__name__}' parameter '{param.name}: {param_type_name}' "
+							f"conflicts with special argument injected by tools: '{param.name}: {expected_type_name}'"
+						)
+				special_params.append(param)
+			else:
+				action_params.append(param)
+
+		# Step 3: Create or validate param model
+		if not param_model_provided:
+			# Type 2: Generate param model from action params
+			if action_params:
+				params_dict = {}
+				for param in action_params:
+					annotation = param.annotation if param.annotation != Parameter.empty else str
+					default = ... if param.default == Parameter.empty else param.default
+					params_dict[param.name] = (annotation, default)
+
+				param_model = create_model(f'{func.__name__}_Params', __base__=ActionModel, **params_dict)
+			else:
+				# No action params, create empty model
+				param_model = create_model(
+					f'{func.__name__}_Params',
+					__base__=ActionModel,
+				)
+		assert param_model is not None, f'param_model is None for {func.__name__}'
+
+		# Step 4: Create normalized wrapper function
+		@functools.wraps(func)
+		async def normalized_wrapper(*args, params: BaseModel | None = None, **kwargs):
+			"""Normalized action that only accepts kwargs"""
+			# Validate no positional args
+			if args:
+				raise TypeError(f'{func.__name__}() does not accept positional arguments, only keyword arguments are allowed')
+
+			# Prepare arguments for original function
+			call_args = []
+			call_kwargs = {}
+
+			# Handle Type 1 pattern (first arg is the param model)
+			if param_model_provided and parameters and parameters[0].name not in special_param_names:
+				if params is None:
+					raise ValueError(f"{func.__name__}() missing required 'params' argument")
+				# For Type 1, we'll use the params object as first argument
+				pass
+			else:
+				# Type 2 pattern - need to unpack params
+				# If params is None, try to create it from kwargs
+				if params is None and action_params:
+					# Extract action params from kwargs
+					action_kwargs = {}
+					for param in action_params:
+						if param.name in kwargs:
+							action_kwargs[param.name] = kwargs[param.name]
+					if action_kwargs:
+						# Use the param_model which has the correct types defined
+						params = param_model(**action_kwargs)
+
+			# Build call_args by iterating through original function parameters in order
+			params_dict = params.model_dump() if params is not None else {}
+
+			for i, param in enumerate(parameters):
+				# Skip first param for Type 1 pattern (it's the model itself)
+				if param_model_provided and i == 0 and param.name not in special_param_names:
+					call_args.append(params)
+				elif param.name in special_param_names:
+					# This is a special parameter
+					if param.name in kwargs:
+						value = kwargs[param.name]
+						# Check if required special param is None
+						if value is None and param.default == Parameter.empty:
+							if param.name == 'browser_session':
+								raise ValueError(f'Action {func.__name__} requires browser_session but none provided.')
+							elif param.name == 'page_extraction_llm':
+								raise ValueError(f'Action {func.__name__} requires page_extraction_llm but none provided.')
+							elif param.name == 'file_system':
+								raise ValueError(f'Action {func.__name__} requires file_system but none provided.')
+							elif param.name == 'page':
+								raise ValueError(f'Action {func.__name__} requires page but none provided.')
+							elif param.name == 'available_file_paths':
+								raise ValueError(f'Action {func.__name__} requires available_file_paths but none provided.')
+							elif param.name == 'file_system':
+								raise ValueError(f'Action {func.__name__} requires file_system but none provided.')
+							else:
+								raise ValueError(f"{func.__name__}() missing required special parameter '{param.name}'")
+						call_args.append(value)
+					elif param.default != Parameter.empty:
+						call_args.append(param.default)
+					else:
+						# Special param is required but not provided
+						if param.name == 'browser_session':
+							raise ValueError(f'Action {func.__name__} requires browser_session but none provided.')
+						elif param.name == 'page_extraction_llm':
+							raise ValueError(f'Action {func.__name__} requires page_extraction_llm but none provided.')
+						elif param.name == 'file_system':
+							raise ValueError(f'Action {func.__name__} requires file_system but none provided.')
+						elif param.name == 'page':
+							raise ValueError(f'Action {func.__name__} requires page but none provided.')
+						elif param.name == 'available_file_paths':
+							raise ValueError(f'Action {func.__name__} requires available_file_paths but none provided.')
+						elif param.name == 'file_system':
+							raise ValueError(f'Action {func.__name__} requires file_system but none provided.')
+						else:
+							raise ValueError(f"{func.__name__}() missing required special parameter '{param.name}'")
+				else:
+					# This is an action parameter
+					if param.name in params_dict:
+						call_args.append(params_dict[param.name])
+					elif param.default != Parameter.empty:
+						call_args.append(param.default)
+					else:
+						raise ValueError(f"{func.__name__}() missing required parameter '{param.name}'")
+
+			# Call original function with positional args
+			if iscoroutinefunction(func):
+				return await func(*call_args)
+			else:
+				return await asyncio.to_thread(func, *call_args)
+
+		# Update wrapper signature to be kwargs-only
+		new_params = [Parameter('params', Parameter.KEYWORD_ONLY, default=None, annotation=Optional[param_model])]
+
+		# Add special params as keyword-only
+		for sp in special_params:
+			new_params.append(Parameter(sp.name, Parameter.KEYWORD_ONLY, default=sp.default, annotation=sp.annotation))
+
+		# Add **kwargs to accept and ignore extra params
+		new_params.append(Parameter('kwargs', Parameter.VAR_KEYWORD))
+
+		normalized_wrapper.__signature__ = sig.replace(parameters=new_params)  # type: ignore[attr-defined]
+
+		return normalized_wrapper, param_model
+
+	# @time_execution_sync('--create_param_model')
+	def _create_param_model(self, function: Callable) -> type[BaseModel]:
+		"""Creates a Pydantic model from function signature"""
+		sig = signature(function)
+		special_param_names = set(SpecialActionParameters.model_fields.keys())
+		params = {
+			name: (param.annotation, ... if param.default == param.empty else param.default)
+			for name, param in sig.parameters.items()
+			if name not in special_param_names
+		}
+		# TODO: make the types here work
+		return create_model(
+			f'{function.__name__}_parameters',
+			__base__=ActionModel,
+			**params,  # type: ignore
+		)
+
+	def action(
+		self,
+		description: str,
+		param_model: type[BaseModel] | None = None,
+		domains: list[str] | None = None,
+		allowed_domains: list[str] | None = None,
+		terminates_sequence: bool = False,
+	):
+		"""Decorator for registering actions"""
+		# Handle aliases: domains and allowed_domains are the same parameter
+		if allowed_domains is not None and domains is not None:
+			raise ValueError("Cannot specify both 'domains' and 'allowed_domains' - they are aliases for the same parameter")
+
+		final_domains = allowed_domains if allowed_domains is not None else domains
+
+		def decorator(func: Callable):
+			# Skip registration if action is in exclude_actions
+			if func.__name__ in self.exclude_actions:
+				return func
+
+			# Normalize the function signature
+			normalized_func, actual_param_model = self._normalize_action_function_signature(func, description, param_model)
+
+			action = RegisteredAction(
+				name=func.__name__,
+				description=description,
+				function=normalized_func,
+				param_model=actual_param_model,
+				domains=final_domains,
+				terminates_sequence=terminates_sequence,
+			)
+			self.registry.actions[func.__name__] = action
+
+			# Return the normalized function so it can be called with kwargs
+			return normalized_func
+
+		return decorator
+
+	@observe_debug(ignore_input=True, ignore_output=True, name='execute_action')
+	@time_execution_async('--execute_action')
+	async def execute_action(
+		self,
+		action_name: str,
+		params: dict,
+		browser_session: BrowserSession | None = None,
+		page_extraction_llm: BaseChatModel | None = None,
+		file_system: FileSystem | None = None,
+		sensitive_data: dict[str, str | dict[str, str]] | None = None,
+		available_file_paths: list[str] | None = None,
+		extraction_schema: dict | None = None,
+	) -> Any:
+		"""Execute a registered action with simplified parameter handling"""
+		if action_name not in self.registry.actions:
+			raise ValueError(f'Action {action_name} not found')
+
+		action = self.registry.actions[action_name]
+		try:
+			# Create the validated Pydantic model
+			try:
+				validated_params = action.param_model(**params)
+			except Exception as e:
+				raise ValueError(f'Invalid parameters {params} for action {action_name}: {type(e)}: {e}') from e
+
+			if sensitive_data:
+				# Get current URL if browser_session is provided
+				current_url = None
+				if browser_session and browser_session.agent_focus_target_id:
+					try:
+						# Get current page info from session_manager
+						target = browser_session.session_manager.get_target(browser_session.agent_focus_target_id)
+						if target:
+							current_url = target.url
+					except Exception:
+						pass
+				validated_params = self._replace_sensitive_data(validated_params, sensitive_data, current_url)
+
+			# Build special context dict
+			special_context = {
+				'browser_session': browser_session,
+				'page_extraction_llm': page_extraction_llm,
+				'available_file_paths': available_file_paths,
+				'has_sensitive_data': action_name == 'input' and bool(sensitive_data),
+				'file_system': file_system,
+				'extraction_schema': extraction_schema,
+			}
+
+			# Only pass sensitive_data to actions that explicitly need it (input)
+			if action_name == 'input':
+				special_context['sensitive_data'] = sensitive_data
+
+			# Add CDP-related parameters if browser_session is available
+			if browser_session:
+				# Add page_url
+				try:
+					special_context['page_url'] = await browser_session.get_current_page_url()
+				except Exception:
+					special_context['page_url'] = None
+
+				# Add cdp_client
+				special_context['cdp_client'] = browser_session.cdp_client
+
+			# All functions are now normalized to accept kwargs only
+			# Call with params and unpacked special context
+			try:
+				return await action.function(params=validated_params, **special_context)
+			except Exception as e:
+				raise
+
+		except BrowserError as e:
+			# BrowserError can carry structured short/long-term memory for the LLM
+			# (e.g. available dropdown options) — let Tools.act format it instead of
+			# flattening it into a generic RuntimeError string. Only errors with
+			# long_term_memory bypass: handle_browser_error re-raises without it,
+			# which would escape Tools.act instead of returning an ActionResult.
+			if e.long_term_memory is not None:
+				raise
+			raise RuntimeError(f'Error executing action {action_name}: {str(e)}') from e
+		except ValueError as e:
+			# Preserve ValueError messages from validation
+			if 'requires browser_session but none provided' in str(e) or 'requires page_extraction_llm but none provided' in str(
+				e
+			):
+				raise RuntimeError(str(e)) from e
+			else:
+				raise RuntimeError(f'Error executing action {action_name}: {str(e)}') from e
+		except TimeoutError as e:
+			raise RuntimeError(f'Error executing action {action_name} due to timeout.') from e
+		except Exception as e:
+			raise RuntimeError(f'Error executing action {action_name}: {str(e)}') from e
+
+	def _log_sensitive_data_usage(self, placeholders_used: set[str], current_url: str | None) -> None:
+		"""Log when sensitive data is being used on a page"""
+		if placeholders_used:
+			url_info = f' on {current_url}' if current_url and not is_new_tab_page(current_url) else ''
+			logger.info(f'🔒 Using sensitive data placeholders: {", ".join(sorted(placeholders_used))}{url_info}')
+
+	def _replace_sensitive_data(
+		self, params: BaseModel, sensitive_data: dict[str, Any], current_url: str | None = None
+	) -> BaseModel:
+		"""
+		Replaces sensitive data placeholders in params with actual values.
+
+		Args:
+			params: The parameter object containing <secret>placeholder</secret> tags
+			sensitive_data: Dictionary of sensitive data, either in old format {key: value}
+						   or new format {domain_pattern: {key: value}}
+			current_url: Optional current URL for domain matching
+
+		Returns:
+			BaseModel: The parameter object with placeholders replaced by actual values
+		"""
+		secret_pattern = re.compile(r'<secret>(.*?)</secret>')
+
+		# Set to track all missing placeholders across the full object
+		all_missing_placeholders = set()
+		# Set to track successfully replaced placeholders
+		replaced_placeholders = set()
+
+		# Process sensitive data based on format and current URL
+		applicable_secrets = {}
+
+		for domain_or_key, content in sensitive_data.items():
+			if isinstance(content, dict):
+				# New format: {domain_pattern: {key: value}}
+				# Only include secrets for domains that match the current URL
+				if current_url and not is_new_tab_page(current_url):
+					# it's a real url, check it using our custom allowed_domains scheme://*.example.com glob matching
+					if match_url_with_domain_pattern(current_url, domain_or_key):
+						applicable_secrets.update(content)
+			else:
+				# Old format: {key: value}, expose to all domains (only allowed for legacy reasons)
+				applicable_secrets[domain_or_key] = content
+
+		# Filter out empty values
+		applicable_secrets = {k: v for k, v in applicable_secrets.items() if v}
+
+		def recursively_replace_secrets(value: str | dict | list) -> str | dict | list:
+			if isinstance(value, str):
+				# 1. Handle tagged secrets: <secret>label</secret>
+				matches = secret_pattern.findall(value)
+				for placeholder in matches:
+					if placeholder in applicable_secrets:
+						# generate a totp code if secret is suffixed with bu_2fa_code
+						if placeholder.endswith('bu_2fa_code'):
+							totp = pyotp.TOTP(applicable_secrets[placeholder], digits=6)
+							replacement_value = totp.now()
+						else:
+							replacement_value = applicable_secrets[placeholder]
+
+						value = value.replace(f'<secret>{placeholder}</secret>', replacement_value)
+						replaced_placeholders.add(placeholder)
+					else:
+						# Keep track of missing placeholders
+						all_missing_placeholders.add(placeholder)
+
+				# 2. Handle literal secrets: "user_name" (no tags)
+				# This handles cases where the LLM forgets to use tags but uses the exact placeholder name
+				if value in applicable_secrets:
+					placeholder_name = value
+					if placeholder_name.endswith('bu_2fa_code'):
+						totp = pyotp.TOTP(applicable_secrets[placeholder_name], digits=6)
+						value = totp.now()
+					else:
+						value = applicable_secrets[placeholder_name]
+					replaced_placeholders.add(placeholder_name)
+
+				return value
+			elif isinstance(value, dict):
+				return {k: recursively_replace_secrets(v) for k, v in value.items()}
+			elif isinstance(value, list):
+				return [recursively_replace_secrets(v) for v in value]
+			return value
+
+		params_dump = params.model_dump()
+		processed_params = recursively_replace_secrets(params_dump)
+
+		# Log sensitive data usage
+		self._log_sensitive_data_usage(replaced_placeholders, current_url)
+
+		# Log a warning if any placeholders are missing
+		if all_missing_placeholders:
+			logger.warning(f'Missing or empty keys in sensitive_data dictionary: {", ".join(all_missing_placeholders)}')
+
+		return type(params).model_validate(processed_params)
+
+	# @time_execution_sync('--create_action_model')
+	def create_action_model(self, include_actions: list[str] | None = None, page_url: str | None = None) -> type[ActionModel]:
+		"""Creates a Union of individual action models from registered actions,
+		used by LLM APIs that support tool calling & enforce a schema.
+
+		Each action model contains only the specific action being used,
+		rather than all actions with most set to None.
+		"""
+		from typing import Union
+
+		# Filter actions based on page_url if provided:
+		#   if page_url is None, only include actions with no filters
+		#   if page_url is provided, only include actions that match the URL
+
+		available_actions: dict[str, RegisteredAction] = {}
+		for name, action in self.registry.actions.items():
+			if include_actions is not None and name not in include_actions:
+				continue
+
+			# If no page_url provided, only include actions with no filters
+			if page_url is None:
+				if action.domains is None:
+					available_actions[name] = action
+				continue
+
+			# Check domain filter if present
+			domain_is_allowed = self.registry._match_domains(action.domains, page_url)
+
+			# Include action if domain filter matches
+			if domain_is_allowed:
+				available_actions[name] = action
+
+		# Create individual action models for each action
+		individual_action_models: list[type[BaseModel]] = []
+
+		for name, action in available_actions.items():
+			# Create an individual model for each action that contains only one field
+			individual_model = create_model(
+				f'{name.title().replace("_", "")}ActionModel',
+				__base__=ActionModel,
+				**{
+					name: (
+						action.param_model,
+						Field(description=action.description),
+					)  # type: ignore
+				},
+			)
+			individual_action_models.append(individual_model)
+
+		# If no actions available, return empty ActionModel
+		if not individual_action_models:
+			return create_model('EmptyActionModel', __base__=ActionModel)
+
+		# Create proper Union type that maintains ActionModel interface
+		if len(individual_action_models) == 1:
+			# If only one action, return it directly (no Union needed)
+			result_model = individual_action_models[0]
+
+		# Meaning the length is more than 1
+		else:
+			# Create a Union type using RootModel that properly delegates ActionModel methods
+			union_type = Union[tuple(individual_action_models)]  # type: ignore : Typing doesn't understand that the length is >= 2 (by design)
+
+			class ActionModelUnion(RootModel[union_type]):  # type: ignore
+				def get_index(self) -> int | None:
+					"""Delegate get_index to the underlying action model"""
+					if hasattr(self.root, 'get_index'):
+						return self.root.get_index()  # type: ignore
+					return None
+
+				def set_index(self, index: int):
+					"""Delegate set_index to the underlying action model"""
+					if hasattr(self.root, 'set_index'):
+						self.root.set_index(index)  # type: ignore
+
+				def model_dump(self, **kwargs):
+					"""Delegate model_dump to the underlying action model"""
+					if hasattr(self.root, 'model_dump'):
+						return self.root.model_dump(**kwargs)  # type: ignore
+					return super().model_dump(**kwargs)
+
+			# Set the name for better debugging
+			ActionModelUnion.__name__ = 'ActionModel'
+			ActionModelUnion.__qualname__ = 'ActionModel'
+
+			result_model = ActionModelUnion
+
+		return result_model  # type:ignore
+
+	def get_prompt_description(self, page_url: str | None = None) -> str:
+		"""Get a description of all actions for the prompt
+
+		If page_url is provided, only include actions that are available for that URL
+		based on their domain filters
+		"""
+		return self.registry.get_prompt_description(page_url=page_url)

browser_use/tools/registry/views.py

@@ -0,0 +1,179 @@
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, ConfigDict
+
+from browser_use.browser import BrowserSession
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.base import BaseChatModel
+
+if TYPE_CHECKING:
+	pass
+
+
+class RegisteredAction(BaseModel):
+	"""Model for a registered action"""
+
+	name: str
+	description: str
+	function: Callable
+	param_model: type[BaseModel]
+
+	# If True, this action is known to change the page (e.g. navigate, search, go_back, switch).
+	# multi_act() will abort remaining queued actions after executing a terminates_sequence action.
+	terminates_sequence: bool = False
+
+	# filters: provide specific domains to determine whether the action should be available on the given URL or not
+	domains: list[str] | None = None  # e.g. ['*.google.com', 'www.bing.com', 'yahoo.*]
+
+	model_config = ConfigDict(arbitrary_types_allowed=True)
+
+	def prompt_description(self) -> str:
+		"""Get a description of the action for the prompt in unstructured format"""
+		schema = self.param_model.model_json_schema()
+		params = []
+
+		if 'properties' in schema:
+			for param_name, param_info in schema['properties'].items():
+				# Build parameter description
+				param_desc = param_name
+
+				# Add type information if available
+				if 'type' in param_info:
+					param_type = param_info['type']
+					param_desc += f'={param_type}'
+
+				# Add description as comment if available
+				if 'description' in param_info:
+					param_desc += f' ({param_info["description"]})'
+
+				params.append(param_desc)
+
+		# Format: action_name: Description. (param1=type, param2=type, ...)
+		if params:
+			return f'{self.name}: {self.description}. ({", ".join(params)})'
+		else:
+			return f'{self.name}: {self.description}'
+
+
+class ActionModel(BaseModel):
+	"""Base model for dynamically created action models"""
+
+	# this will have all the registered actions, e.g.
+	# click_element = param_model = ClickElementParams
+	# done = param_model = None
+	#
+	model_config = ConfigDict(arbitrary_types_allowed=True, extra='forbid')
+
+	def get_index(self) -> int | None:
+		"""Get the index of the action"""
+		# {'clicked_element': {'index':5}}
+		params = self.model_dump(exclude_unset=True).values()
+		if not params:
+			return None
+		for param in params:
+			if param is not None and 'index' in param:
+				return param['index']
+		return None
+
+	def set_index(self, index: int):
+		"""Overwrite the index of the action"""
+		# Get the action name and params
+		action_data = self.model_dump(exclude_unset=True)
+		action_name = next(iter(action_data.keys()))
+		action_params = getattr(self, action_name)
+
+		# Update the index directly on the model
+		if hasattr(action_params, 'index'):
+			action_params.index = index
+
+
+class ActionRegistry(BaseModel):
+	"""Model representing the action registry"""
+
+	actions: dict[str, RegisteredAction] = {}
+
+	@staticmethod
+	def _match_domains(domains: list[str] | None, url: str) -> bool:
+		"""
+		Match a list of domain glob patterns against a URL.
+
+		Args:
+			domains: A list of domain patterns that can include glob patterns (* wildcard)
+			url: The URL to match against
+
+		Returns:
+			True if the URL's domain matches the pattern, False otherwise
+		"""
+
+		if domains is None or not url:
+			return True
+
+		# Use the centralized URL matching logic from utils
+		from browser_use.utils import match_url_with_domain_pattern
+
+		for domain_pattern in domains:
+			if match_url_with_domain_pattern(url, domain_pattern):
+				return True
+		return False
+
+	def get_prompt_description(self, page_url: str | None = None) -> str:
+		"""Get a description of all actions for the prompt
+
+		Args:
+			page_url: If provided, filter actions by URL using domain filters.
+
+		Returns:
+			A string description of available actions.
+			- If page is None: return only actions with no page_filter and no domains (for system prompt)
+			- If page is provided: return only filtered actions that match the current page (excluding unfiltered actions)
+		"""
+		if page_url is None:
+			# For system prompt (no URL provided), include only actions with no filters
+			return '\n'.join(action.prompt_description() for action in self.actions.values() if action.domains is None)
+
+		# only include filtered actions for the current page URL
+		filtered_actions = []
+		for action in self.actions.values():
+			if not action.domains:
+				# skip actions with no filters, they are already included in the system prompt
+				continue
+
+			# Check domain filter
+			if self._match_domains(action.domains, page_url):
+				filtered_actions.append(action)
+
+		return '\n'.join(action.prompt_description() for action in filtered_actions)
+
+
+class SpecialActionParameters(BaseModel):
+	"""Model defining all special parameters that can be injected into actions"""
+
+	model_config = ConfigDict(arbitrary_types_allowed=True)
+
+	# optional user-provided context object passed down from Agent(context=...)
+	# e.g. can contain anything, external db connections, file handles, queues, runtime config objects, etc.
+	# that you might want to be able to access quickly from within many of your actions
+	# browser-use code doesn't use this at all, we just pass it down to your actions for convenience
+	context: Any | None = None
+
+	# browser-use session object, can be used to create new tabs, navigate, access CDP
+	browser_session: BrowserSession | None = None
+
+	# Current page URL for filtering and context
+	page_url: str | None = None
+
+	# CDP client for direct Chrome DevTools Protocol access
+	cdp_client: Any | None = None  # CDPClient type from cdp_use
+
+	# extra injected config if the action asks for these arg names
+	page_extraction_llm: BaseChatModel | None = None
+	file_system: FileSystem | None = None
+	available_file_paths: list[str] | None = None
+	has_sensitive_data: bool = False
+	extraction_schema: dict | None = None
+
+	@classmethod
+	def get_browser_requiring_params(cls) -> set[str]:
+		"""Get parameter names that require browser_session"""
+		return {'browser_session', 'cdp_client', 'page_url'}

browser_use/tools/utils.py

@@ -0,0 +1,82 @@
+"""Utility functions for browser tools."""
+
+from browser_use.dom.service import EnhancedDOMTreeNode
+
+
+def get_click_description(node: EnhancedDOMTreeNode) -> str:
+	"""Get a brief description of the clicked element for memory."""
+	parts = []
+
+	# Tag name
+	parts.append(node.tag_name)
+
+	# Add type for inputs
+	if node.tag_name == 'input' and node.attributes.get('type'):
+		input_type = node.attributes['type']
+		parts.append(f'type={input_type}')
+
+		# For checkboxes, include checked state
+		if input_type == 'checkbox':
+			is_checked = node.attributes.get('checked', 'false').lower() in ['true', 'checked', '']
+			# Also check AX node
+			if node.ax_node and node.ax_node.properties:
+				for prop in node.ax_node.properties:
+					if prop.name == 'checked':
+						is_checked = prop.value is True or prop.value == 'true'
+						break
+			state = 'checked' if is_checked else 'unchecked'
+			parts.append(f'checkbox-state={state}')
+
+	# Add role if present
+	if node.attributes.get('role'):
+		role = node.attributes['role']
+		parts.append(f'role={role}')
+
+		# For role=checkbox, include state
+		if role == 'checkbox':
+			aria_checked = node.attributes.get('aria-checked', 'false').lower()
+			is_checked = aria_checked in ['true', 'checked']
+			if node.ax_node and node.ax_node.properties:
+				for prop in node.ax_node.properties:
+					if prop.name == 'checked':
+						is_checked = prop.value is True or prop.value == 'true'
+						break
+			state = 'checked' if is_checked else 'unchecked'
+			parts.append(f'checkbox-state={state}')
+
+	# For labels/spans/divs, check if related to a hidden checkbox
+	if node.tag_name in ['label', 'span', 'div'] and 'type=' not in ' '.join(parts):
+		# Check children for hidden checkbox
+		for child in node.children:
+			if child.tag_name == 'input' and child.attributes.get('type') == 'checkbox':
+				# Check if hidden
+				is_hidden = False
+				if child.snapshot_node and child.snapshot_node.computed_styles:
+					opacity = child.snapshot_node.computed_styles.get('opacity', '1')
+					if opacity == '0' or opacity == '0.0':
+						is_hidden = True
+
+				if is_hidden or not child.is_visible:
+					# Get checkbox state
+					is_checked = child.attributes.get('checked', 'false').lower() in ['true', 'checked', '']
+					if child.ax_node and child.ax_node.properties:
+						for prop in child.ax_node.properties:
+							if prop.name == 'checked':
+								is_checked = prop.value is True or prop.value == 'true'
+								break
+					state = 'checked' if is_checked else 'unchecked'
+					parts.append(f'checkbox-state={state}')
+					break
+
+	# Add short text content if available
+	text = node.get_all_children_text().strip()
+	if text:
+		short_text = text[:30] + ('...' if len(text) > 30 else '')
+		parts.append(f'"{short_text}"')
+
+	# Add key attributes like id, name, aria-label
+	for attr in ['id', 'name', 'aria-label']:
+		if node.attributes.get(attr):
+			parts.append(f'{attr}={node.attributes[attr][:20]}')
+
+	return ' '.join(parts)

browser_use/tools/views.py

@@ -0,0 +1,205 @@
+from typing import Generic, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.json_schema import SkipJsonSchema
+
+
+# Action Input Models
+class ExtractAction(BaseModel):
+	query: str
+	extract_links: bool = Field(
+		default=False, description='Set True to true if the query requires links, else false to safe tokens'
+	)
+	extract_images: bool = Field(
+		default=False,
+		description='Set True to include image src URLs in extracted markdown. Auto-enabled when query contains image-related keywords.',
+	)
+	start_from_char: int = Field(
+		default=0, description='Use this for long markdowns to start from a specific character (not index in browser_state)'
+	)
+	output_schema: SkipJsonSchema[dict | None] = Field(
+		default=None,
+		description='Optional JSON Schema dict. When provided, extraction returns validated JSON matching this schema instead of free-text.',
+	)
+	already_collected: list[str] = Field(
+		default_factory=list,
+		description='Item identifiers (name, URL, or ID) already collected in prior extract calls on other pages. The extractor will skip items matching these to prevent duplicates. Use when paginating across multiple pages.',
+	)
+
+
+class SearchPageAction(BaseModel):
+	pattern: str = Field(description='Text or regex pattern to search for in page content')
+	regex: bool = Field(default=False, description='Treat pattern as regex (default: literal text match)')
+	case_sensitive: bool = Field(default=False, description='Case-sensitive search (default: case-insensitive)')
+	context_chars: int = Field(default=150, description='Characters of surrounding context per match')
+	css_scope: str | None = Field(default=None, description='CSS selector to limit search scope (e.g. "div#main")')
+	max_results: int = Field(default=25, description='Maximum matches to return')
+
+
+class FindElementsAction(BaseModel):
+	selector: str = Field(description='CSS selector to query elements (e.g. "table tr", "a.link", "div.product")')
+	attributes: list[str] | None = Field(
+		default=None,
+		description='Specific attributes to extract (e.g. ["href", "src", "class"]). If not set, returns tag and text only.',
+	)
+	max_results: int = Field(default=50, description='Maximum elements to return')
+	include_text: bool = Field(default=True, description='Include text content of each element')
+
+
+class SearchAction(BaseModel):
+	query: str
+	engine: str = Field(
+		default='duckduckgo', description='duckduckgo, google, bing (use duckduckgo by default because less captchas)'
+	)
+
+
+# Backward compatibility alias
+SearchAction = SearchAction
+
+
+class NavigateAction(BaseModel):
+	url: str
+	new_tab: bool = Field(default=False)
+
+
+# Backward compatibility alias
+GoToUrlAction = NavigateAction
+
+
+class ClickElementAction(BaseModel):
+	index: int | None = Field(default=None, ge=1, description='Element index from browser_state')
+	coordinate_x: int | None = Field(default=None, description='Horizontal coordinate relative to viewport left edge')
+	coordinate_y: int | None = Field(default=None, description='Vertical coordinate relative to viewport top edge')
+	# expect_download: bool = Field(default=False, description='set True if expecting a download, False otherwise')  # moved to downloads_watchdog.py
+	# click_count: int = 1  # TODO
+
+
+class ClickElementActionIndexOnly(BaseModel):
+	model_config = ConfigDict(title='ClickElementAction')
+
+	index: int = Field(ge=1, description='Element index from browser_state')
+
+
+class InputTextAction(BaseModel):
+	index: int = Field(ge=0, description='from browser_state')
+	text: str = Field(description='Text to enter. With clear=True, text="" clears the field without typing.')
+	clear: bool = Field(default=True, description='Clear existing text before typing. Set to False to append instead.')
+
+
+class DoneAction(BaseModel):
+	text: str = Field(
+		description=(
+			'Final message to the user. '
+			'ONLY report data you directly observed in browser_state, tool outputs, or screenshots during this session. '
+			'Do NOT use training knowledge to fill gaps — if information was not found on the page, say so explicitly. '
+			'Do NOT claim completion of steps from compacted_memory or prior session summaries '
+			'unless you explicitly verified them yourself. '
+			'If uncertain whether a prior step completed, say so explicitly.'
+		)
+	)
+	success: bool = Field(default=True, description='True if user_request completed successfully')
+	files_to_display: list[str] | None = Field(default=[])
+
+
+T = TypeVar('T', bound=BaseModel)
+
+
+def _hide_internal_fields_from_schema(schema: dict) -> None:
+	"""Remove internal fields from the JSON schema to avoid collisions with user models."""
+	props = schema.get('properties', {})
+	props.pop('success', None)
+	props.pop('files_to_display', None)
+
+
+class StructuredOutputAction(BaseModel, Generic[T]):
+	model_config = ConfigDict(json_schema_extra=_hide_internal_fields_from_schema)
+
+	success: bool = Field(default=True, description='True if user_request completed successfully')
+	data: T = Field(description='The actual output data matching the requested schema')
+	files_to_display: list[str] | None = Field(default=[])
+
+
+class SwitchTabAction(BaseModel):
+	tab_id: str = Field(min_length=4, max_length=4, description='4-char id')
+
+
+class CloseTabAction(BaseModel):
+	tab_id: str = Field(min_length=4, max_length=4, description='4-char id')
+
+
+class ScrollAction(BaseModel):
+	down: bool = Field(default=True, description='down=True=scroll down, down=False scroll up')
+	pages: float = Field(default=1.0, description='0.5=half page, 1=full page, 10=to bottom/top')
+	index: int | None = Field(default=None, description='Optional element index to scroll within specific element')
+
+
+class SendKeysAction(BaseModel):
+	keys: str = Field(description='keys (Escape, Enter, PageDown) or shortcuts (Control+o)')
+
+
+class UploadFileAction(BaseModel):
+	index: int
+	path: str
+
+
+class NoParamsAction(BaseModel):
+	model_config = ConfigDict(extra='ignore')
+
+	# Optional field required by Gemini API which errors on empty objects in response_schema
+	description: str | None = Field(None, description='Optional description for the action')
+
+
+class ScreenshotAction(BaseModel):
+	model_config = ConfigDict(extra='ignore')
+
+	file_name: str | None = Field(
+		default=None,
+		description='If provided, saves screenshot to this file and returns path. Otherwise screenshot is included in next observation.',
+	)
+
+
+class SaveAsPdfAction(BaseModel):
+	file_name: str | None = Field(
+		default=None,
+		description='Output PDF filename (without path). Defaults to page title. Extension .pdf is added automatically if missing.',
+	)
+	print_background: bool = Field(default=True, description='Include background graphics and colors')
+	landscape: bool = Field(default=False, description='Use landscape orientation')
+	scale: float = Field(default=1.0, ge=0.1, le=2.0, description='Scale of the webpage rendering (0.1 to 2.0)')
+	paper_format: str = Field(
+		default='Letter',
+		description='Paper size: Letter, Legal, A4, A3, or Tabloid',
+	)
+	display_header_footer: bool = Field(
+		default=True,
+		description=(
+			'Print page metadata into the margins, matching the browser Print dialog default: '
+			'the date in the header and the page URL plus page numbers in the footer. '
+			'Set to False for a clean PDF with no header/footer.'
+		),
+	)
+	header_template: str | None = Field(
+		default=None,
+		description=(
+			'Custom HTML for the page header. Inject values with spans using the classes '
+			'date, title, url, pageNumber, totalPages (e.g. \'<span class="title"></span>\'). '
+			'Set an explicit font-size or the text renders invisibly. Only used when '
+			'display_header_footer is True; defaults to showing the date.'
+		),
+	)
+	footer_template: str | None = Field(
+		default=None,
+		description=(
+			'Custom HTML for the page footer, same format as header_template. Only used when '
+			'display_header_footer is True; defaults to showing the page URL and page numbers.'
+		),
+	)
+
+
+class GetDropdownOptionsAction(BaseModel):
+	index: int
+
+
+class SelectDropdownOptionAction(BaseModel):
+	index: int
+	text: str = Field(description='exact text/value')