4cd2d4af2b
Test Browser Use CLI Install / uv pip install (macos-latest) (push) Waiting to run
Test Browser Use CLI Install / uv pip install (windows-latest) (push) Waiting to run
Test Browser Use CLI Install / uv pip install (ubuntu-latest) (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use from local wheel (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use[cli] from PyPI (push) Failing after 1s
package / pip-install-on-macos-latest-py-3.11 (push) Has been skipped
package / pip-install-on-macos-latest-py-3.13 (push) Has been skipped
package / pip-install-on-ubuntu-latest-py-3.11 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.13 (push) Has been skipped
cloud_evals / trigger_cloud_eval_image_build (push) Failing after 1s
docker / build_publish_image (push) Failing after 1s
Test Browser Use CLI Install / browser-use skill sync (push) Failing after 1s
lint / code-style (push) Failing after 0s
lint / type-checker (push) Failing after 1s
package / pip-build (push) Failing after 1s
lint / syntax-errors (push) Failing after 3s
package / pip-install-on-ubuntu-latest-py-3.13 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.11 (push) Has been skipped
test / ${{ matrix.test_filename }} (push) Has been skipped
test / evaluate-tasks (push) Has been skipped
test / setup-chromium (push) Failing after 2s
test / find_tests (push) Failing after 2s
6801 lines
250 KiB
Python
6801 lines
250 KiB
Python
from __future__ import annotations
|
|
|
|
import ast
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import keyword
|
|
import logging
|
|
import mimetypes
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from collections.abc import Awaitable, Callable
|
|
from contextlib import nullcontext, suppress
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Generic, Literal
|
|
from urllib.parse import urlparse
|
|
|
|
from bubus import EventBus
|
|
from pydantic import BaseModel, ValidationError, create_model
|
|
from typing_extensions import TypeVar
|
|
from uuid_extensions import uuid7str
|
|
|
|
from browser_use.agent.cloud_events import (
|
|
CreateAgentOutputFileEvent,
|
|
CreateAgentSessionEvent,
|
|
CreateAgentStepEvent,
|
|
CreateAgentTaskEvent,
|
|
UpdateAgentTaskEvent,
|
|
)
|
|
from browser_use.agent.judge import construct_judge_messages
|
|
from browser_use.agent.message_manager.service import MessageManager
|
|
from browser_use.agent.message_manager.utils import save_conversation
|
|
from browser_use.agent.prompts import SystemPrompt
|
|
from browser_use.agent.views import (
|
|
ActionResult,
|
|
AgentError,
|
|
AgentHistory,
|
|
AgentHistoryList,
|
|
AgentOutput,
|
|
AgentSettings,
|
|
AgentState,
|
|
AgentStepInfo,
|
|
AgentStructuredOutput,
|
|
JudgementResult,
|
|
MessageCompactionSettings,
|
|
StepMetadata,
|
|
)
|
|
from browser_use.browser import BrowserProfile, BrowserSession
|
|
from browser_use.browser.profile import CHROME_DETERMINISTIC_RENDERING_ARGS, CHROME_DISABLE_SECURITY_ARGS, CHROME_DOCKER_ARGS
|
|
from browser_use.browser.views import BrowserStateHistory, BrowserStateSummary, TabInfo
|
|
from browser_use.filesystem.file_system import FileSystem
|
|
from browser_use.llm.base import BaseChatModel
|
|
from browser_use.llm.messages import BaseMessage, ContentPartImageParam, ContentPartTextParam
|
|
from browser_use.llm.views import ChatInvokeUsage
|
|
from browser_use.observability import observe
|
|
from browser_use.screenshots.service import ScreenshotService
|
|
from browser_use.telemetry.service import ProductTelemetry
|
|
from browser_use.telemetry.views import AgentTelemetryEvent
|
|
from browser_use.tokens.custom_pricing import CUSTOM_MODEL_PRICING
|
|
from browser_use.tokens.service import TokenCost
|
|
from browser_use.tokens.views import ModelUsageStats, UsageSummary
|
|
from browser_use.tools.registry.views import ActionModel
|
|
from browser_use.tools.service import Tools
|
|
from browser_use.utils import (
|
|
URL_PATTERN,
|
|
SignalHandler,
|
|
_log_pretty_path,
|
|
check_latest_browser_use_version,
|
|
get_browser_use_version,
|
|
get_git_info,
|
|
is_placeholder_url,
|
|
sanitize_url_candidate,
|
|
)
|
|
|
|
Context = TypeVar('Context')
|
|
AgentHookFunc = Callable[['Agent'], Awaitable[None]]
|
|
AgentNewStepCallback = (
|
|
Callable[[BrowserStateSummary, AgentOutput, int], None] | Callable[[BrowserStateSummary, AgentOutput, int], Awaitable[None]]
|
|
)
|
|
AgentDoneCallback = Callable[[AgentHistoryList], Awaitable[None]] | Callable[[AgentHistoryList], None]
|
|
logger = logging.getLogger(__name__)
|
|
TERMINAL_INSTALL_COMMAND = 'curl -fsSL https://browser-use.com/terminal/install.sh | sh'
|
|
AGENT_TOOLS_DIR_ENV = 'BUT_AGENT_TOOLS_DIR'
|
|
|
|
try:
|
|
from lmnr import Laminar # type: ignore
|
|
except ImportError:
|
|
Laminar = None # type: ignore
|
|
|
|
|
|
class BetaAgentError(RuntimeError):
|
|
"""Raised when the beta agent cannot run a task."""
|
|
|
|
|
|
def _laminar_ready() -> bool:
|
|
if Laminar is None:
|
|
return False
|
|
try:
|
|
return bool(Laminar.is_initialized())
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _laminar_preview(value: Any, limit: int = 2000) -> Any:
|
|
if value is None or isinstance(value, (bool, int, float)):
|
|
return value
|
|
text = value if isinstance(value, str) else json.dumps(value, default=str)
|
|
if len(text) <= limit:
|
|
return text
|
|
return text[:limit] + f'...[truncated {len(text) - limit} chars]'
|
|
|
|
|
|
def _laminar_set_span_attributes(attributes: dict[str, Any]) -> None:
|
|
if not _laminar_ready():
|
|
return
|
|
safe_attributes = {key: value for key, value in attributes.items() if isinstance(value, (str, bool, int, float))}
|
|
if not safe_attributes:
|
|
return
|
|
try:
|
|
Laminar.set_span_attributes(safe_attributes)
|
|
except Exception:
|
|
logger.debug('Failed to set Laminar span attributes', exc_info=True)
|
|
|
|
|
|
def _laminar_set_span_output(output: Any) -> None:
|
|
if not _laminar_ready():
|
|
return
|
|
try:
|
|
Laminar.set_span_output(output)
|
|
except Exception:
|
|
logger.debug('Failed to set Laminar span output', exc_info=True)
|
|
|
|
|
|
def _laminar_event(name: str, attributes: dict[str, Any] | None = None) -> None:
|
|
if not _laminar_ready():
|
|
return
|
|
safe_attributes = {key: value for key, value in (attributes or {}).items() if isinstance(value, (str, bool, int, float))}
|
|
try:
|
|
Laminar.event(name, safe_attributes or None)
|
|
except Exception:
|
|
logger.debug('Failed to emit Laminar event %s', name, exc_info=True)
|
|
|
|
|
|
def _laminar_start_span(name: str, *, input: Any = None, span_type: str = 'DEFAULT'):
|
|
if not _laminar_ready():
|
|
return nullcontext()
|
|
try:
|
|
return Laminar.start_as_current_span(name=name, input=input, span_type=span_type)
|
|
except Exception:
|
|
logger.debug('Failed to start Laminar span %s', name, exc_info=True)
|
|
return nullcontext()
|
|
|
|
|
|
def _laminar_force_flush() -> None:
|
|
if not _laminar_ready():
|
|
return
|
|
for method_name in ('flush', 'force_flush'):
|
|
method = getattr(Laminar, method_name, None)
|
|
if method is None:
|
|
continue
|
|
try:
|
|
method()
|
|
return
|
|
except Exception:
|
|
logger.debug('Failed to flush Laminar via %s', method_name, exc_info=True)
|
|
|
|
|
|
def _laminar_current_trace_id() -> str | None:
|
|
if not _laminar_ready():
|
|
return None
|
|
try:
|
|
trace_id = Laminar.get_trace_id()
|
|
except Exception:
|
|
return None
|
|
return str(trace_id) if trace_id else None
|
|
|
|
|
|
def find_browser_use_terminal_binary() -> str:
|
|
"""Find the terminal binary used by the Rust-backed Browser Use Agent."""
|
|
env_path = os.environ.get('BROWSER_USE_TERMINAL_BINARY')
|
|
if env_path:
|
|
return env_path
|
|
packaged_path = _find_packaged_browser_use_terminal_binary()
|
|
if packaged_path:
|
|
return packaged_path
|
|
but_home = Path(os.environ.get('BUT_HOME', '~/.browser-use-terminal')).expanduser()
|
|
but_install_dir = Path(os.environ.get('BUT_INSTALL_DIR', '~/.local/bin')).expanduser()
|
|
candidates = [
|
|
but_home / 'packages' / 'standalone' / 'current' / 'bin' / 'browser-use-terminal',
|
|
but_install_dir / 'browser-use-terminal',
|
|
]
|
|
for candidate in candidates:
|
|
if candidate.exists() and _terminal_supports_sdk_server(candidate):
|
|
return str(candidate)
|
|
path_binary = shutil.which('browser-use-terminal')
|
|
if path_binary and _terminal_supports_sdk_server(Path(path_binary)):
|
|
return path_binary
|
|
raise BetaAgentError(
|
|
f'Could not find browser-use-terminal. Install Browser Use Terminal with `{TERMINAL_INSTALL_COMMAND}`, '
|
|
'install browser-use-core, or set BROWSER_USE_TERMINAL_BINARY to a built terminal CLI.'
|
|
)
|
|
|
|
|
|
def _find_packaged_browser_use_terminal_binary() -> str | None:
|
|
try:
|
|
from browser_use_core import binary_path
|
|
except Exception:
|
|
return None
|
|
try:
|
|
return binary_path('browser-use-terminal')
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _apply_agent_tools_env(env: dict[str, str]) -> None:
|
|
agent_tools_dir = _find_agent_tools_dir(env.get(AGENT_TOOLS_DIR_ENV))
|
|
if agent_tools_dir is None:
|
|
return
|
|
env[AGENT_TOOLS_DIR_ENV] = str(agent_tools_dir)
|
|
_prepend_env_path(env, agent_tools_dir)
|
|
|
|
|
|
def _find_agent_tools_dir(preferred_dir: str | None = None) -> Path | None:
|
|
if preferred_dir:
|
|
preferred_path = Path(preferred_dir).expanduser()
|
|
return preferred_path if _agent_tools_dir_contains_ripgrep(preferred_path) else None
|
|
|
|
env_binary = os.environ.get('BROWSER_USE_TERMINAL_BINARY')
|
|
if env_binary:
|
|
agent_tools_dir = _agent_tools_dir_for_terminal_binary(env_binary)
|
|
if agent_tools_dir:
|
|
return agent_tools_dir
|
|
|
|
packaged_dir = _find_packaged_agent_tools_dir()
|
|
if packaged_dir:
|
|
return packaged_dir
|
|
|
|
try:
|
|
terminal_binary = find_browser_use_terminal_binary()
|
|
except BetaAgentError:
|
|
return None
|
|
return _agent_tools_dir_for_terminal_binary(terminal_binary)
|
|
|
|
|
|
def _find_packaged_agent_tools_dir() -> Path | None:
|
|
try:
|
|
from browser_use_core import agent_tools_dir
|
|
except Exception:
|
|
agent_tools_dir = None
|
|
|
|
if agent_tools_dir is not None:
|
|
try:
|
|
candidate = Path(agent_tools_dir())
|
|
except Exception:
|
|
candidate = None
|
|
if candidate and _agent_tools_dir_contains_ripgrep(candidate):
|
|
return candidate
|
|
|
|
try:
|
|
from browser_use_core import binary_path
|
|
except Exception:
|
|
return None
|
|
|
|
try:
|
|
binary = Path(binary_path('browser-use-terminal'))
|
|
except Exception:
|
|
return None
|
|
candidate = binary.parent / 'agent-tools'
|
|
return candidate if _agent_tools_dir_contains_ripgrep(candidate) else None
|
|
|
|
|
|
def _agent_tools_dir_for_terminal_binary(binary_path: str | Path) -> Path | None:
|
|
candidate = Path(binary_path).expanduser().parent / 'agent-tools'
|
|
return candidate if _agent_tools_dir_contains_ripgrep(candidate) else None
|
|
|
|
|
|
def _agent_tools_dir_contains_ripgrep(directory: Path) -> bool:
|
|
return (directory / _agent_tools_ripgrep_name()).exists()
|
|
|
|
|
|
def _agent_tools_ripgrep_name() -> str:
|
|
return 'rg.exe' if os.name == 'nt' else 'rg'
|
|
|
|
|
|
def _prepend_env_path(env: dict[str, str], directory: Path) -> None:
|
|
directory_str = str(directory)
|
|
path_parts = [part for part in env.get('PATH', '').split(os.pathsep) if part and part != directory_str]
|
|
env['PATH'] = os.pathsep.join([directory_str, *path_parts])
|
|
|
|
|
|
def _terminal_supports_sdk_server(binary: Path) -> bool:
|
|
"""Return whether a terminal binary supports the SDK server subcommand required by this wrapper."""
|
|
try:
|
|
result = subprocess.run(
|
|
[str(binary), '--help'],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return False
|
|
return 'sdk-server' in f'{result.stdout}\n{result.stderr}'
|
|
|
|
|
|
class RustSdkJsonRpcError(BetaAgentError):
|
|
"""Raised when the Rust SDK server returns a JSON-RPC error."""
|
|
|
|
def __init__(self, code: int, message: str) -> None:
|
|
super().__init__(message)
|
|
self.code = code
|
|
self.message = message
|
|
|
|
|
|
class RustSdkClient:
|
|
"""Minimal stdio JSON-RPC client for browser-use-terminal sdk-server."""
|
|
|
|
def __init__(self, command: list[str], env: dict[str, str]) -> None:
|
|
self.command = list(command)
|
|
self.env = dict(env)
|
|
self.process: asyncio.subprocess.Process | None = None
|
|
self._reader_task: asyncio.Task[Any] | None = None
|
|
self._stderr_task: asyncio.Task[Any] | None = None
|
|
self._next_id = 1
|
|
self._pending: dict[int, asyncio.Future[Any]] = {}
|
|
self._write_lock = asyncio.Lock()
|
|
self.stderr_lines: list[str] = []
|
|
self.notifications: list[dict[str, Any]] = []
|
|
self.notification_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
|
self.stream_limit = int(os.environ.get('BROWSER_USE_SDK_STREAM_LIMIT_BYTES', str(64 * 1024 * 1024)))
|
|
self.read_chunk_size = int(os.environ.get('BROWSER_USE_SDK_READ_CHUNK_BYTES', str(1024 * 1024)))
|
|
self.max_line_bytes = int(os.environ.get('BROWSER_USE_SDK_MAX_LINE_BYTES', str(512 * 1024 * 1024)))
|
|
|
|
async def start(self) -> None:
|
|
if self.process is not None and self.process.returncode is None:
|
|
return
|
|
try:
|
|
self.process = await asyncio.create_subprocess_exec(
|
|
*self.command,
|
|
stdin=asyncio.subprocess.PIPE,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
env=self.env,
|
|
limit=self.stream_limit,
|
|
)
|
|
except (FileNotFoundError, PermissionError) as exc:
|
|
command = self.command[0] if self.command else 'browser-use-terminal'
|
|
raise BetaAgentError(
|
|
f'Could not start Rust SDK server command {command!r}. '
|
|
f'Install Browser Use Terminal with `{TERMINAL_INSTALL_COMMAND}`, '
|
|
'or set BROWSER_USE_TERMINAL_BINARY to a built terminal CLI.'
|
|
) from exc
|
|
self._reader_task = asyncio.create_task(self._read_stdout())
|
|
self._stderr_task = asyncio.create_task(self._read_stderr())
|
|
|
|
async def call(self, method: str, params: dict[str, Any] | None = None) -> Any:
|
|
await self.start()
|
|
if self.process is None or self.process.stdin is None:
|
|
raise BetaAgentError('Rust SDK server stdin is unavailable')
|
|
loop = asyncio.get_running_loop()
|
|
async with self._write_lock:
|
|
request_id = self._next_id
|
|
self._next_id += 1
|
|
future: asyncio.Future[Any] = loop.create_future()
|
|
self._pending[request_id] = future
|
|
request = {
|
|
'jsonrpc': '2.0',
|
|
'id': request_id,
|
|
'method': method,
|
|
'params': params or {},
|
|
}
|
|
try:
|
|
self.process.stdin.write((json.dumps(request) + '\n').encode('utf-8'))
|
|
await self.process.stdin.drain()
|
|
except (BrokenPipeError, ConnectionResetError) as exc:
|
|
self._pending.pop(request_id, None)
|
|
raise BetaAgentError(f'Rust SDK server pipe closed while sending {method}') from exc
|
|
try:
|
|
return await future
|
|
except asyncio.CancelledError:
|
|
self._pending.pop(request_id, None)
|
|
raise
|
|
|
|
async def close(self) -> None:
|
|
process = self.process
|
|
if process is None:
|
|
return
|
|
if process.stdin is not None:
|
|
process.stdin.close()
|
|
with suppress(BrokenPipeError, ConnectionResetError):
|
|
await process.stdin.wait_closed()
|
|
if process.returncode is None:
|
|
process.terminate()
|
|
try:
|
|
await asyncio.wait_for(process.wait(), timeout=2)
|
|
except TimeoutError:
|
|
process.kill()
|
|
await process.wait()
|
|
for task in (self._reader_task, self._stderr_task):
|
|
if task is not None:
|
|
task.cancel()
|
|
await asyncio.gather(
|
|
*(task for task in (self._reader_task, self._stderr_task) if task is not None and task is not asyncio.current_task()),
|
|
return_exceptions=True,
|
|
)
|
|
self._fail_all(BetaAgentError('Rust SDK server closed'))
|
|
self.process = None
|
|
|
|
async def _read_stdout(self) -> None:
|
|
assert self.process is not None
|
|
assert self.process.stdout is not None
|
|
buffer = bytearray()
|
|
try:
|
|
while True:
|
|
chunk = await self.process.stdout.read(self.read_chunk_size)
|
|
if not chunk:
|
|
if buffer and not self._handle_stdout_line(bytes(buffer)):
|
|
return
|
|
return
|
|
buffer.extend(chunk)
|
|
while True:
|
|
newline_index = buffer.find(b'\n')
|
|
if newline_index < 0:
|
|
break
|
|
raw_line = bytes(buffer[:newline_index])
|
|
del buffer[: newline_index + 1]
|
|
if not self._handle_stdout_line(raw_line):
|
|
return
|
|
if len(buffer) > self.max_line_bytes:
|
|
self._fail_all(BetaAgentError(f'Rust SDK JSON-RPC line exceeded {self.max_line_bytes} bytes without newline'))
|
|
return
|
|
except Exception as exc:
|
|
message = f'Rust SDK stdout reader failed: {exc}'
|
|
self.stderr_lines.append(message)
|
|
self._fail_all(BetaAgentError(message))
|
|
finally:
|
|
if self._pending:
|
|
detail = '\n'.join(self.stderr_lines[-20:])
|
|
message = 'Rust SDK server exited before responding'
|
|
if detail:
|
|
message = f'{message}: {detail}'
|
|
self._fail_all(BetaAgentError(message))
|
|
|
|
def _handle_stdout_line(self, raw_line: bytes) -> bool:
|
|
line = raw_line.decode('utf-8', errors='replace').strip()
|
|
if not line:
|
|
return True
|
|
try:
|
|
message = json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
self._fail_all(BetaAgentError(f'Invalid Rust SDK JSON-RPC line: {line}: {exc}'))
|
|
return False
|
|
self._handle_message(message)
|
|
return True
|
|
|
|
async def _read_stderr(self) -> None:
|
|
assert self.process is not None
|
|
assert self.process.stderr is not None
|
|
async for raw_line in self.process.stderr:
|
|
self.stderr_lines.append(raw_line.decode('utf-8', errors='replace').rstrip())
|
|
del self.stderr_lines[:-500]
|
|
|
|
def _handle_message(self, message: Any) -> None:
|
|
if not isinstance(message, dict):
|
|
self._fail_all(BetaAgentError(f'Rust SDK server emitted non-object JSON-RPC message: {message!r}'))
|
|
return
|
|
method = message.get('method')
|
|
if method in {'agent.event', 'agent.projected_event'}:
|
|
notification = {
|
|
'method': method,
|
|
'params': message.get('params') if isinstance(message.get('params'), dict) else {},
|
|
}
|
|
self.notifications.append(notification)
|
|
del self.notifications[:-2000]
|
|
self.notification_queue.put_nowait(notification)
|
|
return
|
|
if 'id' not in message:
|
|
return
|
|
request_id = message.get('id')
|
|
if not isinstance(request_id, int):
|
|
self._fail_all(BetaAgentError('Rust SDK JSON-RPC response id must be an integer'))
|
|
return
|
|
future = self._pending.pop(request_id, None)
|
|
if future is None or future.done():
|
|
return
|
|
if 'error' in message:
|
|
error = message.get('error') or {}
|
|
if not isinstance(error, dict):
|
|
error = {}
|
|
future.set_exception(
|
|
RustSdkJsonRpcError(
|
|
int(error.get('code', -32000)),
|
|
str(error.get('message') or 'Rust SDK server error'),
|
|
)
|
|
)
|
|
return
|
|
future.set_result(message.get('result'))
|
|
|
|
def _fail_all(self, error: BaseException) -> None:
|
|
for future in self._pending.values():
|
|
if not future.done():
|
|
future.set_exception(error)
|
|
self._pending.clear()
|
|
|
|
|
|
def _sdk_preview(value: Any, limit: int = 180) -> str | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, str):
|
|
text = value
|
|
else:
|
|
try:
|
|
text = json.dumps(value, default=str)
|
|
except Exception:
|
|
text = str(value)
|
|
text = ' '.join(text.split())
|
|
if not text:
|
|
return None
|
|
if len(text) <= limit:
|
|
return text
|
|
return text[:limit] + f'...[{len(text) - limit} more chars]'
|
|
|
|
|
|
def _sdk_payload_label(payload: dict[str, Any]) -> str | None:
|
|
for key in ('name', 'tool_name', 'tool', 'task', 'command', 'url', 'title', 'result', 'error', 'message'):
|
|
value = _sdk_preview(payload.get(key))
|
|
if value:
|
|
return f'{key}={value}'
|
|
arguments = payload.get('arguments')
|
|
if isinstance(arguments, dict):
|
|
for key in ('name', 'cmd', 'command', 'code', 'url'):
|
|
value = _sdk_preview(arguments.get(key))
|
|
if value:
|
|
return f'{key}={value}'
|
|
return None
|
|
|
|
|
|
def _sdk_notification_summary(notification: dict[str, Any]) -> str | None:
|
|
method = notification.get('method')
|
|
params = notification.get('params')
|
|
if not isinstance(params, dict):
|
|
return None
|
|
event = params.get('event')
|
|
if not isinstance(event, dict):
|
|
return None
|
|
payload = event.get('payload')
|
|
if not isinstance(payload, dict):
|
|
payload = {}
|
|
kind = event.get('kind') or event.get('event_type') or event.get('type')
|
|
if isinstance(kind, dict):
|
|
kind = kind.get('type') or kind.get('name')
|
|
if not isinstance(kind, str) or not kind:
|
|
kind = str(method or 'sdk.notification')
|
|
|
|
observed_type = payload.get('event_type')
|
|
observed_payload = payload.get('payload')
|
|
if isinstance(observed_type, str) and observed_type:
|
|
kind = observed_type
|
|
if isinstance(observed_payload, dict):
|
|
payload = observed_payload
|
|
|
|
if kind in {
|
|
'model.stream_delta',
|
|
'model.thinking_delta',
|
|
'tool.output_delta',
|
|
'browser.script.output_delta',
|
|
'python.output_delta',
|
|
'exec_command.output_delta',
|
|
}:
|
|
return None
|
|
|
|
label = _sdk_payload_label(payload)
|
|
if method == 'agent.projected_event':
|
|
projected_kind = event.get('kind')
|
|
if isinstance(projected_kind, str) and projected_kind:
|
|
kind = f'projected.{projected_kind}'
|
|
return f'{kind} {label}'.strip() if label else kind
|
|
|
|
|
|
def _sdk_notification_events(sdk: Any) -> list[dict[str, Any]]:
|
|
events: list[dict[str, Any]] = []
|
|
seen: set[tuple[Any, Any, Any]] = set()
|
|
for fallback_index, notification in enumerate(list(getattr(sdk, 'notifications', []) or [])):
|
|
if not isinstance(notification, dict) or notification.get('method') not in {'agent.event', 'agent.projected_event'}:
|
|
continue
|
|
params = notification.get('params')
|
|
if not isinstance(params, dict):
|
|
continue
|
|
event = params.get('event')
|
|
if not isinstance(event, dict):
|
|
continue
|
|
if not isinstance(event.get('event_type'), str):
|
|
payload = event.get('payload')
|
|
if not isinstance(payload, dict) or not isinstance(payload.get('event_type'), str):
|
|
continue
|
|
event = {
|
|
'seq': payload.get('seq', event.get('seq')),
|
|
'id': payload.get('id', event.get('id')),
|
|
'session_id': payload.get('session_id', event.get('session_id')),
|
|
'ts_ms': payload.get('ts_ms', event.get('ts_ms')),
|
|
'event_type': payload.get('event_type'),
|
|
'payload': payload.get('payload') if isinstance(payload.get('payload'), dict) else {},
|
|
}
|
|
elif notification.get('method') == 'agent.projected_event':
|
|
event = {
|
|
'seq': event.get('seq'),
|
|
'id': event.get('id'),
|
|
'session_id': event.get('session_id'),
|
|
'ts_ms': event.get('ts_ms'),
|
|
'event_type': event.get('event_type'),
|
|
'payload': event.get('payload') if isinstance(event.get('payload'), dict) else {},
|
|
}
|
|
identity = (event.get('seq'), event.get('id'), event.get('event_type'))
|
|
if identity[0] is None and identity[1] is None:
|
|
identity = (fallback_index, notification.get('method'), event.get('event_type'))
|
|
if identity in seen:
|
|
continue
|
|
seen.add(identity)
|
|
events.append(event)
|
|
return _dedupe_sdk_events(events)
|
|
|
|
|
|
def _event_payload_fingerprint(event: dict[str, Any]) -> str:
|
|
try:
|
|
return json.dumps(_event_payload(event), sort_keys=True, default=str)
|
|
except Exception:
|
|
return repr(_event_payload(event))
|
|
|
|
|
|
def _sdk_event_dedupe_identity(event: dict[str, Any], fallback_index: int) -> tuple[Any, ...]:
|
|
event_type = _event_type(event)
|
|
seq = event.get('seq')
|
|
if seq is not None:
|
|
return ('seq', seq, event_type, _event_payload_fingerprint(event))
|
|
event_id = event.get('id') or event.get('event_id')
|
|
if event_id is not None:
|
|
return ('id', event_id, event_type)
|
|
return ('fallback', fallback_index)
|
|
|
|
|
|
def _dedupe_sdk_events(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Remove SDK response/projected duplicates while preserving first-seen order."""
|
|
deduped: dict[tuple[Any, ...], dict[str, Any]] = {}
|
|
order: list[tuple[Any, ...]] = []
|
|
for index, event in enumerate(events):
|
|
key = _sdk_event_dedupe_identity(event, index)
|
|
existing = deduped.get(key)
|
|
if existing is None:
|
|
deduped[key] = event
|
|
order.append(key)
|
|
continue
|
|
if not (existing.get('id') or existing.get('event_id')) and (event.get('id') or event.get('event_id')):
|
|
deduped[key] = event
|
|
return [deduped[key] for key in order]
|
|
|
|
|
|
def _sdk_events_truncated_for_transport(events: list[dict[str, Any]]) -> bool:
|
|
return any(_event_type(event) == 'sdk.transport.truncated' for event in events)
|
|
|
|
|
|
def _sdk_transport_error_after_final_result(process_error: str | None) -> bool:
|
|
if not process_error:
|
|
return False
|
|
return any(
|
|
fragment in process_error
|
|
for fragment in (
|
|
'Rust SDK JSON-RPC line exceeded',
|
|
'Rust SDK stdout reader failed',
|
|
'Rust SDK server emitted non-object JSON-RPC message',
|
|
'Invalid Rust SDK JSON-RPC line',
|
|
)
|
|
)
|
|
|
|
|
|
def _model_name(llm: Any | None) -> str:
|
|
for attr in ('model', 'model_name', 'name'):
|
|
value = getattr(llm, attr, None)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return os.environ.get('BROWSER_USE_RUST_MODEL', 'gpt-5.3-codex-spark')
|
|
|
|
|
|
def _llm_timeout_for_model(llm: Any | None) -> int:
|
|
model_name = str(getattr(llm, 'model', '') or '').lower()
|
|
if 'gemini' in model_name:
|
|
if '3-pro' in model_name:
|
|
return 90
|
|
return 75
|
|
if 'groq' in model_name:
|
|
return 30
|
|
if 'o3' in model_name or 'claude' in model_name or 'sonnet' in model_name or 'deepseek' in model_name:
|
|
return 90
|
|
return 75
|
|
|
|
|
|
def _resolve_default_llm(llm: BaseChatModel | None) -> BaseChatModel:
|
|
if llm is not None:
|
|
return llm
|
|
try:
|
|
from browser_use.config import CONFIG
|
|
|
|
default_llm_name = CONFIG.DEFAULT_LLM
|
|
except Exception:
|
|
default_llm_name = ''
|
|
if default_llm_name:
|
|
from browser_use.llm.models import get_llm_by_name
|
|
|
|
return get_llm_by_name(default_llm_name)
|
|
from browser_use import ChatBrowserUse
|
|
|
|
return ChatBrowserUse()
|
|
|
|
|
|
def _extract_cdp_url(browser_session: BrowserSession | None) -> str | None:
|
|
if browser_session is None:
|
|
return None
|
|
for attr in ('cdp_url',):
|
|
value = getattr(browser_session, attr, None)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
profile = getattr(browser_session, 'browser_profile', None)
|
|
value = getattr(profile, 'cdp_url', None)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _extract_profile_cdp_url(browser_profile: BrowserProfile | None) -> str | None:
|
|
if browser_profile is None:
|
|
return None
|
|
value = getattr(browser_profile, 'cdp_url', None)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _extract_headless_preference(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> bool | None:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile):
|
|
value = getattr(profile, 'headless', None)
|
|
if isinstance(value, bool):
|
|
return value
|
|
value = getattr(browser_session, 'headless', None)
|
|
if isinstance(value, bool):
|
|
return value
|
|
return None
|
|
|
|
|
|
def _extract_cloud_preference(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> bool:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
for attr in ('use_cloud', 'cloud_browser'):
|
|
value = getattr(profile, attr, None)
|
|
if isinstance(value, bool) and value:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _value_from_object(value: Any, key: str) -> Any:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, dict):
|
|
return value.get(key)
|
|
return getattr(value, key, None)
|
|
|
|
|
|
def _append_unique(items: list[str], seen: set[str], value: Any) -> None:
|
|
if not isinstance(value, str) or not value:
|
|
return
|
|
if value in seen:
|
|
return
|
|
items.append(value)
|
|
seen.add(value)
|
|
|
|
|
|
def _window_size_arg(window_size: Any) -> str | None:
|
|
if window_size is None:
|
|
return None
|
|
if isinstance(window_size, (list, tuple)) and len(window_size) >= 2:
|
|
width, height = window_size[0], window_size[1]
|
|
else:
|
|
width = _value_from_object(window_size, 'width')
|
|
height = _value_from_object(window_size, 'height')
|
|
if not isinstance(width, int) or not isinstance(height, int) or width <= 0 or height <= 0:
|
|
return None
|
|
return f'--window-size={width},{height}'
|
|
|
|
|
|
def _window_position_arg(window_position: Any) -> str | None:
|
|
if window_position is None:
|
|
return None
|
|
if isinstance(window_position, (list, tuple)) and len(window_position) >= 2:
|
|
x, y = window_position[0], window_position[1]
|
|
else:
|
|
x = _value_from_object(window_position, 'width')
|
|
y = _value_from_object(window_position, 'height')
|
|
if not isinstance(x, int) or not isinstance(y, int):
|
|
return None
|
|
return f'--window-position={x},{y}'
|
|
|
|
|
|
def _managed_browser_launch_args(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> list[str]:
|
|
args: list[str] = []
|
|
seen: set[str] = set()
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile):
|
|
raw_args = getattr(profile, 'args', None)
|
|
if isinstance(raw_args, set):
|
|
raw_args = sorted(raw_args)
|
|
if isinstance(raw_args, (list, tuple)):
|
|
for arg in raw_args:
|
|
_append_unique(args, seen, arg)
|
|
if getattr(profile, 'disable_security', False) is True:
|
|
for arg in CHROME_DISABLE_SECURITY_ARGS:
|
|
_append_unique(args, seen, arg)
|
|
if getattr(profile, 'deterministic_rendering', False) is True:
|
|
for arg in CHROME_DETERMINISTIC_RENDERING_ARGS:
|
|
_append_unique(args, seen, arg)
|
|
if getattr(profile, 'chromium_sandbox', None) is False:
|
|
for arg in CHROME_DOCKER_ARGS:
|
|
_append_unique(args, seen, arg)
|
|
if getattr(profile, 'devtools', False) is True:
|
|
_append_unique(args, seen, '--auto-open-devtools-for-tabs')
|
|
_append_unique(args, seen, _window_size_arg(getattr(profile, 'window_size', None)))
|
|
_append_unique(args, seen, _window_position_arg(getattr(profile, 'window_position', None)))
|
|
proxy = getattr(profile, 'proxy', None)
|
|
proxy_server = _value_from_object(proxy, 'server')
|
|
if isinstance(proxy_server, str) and proxy_server:
|
|
_append_unique(args, seen, f'--proxy-server={proxy_server}')
|
|
proxy_bypass = _value_from_object(proxy, 'bypass')
|
|
if isinstance(proxy_bypass, str) and proxy_bypass:
|
|
_append_unique(args, seen, f'--proxy-bypass-list={proxy_bypass}')
|
|
user_agent = getattr(profile, 'user_agent', None)
|
|
if isinstance(user_agent, str) and user_agent:
|
|
_append_unique(args, seen, f'--user-agent={user_agent}')
|
|
profile_directory = getattr(profile, 'profile_directory', None)
|
|
if isinstance(profile_directory, str) and profile_directory:
|
|
_append_unique(args, seen, f'--profile-directory={profile_directory}')
|
|
return args
|
|
|
|
|
|
def _managed_browser_profile_dir(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> str | None:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
value = getattr(profile, 'user_data_dir', None)
|
|
if isinstance(value, (str, os.PathLike)) and str(value):
|
|
return str(Path(value).expanduser())
|
|
return None
|
|
|
|
|
|
def _managed_browser_executable_path(
|
|
browser_session: BrowserSession | None, browser_profile: BrowserProfile | None
|
|
) -> str | None:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
value = getattr(profile, 'executable_path', None)
|
|
if isinstance(value, (str, os.PathLike)) and str(value):
|
|
return str(Path(value).expanduser())
|
|
return None
|
|
|
|
|
|
def _env_value_to_str(value: Any) -> str | None:
|
|
if isinstance(value, bool):
|
|
return 'true' if value else 'false'
|
|
if isinstance(value, (str, int, float, os.PathLike)):
|
|
return str(value)
|
|
return None
|
|
|
|
|
|
def _managed_browser_env(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> dict[str, str]:
|
|
env: dict[str, str] = {}
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
raw_env = getattr(profile, 'env', None)
|
|
if not isinstance(raw_env, dict):
|
|
continue
|
|
for key, value in raw_env.items():
|
|
env_value = _env_value_to_str(value)
|
|
if isinstance(key, str) and key and env_value is not None:
|
|
env[key] = env_value
|
|
return env
|
|
|
|
|
|
def _extract_cdp_headers(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> dict[str, str]:
|
|
headers: dict[str, str] = {}
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
raw_headers = getattr(profile, 'headers', None)
|
|
if not isinstance(raw_headers, dict):
|
|
continue
|
|
for key, value in raw_headers.items():
|
|
header_value = _env_value_to_str(value)
|
|
if isinstance(key, str) and key and header_value is not None:
|
|
headers[key] = header_value
|
|
return headers
|
|
|
|
|
|
def _extract_user_agent(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> str | None:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
value = getattr(profile, 'user_agent', None)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _extract_highlight_settings(
|
|
browser_session: BrowserSession | None, browser_profile: BrowserProfile | None
|
|
) -> tuple[bool | None, str | None, int | None]:
|
|
enabled: bool | None = None
|
|
color: str | None = None
|
|
duration_ms: int | None = None
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
if enabled is None:
|
|
if getattr(profile, 'dom_highlight_elements', None) is True:
|
|
enabled = False
|
|
else:
|
|
value = getattr(profile, 'highlight_elements', None)
|
|
if isinstance(value, bool):
|
|
enabled = value
|
|
if color is None:
|
|
value = getattr(profile, 'interaction_highlight_color', None)
|
|
if isinstance(value, str) and value:
|
|
color = value
|
|
if duration_ms is None:
|
|
value = getattr(profile, 'interaction_highlight_duration', None)
|
|
if isinstance(value, (int, float)) and value >= 0:
|
|
duration_ms = int(value * 1000)
|
|
return enabled, color, duration_ms
|
|
|
|
|
|
def _seconds_to_ms(value: Any) -> int | None:
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
|
|
return None
|
|
return int(value * 1000)
|
|
|
|
|
|
def _extract_wait_timing_settings(
|
|
browser_session: BrowserSession | None, browser_profile: BrowserProfile | None
|
|
) -> dict[str, str]:
|
|
settings: dict[str, str] = {}
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
mappings = (
|
|
('minimum_wait_page_load_time', 'BU_BROWSER_MINIMUM_WAIT_PAGE_LOAD_MS'),
|
|
('wait_for_network_idle_page_load_time', 'BU_BROWSER_NETWORK_IDLE_PAGE_LOAD_MS'),
|
|
('wait_between_actions', 'BU_BROWSER_WAIT_BETWEEN_ACTIONS_MS'),
|
|
)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
for attr, env_name in mappings:
|
|
if env_name in settings:
|
|
continue
|
|
value_ms = _seconds_to_ms(getattr(profile, attr, None))
|
|
if value_ms is not None:
|
|
settings[env_name] = str(value_ms)
|
|
return settings
|
|
|
|
|
|
def _extract_block_ip_addresses(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> bool | None:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
value = getattr(profile, 'block_ip_addresses', None)
|
|
if isinstance(value, bool):
|
|
return value
|
|
return None
|
|
|
|
|
|
def _extract_profile_permissions(browser_session: BrowserSession | None, browser_profile: BrowserProfile | None) -> list[str]:
|
|
values: list[str] = []
|
|
seen: set[str] = set()
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile):
|
|
raw_permissions = getattr(profile, 'permissions', None)
|
|
if isinstance(raw_permissions, set):
|
|
raw_permissions = sorted(raw_permissions)
|
|
if not isinstance(raw_permissions, (list, tuple)):
|
|
continue
|
|
for permission in raw_permissions:
|
|
if not isinstance(permission, str) or not permission or permission in seen:
|
|
continue
|
|
values.append(permission)
|
|
seen.add(permission)
|
|
return values
|
|
|
|
|
|
def _extract_browser_downloads(
|
|
browser_session: BrowserSession | None, browser_profile: BrowserProfile | None
|
|
) -> tuple[bool | None, str | None]:
|
|
accept_downloads: bool | None = None
|
|
downloads_path: str | None = None
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
if accept_downloads is None:
|
|
value = getattr(profile, 'accept_downloads', None)
|
|
if isinstance(value, bool):
|
|
accept_downloads = value
|
|
if downloads_path is None:
|
|
value = getattr(profile, 'downloads_path', None)
|
|
if isinstance(value, (str, os.PathLike)) and str(value):
|
|
downloads_path = str(Path(value).expanduser())
|
|
return accept_downloads, downloads_path
|
|
|
|
|
|
def _viewport_size(value: Any) -> tuple[int, int] | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, (list, tuple)) and len(value) >= 2:
|
|
width, height = value[0], value[1]
|
|
else:
|
|
width = _value_from_object(value, 'width')
|
|
height = _value_from_object(value, 'height')
|
|
if not isinstance(width, int) or not isinstance(height, int) or width <= 0 or height <= 0:
|
|
return None
|
|
return width, height
|
|
|
|
|
|
def _extract_browser_viewport(
|
|
browser_session: BrowserSession | None, browser_profile: BrowserProfile | None
|
|
) -> tuple[bool | None, dict[str, int | float] | None]:
|
|
no_viewport: bool | None = None
|
|
viewport_size: tuple[int, int] | None = None
|
|
screen_size: tuple[int, int] | None = None
|
|
device_scale_factor: int | float | None = None
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
if no_viewport is None:
|
|
value = getattr(profile, 'no_viewport', None)
|
|
if isinstance(value, bool):
|
|
no_viewport = value
|
|
if viewport_size is None:
|
|
viewport_size = _viewport_size(getattr(profile, 'viewport', None))
|
|
if screen_size is None:
|
|
screen_size = _viewport_size(getattr(profile, 'screen', None))
|
|
if device_scale_factor is None:
|
|
value = getattr(profile, 'device_scale_factor', None)
|
|
if isinstance(value, (int, float)) and value >= 0:
|
|
device_scale_factor = value
|
|
if no_viewport is True:
|
|
return no_viewport, None
|
|
if viewport_size is None and no_viewport is False:
|
|
viewport_size = screen_size
|
|
if viewport_size is None:
|
|
return no_viewport, None
|
|
width, height = viewport_size
|
|
viewport: dict[str, int | float] = {
|
|
'width': width,
|
|
'height': height,
|
|
'deviceScaleFactor': 1 if device_scale_factor is None else device_scale_factor,
|
|
}
|
|
if screen_size is not None:
|
|
screen_width, screen_height = screen_size
|
|
viewport['screenWidth'] = screen_width
|
|
viewport['screenHeight'] = screen_height
|
|
return no_viewport, viewport
|
|
|
|
|
|
def _extract_browser_window_size(
|
|
browser_session: BrowserSession | None, browser_profile: BrowserProfile | None
|
|
) -> dict[str, int] | None:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
window_size = _viewport_size(getattr(profile, 'window_size', None))
|
|
if window_size is not None:
|
|
width, height = window_size
|
|
return {'width': width, 'height': height}
|
|
return None
|
|
|
|
|
|
def _storage_state_value(value: Any) -> dict[str, Any] | None:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if hasattr(value, 'model_dump'):
|
|
dumped = value.model_dump()
|
|
return dumped if isinstance(dumped, dict) else None
|
|
if isinstance(value, (str, os.PathLike)) and str(value):
|
|
path = Path(value).expanduser()
|
|
if not path.exists():
|
|
return None
|
|
try:
|
|
loaded = json.loads(path.read_text())
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
return loaded if isinstance(loaded, dict) else None
|
|
return None
|
|
|
|
|
|
def _extract_browser_storage_state(
|
|
browser_session: BrowserSession | None, browser_profile: BrowserProfile | None
|
|
) -> dict[str, Any] | None:
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile, browser_session):
|
|
value = _storage_state_value(getattr(profile, 'storage_state', None))
|
|
if value is not None:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _is_managed_browser_mode(mode: str) -> bool:
|
|
normalized = mode.strip().lower().replace('_', '-').replace(' ', '-')
|
|
return normalized in {'managed-headless', 'headless', 'headless-chromium', 'managed-headed', 'managed', 'headed'}
|
|
|
|
|
|
def _domain_list(value: Any) -> list[str]:
|
|
if value is None or isinstance(value, str):
|
|
return []
|
|
if isinstance(value, set):
|
|
items = sorted(value)
|
|
elif isinstance(value, list):
|
|
items = value
|
|
else:
|
|
return []
|
|
return [item for item in items if isinstance(item, str) and item]
|
|
|
|
|
|
def _extract_profile_domains(
|
|
browser_session: BrowserSession | None,
|
|
browser_profile: BrowserProfile | None,
|
|
attr: str,
|
|
) -> list[str]:
|
|
values: list[str] = []
|
|
seen: set[str] = set()
|
|
session_profile = getattr(browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, browser_profile):
|
|
for domain in _domain_list(getattr(profile, attr, None)):
|
|
if domain in seen:
|
|
continue
|
|
values.append(domain)
|
|
seen.add(domain)
|
|
return values
|
|
|
|
|
|
def _sensitive_data_context(sensitive_data: dict[str, str | dict[str, str]] | None) -> dict[str, Any]:
|
|
if not sensitive_data:
|
|
return {'global_placeholders': [], 'domain_placeholders': {}}
|
|
global_placeholders: list[str] = []
|
|
domain_placeholders: dict[str, list[str]] = {}
|
|
for key, value in sensitive_data.items():
|
|
if isinstance(value, dict):
|
|
placeholders = sorted(name for name, secret in value.items() if isinstance(name, str) and name and secret)
|
|
if placeholders:
|
|
domain_placeholders[key] = placeholders
|
|
elif isinstance(value, str) and value:
|
|
global_placeholders.append(key)
|
|
return {
|
|
'global_placeholders': sorted(global_placeholders),
|
|
'domain_placeholders': domain_placeholders,
|
|
}
|
|
|
|
|
|
def _sensitive_domain_is_allowed(domain_pattern: str, allowed_domain: str) -> bool:
|
|
if domain_pattern == allowed_domain or allowed_domain == '*':
|
|
return True
|
|
pattern_domain = domain_pattern.split('://')[-1] if '://' in domain_pattern else domain_pattern
|
|
allowed_domain_part = allowed_domain.split('://')[-1] if '://' in allowed_domain else allowed_domain
|
|
return pattern_domain == allowed_domain_part or (
|
|
allowed_domain_part.startswith('*.')
|
|
and (pattern_domain == allowed_domain_part[2:] or pattern_domain.endswith('.' + allowed_domain_part[2:]))
|
|
)
|
|
|
|
|
|
def _warn_sensitive_data_domain_constraints(
|
|
logger: logging.Logger,
|
|
sensitive_data: dict[str, str | dict[str, str]] | None,
|
|
allowed_domains: list[str],
|
|
) -> None:
|
|
if not sensitive_data:
|
|
return
|
|
if not allowed_domains:
|
|
logger.warning(
|
|
'⚠️ Agent(sensitive_data=••••••••) was provided but Browser(allowed_domains=[...]) is not locked down! ⚠️\n'
|
|
' ☠️ If the agent visits a malicious website and encounters a prompt-injection attack, your sensitive_data may be exposed!\n\n'
|
|
' \n'
|
|
)
|
|
return
|
|
for domain_pattern, value in sensitive_data.items():
|
|
if not isinstance(value, dict):
|
|
continue
|
|
if not any(_sensitive_domain_is_allowed(domain_pattern, allowed_domain) for allowed_domain in allowed_domains):
|
|
logger.warning(
|
|
f'⚠️ Domain pattern "{domain_pattern}" in sensitive_data is not covered by any pattern in allowed_domains={allowed_domains}\n'
|
|
f' This may be a security risk as credentials could be used on unintended domains.'
|
|
)
|
|
|
|
|
|
def _string_env_value(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
return text or None
|
|
|
|
|
|
def _direct_initial_navigation_enabled() -> bool:
|
|
raw = os.getenv('BROWSER_USE_RUST_DIRECT_INITIAL_NAVIGATION')
|
|
if raw is None:
|
|
return True
|
|
return raw.strip().lower() not in {'0', 'false', 'no', 'off'}
|
|
|
|
|
|
def _llm_provider_name(llm: Any) -> str | None:
|
|
provider = getattr(llm, 'provider', None)
|
|
if callable(provider):
|
|
try:
|
|
provider = provider()
|
|
except TypeError:
|
|
provider = None
|
|
text = _string_env_value(provider)
|
|
return text.lower() if text else None
|
|
|
|
|
|
def _llm_env_overrides(llm: Any) -> dict[str, str]:
|
|
provider = _llm_provider_name(llm)
|
|
if provider is None:
|
|
return {}
|
|
api_key = _string_env_value(getattr(llm, 'api_key', None))
|
|
base_url = _string_env_value(getattr(llm, 'base_url', None))
|
|
overrides: dict[str, str] = {}
|
|
if provider == 'openai':
|
|
if api_key:
|
|
overrides['LLM_BROWSER_OPENAI_API_KEY'] = api_key
|
|
if base_url:
|
|
overrides['LLM_BROWSER_OPENAI_BASE_URL'] = base_url
|
|
elif provider == 'anthropic':
|
|
if api_key:
|
|
overrides['LLM_BROWSER_ANTHROPIC_API_KEY'] = api_key
|
|
if base_url:
|
|
overrides['LLM_BROWSER_ANTHROPIC_BASE_URL'] = base_url
|
|
elif provider == 'openrouter':
|
|
if api_key:
|
|
overrides['OPENROUTER_API_KEY'] = api_key
|
|
if base_url:
|
|
overrides['OPENROUTER_BASE_URL'] = base_url
|
|
elif provider == 'deepseek' and api_key:
|
|
overrides['DEEPSEEK_API_KEY'] = api_key
|
|
elif provider == 'browser-use':
|
|
if api_key:
|
|
overrides['LLM_BROWSER_BROWSER_USE_API_KEY'] = api_key
|
|
if base_url:
|
|
overrides['LLM_BROWSER_BROWSER_USE_BASE_URL'] = base_url.rstrip('/').removesuffix('/v1')
|
|
return overrides
|
|
|
|
|
|
def _navigation_url_from_action(action: Any) -> str | None:
|
|
params = _initial_navigation_params_from_action(action)
|
|
return params[0] if params is not None else None
|
|
|
|
|
|
def _initial_navigation_params_from_action(action: Any) -> tuple[str, bool] | None:
|
|
if not isinstance(action, dict):
|
|
return None
|
|
for name, payload in action.items():
|
|
if name in ('open_tab', 'go_to_url', 'navigate') and isinstance(payload, dict):
|
|
url = payload.get('url')
|
|
if isinstance(url, str) and url:
|
|
new_tab = name == 'open_tab' or bool(payload.get('new_tab'))
|
|
return url, new_tab
|
|
return None
|
|
|
|
|
|
def _initial_navigation_url(initial_actions: Any) -> str | None:
|
|
if not isinstance(initial_actions, list):
|
|
return None
|
|
for action in initial_actions:
|
|
url = _navigation_url_from_action(action)
|
|
if url:
|
|
return url
|
|
return None
|
|
|
|
|
|
def _task_with_initial_actions(task: str, initial_actions: Any) -> str:
|
|
if not isinstance(initial_actions, list) or not initial_actions:
|
|
return task
|
|
if len(initial_actions) == 1:
|
|
url = _navigation_url_from_action(initial_actions[0])
|
|
if url:
|
|
return f'First navigate to {url!r}, then complete the task.\n\n{task}'
|
|
actions = json.dumps(initial_actions, indent=2, default=str)
|
|
return f'Before the task, perform these Browser Use initial actions in order:\n{actions}\n\nThen complete the task.\n\n{task}'
|
|
|
|
|
|
def _initial_navigation_state_lines(completed_states: list[dict[str, Any]] | None) -> list[str]:
|
|
lines: list[str] = []
|
|
for state in completed_states or []:
|
|
requested_url = str(state.get('requested_url') or '')
|
|
current_url = str(state.get('url') or '')
|
|
title = str(state.get('title') or '')
|
|
if not requested_url and not current_url and not title:
|
|
continue
|
|
parts = []
|
|
if requested_url:
|
|
parts.append(f'requested={requested_url!r}')
|
|
if current_url:
|
|
parts.append(f'current_url={current_url!r}')
|
|
if title:
|
|
parts.append(f'title={title!r}')
|
|
lines.append('- ' + ', '.join(parts))
|
|
return lines
|
|
|
|
|
|
def _task_with_completed_initial_navigation_context(
|
|
task: str,
|
|
completed_urls: list[str],
|
|
initial_actions: Any = None,
|
|
completed_states: list[dict[str, Any]] | None = None,
|
|
) -> str:
|
|
urls = [url for url in completed_urls if isinstance(url, str) and url]
|
|
if not urls:
|
|
return task
|
|
cleaned_task = task
|
|
state_lines = _initial_navigation_state_lines(completed_states)
|
|
state_context = ''
|
|
if state_lines:
|
|
state_context = '\nObserved current page after the completed initial navigation:\n' + '\n'.join(state_lines)
|
|
if len(urls) == 1:
|
|
prefix = f'First navigate to {urls[0]!r}, then complete the task.\n\n'
|
|
if cleaned_task.startswith(prefix):
|
|
cleaned_task = cleaned_task[len(prefix) :]
|
|
return (
|
|
f'The browser session is already open at {urls[0]!r}. '
|
|
'Continue from the current page. Your first browser step should inspect or extract from the current page before any repeat navigation. '
|
|
'Do not navigate to that same start URL again unless browser status or page_info shows a different URL.'
|
|
f'{state_context}'
|
|
f'\n\n{cleaned_task}'
|
|
)
|
|
initial_action_urls: list[str] = []
|
|
if isinstance(initial_actions, list):
|
|
for action in initial_actions:
|
|
url = _navigation_url_from_action(action)
|
|
if url:
|
|
initial_action_urls.append(url)
|
|
if initial_action_urls == urls:
|
|
actions = json.dumps(initial_actions, indent=2, default=str)
|
|
prefix = f'Before the task, perform these Browser Use initial actions in order:\n{actions}\n\nThen complete the task.\n\n'
|
|
if cleaned_task.startswith(prefix):
|
|
cleaned_task = cleaned_task[len(prefix) :]
|
|
completed = ', '.join(repr(url) for url in urls)
|
|
return (
|
|
f'The browser session has already completed these initial navigations: {completed}. '
|
|
'Continue from the current browser state. Do not repeat those navigation steps unless browser status shows a different page.'
|
|
f'{state_context}'
|
|
f'\n\n{cleaned_task}'
|
|
)
|
|
|
|
|
|
def _task_with_schema(task: str, output_model_schema: type[BaseModel] | None) -> str:
|
|
if output_model_schema is None:
|
|
return task
|
|
schema = json.dumps(output_model_schema.model_json_schema(), indent=2)
|
|
return f'{task}\n\nExpected output JSON schema for the final answer:\n{schema}'
|
|
|
|
|
|
def _task_with_available_files(task: str, available_file_paths: list[str] | None) -> str:
|
|
if not available_file_paths:
|
|
return task
|
|
files = '\n'.join(f'- {Path(path).expanduser()}' for path in available_file_paths)
|
|
return f'{task}\n\nAvailable local files:\n{files}'
|
|
|
|
|
|
def _task_with_domain_constraints(task: str, allowed_domains: list[str], prohibited_domains: list[str]) -> str:
|
|
if not allowed_domains and not prohibited_domains:
|
|
return task
|
|
sections = ['Browser profile navigation constraints:']
|
|
if allowed_domains:
|
|
sections.append('Allowed domains:')
|
|
sections.extend(f'- {domain}' for domain in allowed_domains)
|
|
if prohibited_domains:
|
|
sections.append('Prohibited domains:')
|
|
sections.extend(f'- {domain}' for domain in prohibited_domains)
|
|
sections.append('Respect these BrowserProfile domain constraints when navigating.')
|
|
return f'{task}\n\n' + '\n'.join(sections)
|
|
|
|
|
|
def _task_with_sensitive_data_context(task: str, sensitive_context: dict[str, Any]) -> str:
|
|
global_placeholders = sensitive_context.get('global_placeholders') or []
|
|
domain_placeholders = sensitive_context.get('domain_placeholders') or {}
|
|
if not global_placeholders and not domain_placeholders:
|
|
return task
|
|
sections = ['Sensitive data placeholders are available. Use <secret>placeholder</secret> when a matching secret is needed.']
|
|
if global_placeholders:
|
|
sections.append('Global placeholders:')
|
|
sections.extend(f'- {placeholder}' for placeholder in global_placeholders)
|
|
if domain_placeholders:
|
|
sections.append('Domain-scoped placeholders:')
|
|
for domain, placeholders in domain_placeholders.items():
|
|
sections.append(f'- {domain}: {", ".join(placeholders)}')
|
|
sections.append('Do not reveal placeholder values in the final answer.')
|
|
return f'{task}\n\n' + '\n'.join(sections)
|
|
|
|
|
|
def _extract_start_url(task: str) -> str | None:
|
|
task_without_emails = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '', task)
|
|
patterns = [
|
|
r'https?://[^\s<>"\']+',
|
|
r'(?:www\.)?[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,}(?:/[^\s<>"\']*)?',
|
|
]
|
|
excluded_extensions = {
|
|
'pdf',
|
|
'doc',
|
|
'docx',
|
|
'xls',
|
|
'xlsx',
|
|
'ppt',
|
|
'pptx',
|
|
'odt',
|
|
'ods',
|
|
'odp',
|
|
'txt',
|
|
'md',
|
|
'csv',
|
|
'json',
|
|
'xml',
|
|
'yaml',
|
|
'yml',
|
|
'zip',
|
|
'rar',
|
|
'7z',
|
|
'tar',
|
|
'gz',
|
|
'bz2',
|
|
'xz',
|
|
'jpg',
|
|
'jpeg',
|
|
'png',
|
|
'gif',
|
|
'bmp',
|
|
'svg',
|
|
'webp',
|
|
'ico',
|
|
'mp3',
|
|
'mp4',
|
|
'avi',
|
|
'mkv',
|
|
'mov',
|
|
'wav',
|
|
'flac',
|
|
'ogg',
|
|
'py',
|
|
'js',
|
|
'css',
|
|
'java',
|
|
'cpp',
|
|
'bib',
|
|
'bibtex',
|
|
'tex',
|
|
'latex',
|
|
'cls',
|
|
'sty',
|
|
'exe',
|
|
'msi',
|
|
'dmg',
|
|
'pkg',
|
|
'deb',
|
|
'rpm',
|
|
'iso',
|
|
}
|
|
excluded_words = {'never', 'dont', 'not', "don't"}
|
|
|
|
found_urls = []
|
|
for pattern in patterns:
|
|
for match in re.finditer(pattern, task_without_emails):
|
|
url = sanitize_url_candidate(match.group(0))
|
|
url_lower = url.lower()
|
|
if is_placeholder_url(url):
|
|
continue
|
|
if any(f'.{ext}' in url_lower for ext in excluded_extensions):
|
|
continue
|
|
context_start = max(0, match.start() - 20)
|
|
context_text = task_without_emails[context_start : match.start()]
|
|
if any(word in context_text.lower() for word in excluded_words):
|
|
continue
|
|
if not url.startswith(('http://', 'https://')):
|
|
url = 'https://' + url
|
|
found_urls.append(url)
|
|
|
|
unique_urls = list(set(found_urls))
|
|
return unique_urls[0] if len(unique_urls) == 1 else None
|
|
|
|
|
|
def _event_type(event: dict[str, Any]) -> str:
|
|
return str(event.get('event_type') or event.get('type') or '')
|
|
|
|
|
|
def _event_payload(event: dict[str, Any]) -> dict[str, Any]:
|
|
payload = event.get('payload')
|
|
return payload if isinstance(payload, dict) else {}
|
|
|
|
|
|
def _event_seq(event: dict[str, Any]) -> int | None:
|
|
for key in ('seq', 'event_seq', 'sequence'):
|
|
value = event.get(key)
|
|
if isinstance(value, bool):
|
|
continue
|
|
if isinstance(value, int):
|
|
return value
|
|
return None
|
|
|
|
|
|
def _rollback_turn_count(payload: dict[str, Any]) -> int:
|
|
for key in ('num_turns', 'turns', 'n'):
|
|
value = payload.get(key)
|
|
if isinstance(value, bool):
|
|
continue
|
|
if isinstance(value, int) and value >= 0:
|
|
return value
|
|
return 1
|
|
|
|
|
|
def _is_terminal_user_turn_event(event: dict[str, Any]) -> bool:
|
|
event_type = _event_type(event)
|
|
if event_type in ('session.input', 'session.followup'):
|
|
return True
|
|
if event_type in ('agent.message', 'agent.mailbox_input'):
|
|
content = _event_payload(event).get('content')
|
|
return isinstance(content, str) and bool(content.strip())
|
|
return False
|
|
|
|
|
|
def _contextual_event_targets_turn(event: dict[str, Any], target_seq: int | None) -> bool:
|
|
if target_seq is None:
|
|
return False
|
|
if _event_type(event) not in (
|
|
'workspace.context',
|
|
'model.switch_context',
|
|
'model.personality_context',
|
|
'model.collaboration_context',
|
|
'model.generated_image_context',
|
|
):
|
|
return False
|
|
before_seq = _event_payload(event).get('before_seq')
|
|
return isinstance(before_seq, int) and before_seq == target_seq
|
|
|
|
|
|
def _compaction_replay_start_seq(event: dict[str, Any]) -> int | None:
|
|
replay_from_seq = _event_payload(event).get('replay_from_seq')
|
|
if isinstance(replay_from_seq, bool):
|
|
return None
|
|
if isinstance(replay_from_seq, int):
|
|
return replay_from_seq
|
|
return _event_seq(event)
|
|
|
|
|
|
def _events_after_terminal_compaction(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
compaction_index = next(
|
|
(index for index in range(len(events) - 1, -1, -1) if _event_type(events[index]) == 'session.compacted'), None
|
|
)
|
|
if compaction_index is None:
|
|
return events
|
|
replay_start_seq = _compaction_replay_start_seq(events[compaction_index])
|
|
if replay_start_seq is None:
|
|
return events[compaction_index + 1 :]
|
|
replay_events: list[dict[str, Any]] = []
|
|
for index, event in enumerate(events):
|
|
seq = _event_seq(event)
|
|
if seq is not None:
|
|
if seq > replay_start_seq:
|
|
replay_events.append(event)
|
|
elif index > compaction_index:
|
|
replay_events.append(event)
|
|
return replay_events
|
|
|
|
|
|
def _rollback_last_terminal_user_turn(events: list[dict[str, Any]]) -> bool:
|
|
user_pos = next((index for index in range(len(events) - 1, -1, -1) if _is_terminal_user_turn_event(events[index])), None)
|
|
if user_pos is None:
|
|
return False
|
|
target_seq = _event_seq(events[user_pos])
|
|
truncate_at = user_pos
|
|
while truncate_at > 0 and _contextual_event_targets_turn(events[truncate_at - 1], target_seq):
|
|
truncate_at -= 1
|
|
del events[truncate_at:]
|
|
return True
|
|
|
|
|
|
def _events_after_terminal_rollbacks(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
if not any(_event_type(event) == 'session.rollback' for event in events):
|
|
return events
|
|
replay_events: list[dict[str, Any]] = []
|
|
for event in events:
|
|
if _event_type(event) == 'session.rollback':
|
|
for _ in range(_rollback_turn_count(_event_payload(event))):
|
|
if not _rollback_last_terminal_user_turn(replay_events):
|
|
break
|
|
continue
|
|
replay_events.append(event)
|
|
return replay_events
|
|
|
|
|
|
def _result_file_pointer(payload: dict[str, Any]) -> str | None:
|
|
result_file = payload.get('result_file')
|
|
if isinstance(result_file, dict):
|
|
for key in ('url', 'path'):
|
|
value = result_file.get(key)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
if isinstance(result_file, str) and result_file:
|
|
return result_file
|
|
for key in ('result_file_url', 'result_file_path', 'result_file'):
|
|
value = payload.get(key)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _artifact_attachment_pointer(value: Any) -> str | None:
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
if not isinstance(value, dict):
|
|
return None
|
|
kind = value.get('kind')
|
|
mime = value.get('mime_type') or value.get('mime')
|
|
if isinstance(kind, str) and kind.lower() == 'image':
|
|
return None
|
|
if isinstance(mime, str) and mime.lower().startswith('image/'):
|
|
return None
|
|
for key in ('url', 'file_url', 'path'):
|
|
pointer = value.get(key)
|
|
if isinstance(pointer, str) and pointer:
|
|
return pointer
|
|
return None
|
|
|
|
|
|
def _done_tool_result_from_events(events: list[dict[str, Any]]) -> str | None:
|
|
for event in reversed(events):
|
|
event_type = _event_type(event)
|
|
if event_type not in ('tool.started', 'tool.output', 'tool.finished'):
|
|
continue
|
|
payload = _event_payload(event)
|
|
if payload.get('name') != 'done':
|
|
continue
|
|
arguments = payload.get('arguments')
|
|
if isinstance(arguments, dict):
|
|
for key in ('text', 'result', 'answer'):
|
|
value = arguments.get(key)
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()
|
|
text = _tool_result_text(payload)
|
|
if not text:
|
|
continue
|
|
text = text.strip()
|
|
if text.lower().startswith('done:'):
|
|
text = text[5:].strip()
|
|
if text:
|
|
return text
|
|
return None
|
|
|
|
|
|
def _result_from_events(events: list[dict[str, Any]]) -> str | None:
|
|
done_result = _done_tool_result_from_events(events)
|
|
if done_result:
|
|
return done_result
|
|
for event in reversed(events):
|
|
if _event_type(event) != 'session.done':
|
|
continue
|
|
payload = _event_payload(event)
|
|
result = payload.get('result')
|
|
if isinstance(result, str) and result.strip():
|
|
return result.strip()
|
|
result_file = _result_file_pointer(payload)
|
|
if result_file:
|
|
return f'Saved result file.\n\nFile:\n{result_file}'
|
|
for event in reversed(events):
|
|
if _event_type(event) != 'agent.completed':
|
|
continue
|
|
payload = _event_payload(event).get('payload')
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
result = payload.get('result')
|
|
if isinstance(result, str) and result.strip():
|
|
return result.strip()
|
|
return None
|
|
|
|
|
|
def _last_streamed_assistant_text_from_events(events: list[dict[str, Any]]) -> str | None:
|
|
last_request_index = next(
|
|
(index for index in range(len(events) - 1, -1, -1) if _event_type(events[index]) == 'model.turn.request'),
|
|
-1,
|
|
)
|
|
candidates = events[last_request_index + 1 :] if last_request_index >= 0 else events
|
|
text = _streaming_text_from_events(candidates, ('model.delta', 'model.stream_delta', 'model.response.output_item'))
|
|
if not isinstance(text, str):
|
|
return None
|
|
text = text.strip()
|
|
return text or None
|
|
|
|
|
|
def _attachments_from_events(events: list[dict[str, Any]]) -> list[str] | None:
|
|
attachments: list[str] = []
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
pointers: list[str] = []
|
|
if event_type == 'session.done':
|
|
result_file = _result_file_pointer(payload)
|
|
if result_file:
|
|
pointers.append(result_file)
|
|
elif event_type in ('artifact.created', 'tool.output_spilled'):
|
|
pointer = _artifact_attachment_pointer(payload.get('artifact'))
|
|
if pointer:
|
|
pointers.append(pointer)
|
|
elif event_type == 'capture.curation':
|
|
gif_path = payload.get('gif_path')
|
|
if isinstance(gif_path, str) and gif_path:
|
|
pointers.append(gif_path)
|
|
elif event_type in ('tool.output', 'tool.failed'):
|
|
text_artifact = _artifact_attachment_pointer(payload.get('text_artifact'))
|
|
if text_artifact:
|
|
pointers.append(text_artifact)
|
|
artifacts = payload.get('artifacts')
|
|
if isinstance(artifacts, list):
|
|
for artifact in artifacts:
|
|
pointer = _artifact_attachment_pointer(artifact)
|
|
if pointer:
|
|
pointers.append(pointer)
|
|
for pointer in pointers:
|
|
if pointer not in attachments:
|
|
attachments.append(pointer)
|
|
return attachments or None
|
|
|
|
|
|
def _json_result_candidates(text: str) -> list[str]:
|
|
candidates = [text.strip()]
|
|
candidates.extend(
|
|
match.group(1).strip() for match in re.finditer(r'```(?:json)?\s*(.*?)```', text, re.IGNORECASE | re.DOTALL)
|
|
)
|
|
decoder = json.JSONDecoder()
|
|
for index, char in enumerate(text):
|
|
if char not in '{[':
|
|
continue
|
|
try:
|
|
parsed, _end = decoder.raw_decode(text[index:])
|
|
except json.JSONDecodeError:
|
|
continue
|
|
candidates.append(json.dumps(parsed))
|
|
seen = set()
|
|
unique = []
|
|
for candidate in candidates:
|
|
if not candidate or candidate in seen:
|
|
continue
|
|
seen.add(candidate)
|
|
unique.append(candidate)
|
|
return unique
|
|
|
|
|
|
def _structured_result_text(
|
|
result: str | None,
|
|
output_model_schema: type[BaseModel] | None,
|
|
) -> str | None:
|
|
if result is None or output_model_schema is None:
|
|
return result
|
|
for candidate in _json_result_candidates(result):
|
|
try:
|
|
output_model_schema.model_validate_json(candidate)
|
|
except (ValidationError, ValueError):
|
|
continue
|
|
return candidate
|
|
return result
|
|
|
|
|
|
def _failure_from_events(events: list[dict[str, Any]]) -> str | None:
|
|
for event in reversed(events):
|
|
event_type = _event_type(event)
|
|
if event_type in ('session.failed', 'stream_error'):
|
|
payload = _event_payload(event)
|
|
error = payload.get('error') or payload.get('message')
|
|
if isinstance(error, str) and error:
|
|
return error
|
|
if event_type == 'session.cancelled':
|
|
payload = _event_payload(event)
|
|
reason = payload.get('reason') or payload.get('message') or payload.get('error')
|
|
if isinstance(reason, str) and reason.strip():
|
|
return f'Rust terminal session was cancelled: {reason.strip()}'
|
|
return 'Rust terminal session was cancelled.'
|
|
if event_type == 'session.interrupted':
|
|
payload = _event_payload(event)
|
|
reason = payload.get('reason') or payload.get('message') or payload.get('error')
|
|
if isinstance(reason, str) and reason.strip():
|
|
return f'Rust terminal session was interrupted: {reason.strip()}'
|
|
return 'Rust terminal session was interrupted.'
|
|
if event_type in ('agent.failed', 'agent.cancelled'):
|
|
payload = _event_payload(event)
|
|
inner_payload = payload.get('payload')
|
|
if not isinstance(inner_payload, dict):
|
|
inner_payload = payload
|
|
detail = _payload_failure_detail(inner_payload)
|
|
if detail:
|
|
return detail
|
|
if event_type == 'agent.cancelled':
|
|
return 'Subagent was cancelled.'
|
|
return 'Subagent failed.'
|
|
return None
|
|
|
|
|
|
def _recoverable_failure_from_events(events: list[dict[str, Any]]) -> str | None:
|
|
for event in reversed(events):
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
if event_type == 'tool.failed':
|
|
abort_payload = _matching_tool_abort_payload(events, payload)
|
|
error = _tool_abort_message(abort_payload) if abort_payload is not None else _tool_failure_message(payload)
|
|
if error:
|
|
return error
|
|
if event_type == 'tool.aborted':
|
|
error = _tool_abort_message(payload)
|
|
if error:
|
|
return error
|
|
if event_type == 'exec_command.end' and not _matching_tool_result_payload(
|
|
events, payload, ('tool.failed', 'tool.aborted')
|
|
):
|
|
error = _exec_command_end_failure_message(payload)
|
|
if error:
|
|
return error
|
|
if event_type in ('model.turn.error', 'model.turn.context_overflow'):
|
|
error = payload.get('error') or payload.get('message') or payload.get('reason')
|
|
if isinstance(error, str) and error.strip():
|
|
return error.strip()
|
|
error = _terminal_operational_failure_message(event_type, payload)
|
|
if error:
|
|
return error
|
|
return None
|
|
|
|
|
|
def _tool_failure_message(payload: dict[str, Any]) -> str | None:
|
|
error = payload.get('error')
|
|
if not isinstance(error, str) or not error.strip():
|
|
return None
|
|
name = payload.get('name')
|
|
return f'{name} failed: {error.strip()}' if isinstance(name, str) and name else error.strip()
|
|
|
|
|
|
def _tool_abort_message(payload: dict[str, Any]) -> str | None:
|
|
error = payload.get('error') or payload.get('reason') or payload.get('message')
|
|
if not isinstance(error, str) or not error.strip():
|
|
error = 'aborted'
|
|
name = payload.get('name')
|
|
return f'{name} aborted: {error.strip()}' if isinstance(name, str) and name else error.strip()
|
|
|
|
|
|
def _exec_command_end_failure_message(payload: dict[str, Any]) -> str | None:
|
|
exit_code = _int_value(payload.get('exit_code'))
|
|
if exit_code == 0:
|
|
return None
|
|
name = payload.get('name')
|
|
tool_name = name if isinstance(name, str) and name else 'exec_command'
|
|
detail = f'exit code {exit_code}'
|
|
text = _tool_result_text(payload)
|
|
if text:
|
|
detail = f'{detail}: {text}'
|
|
return f'{tool_name} failed: {detail}'
|
|
|
|
|
|
def _payload_failure_detail(payload: dict[str, Any]) -> str | None:
|
|
for key in ('error', 'failure', 'message', 'reason', 'detail', 'details'):
|
|
value = payload.get(key)
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()
|
|
if isinstance(payload.get('errors'), list):
|
|
errors = [str(error).strip() for error in payload['errors'] if str(error).strip()]
|
|
if errors:
|
|
return '; '.join(errors)
|
|
return None
|
|
|
|
|
|
def _terminal_operational_failure_message(event_type: str, payload: dict[str, Any]) -> str | None:
|
|
if event_type == 'session.final_answer_not_ready_at_max_turns':
|
|
return _payload_failure_detail(payload) or 'final answer artifact is not ready'
|
|
if event_type == 'browser.cleanup_timed_out':
|
|
timeout_ms = _int_value(payload.get('timeout_ms'))
|
|
if timeout_ms:
|
|
return f'browser cleanup timed out after {timeout_ms}ms'
|
|
return 'browser cleanup timed out'
|
|
labels = {
|
|
'browser.cloud_shutdown_failed': 'browser cloud shutdown failed',
|
|
'browser.cleanup_failed': 'browser cleanup failed',
|
|
'browser.bridge_errors': 'browser bridge failed',
|
|
'command.write_error': 'command write failed',
|
|
'session.compaction_failed': 'session compaction failed',
|
|
}
|
|
label = labels.get(event_type)
|
|
if label is None:
|
|
return None
|
|
detail = _payload_failure_detail(payload)
|
|
return f'{label}: {detail}' if detail else label
|
|
|
|
|
|
def _command_waiting_text(payload: dict[str, Any]) -> str:
|
|
session_id = payload.get('session_id') or payload.get('process_id')
|
|
if session_id is not None and str(session_id):
|
|
return f'Process running with session ID {session_id}'
|
|
return 'Process running'
|
|
|
|
|
|
def _command_waiting_payload(payload: dict[str, Any], previous_payload: dict[str, Any] | None) -> dict[str, Any]:
|
|
waiting_text = _command_waiting_text(payload)
|
|
previous_text = _tool_result_text(previous_payload) if previous_payload is not None else None
|
|
if previous_text and waiting_text not in previous_text:
|
|
waiting_text = f'{previous_text}\n\n{waiting_text}'
|
|
next_payload = dict(payload)
|
|
next_payload['text'] = waiting_text
|
|
return next_payload
|
|
|
|
|
|
_TOOL_OUTPUT_DELTA_EVENTS = ('tool.output_delta', 'exec_command.output_delta', 'browser_script.output_delta')
|
|
_BROWSER_SCRIPT_RESULT_EVENTS = ('browser_script.completed', 'browser_script.cancelled', 'browser_script.failed')
|
|
|
|
|
|
_TEXTUAL_TOOL_RESULT_EVENTS = (
|
|
'tool.output',
|
|
'tool.output_delta',
|
|
'exec_command.output_delta',
|
|
'browser_script.output_delta',
|
|
'command.waiting',
|
|
'exec_command.end',
|
|
'tool.finished',
|
|
'model.response.input_item',
|
|
'browser_script.completed',
|
|
'browser_script.cancelled',
|
|
)
|
|
_MAX_TERMINAL_LONG_TERM_TEXT_LENGTH = 1000
|
|
|
|
|
|
def _is_redundant_paired_output_delta(
|
|
previous_event_type: str,
|
|
previous_payload: dict[str, Any],
|
|
event_type: str,
|
|
payload: dict[str, Any],
|
|
delta_text: str,
|
|
) -> bool:
|
|
if previous_event_type not in _TOOL_OUTPUT_DELTA_EVENTS or previous_event_type == event_type:
|
|
return False
|
|
previous_text = previous_payload.get('text')
|
|
if not isinstance(previous_text, str) or not previous_text.endswith(delta_text):
|
|
return False
|
|
previous_stream = previous_payload.get('stream')
|
|
stream = payload.get('stream')
|
|
if previous_stream is not None and stream is not None and previous_stream != stream:
|
|
return False
|
|
previous_session_id = previous_payload.get('session_id') or previous_payload.get('process_id')
|
|
session_id = payload.get('session_id') or payload.get('process_id')
|
|
if previous_session_id is not None and session_id is not None and str(previous_session_id) != str(session_id):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _tool_result_key(payload: dict[str, Any]) -> tuple[str, str] | None:
|
|
call_id = payload.get('tool_call_id') or payload.get('call_id')
|
|
if call_id:
|
|
return ('call_id', str(call_id))
|
|
name = payload.get('name')
|
|
if isinstance(name, str) and name:
|
|
return ('name', name)
|
|
return None
|
|
|
|
|
|
def _matching_tool_result_payload(
|
|
events: list[dict[str, Any]], payload: dict[str, Any], event_types: tuple[str, ...]
|
|
) -> dict[str, Any] | None:
|
|
key = _tool_result_key(payload)
|
|
if key is None:
|
|
return None
|
|
for event in events:
|
|
if _event_type(event) not in event_types:
|
|
continue
|
|
result_payload = _event_payload(event)
|
|
if _tool_result_key(result_payload) == key:
|
|
return result_payload
|
|
return None
|
|
|
|
|
|
def _matching_tool_abort_payload(events: list[dict[str, Any]], payload: dict[str, Any]) -> dict[str, Any] | None:
|
|
return _matching_tool_result_payload(events, payload, ('tool.aborted',))
|
|
|
|
|
|
def _safe_tool_action_name(value: Any) -> str | None:
|
|
if not isinstance(value, str):
|
|
return None
|
|
name = value.strip()
|
|
if not name or name.startswith('_') or not name.isidentifier() or keyword.iskeyword(name):
|
|
return None
|
|
return name
|
|
|
|
|
|
def _tool_arguments(value: Any) -> dict[str, Any]:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
parsed = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return {'value': value}
|
|
return parsed if isinstance(parsed, dict) else {'value': parsed}
|
|
if value is None:
|
|
return {}
|
|
return {'value': value}
|
|
|
|
|
|
def _tool_call_from_payload(payload: dict[str, Any], fallback_id: str) -> dict[str, Any] | None:
|
|
function = payload.get('function')
|
|
if not isinstance(function, dict):
|
|
function = {}
|
|
name = _safe_tool_action_name(payload.get('name') or function.get('name'))
|
|
if name is None:
|
|
return None
|
|
raw_call_id = payload.get('tool_call_id') or payload.get('call_id') or payload.get('id')
|
|
call_id = raw_call_id or fallback_id
|
|
arguments = payload.get('arguments')
|
|
if arguments is None:
|
|
arguments = function.get('arguments')
|
|
return {
|
|
'name': name,
|
|
'tool_call_id': str(call_id),
|
|
'_has_explicit_tool_call_id': raw_call_id is not None,
|
|
'arguments': _tool_arguments(arguments),
|
|
}
|
|
|
|
|
|
def _tool_call_from_response_item(payload: dict[str, Any], fallback_id: str) -> dict[str, Any] | None:
|
|
item = payload.get('item')
|
|
if not isinstance(item, dict):
|
|
return None
|
|
if item.get('type') not in ('function_call', 'custom_tool_call'):
|
|
return None
|
|
return _tool_call_from_payload(item, fallback_id)
|
|
|
|
|
|
def _tool_started_calls(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
tool_calls: list[dict[str, Any]] = []
|
|
seen_call_ids: set[str] = set()
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
if event_type in ('tool.started', 'model.tool_call'):
|
|
tool_call = _tool_call_from_payload(payload, f'tool-{len(tool_calls)}')
|
|
elif event_type == 'model.response.output_item':
|
|
tool_call = _tool_call_from_response_item(payload, f'tool-{len(tool_calls)}')
|
|
else:
|
|
continue
|
|
if tool_call is None:
|
|
continue
|
|
call_id = tool_call['tool_call_id']
|
|
if call_id in seen_call_ids:
|
|
continue
|
|
seen_call_ids.add(call_id)
|
|
tool_calls.append(tool_call)
|
|
return tool_calls
|
|
|
|
|
|
def _tool_calls_with_final_done(
|
|
tool_calls: list[dict[str, Any]],
|
|
*,
|
|
final_result: str | None,
|
|
attachments: list[str] | None,
|
|
is_done: bool,
|
|
) -> list[dict[str, Any]]:
|
|
if not is_done or final_result is None:
|
|
return tool_calls
|
|
if tool_calls and tool_calls[-1].get('name') == 'done':
|
|
return tool_calls
|
|
done_arguments: dict[str, Any] = {'text': final_result, 'success': True}
|
|
if attachments:
|
|
done_arguments['files_to_display'] = attachments
|
|
return [
|
|
*tool_calls,
|
|
{
|
|
'name': 'done',
|
|
'tool_call_id': 'session.done',
|
|
'arguments': done_arguments,
|
|
},
|
|
]
|
|
|
|
|
|
def _tool_results_by_call_id(events: list[dict[str, Any]]) -> dict[str, tuple[str, dict[str, Any]]]:
|
|
results: dict[str, tuple[str, dict[str, Any]]] = {}
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
if event_type not in (
|
|
'tool.output',
|
|
'tool.output_delta',
|
|
'exec_command.output_delta',
|
|
'browser_script.output_delta',
|
|
'command.waiting',
|
|
'exec_command.end',
|
|
'tool.failed',
|
|
'tool.aborted',
|
|
'tool.finished',
|
|
'model.response.input_item',
|
|
*_BROWSER_SCRIPT_RESULT_EVENTS,
|
|
):
|
|
continue
|
|
payload = _event_payload(event)
|
|
if event_type == 'model.response.input_item':
|
|
payload = _response_input_item_tool_payload(payload)
|
|
if event_type in _TOOL_OUTPUT_DELTA_EVENTS:
|
|
delta_text = _tool_output_delta_text(payload)
|
|
if not delta_text:
|
|
continue
|
|
if event_type == 'exec_command.end' and not _tool_result_text(payload):
|
|
continue
|
|
call_id = payload.get('tool_call_id') or payload.get('call_id')
|
|
if call_id:
|
|
key = str(call_id)
|
|
previous = results.get(key)
|
|
if event_type in _TOOL_OUTPUT_DELTA_EVENTS:
|
|
if previous is not None and previous[0] not in (*_TOOL_OUTPUT_DELTA_EVENTS, 'tool.finished'):
|
|
continue
|
|
if (
|
|
previous is not None
|
|
and delta_text
|
|
and _is_redundant_paired_output_delta(previous[0], previous[1], event_type, payload, delta_text)
|
|
):
|
|
continue
|
|
merged_payload = (
|
|
dict(previous[1]) if previous is not None and previous[0] in _TOOL_OUTPUT_DELTA_EVENTS else dict(payload)
|
|
)
|
|
previous_text = (
|
|
merged_payload.get('text') if previous is not None and previous[0] in _TOOL_OUTPUT_DELTA_EVENTS else ''
|
|
)
|
|
for payload_key, payload_value in payload.items():
|
|
if payload_key != 'text':
|
|
merged_payload[payload_key] = payload_value
|
|
merged_payload['text'] = f'{previous_text if isinstance(previous_text, str) else ""}{delta_text}'
|
|
results[key] = (event_type, merged_payload)
|
|
continue
|
|
if event_type == 'command.waiting':
|
|
if previous is not None and previous[0] in (
|
|
'tool.output',
|
|
'tool.failed',
|
|
'tool.aborted',
|
|
'exec_command.end',
|
|
'model.response.input_item',
|
|
):
|
|
continue
|
|
results[key] = (event_type, _command_waiting_payload(payload, previous[1] if previous is not None else None))
|
|
continue
|
|
if (
|
|
event_type == 'exec_command.end'
|
|
and previous is not None
|
|
and previous[0]
|
|
in (
|
|
'tool.output',
|
|
'tool.failed',
|
|
'tool.aborted',
|
|
'model.response.input_item',
|
|
)
|
|
):
|
|
continue
|
|
if event_type == 'tool.failed' and previous is not None and previous[0] == 'tool.aborted':
|
|
continue
|
|
if event_type == 'tool.finished' and previous is not None and previous[0] != 'tool.finished':
|
|
continue
|
|
results[key] = (event_type, payload)
|
|
return results
|
|
|
|
|
|
def _unkeyed_tool_results(events: list[dict[str, Any]]) -> list[tuple[str, dict[str, Any]]]:
|
|
results: list[tuple[str, dict[str, Any]]] = []
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
if event_type not in (
|
|
'tool.output',
|
|
'tool.output_delta',
|
|
'exec_command.output_delta',
|
|
'browser_script.output_delta',
|
|
'command.waiting',
|
|
'exec_command.end',
|
|
'tool.failed',
|
|
'tool.aborted',
|
|
'tool.finished',
|
|
'model.response.input_item',
|
|
*_BROWSER_SCRIPT_RESULT_EVENTS,
|
|
):
|
|
continue
|
|
payload = _event_payload(event)
|
|
if event_type == 'model.response.input_item':
|
|
payload = _response_input_item_tool_payload(payload)
|
|
if payload.get('tool_call_id') or payload.get('call_id'):
|
|
continue
|
|
if event_type in _TOOL_OUTPUT_DELTA_EVENTS:
|
|
delta_text = _tool_output_delta_text(payload)
|
|
if not delta_text:
|
|
continue
|
|
name = payload.get('name')
|
|
previous = next(
|
|
(
|
|
result
|
|
for result in reversed(results)
|
|
if isinstance(name, str) and name and result[0] in _TOOL_OUTPUT_DELTA_EVENTS and result[1].get('name') == name
|
|
),
|
|
None,
|
|
)
|
|
if previous is not None:
|
|
previous_text = previous[1].get('text')
|
|
previous[1]['text'] = f'{previous_text if isinstance(previous_text, str) else ""}{delta_text}'
|
|
continue
|
|
payload = dict(payload)
|
|
payload['text'] = delta_text
|
|
elif event_type == 'tool.output':
|
|
name = payload.get('name')
|
|
previous_delta_index = next(
|
|
(
|
|
index
|
|
for index in range(len(results) - 1, -1, -1)
|
|
if isinstance(name, str)
|
|
and name
|
|
and results[index][0] in _TOOL_OUTPUT_DELTA_EVENTS
|
|
and results[index][1].get('name') == name
|
|
),
|
|
None,
|
|
)
|
|
if previous_delta_index is not None and payload.get('stream') is not True:
|
|
results[previous_delta_index] = (event_type, payload)
|
|
continue
|
|
elif event_type == 'command.waiting':
|
|
name = payload.get('name')
|
|
previous_result_index = next(
|
|
(
|
|
index
|
|
for index in range(len(results) - 1, -1, -1)
|
|
if isinstance(name, str) and name and results[index][1].get('name') == name
|
|
),
|
|
None,
|
|
)
|
|
if previous_result_index is not None:
|
|
previous_event_type, previous_payload = results[previous_result_index]
|
|
if previous_event_type in (
|
|
'tool.output',
|
|
'tool.failed',
|
|
'tool.aborted',
|
|
'exec_command.end',
|
|
'model.response.input_item',
|
|
):
|
|
continue
|
|
results[previous_result_index] = (event_type, _command_waiting_payload(payload, previous_payload))
|
|
continue
|
|
payload = _command_waiting_payload(payload, None)
|
|
elif event_type == 'exec_command.end':
|
|
if not _tool_result_text(payload):
|
|
continue
|
|
name = payload.get('name')
|
|
previous_result_index = next(
|
|
(
|
|
index
|
|
for index in range(len(results) - 1, -1, -1)
|
|
if isinstance(name, str) and name and results[index][1].get('name') == name
|
|
),
|
|
None,
|
|
)
|
|
if previous_result_index is not None:
|
|
previous_event_type = results[previous_result_index][0]
|
|
if previous_event_type in ('tool.output', 'tool.failed', 'tool.aborted', 'model.response.input_item'):
|
|
continue
|
|
results[previous_result_index] = (event_type, payload)
|
|
continue
|
|
if event_type == 'tool.output' and payload.get('stream') is True:
|
|
continue
|
|
if event_type == 'tool.failed':
|
|
name = payload.get('name')
|
|
previous_abort = next(
|
|
(
|
|
result
|
|
for result in reversed(results)
|
|
if isinstance(name, str) and name and result[0] == 'tool.aborted' and result[1].get('name') == name
|
|
),
|
|
None,
|
|
)
|
|
if previous_abort is not None:
|
|
continue
|
|
if event_type == 'tool.finished':
|
|
name = payload.get('name')
|
|
previous = next(
|
|
(
|
|
result
|
|
for result in reversed(results)
|
|
if isinstance(name, str) and name and result[0] != 'tool.finished' and result[1].get('name') == name
|
|
),
|
|
None,
|
|
)
|
|
if previous is not None:
|
|
continue
|
|
results.append((event_type, payload))
|
|
return results
|
|
|
|
|
|
def _matching_unkeyed_tool_result_index(
|
|
results: list[tuple[str, dict[str, Any]]],
|
|
used_indices: set[int],
|
|
name: Any,
|
|
*,
|
|
allow_any: bool,
|
|
) -> int | None:
|
|
if isinstance(name, str) and name:
|
|
for index, (_event_type, payload) in enumerate(results):
|
|
if index not in used_indices and payload.get('name') == name:
|
|
return index
|
|
if not allow_any:
|
|
return None
|
|
for index in range(len(results)):
|
|
if index not in used_indices:
|
|
return index
|
|
return None
|
|
|
|
|
|
def _matching_unkeyed_tool_attachment_index(
|
|
attachments: list[tuple[str, list[str]]],
|
|
used_indices: set[int],
|
|
name: Any,
|
|
*,
|
|
allow_any: bool,
|
|
) -> int | None:
|
|
if isinstance(name, str) and name:
|
|
for index, (attachment_name, _paths) in enumerate(attachments):
|
|
if index not in used_indices and attachment_name == name:
|
|
return index
|
|
if not allow_any:
|
|
return None
|
|
for index in range(len(attachments)):
|
|
if index not in used_indices:
|
|
return index
|
|
return None
|
|
|
|
|
|
def _response_input_item_tool_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
|
item = payload.get('item')
|
|
if not isinstance(item, dict):
|
|
item = {}
|
|
call_id = item.get('call_id') or item.get('id') or payload.get('call_id') or payload.get('id')
|
|
result_payload: dict[str, Any] = dict(payload)
|
|
if call_id:
|
|
result_payload['tool_call_id'] = call_id
|
|
name = item.get('name') or payload.get('name')
|
|
if isinstance(name, str) and name:
|
|
result_payload['name'] = name
|
|
if 'output' in item:
|
|
result_payload['output'] = item.get('output')
|
|
elif 'content' in item:
|
|
result_payload['content'] = item.get('content')
|
|
return result_payload
|
|
|
|
|
|
def _tool_output_text(value: Any) -> str | None:
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()
|
|
if isinstance(value, list):
|
|
text_parts = []
|
|
for part in value:
|
|
if isinstance(part, str):
|
|
text_parts.append(part)
|
|
elif isinstance(part, dict):
|
|
text = part.get('text')
|
|
if isinstance(text, str):
|
|
text_parts.append(text)
|
|
text = ''.join(text_parts).strip()
|
|
return text or None
|
|
if isinstance(value, dict):
|
|
return json.dumps(value, ensure_ascii=False, default=str)
|
|
return None
|
|
|
|
|
|
def _tool_output_delta_text(payload: dict[str, Any]) -> str | None:
|
|
for key in ('text', 'delta', 'chunk'):
|
|
value = payload.get(key)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _structured_tool_output_text(value: Any) -> str | None:
|
|
if isinstance(value, str) and value.strip():
|
|
return value.strip()
|
|
if isinstance(value, (dict, list)) and value:
|
|
return json.dumps(value, ensure_ascii=False, default=str)
|
|
return None
|
|
|
|
|
|
def _tool_result_text(payload: dict[str, Any], *, include_completion_fallback: bool = True) -> str | None:
|
|
for key in ('text', 'output', 'result'):
|
|
text = _tool_output_text(payload.get(key))
|
|
if text:
|
|
return text
|
|
content = payload.get('content')
|
|
text = _tool_output_text(content)
|
|
if text:
|
|
return text
|
|
text = _browser_script_running_tool_text(payload)
|
|
if text:
|
|
return text
|
|
for key in ('summary', 'data', 'outputs'):
|
|
text = _structured_tool_output_text(payload.get(key))
|
|
if text:
|
|
return text
|
|
if include_completion_fallback and payload.get('name') == 'browser_script' and payload.get('ok') is True:
|
|
status = payload.get('status')
|
|
if status in ('finished', 'completed') or status is None:
|
|
return 'browser_script completed'
|
|
return None
|
|
|
|
|
|
def _browser_script_running_tool_text(payload: dict[str, Any]) -> str | None:
|
|
if payload.get('name') != 'browser_script' or payload.get('status') != 'running':
|
|
return None
|
|
parts = ['browser_script is still running.']
|
|
run_id = payload.get('run_id')
|
|
if isinstance(run_id, str) and run_id:
|
|
parts.append(f'run_id: {run_id}')
|
|
observe_ms = _int_value(payload.get('next_observe_ms')) or 1000
|
|
parts.append(
|
|
f'Next step: call browser_script with action="observe", run_id="{run_id}", and observe_timeout_ms={observe_ms}.'
|
|
)
|
|
return '\n'.join(parts)
|
|
|
|
|
|
def _synthetic_tool_result_text(name: str) -> str:
|
|
if name == 'update_plan':
|
|
return 'Plan updated'
|
|
if name == 'done':
|
|
return 'done'
|
|
return f'{name} completed'
|
|
|
|
|
|
def _terminal_tool_memory(name: str, text: str) -> tuple[str, bool]:
|
|
if len(text) < _MAX_TERMINAL_LONG_TERM_TEXT_LENGTH:
|
|
return text, False
|
|
tool_name = name or 'tool'
|
|
return (
|
|
f'{tool_name} returned {len(text):,} characters. Full output was included once in <read_state> for that step.',
|
|
True,
|
|
)
|
|
|
|
|
|
def _append_streaming_text_delta(current: str, incoming: str) -> str:
|
|
if not incoming:
|
|
return current
|
|
if not current:
|
|
return incoming
|
|
if incoming == current or incoming.strip() == current.strip():
|
|
return current
|
|
if incoming.startswith(current):
|
|
return current + incoming[len(current) :]
|
|
if len(incoming) >= 24 and current.endswith(incoming):
|
|
return current
|
|
return current + incoming
|
|
|
|
|
|
def _response_content_text(content: Any, part_types: tuple[str, ...]) -> str | None:
|
|
if isinstance(content, str) and content.strip():
|
|
return content
|
|
if not isinstance(content, list):
|
|
return None
|
|
text_parts = []
|
|
for part in content:
|
|
if isinstance(part, str):
|
|
text_parts.append(part)
|
|
continue
|
|
if isinstance(part, dict):
|
|
part_type = part.get('type')
|
|
if part_type in part_types or (part_type is None and 'text' in part):
|
|
text = part.get('text')
|
|
if isinstance(text, str):
|
|
text_parts.append(text)
|
|
text = ''.join(text_parts)
|
|
return text if text.strip() else None
|
|
|
|
|
|
def _response_output_item_text(payload: dict[str, Any]) -> str | None:
|
|
item = payload.get('item')
|
|
if not isinstance(item, dict):
|
|
return None
|
|
if item.get('type') != 'message':
|
|
return None
|
|
if item.get('role', 'user') != 'assistant':
|
|
return None
|
|
return _response_content_text(item.get('content'), ('output_text', 'text', 'input_text'))
|
|
|
|
|
|
def _response_output_item_reasoning_text(payload: dict[str, Any]) -> str | None:
|
|
item = payload.get('item')
|
|
if not isinstance(item, dict):
|
|
return None
|
|
for key in ('reasoning_content', 'thinking', 'reasoning'):
|
|
text = item.get(key)
|
|
if isinstance(text, str) and text.strip():
|
|
return text
|
|
if item.get('type') != 'reasoning':
|
|
return None
|
|
for key in ('text', 'content', 'summary'):
|
|
text = _response_content_text(item.get(key), ('summary_text', 'text', 'output_text', 'input_text'))
|
|
if text:
|
|
return text
|
|
return None
|
|
|
|
|
|
def _streaming_text_from_events(
|
|
events: list[dict[str, Any]],
|
|
stream_event_types: tuple[str, ...],
|
|
*,
|
|
response_item_extractor: Callable[[dict[str, Any]], str | None] = _response_output_item_text,
|
|
) -> str | None:
|
|
text = ''
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
if event_type in ('model.turn.request', 'model.turn.retry', 'model.turn.error'):
|
|
text = ''
|
|
continue
|
|
payload = _event_payload(event)
|
|
if event_type == 'model.response.output_item' and event_type in stream_event_types:
|
|
incoming = response_item_extractor(payload)
|
|
elif event_type in stream_event_types:
|
|
incoming = payload.get('text') or payload.get('delta')
|
|
else:
|
|
continue
|
|
if isinstance(incoming, str):
|
|
text = _append_streaming_text_delta(text, incoming)
|
|
return text if text.strip() else None
|
|
|
|
|
|
def _model_output_from_tool_calls(tool_calls: list[dict[str, Any]], events: list[dict[str, Any]]) -> AgentOutput | None:
|
|
if not tool_calls:
|
|
return None
|
|
action_names = list(dict.fromkeys(call['name'] for call in tool_calls if isinstance(call.get('name'), str)))
|
|
if not action_names:
|
|
return None
|
|
action_fields = {name: (dict[str, Any] | None, None) for name in action_names}
|
|
recovered_action_model = create_model('RustTerminalActionModel', __base__=ActionModel, **action_fields)
|
|
recovered_output_model = AgentOutput.type_with_custom_actions(recovered_action_model)
|
|
actions = []
|
|
for tool_call in tool_calls:
|
|
try:
|
|
actions.append(recovered_action_model(**{tool_call['name']: tool_call['arguments']}))
|
|
except (TypeError, ValueError, ValidationError):
|
|
continue
|
|
if not actions:
|
|
return None
|
|
streamed_text = _streaming_text_from_events(events, ('model.delta', 'model.stream_delta', 'model.response.output_item'))
|
|
thinking_text = _streaming_text_from_events(
|
|
events,
|
|
('model.thinking_delta', 'model.response.output_item'),
|
|
response_item_extractor=_response_output_item_reasoning_text,
|
|
)
|
|
return recovered_output_model(
|
|
thinking=thinking_text,
|
|
evaluation_previous_goal='',
|
|
memory=streamed_text or '',
|
|
next_goal='',
|
|
action=actions,
|
|
)
|
|
|
|
|
|
def _action_results_from_tool_calls(
|
|
tool_calls: list[dict[str, Any]],
|
|
events: list[dict[str, Any]],
|
|
*,
|
|
final_result: str | None,
|
|
attachments: list[str] | None,
|
|
failure: str | None,
|
|
is_done: bool,
|
|
append_terminal_result: bool = True,
|
|
) -> list[ActionResult]:
|
|
tool_results = _tool_results_by_call_id(events)
|
|
unkeyed_results = _unkeyed_tool_results(events)
|
|
image_attachments_by_call_id = _tool_image_attachments_by_call_id(events)
|
|
unkeyed_image_attachments = _tool_image_attachments_by_name(events)
|
|
used_unkeyed_result_indices: set[int] = set()
|
|
used_unkeyed_image_attachment_indices: set[int] = set()
|
|
action_results: list[ActionResult] = []
|
|
for tool_call in tool_calls:
|
|
tool_call_id = str(tool_call.get('tool_call_id'))
|
|
event_type, payload = tool_results.get(tool_call_id, ('', {}))
|
|
if not event_type:
|
|
unkeyed_result_index = _matching_unkeyed_tool_result_index(
|
|
unkeyed_results,
|
|
used_unkeyed_result_indices,
|
|
tool_call.get('name'),
|
|
allow_any=not tool_call.get('_has_explicit_tool_call_id', True),
|
|
)
|
|
if unkeyed_result_index is not None:
|
|
used_unkeyed_result_indices.add(unkeyed_result_index)
|
|
event_type, payload = unkeyed_results[unkeyed_result_index]
|
|
tool_attachments = image_attachments_by_call_id.get(tool_call_id)
|
|
if not tool_attachments:
|
|
unkeyed_attachment_index = _matching_unkeyed_tool_attachment_index(
|
|
unkeyed_image_attachments,
|
|
used_unkeyed_image_attachment_indices,
|
|
tool_call.get('name'),
|
|
allow_any=not tool_call.get('_has_explicit_tool_call_id', True),
|
|
)
|
|
if unkeyed_attachment_index is not None:
|
|
used_unkeyed_image_attachment_indices.add(unkeyed_attachment_index)
|
|
tool_attachments = unkeyed_image_attachments[unkeyed_attachment_index][1]
|
|
text = _tool_result_text(payload)
|
|
if event_type == 'tool.finished' and not text:
|
|
name = payload.get('name') or tool_call.get('name') or 'tool'
|
|
text = _synthetic_tool_result_text(str(name))
|
|
if event_type == 'tool.failed':
|
|
error = _tool_failure_message(payload)
|
|
elif event_type == 'browser_script.failed':
|
|
error = _tool_failure_message(payload) or 'browser_script failed'
|
|
elif event_type == 'tool.aborted':
|
|
error = _tool_abort_message(payload)
|
|
elif event_type == 'exec_command.end':
|
|
error = _exec_command_end_failure_message(payload)
|
|
else:
|
|
error = None
|
|
long_term_memory = None
|
|
include_extracted_content_only_once = False
|
|
if event_type in _TEXTUAL_TOOL_RESULT_EVENTS and text:
|
|
name = str(payload.get('name') or tool_call.get('name') or 'tool')
|
|
long_term_memory, include_extracted_content_only_once = _terminal_tool_memory(name, text)
|
|
action_results.append(
|
|
ActionResult(
|
|
error=error,
|
|
attachments=list(tool_attachments) if tool_attachments else None,
|
|
extracted_content=text if event_type in _TEXTUAL_TOOL_RESULT_EVENTS else None,
|
|
long_term_memory=long_term_memory,
|
|
include_extracted_content_only_once=include_extracted_content_only_once,
|
|
)
|
|
)
|
|
|
|
if not append_terminal_result:
|
|
return action_results
|
|
|
|
final_action = ActionResult(
|
|
is_done=is_done,
|
|
success=True if is_done else None,
|
|
error=failure,
|
|
attachments=attachments,
|
|
extracted_content=final_result,
|
|
long_term_memory=final_result,
|
|
)
|
|
if action_results and tool_calls[-1].get('name') == 'done':
|
|
action_results[-1] = final_action
|
|
else:
|
|
action_results.append(final_action)
|
|
return action_results
|
|
|
|
|
|
def _terminal_turn_spans(events: list[dict[str, Any]]) -> list[tuple[int, int]]:
|
|
starts = [index for index, event in enumerate(events) if _event_type(event) == 'model.turn.request']
|
|
if len(starts) <= 1:
|
|
return []
|
|
return [(start, starts[index + 1] if index + 1 < len(starts) else len(events)) for index, start in enumerate(starts)]
|
|
|
|
|
|
def _event_time_seconds(event: dict[str, Any], fallback: float) -> float:
|
|
ts_ms = event.get('ts_ms')
|
|
if isinstance(ts_ms, (int, float)):
|
|
return float(ts_ms) / 1000.0
|
|
ts = event.get('timestamp') or event.get('time')
|
|
if isinstance(ts, (int, float)):
|
|
return float(ts)
|
|
return fallback
|
|
|
|
|
|
def _history_items_from_terminal_turns(
|
|
events: list[dict[str, Any]],
|
|
*,
|
|
started: float,
|
|
finished: float,
|
|
final_result: str | None,
|
|
attachments: list[str] | None,
|
|
failure: str | None,
|
|
is_done: bool,
|
|
) -> list[AgentHistory] | None:
|
|
spans = _terminal_turn_spans(events)
|
|
if not spans:
|
|
return None
|
|
history: list[AgentHistory] = []
|
|
for step_number, (start_index, end_index) in enumerate(spans, start=1):
|
|
step_events = events[start_index:end_index]
|
|
if not step_events:
|
|
continue
|
|
is_final_step = step_number == len(spans)
|
|
step_final_result = final_result if is_final_step else None
|
|
step_failure = failure if is_final_step else None
|
|
step_is_done = is_done if is_final_step else False
|
|
step_attachments = attachments if is_final_step else None
|
|
tool_calls = _tool_calls_with_final_done(
|
|
_tool_started_calls(step_events),
|
|
final_result=step_final_result,
|
|
attachments=step_attachments,
|
|
is_done=step_is_done,
|
|
)
|
|
if not tool_calls and step_final_result is None and step_failure is None:
|
|
continue
|
|
state_events = events[:end_index]
|
|
step_start = _event_time_seconds(step_events[0], started)
|
|
step_end = _event_time_seconds(step_events[-1], finished if is_final_step else step_start)
|
|
history.append(
|
|
AgentHistory(
|
|
model_output=_model_output_from_tool_calls(tool_calls, step_events),
|
|
result=_action_results_from_tool_calls(
|
|
tool_calls,
|
|
step_events,
|
|
final_result=step_final_result,
|
|
attachments=step_attachments,
|
|
failure=step_failure,
|
|
is_done=step_is_done,
|
|
append_terminal_result=is_final_step,
|
|
),
|
|
state=_browser_state_from_events(state_events),
|
|
metadata=StepMetadata(step_start_time=step_start, step_end_time=step_end, step_number=step_number),
|
|
)
|
|
)
|
|
return history or None
|
|
|
|
|
|
def _browser_url(value: Any) -> str:
|
|
if not isinstance(value, str):
|
|
return ''
|
|
value = value.strip()
|
|
if value.startswith(('http://', 'https://', 'about:', 'file://')):
|
|
return value
|
|
return ''
|
|
|
|
|
|
def _internal_browser_endpoint_url(url: str) -> bool:
|
|
parsed = urlparse(url)
|
|
return (
|
|
parsed.scheme in {'http', 'https'}
|
|
and parsed.hostname in {'127.0.0.1', 'localhost', '::1'}
|
|
and parsed.path
|
|
in {
|
|
'',
|
|
'/',
|
|
}
|
|
)
|
|
|
|
|
|
def _browser_state_candidates(value: Any) -> list[tuple[str, str, str]]:
|
|
candidates: list[tuple[str, str, str]] = []
|
|
if isinstance(value, dict):
|
|
url = _browser_url(value.get('url'))
|
|
if url:
|
|
title = str(value.get('title') or '')
|
|
raw_target = value.get('target')
|
|
nested_target_id = ''
|
|
if isinstance(raw_target, dict):
|
|
nested_target_id = str(
|
|
raw_target.get('target_id')
|
|
or raw_target.get('targetId')
|
|
or raw_target.get('tab_id')
|
|
or raw_target.get('tabId')
|
|
or ''
|
|
)
|
|
target_id = str(
|
|
value.get('target_id')
|
|
or value.get('targetId')
|
|
or value.get('tab_id')
|
|
or value.get('tabId')
|
|
or nested_target_id
|
|
or 'tab-0'
|
|
)
|
|
candidates.append((url, title, target_id))
|
|
for key, child in value.items():
|
|
if key in ('url', 'live_url', 'title'):
|
|
continue
|
|
candidates.extend(_browser_state_candidates(child))
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
candidates.extend(_browser_state_candidates(child))
|
|
elif isinstance(value, str):
|
|
text = value.strip()
|
|
url = _browser_url(text)
|
|
if url:
|
|
candidates.append((url, '', 'tab-0'))
|
|
if len(text) > 50_000:
|
|
return candidates
|
|
segments = [text]
|
|
if '\n' in text:
|
|
segments.extend(line.strip() for line in text.splitlines() if line.strip().startswith(('{', '[')))
|
|
seen_segments: set[str] = set()
|
|
for segment in segments:
|
|
if segment in seen_segments or not segment.startswith(('{', '[')):
|
|
continue
|
|
seen_segments.add(segment)
|
|
parsed = None
|
|
for parser in (json.loads, ast.literal_eval):
|
|
try:
|
|
parsed = parser(segment)
|
|
break
|
|
except (SyntaxError, ValueError, TypeError, json.JSONDecodeError):
|
|
continue
|
|
if isinstance(parsed, (dict, list)):
|
|
candidates.extend(_browser_state_candidates(parsed))
|
|
return candidates
|
|
|
|
|
|
def _image_path(value: Any) -> str | None:
|
|
if not isinstance(value, dict):
|
|
return None
|
|
path = value.get('path')
|
|
if isinstance(path, str) and path:
|
|
return path
|
|
url = value.get('url')
|
|
if isinstance(url, str) and url.startswith('file://'):
|
|
return url.removeprefix('file://')
|
|
return None
|
|
|
|
|
|
def _append_unique_attachment(attachments: list[str], pointer: str | None) -> None:
|
|
if pointer and pointer not in attachments:
|
|
attachments.append(pointer)
|
|
|
|
|
|
def _tool_image_attachments_by_call_id(events: list[dict[str, Any]]) -> dict[str, list[str]]:
|
|
attachments: dict[str, list[str]] = {}
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
if event_type == 'tool.image':
|
|
image_path = _image_path(payload.get('image'))
|
|
call_id = payload.get('tool_call_id') or payload.get('call_id')
|
|
if image_path and call_id:
|
|
_append_unique_attachment(attachments.setdefault(str(call_id), []), image_path)
|
|
continue
|
|
if event_type not in ('tool.output', 'tool.failed'):
|
|
continue
|
|
call_id = payload.get('tool_call_id') or payload.get('call_id')
|
|
if not call_id:
|
|
continue
|
|
images = payload.get('images')
|
|
if not isinstance(images, list):
|
|
continue
|
|
for image in images:
|
|
image_path = _image_path(image)
|
|
if image_path:
|
|
_append_unique_attachment(attachments.setdefault(str(call_id), []), image_path)
|
|
return attachments
|
|
|
|
|
|
def _tool_image_attachments_by_name(events: list[dict[str, Any]]) -> list[tuple[str, list[str]]]:
|
|
attachments: list[tuple[str, list[str]]] = []
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
if payload.get('tool_call_id') or payload.get('call_id'):
|
|
continue
|
|
name = payload.get('name')
|
|
if not isinstance(name, str) or not name:
|
|
continue
|
|
image_paths: list[str] = []
|
|
if event_type == 'tool.image':
|
|
_append_unique_attachment(image_paths, _image_path(payload.get('image')))
|
|
elif event_type in ('tool.output', 'tool.failed', *_BROWSER_SCRIPT_RESULT_EVENTS):
|
|
images = payload.get('images')
|
|
if isinstance(images, list):
|
|
for image in images:
|
|
_append_unique_attachment(image_paths, _image_path(image))
|
|
if image_paths:
|
|
attachments.append((name, image_paths))
|
|
return attachments
|
|
|
|
|
|
def _screenshot_path_from_events(events: list[dict[str, Any]]) -> str | None:
|
|
screenshot_path = None
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
if event_type == 'tool.image':
|
|
image_path = _image_path(payload.get('image'))
|
|
if image_path:
|
|
screenshot_path = image_path
|
|
continue
|
|
if event_type not in ('tool.output', 'tool.failed', *_BROWSER_SCRIPT_RESULT_EVENTS):
|
|
continue
|
|
images = payload.get('images')
|
|
if not isinstance(images, list):
|
|
continue
|
|
for image in images:
|
|
image_path = _image_path(image)
|
|
if image_path:
|
|
screenshot_path = image_path
|
|
return screenshot_path
|
|
|
|
|
|
def _browser_script_navigation_candidates(payload: dict[str, Any]) -> list[tuple[str, str, str]]:
|
|
if payload.get('name') != 'browser_script':
|
|
return []
|
|
arguments = payload.get('arguments')
|
|
if not isinstance(arguments, dict):
|
|
return []
|
|
code = arguments.get('code')
|
|
if not isinstance(code, str) or not re.search(r'\b(?:goto_url|new_tab|open_tab|navigate)\s*\(', code):
|
|
return []
|
|
candidates = []
|
|
for match in URL_PATTERN.finditer(code):
|
|
url = _browser_url(match.group(0))
|
|
if url:
|
|
candidates.append((url, '', 'tab-0'))
|
|
return candidates
|
|
|
|
|
|
def _browser_state_from_events(events: list[dict[str, Any]]) -> BrowserStateHistory:
|
|
url = ''
|
|
title = ''
|
|
tabs: list[TabInfo] = []
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
if event_type in ('tool.output', 'tool.started', *_BROWSER_SCRIPT_RESULT_EVENTS):
|
|
payload = _event_payload(event)
|
|
if event_type == 'tool.started':
|
|
candidates = _browser_script_navigation_candidates(payload)
|
|
else:
|
|
candidates = _browser_state_candidates(payload)
|
|
for candidate_url, candidate_title, candidate_target_id in candidates:
|
|
if payload.get('name') == 'browser' and _internal_browser_endpoint_url(candidate_url):
|
|
continue
|
|
url = candidate_url
|
|
title = candidate_title or title
|
|
tabs = [TabInfo(url=url, title=title, target_id=candidate_target_id)]
|
|
continue
|
|
if event_type not in (
|
|
'browser.connected',
|
|
'browser.reconnected',
|
|
'browser.target_changed',
|
|
'browser.live_url',
|
|
'browser.page',
|
|
'browser.state',
|
|
):
|
|
continue
|
|
payload = _event_payload(event)
|
|
next_url = _browser_url(payload.get('live_url')) or _browser_url(payload.get('url'))
|
|
if event_type in {'browser.connected', 'browser.reconnected'} and _internal_browser_endpoint_url(next_url):
|
|
next_url = ''
|
|
url = next_url or url
|
|
title = str(payload.get('title') or title)
|
|
raw_tabs = payload.get('tabs')
|
|
if isinstance(raw_tabs, list):
|
|
next_tabs = []
|
|
for idx, raw in enumerate(raw_tabs):
|
|
if isinstance(raw, dict):
|
|
next_tabs.append(
|
|
TabInfo(
|
|
url=str(raw.get('url') or ''),
|
|
title=str(raw.get('title') or ''),
|
|
target_id=str(raw.get('target_id') or raw.get('tab_id') or f'tab-{idx}'),
|
|
)
|
|
)
|
|
if next_tabs:
|
|
tabs = next_tabs
|
|
if not tabs and (url or title):
|
|
tabs = [TabInfo(url=url, title=title, target_id='tab-0')]
|
|
return BrowserStateHistory(
|
|
url=url,
|
|
title=title,
|
|
tabs=tabs,
|
|
interacted_element=[],
|
|
screenshot_path=_screenshot_path_from_events(events),
|
|
)
|
|
|
|
|
|
def _int_value(value: Any) -> int:
|
|
try:
|
|
return int(value or 0)
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
def _float_value(value: Any) -> float:
|
|
try:
|
|
return float(value or 0.0)
|
|
except (TypeError, ValueError):
|
|
return 0.0
|
|
|
|
|
|
def _int_field(usage: dict[str, Any], keys: tuple[str, ...]) -> int:
|
|
for key in keys:
|
|
if key in usage:
|
|
return _int_value(usage.get(key))
|
|
return 0
|
|
|
|
|
|
def _token_count_usage(payload: dict[str, Any]) -> dict[str, Any] | None:
|
|
info = payload.get('info')
|
|
if isinstance(info, dict):
|
|
raw_usage = info.get('total_token_usage') or info.get('last_token_usage')
|
|
else:
|
|
raw_usage = payload.get('total_token_usage') or payload.get('last_token_usage')
|
|
return raw_usage if isinstance(raw_usage, dict) else None
|
|
|
|
|
|
def _token_count_last_usage(payload: dict[str, Any]) -> dict[str, Any] | None:
|
|
info = payload.get('info')
|
|
if isinstance(info, dict):
|
|
raw_usage = info.get('last_token_usage') or info.get('total_token_usage')
|
|
else:
|
|
raw_usage = payload.get('last_token_usage') or payload.get('total_token_usage')
|
|
return raw_usage if isinstance(raw_usage, dict) else None
|
|
|
|
|
|
def _model_usage_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
|
usage = payload.get('usage')
|
|
return usage if isinstance(usage, dict) else payload
|
|
|
|
|
|
def _optional_positive_int(value: Any) -> int | None:
|
|
integer = _int_value(value)
|
|
return integer if integer > 0 else None
|
|
|
|
|
|
def _cache_creation_usage_tokens(usage: dict[str, Any]) -> tuple[int, int, int]:
|
|
cache_creation = usage.get('cache_creation')
|
|
if isinstance(cache_creation, dict):
|
|
cache_creation_5m_tokens = _int_value(cache_creation.get('ephemeral_5m_input_tokens'))
|
|
cache_creation_1h_tokens = _int_value(cache_creation.get('ephemeral_1h_input_tokens'))
|
|
return cache_creation_5m_tokens + cache_creation_1h_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens
|
|
|
|
cache_creation_tokens = _int_field(
|
|
usage, ('cache_creation_input_tokens', 'prompt_cache_creation_tokens', 'input_cache_creation_tokens')
|
|
)
|
|
return cache_creation_tokens, 0, 0
|
|
|
|
|
|
def _input_usage_buckets(usage: dict[str, Any]) -> tuple[int, int, int, int, int]:
|
|
cache_read_tokens = _int_field(usage, ('cache_read_input_tokens', 'input_cached_tokens', 'cached_input_tokens'))
|
|
cache_creation_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens = _cache_creation_usage_tokens(usage)
|
|
input_tokens = _int_value(usage.get('input_tokens'))
|
|
if 'cache_read_input_tokens' in usage or 'cache_creation_input_tokens' in usage:
|
|
input_tokens += cache_read_tokens
|
|
return input_tokens, cache_read_tokens, cache_creation_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens
|
|
|
|
|
|
def _chat_invoke_usage_from_payload(usage: dict[str, Any]) -> ChatInvokeUsage | None:
|
|
input_tokens, cached_input_tokens, cache_creation_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens = (
|
|
_input_usage_buckets(usage)
|
|
)
|
|
completion_tokens = _usage_completion_tokens(usage)
|
|
total_tokens = _usage_total_tokens(usage, input_tokens, cache_creation_tokens, completion_tokens)
|
|
if input_tokens == 0 and completion_tokens == 0 and total_tokens == 0:
|
|
return None
|
|
pricing_multiplier = _float_value(usage.get('pricing_multiplier'))
|
|
if not pricing_multiplier and usage.get('inference_geo') == 'us':
|
|
pricing_multiplier = 1.1
|
|
return ChatInvokeUsage(
|
|
prompt_tokens=input_tokens,
|
|
prompt_cached_tokens=cached_input_tokens or None,
|
|
prompt_cache_creation_tokens=cache_creation_tokens or None,
|
|
prompt_cache_creation_5m_tokens=cache_creation_5m_tokens or None,
|
|
prompt_cache_creation_1h_tokens=cache_creation_1h_tokens or None,
|
|
prompt_image_tokens=_optional_positive_int(usage.get('prompt_image_tokens') or usage.get('image_tokens')),
|
|
completion_tokens=completion_tokens,
|
|
total_tokens=total_tokens,
|
|
pricing_multiplier=pricing_multiplier or None,
|
|
)
|
|
|
|
|
|
def _reasoning_output_tokens(usage: dict[str, Any]) -> int:
|
|
return _int_value(
|
|
usage.get('reasoning_output_tokens') or usage.get('output_reasoning_tokens') or usage.get('reasoning_tokens')
|
|
)
|
|
|
|
|
|
def _usage_completion_tokens(usage: dict[str, Any]) -> int:
|
|
return _int_value(usage.get('output_tokens')) + _reasoning_output_tokens(usage)
|
|
|
|
|
|
def _usage_total_tokens(usage: dict[str, Any], input_tokens: int, cache_creation_tokens: int, completion_tokens: int) -> int:
|
|
computed_total = input_tokens + cache_creation_tokens + completion_tokens
|
|
if 'cache_read_input_tokens' in usage or 'cache_creation_input_tokens' in usage:
|
|
return computed_total
|
|
reported_total = usage.get('total_tokens')
|
|
if reported_total is not None:
|
|
total_tokens = _int_value(reported_total)
|
|
if total_tokens > 0 or computed_total == 0:
|
|
return total_tokens
|
|
return computed_total
|
|
|
|
|
|
def _usage_from_events(events: list[dict[str, Any]], model: str) -> UsageSummary:
|
|
input_tokens = 0
|
|
cached_input_tokens = 0
|
|
cache_creation_tokens = 0
|
|
completion_tokens = 0
|
|
total_tokens = 0
|
|
cost = 0.0
|
|
invocations = 0
|
|
token_count_invocations = 0
|
|
token_count_input_tokens = 0
|
|
token_count_cached_input_tokens = 0
|
|
token_count_cache_creation_tokens = 0
|
|
token_count_completion_tokens = 0
|
|
token_count_total_tokens = 0
|
|
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
if event_type == 'model.usage':
|
|
usage = _model_usage_payload(payload)
|
|
event_input_tokens, event_cached_input_tokens, event_cache_creation_tokens, _, _ = _input_usage_buckets(usage)
|
|
event_completion_tokens = _usage_completion_tokens(usage)
|
|
input_tokens += event_input_tokens
|
|
cached_input_tokens += event_cached_input_tokens
|
|
cache_creation_tokens += event_cache_creation_tokens
|
|
completion_tokens += event_completion_tokens
|
|
total_tokens += _usage_total_tokens(usage, event_input_tokens, event_cache_creation_tokens, event_completion_tokens)
|
|
cost += _float_value(usage.get('cost_usd') or usage.get('cost') or payload.get('cost_usd') or payload.get('cost'))
|
|
invocations += 1
|
|
continue
|
|
if event_type == 'token_count':
|
|
token_usage = _token_count_usage(payload)
|
|
if token_usage is None:
|
|
continue
|
|
total_input_tokens, total_cached_input_tokens, total_cache_creation_tokens, _, _ = _input_usage_buckets(token_usage)
|
|
total_completion_tokens = _usage_completion_tokens(token_usage)
|
|
total_usage_tokens = _usage_total_tokens(
|
|
token_usage, total_input_tokens, total_cache_creation_tokens, total_completion_tokens
|
|
)
|
|
last_usage = _token_count_last_usage(payload)
|
|
if isinstance(last_usage, dict):
|
|
last_input_tokens, last_cached_input_tokens, last_cache_creation_tokens, _, _ = _input_usage_buckets(last_usage)
|
|
last_completion_tokens = _usage_completion_tokens(last_usage)
|
|
token_count_input_tokens += last_input_tokens
|
|
token_count_cached_input_tokens += last_cached_input_tokens
|
|
token_count_cache_creation_tokens += last_cache_creation_tokens
|
|
token_count_completion_tokens += last_completion_tokens
|
|
token_count_total_tokens += _usage_total_tokens(
|
|
last_usage, last_input_tokens, last_cache_creation_tokens, last_completion_tokens
|
|
)
|
|
elif total_cache_creation_tokens:
|
|
token_count_cache_creation_tokens = total_cache_creation_tokens
|
|
input_tokens = max(input_tokens, total_input_tokens, token_count_input_tokens)
|
|
cached_input_tokens = max(cached_input_tokens, total_cached_input_tokens, token_count_cached_input_tokens)
|
|
cache_creation_tokens = max(cache_creation_tokens, total_cache_creation_tokens, token_count_cache_creation_tokens)
|
|
completion_tokens = max(completion_tokens, total_completion_tokens, token_count_completion_tokens)
|
|
total_tokens = max(total_tokens, total_usage_tokens, token_count_total_tokens)
|
|
token_count_invocations += 1
|
|
|
|
invocations = max(invocations, token_count_invocations)
|
|
by_model = {
|
|
model: ModelUsageStats(
|
|
model=model,
|
|
prompt_tokens=input_tokens,
|
|
completion_tokens=completion_tokens,
|
|
total_tokens=total_tokens,
|
|
cost=cost,
|
|
invocations=invocations,
|
|
)
|
|
}
|
|
return UsageSummary(
|
|
total_prompt_tokens=input_tokens,
|
|
total_prompt_cost=0.0,
|
|
total_prompt_cached_tokens=cached_input_tokens,
|
|
total_prompt_cached_cost=0.0,
|
|
total_prompt_cache_creation_tokens=cache_creation_tokens,
|
|
total_prompt_cache_creation_cost=0.0,
|
|
total_completion_tokens=completion_tokens,
|
|
total_completion_cost=0.0,
|
|
total_tokens=total_tokens,
|
|
total_cost=cost,
|
|
entry_count=invocations,
|
|
by_model=by_model,
|
|
)
|
|
|
|
|
|
def _usage_tokens(summary: UsageSummary | None) -> int:
|
|
if summary is None:
|
|
return 0
|
|
return max(
|
|
_int_value(summary.total_tokens),
|
|
_int_value(summary.total_prompt_tokens)
|
|
+ _int_value(summary.total_prompt_cache_creation_tokens)
|
|
+ _int_value(summary.total_completion_tokens),
|
|
)
|
|
|
|
|
|
def _usage_event_from_sdk_history_usage(raw_usage: Any) -> dict[str, Any] | None:
|
|
if not isinstance(raw_usage, dict):
|
|
return None
|
|
if _chat_invoke_usage_from_payload(raw_usage) is None:
|
|
return None
|
|
return {
|
|
'event_type': 'token_count',
|
|
'payload': {
|
|
'info': {
|
|
'last_token_usage': raw_usage,
|
|
'total_token_usage': raw_usage,
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
async def _usage_from_events_with_costs(
|
|
events: list[dict[str, Any]],
|
|
model: str,
|
|
token_cost_service: TokenCost,
|
|
) -> UsageSummary:
|
|
"""Reconstruct usage from terminal events and price per-call usage when enabled."""
|
|
summary = _usage_from_events(events, model)
|
|
if not token_cost_service.include_cost:
|
|
return summary
|
|
|
|
total_prompt_cost = 0.0
|
|
total_completion_cost = 0.0
|
|
total_prompt_cached_cost = 0.0
|
|
total_prompt_cache_creation_cost = 0.0
|
|
total_cost = 0.0
|
|
priced_invocations = 0
|
|
summed_prompt_tokens = 0
|
|
summed_prompt_cached_tokens = 0
|
|
summed_prompt_cache_creation_tokens = 0
|
|
summed_completion_tokens = 0
|
|
summed_total_tokens = 0
|
|
summed_invocations = 0
|
|
has_model_usage = any(_event_type(event) == 'model.usage' for event in events)
|
|
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
raw_usage = None
|
|
if event_type == 'model.usage':
|
|
raw_usage = _model_usage_payload(payload)
|
|
elif event_type == 'token_count':
|
|
if has_model_usage:
|
|
continue
|
|
raw_usage = _token_count_last_usage(payload)
|
|
if not isinstance(raw_usage, dict):
|
|
continue
|
|
chat_usage = _chat_invoke_usage_from_payload(raw_usage)
|
|
if chat_usage is None:
|
|
continue
|
|
summed_invocations += 1
|
|
summed_prompt_tokens += chat_usage.prompt_tokens
|
|
summed_prompt_cached_tokens += chat_usage.prompt_cached_tokens or 0
|
|
summed_prompt_cache_creation_tokens += chat_usage.prompt_cache_creation_tokens or 0
|
|
summed_completion_tokens += chat_usage.completion_tokens
|
|
summed_total_tokens += chat_usage.total_tokens
|
|
cost = await token_cost_service.calculate_cost(model, chat_usage)
|
|
if cost is None:
|
|
continue
|
|
priced_invocations += 1
|
|
total_prompt_cost += cost.prompt_cost
|
|
total_completion_cost += cost.completion_cost
|
|
total_prompt_cached_cost += cost.prompt_read_cached_cost or 0.0
|
|
total_prompt_cache_creation_cost += cost.prompt_cache_creation_cost or 0.0
|
|
total_cost += cost.total_cost
|
|
|
|
if priced_invocations == 0 and summed_invocations == 0:
|
|
return summary
|
|
|
|
model_stats = dict(summary.by_model)
|
|
stats = model_stats.get(model) or ModelUsageStats(model=model)
|
|
if summed_invocations:
|
|
stats.prompt_tokens = summed_prompt_tokens
|
|
stats.completion_tokens = summed_completion_tokens
|
|
stats.total_tokens = summed_total_tokens
|
|
stats.invocations = max(stats.invocations, summed_invocations)
|
|
if priced_invocations:
|
|
stats.cost = total_cost
|
|
model_stats[model] = stats
|
|
|
|
update: dict[str, Any] = {'by_model': model_stats}
|
|
if summed_invocations:
|
|
update.update(
|
|
{
|
|
'total_prompt_tokens': summed_prompt_tokens,
|
|
'total_prompt_cached_tokens': summed_prompt_cached_tokens,
|
|
'total_prompt_cache_creation_tokens': summed_prompt_cache_creation_tokens,
|
|
'total_completion_tokens': summed_completion_tokens,
|
|
'total_tokens': summed_total_tokens,
|
|
'entry_count': max(summary.entry_count, summed_invocations),
|
|
}
|
|
)
|
|
if priced_invocations:
|
|
update.update(
|
|
{
|
|
'total_prompt_cost': total_prompt_cost,
|
|
'total_prompt_cached_cost': total_prompt_cached_cost,
|
|
'total_prompt_cache_creation_cost': total_prompt_cache_creation_cost,
|
|
'total_completion_cost': total_completion_cost,
|
|
'total_cost': total_cost,
|
|
}
|
|
)
|
|
return summary.model_copy(update=update)
|
|
|
|
|
|
def _terminal_model_turn_event_ranges(events: list[dict[str, Any]]) -> list[tuple[int, int]]:
|
|
starts = [index for index, event in enumerate(events) if _event_type(event) == 'model.turn.request']
|
|
return [(start, starts[index + 1] if index + 1 < len(starts) else len(events)) for index, start in enumerate(starts)]
|
|
|
|
|
|
def _terminal_turn_usage_payload(events: list[dict[str, Any]]) -> dict[str, Any] | None:
|
|
raw_usage = None
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
payload = _event_payload(event)
|
|
if event_type == 'model.usage':
|
|
raw_usage = _model_usage_payload(payload)
|
|
elif event_type == 'token_count':
|
|
raw_usage = _token_count_last_usage(payload)
|
|
return raw_usage if isinstance(raw_usage, dict) else None
|
|
|
|
|
|
def _terminal_laminar_usage_summary(raw_usage: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
if raw_usage is None:
|
|
return None
|
|
input_tokens, cached_input_tokens, cache_creation_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens = (
|
|
_input_usage_buckets(raw_usage)
|
|
)
|
|
output_tokens = _usage_completion_tokens(raw_usage)
|
|
total_tokens = _usage_total_tokens(raw_usage, input_tokens, cache_creation_tokens, output_tokens)
|
|
cost = _float_value(raw_usage.get('cost_usd') or raw_usage.get('cost') or raw_usage.get('total_cost'))
|
|
summary: dict[str, Any] = {
|
|
'input_tokens': input_tokens,
|
|
'cached_input_tokens': cached_input_tokens,
|
|
'output_tokens': output_tokens,
|
|
'total_tokens': total_tokens,
|
|
}
|
|
if cache_creation_tokens:
|
|
summary['cache_creation_input_tokens'] = cache_creation_tokens
|
|
if cache_creation_5m_tokens or cache_creation_1h_tokens:
|
|
summary['cache_creation'] = {
|
|
'ephemeral_5m_input_tokens': cache_creation_5m_tokens,
|
|
'ephemeral_1h_input_tokens': cache_creation_1h_tokens,
|
|
}
|
|
reasoning_output_tokens = _reasoning_output_tokens(raw_usage)
|
|
if reasoning_output_tokens:
|
|
summary['reasoning_output_tokens'] = reasoning_output_tokens
|
|
if cost:
|
|
summary['cost_usd'] = cost
|
|
if raw_usage.get('inference_geo') is not None:
|
|
summary['inference_geo'] = raw_usage['inference_geo']
|
|
if raw_usage.get('pricing_multiplier') is not None:
|
|
summary['pricing_multiplier'] = raw_usage['pricing_multiplier']
|
|
return summary
|
|
|
|
|
|
def _terminal_laminar_usage_cost(model: str, usage: dict[str, Any] | None) -> dict[str, float] | None:
|
|
if not usage:
|
|
return None
|
|
pricing = CUSTOM_MODEL_PRICING.get(model) or CUSTOM_MODEL_PRICING.get(model.replace('.', '-'))
|
|
if not pricing:
|
|
return None
|
|
input_tokens = int(usage.get('input_tokens') or 0)
|
|
cached_tokens = int(usage.get('cached_input_tokens') or 0)
|
|
cache_creation_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens = _cache_creation_usage_tokens(usage)
|
|
output_tokens = int(usage.get('output_tokens') or 0)
|
|
uncached_tokens = max(0, input_tokens - cached_tokens)
|
|
input_cost = uncached_tokens * float(pricing.get('input_cost_per_token') or 0.0)
|
|
cache_read_cost = cached_tokens * float(pricing.get('cache_read_input_token_cost') or 0.0)
|
|
if cache_creation_5m_tokens or cache_creation_1h_tokens:
|
|
cache_creation_cost = cache_creation_5m_tokens * float(pricing.get('cache_creation_input_token_cost') or 0.0)
|
|
cache_creation_cost += cache_creation_1h_tokens * float(
|
|
pricing.get('cache_creation_1h_input_token_cost') or pricing.get('cache_creation_input_token_cost') or 0.0
|
|
)
|
|
else:
|
|
cache_creation_cost = cache_creation_tokens * float(pricing.get('cache_creation_input_token_cost') or 0.0)
|
|
output_cost = output_tokens * float(pricing.get('output_cost_per_token') or 0.0)
|
|
pricing_multiplier = _float_value(usage.get('pricing_multiplier'))
|
|
if not pricing_multiplier and usage.get('inference_geo') == 'us':
|
|
pricing_multiplier = 1.1
|
|
if pricing_multiplier:
|
|
input_cost *= pricing_multiplier
|
|
cache_read_cost *= pricing_multiplier
|
|
cache_creation_cost *= pricing_multiplier
|
|
output_cost *= pricing_multiplier
|
|
total_cost = input_cost + cache_read_cost + cache_creation_cost + output_cost
|
|
if total_cost <= 0:
|
|
return None
|
|
return {
|
|
'input_cost_usd': input_cost,
|
|
'input_cached_cost_usd': cache_read_cost,
|
|
'input_cache_creation_cost_usd': cache_creation_cost,
|
|
'output_cost_usd': output_cost,
|
|
'cost_usd': total_cost,
|
|
}
|
|
|
|
|
|
def _terminal_laminar_turn_input(
|
|
request_payload: dict[str, Any],
|
|
*,
|
|
default_model: str,
|
|
default_provider: str,
|
|
) -> dict[str, Any]:
|
|
composition = request_payload.get('composition')
|
|
if not isinstance(composition, dict):
|
|
composition = {}
|
|
raw_tools = composition.get('tools')
|
|
tools = raw_tools if isinstance(raw_tools, list) else []
|
|
llm_input = request_payload.get('llm_input')
|
|
if not isinstance(llm_input, dict):
|
|
llm_input = {}
|
|
llm_tools = llm_input.get('tools')
|
|
if isinstance(llm_tools, list):
|
|
tools = [tool for tool in llm_tools if isinstance(tool, dict)]
|
|
tool_names = []
|
|
for tool in tools:
|
|
if isinstance(tool, dict) and isinstance(tool.get('name'), str):
|
|
tool_names.append(tool['name'])
|
|
messages = llm_input.get('messages')
|
|
if not isinstance(messages, list):
|
|
messages = []
|
|
system = llm_input.get('system')
|
|
if not isinstance(system, list):
|
|
system = []
|
|
return {
|
|
'model': request_payload.get('model') or default_model,
|
|
'provider': request_payload.get('provider') or default_provider,
|
|
'turn_idx': request_payload.get('turn_idx'),
|
|
'attempt': request_payload.get('attempt'),
|
|
'system_prompt_tokens': composition.get('system_prompt_tokens'),
|
|
'tools_count': len(tools),
|
|
'tool_names': tool_names,
|
|
'tools': tools,
|
|
'system': system,
|
|
'messages': messages,
|
|
'message_count': _int_value(llm_input.get('message_count')),
|
|
'omitted_earlier_messages': _int_value(llm_input.get('omitted_earlier_messages')),
|
|
'truncated': bool(llm_input.get('truncated')),
|
|
}
|
|
|
|
|
|
def _terminal_laminar_turn_output(events: list[dict[str, Any]], raw_usage: dict[str, Any] | None) -> dict[str, Any]:
|
|
assistant_text = _streaming_text_from_events(events, ('model.delta', 'model.stream_delta', 'model.response.output_item'))
|
|
thinking_text = _streaming_text_from_events(
|
|
events,
|
|
('model.thinking_delta', 'model.response.output_item'),
|
|
response_item_extractor=_response_output_item_reasoning_text,
|
|
)
|
|
tool_calls = _tool_started_calls(events)
|
|
assistant_preview = _laminar_preview(assistant_text, limit=2000)
|
|
thinking_preview = _laminar_preview(thinking_text, limit=1200)
|
|
output_messages = []
|
|
if assistant_text:
|
|
output_messages.append({'role': 'assistant', 'content': [{'type': 'text', 'text': assistant_text}]})
|
|
if tool_calls:
|
|
output_messages.append(
|
|
{
|
|
'role': 'assistant',
|
|
'tool_calls': [
|
|
{
|
|
'name': call.get('name'),
|
|
'id': call.get('tool_call_id'),
|
|
'arguments': call.get('arguments'),
|
|
}
|
|
for call in tool_calls[:20]
|
|
],
|
|
}
|
|
)
|
|
return {
|
|
'messages': output_messages,
|
|
'assistant_output_preview': assistant_preview,
|
|
'thinking_preview': thinking_preview,
|
|
'tool_calls_count': len(tool_calls),
|
|
'tool_call_names': [call['name'] for call in tool_calls[:20] if isinstance(call.get('name'), str)],
|
|
'usage': _terminal_laminar_usage_summary(raw_usage),
|
|
}
|
|
|
|
|
|
def _terminal_laminar_system_messages(span_input: dict[str, Any]) -> list[dict[str, Any]]:
|
|
system_messages = []
|
|
system_parts = span_input.get('system')
|
|
if not isinstance(system_parts, list):
|
|
return system_messages
|
|
for part in system_parts:
|
|
if isinstance(part, dict):
|
|
text = part.get('text') or part.get('content')
|
|
else:
|
|
text = part
|
|
if isinstance(text, str) and text:
|
|
system_messages.append({'role': 'system', 'content': [{'type': 'text', 'text': text}]})
|
|
return system_messages
|
|
|
|
|
|
def _terminal_laminar_content_part_for_span(part: Any) -> Any:
|
|
if not isinstance(part, dict):
|
|
return part
|
|
part_type = part.get('type')
|
|
if part_type == 'tool_result':
|
|
content = part.get('content')
|
|
if isinstance(content, list):
|
|
return [_terminal_laminar_content_part_for_span(item) for item in content]
|
|
if isinstance(content, str):
|
|
return {'type': 'text', 'text': content}
|
|
if part_type in {'input_text', 'output_text'}:
|
|
text = part.get('text')
|
|
if isinstance(text, str):
|
|
return {'type': 'text', 'text': text}
|
|
if part_type == 'media':
|
|
mime_type = part.get('mime_type')
|
|
mime_type = mime_type if isinstance(mime_type, str) and mime_type else 'application/octet-stream'
|
|
url = part.get('url')
|
|
data = part.get('data')
|
|
resolved_url = None
|
|
if isinstance(url, str) and url:
|
|
resolved_url = url
|
|
elif isinstance(data, str) and data:
|
|
resolved_url = f'data:{mime_type};base64,{data}'
|
|
if resolved_url and mime_type.startswith('image/'):
|
|
image_url: dict[str, Any] = {'url': resolved_url}
|
|
detail = part.get('detail')
|
|
if isinstance(detail, str) and detail:
|
|
image_url['detail'] = detail
|
|
return {'type': 'image_url', 'image_url': image_url}
|
|
if resolved_url:
|
|
return {'type': 'file', 'file_data': resolved_url}
|
|
if part_type in {'input_image', 'image'}:
|
|
image_url = part.get('image_url')
|
|
if isinstance(image_url, str) and image_url:
|
|
resolved_image: dict[str, Any] = {'url': image_url}
|
|
detail = part.get('detail')
|
|
if isinstance(detail, str) and detail:
|
|
resolved_image['detail'] = detail
|
|
return {'type': 'image_url', 'image_url': resolved_image}
|
|
content = part.get('content')
|
|
if isinstance(content, list):
|
|
normalized = dict(part)
|
|
normalized['content'] = _terminal_laminar_content_parts_for_span(content)
|
|
return normalized
|
|
return part
|
|
|
|
|
|
def _terminal_laminar_content_parts_for_span(parts: list[Any]) -> list[Any]:
|
|
normalized_parts: list[Any] = []
|
|
for part in parts:
|
|
normalized = _terminal_laminar_content_part_for_span(part)
|
|
if isinstance(normalized, list):
|
|
normalized_parts.extend(normalized)
|
|
else:
|
|
normalized_parts.append(normalized)
|
|
return normalized_parts
|
|
|
|
|
|
def _terminal_laminar_message_for_span(message: Any) -> Any:
|
|
if not isinstance(message, dict):
|
|
return message
|
|
content = message.get('content')
|
|
if not isinstance(content, list):
|
|
return message
|
|
normalized = dict(message)
|
|
normalized['content'] = _terminal_laminar_content_parts_for_span(content)
|
|
return normalized
|
|
|
|
|
|
def _terminal_laminar_span_input_messages(span_input: dict[str, Any]) -> list[dict[str, Any]]:
|
|
messages = span_input.get('messages')
|
|
if not isinstance(messages, list):
|
|
messages = []
|
|
normalized_messages = [_terminal_laminar_message_for_span(message) for message in messages]
|
|
return [*_terminal_laminar_system_messages(span_input), *normalized_messages]
|
|
|
|
|
|
def _terminal_laminar_tools_for_span(span_input: dict[str, Any]) -> list[dict[str, Any]]:
|
|
tools = span_input.get('tools')
|
|
if not isinstance(tools, list):
|
|
return []
|
|
return [tool for tool in tools if isinstance(tool, dict)]
|
|
|
|
|
|
def _terminal_laminar_span_input_payload(span_input: dict[str, Any]) -> Any:
|
|
return _terminal_laminar_span_input_messages(span_input)
|
|
|
|
|
|
def _terminal_laminar_span_output_messages(span_output: dict[str, Any]) -> list[dict[str, Any]]:
|
|
messages = span_output.get('messages')
|
|
return messages if isinstance(messages, list) else []
|
|
|
|
|
|
def _terminal_laminar_message_text(message: dict[str, Any]) -> str:
|
|
content = message.get('content')
|
|
if isinstance(content, str):
|
|
return content
|
|
if not isinstance(content, list):
|
|
return _laminar_preview(content, limit=4000) if content else ''
|
|
parts = []
|
|
for part in content:
|
|
if isinstance(part, dict):
|
|
text = part.get('text') or part.get('content')
|
|
if isinstance(text, str):
|
|
parts.append(text)
|
|
elif part.get('type') in {'image', 'image_url', 'media', 'input_image'}:
|
|
parts.append('[image]')
|
|
elif part:
|
|
parts.append(str(_laminar_preview(part, limit=1000)))
|
|
elif isinstance(part, str):
|
|
parts.append(part)
|
|
return '\n'.join(parts)
|
|
|
|
|
|
def _terminal_laminar_semconv_content_part(part: Any, *, inline_image_data: bool = True) -> dict[str, Any]:
|
|
if not isinstance(part, dict):
|
|
return {'type': 'text', 'content': str(part)}
|
|
part_type = part.get('type')
|
|
if part_type in {'text', 'input_text', 'output_text'}:
|
|
text = part.get('text') or part.get('content')
|
|
if isinstance(text, str):
|
|
return {'type': 'text', 'content': text}
|
|
if part_type == 'image_url':
|
|
image_url = part.get('image_url')
|
|
url = image_url.get('url') if isinstance(image_url, dict) else None
|
|
if isinstance(url, str) and url:
|
|
if url.startswith('data:') and ';base64,' in url:
|
|
header, blob = url.split(';base64,', 1)
|
|
mime_type = header.removeprefix('data:') or 'application/octet-stream'
|
|
if not inline_image_data:
|
|
return {'type': 'blob', 'content': '[image in span input]', 'mimeType': mime_type}
|
|
return {'type': 'blob', 'blob': blob, 'mimeType': mime_type}
|
|
return {'type': 'uri', 'uri': url}
|
|
if part_type in {'tool_use', 'tool_call'}:
|
|
tool_call: dict[str, Any] = {'type': 'tool_call'}
|
|
tool_id = part.get('id') or part.get('tool_call_id')
|
|
name = part.get('name')
|
|
arguments = part.get('arguments') if 'arguments' in part else part.get('input')
|
|
if isinstance(tool_id, str):
|
|
tool_call['id'] = tool_id
|
|
if isinstance(name, str):
|
|
tool_call['name'] = name
|
|
if arguments is not None:
|
|
tool_call['arguments'] = arguments
|
|
return tool_call
|
|
if part_type == 'tool_result':
|
|
response: dict[str, Any] = {'type': 'tool_call_response'}
|
|
tool_id = part.get('tool_use_id') or part.get('tool_call_id') or part.get('id')
|
|
if isinstance(tool_id, str):
|
|
response['id'] = tool_id
|
|
if 'content' in part:
|
|
response['response'] = part.get('content')
|
|
return response
|
|
content = part.get('content')
|
|
if isinstance(content, str):
|
|
return {'type': 'text', 'content': content}
|
|
return {'type': 'text', 'content': _terminal_laminar_json_attribute(part)}
|
|
|
|
|
|
def _terminal_laminar_semconv_messages(
|
|
messages: list[dict[str, Any]],
|
|
*,
|
|
inline_image_data: bool = True,
|
|
) -> list[dict[str, Any]]:
|
|
semconv_messages: list[dict[str, Any]] = []
|
|
for message in messages:
|
|
if not isinstance(message, dict):
|
|
continue
|
|
role = message.get('role')
|
|
if not isinstance(role, str):
|
|
continue
|
|
content = message.get('content')
|
|
parts: list[dict[str, Any]]
|
|
if isinstance(content, list):
|
|
parts = [_terminal_laminar_semconv_content_part(part, inline_image_data=inline_image_data) for part in content]
|
|
elif isinstance(content, str):
|
|
parts = [{'type': 'text', 'content': content}]
|
|
else:
|
|
parts = []
|
|
tool_calls = message.get('tool_calls')
|
|
if isinstance(tool_calls, list):
|
|
for tool_call in tool_calls:
|
|
parts.append(_terminal_laminar_semconv_content_part({'type': 'tool_call', **tool_call}))
|
|
semconv_messages.append({'role': role, 'parts': parts})
|
|
return semconv_messages
|
|
|
|
|
|
def _terminal_laminar_system_instructions(span_input: dict[str, Any]) -> str:
|
|
system_parts = span_input.get('system')
|
|
if not isinstance(system_parts, list):
|
|
return ''
|
|
texts: list[str] = []
|
|
for part in system_parts:
|
|
if isinstance(part, dict):
|
|
text = part.get('text') or part.get('content')
|
|
if isinstance(text, str):
|
|
texts.append(text)
|
|
elif part:
|
|
texts.append(_terminal_laminar_json_attribute(part))
|
|
elif isinstance(part, str):
|
|
texts.append(part)
|
|
return '\n\n'.join(texts)
|
|
|
|
|
|
def _terminal_laminar_indexed_message_attributes(
|
|
messages: list[dict[str, Any]], prefix: str, *, max_messages: int = 20
|
|
) -> dict[str, Any]:
|
|
attributes: dict[str, Any] = {}
|
|
for index, message in enumerate(messages[:max_messages]):
|
|
if not isinstance(message, dict):
|
|
continue
|
|
role = message.get('role')
|
|
if isinstance(role, str):
|
|
attributes[f'{prefix}.{index}.role'] = role
|
|
content = _terminal_laminar_message_text(message)
|
|
if content:
|
|
attributes[f'{prefix}.{index}.content'] = _laminar_preview(content, limit=12_000)
|
|
if len(messages) > max_messages:
|
|
attributes[f'{prefix}.truncated_count'] = len(messages) - max_messages
|
|
return attributes
|
|
|
|
|
|
def _terminal_laminar_json_attribute(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, default=str)
|
|
|
|
|
|
def _terminal_laminar_tool_definition_attributes(tools: list[dict[str, Any]]) -> dict[str, Any]:
|
|
if not tools:
|
|
return {}
|
|
attributes: dict[str, Any] = {
|
|
'gen_ai.tool.definitions': _terminal_laminar_json_attribute(tools),
|
|
'gen_ai.request.tools': _terminal_laminar_json_attribute(tools),
|
|
'llm.request.functions': _terminal_laminar_json_attribute(tools),
|
|
}
|
|
for index, tool in enumerate(tools):
|
|
name = tool.get('name')
|
|
description = tool.get('description')
|
|
input_schema = tool.get('input_schema') or tool.get('parameters')
|
|
output_schema = tool.get('output_schema')
|
|
namespace = tool.get('namespace')
|
|
namespace_description = tool.get('namespace_description')
|
|
prefix = f'llm.request.functions.{index}'
|
|
if isinstance(name, str):
|
|
attributes[f'{prefix}.name'] = name
|
|
attributes[f'gen_ai.request.tools.{index}.name'] = name
|
|
if isinstance(description, str):
|
|
attributes[f'{prefix}.description'] = description
|
|
attributes[f'gen_ai.request.tools.{index}.description'] = description
|
|
if input_schema is not None:
|
|
schema = _terminal_laminar_json_attribute(input_schema)
|
|
attributes[f'{prefix}.input_schema'] = schema
|
|
attributes[f'{prefix}.parameters'] = schema
|
|
attributes[f'gen_ai.request.tools.{index}.input_schema'] = schema
|
|
if output_schema is not None:
|
|
attributes[f'{prefix}.output_schema'] = _terminal_laminar_json_attribute(output_schema)
|
|
if isinstance(namespace, str):
|
|
attributes[f'{prefix}.namespace'] = namespace
|
|
if isinstance(namespace_description, str):
|
|
attributes[f'{prefix}.namespace_description'] = namespace_description
|
|
return attributes
|
|
|
|
|
|
def _terminal_laminar_gen_ai_attributes(
|
|
span_input: dict[str, Any],
|
|
span_output: dict[str, Any],
|
|
usage: dict[str, int | float] | None,
|
|
) -> dict[str, Any]:
|
|
input_messages = _terminal_laminar_span_input_messages(span_input)
|
|
output_messages = _terminal_laminar_span_output_messages(span_output)
|
|
attributes: dict[str, Any] = {
|
|
'gen_ai.operation.name': 'chat',
|
|
'gen_ai.request.model': str(span_input.get('model') or ''),
|
|
'gen_ai.system': str(span_input.get('provider') or ''),
|
|
'gen_ai.system_instructions': _terminal_laminar_system_instructions(span_input),
|
|
# Keep the full image-bearing messages in the span input/output. Repeating
|
|
# base64 blobs in attributes makes OTLP exports exceed Laminar limits.
|
|
'gen_ai.input.messages': _terminal_laminar_json_attribute(
|
|
_terminal_laminar_semconv_messages(input_messages, inline_image_data=False)
|
|
),
|
|
'gen_ai.output.messages': _terminal_laminar_json_attribute(
|
|
_terminal_laminar_semconv_messages(output_messages, inline_image_data=False)
|
|
),
|
|
}
|
|
attributes.update(_terminal_laminar_indexed_message_attributes(input_messages, 'gen_ai.prompt'))
|
|
attributes.update(_terminal_laminar_indexed_message_attributes(output_messages, 'gen_ai.completion'))
|
|
attributes.update(_terminal_laminar_tool_definition_attributes(_terminal_laminar_tools_for_span(span_input)))
|
|
if usage:
|
|
attributes.update(
|
|
{
|
|
'gen_ai.usage.input_tokens': usage.get('input_tokens'),
|
|
'gen_ai.usage.input_cached_tokens': usage.get('cached_input_tokens'),
|
|
'gen_ai.usage.cached_input_tokens': usage.get('cached_input_tokens'),
|
|
'gen_ai.usage.cache_read_input_tokens': usage.get('cached_input_tokens'),
|
|
'gen_ai.usage.cache_creation_input_tokens': usage.get('cache_creation_input_tokens') or 0,
|
|
'gen_ai.usage.input_cache_creation_tokens': usage.get('cache_creation_input_tokens') or 0,
|
|
'gen_ai.usage.output_tokens': usage.get('output_tokens'),
|
|
'gen_ai.usage.total_tokens': usage.get('total_tokens'),
|
|
'llm.usage.input_tokens': usage.get('input_tokens'),
|
|
'llm.usage.output_tokens': usage.get('output_tokens'),
|
|
'llm.usage.input_cached_tokens': usage.get('cached_input_tokens'),
|
|
'llm.usage.cache_creation_input_tokens': usage.get('cache_creation_input_tokens') or 0,
|
|
'llm.usage.total_tokens': usage.get('total_tokens'),
|
|
}
|
|
)
|
|
if usage.get('cost_usd'):
|
|
attributes['gen_ai.usage.cost'] = usage.get('cost_usd')
|
|
return attributes
|
|
|
|
|
|
def _terminal_laminar_tool_result_payload(events: list[dict[str, Any]], tool_call_id: str) -> tuple[str, dict[str, Any]] | None:
|
|
for event in events:
|
|
event_type = _event_type(event)
|
|
if event_type not in ('tool.output', 'tool.failed', 'tool.aborted', 'tool.finished', 'exec_command.end'):
|
|
continue
|
|
payload = _event_payload(event)
|
|
call_id = payload.get('tool_call_id') or payload.get('call_id')
|
|
if call_id is not None and str(call_id) == tool_call_id:
|
|
return event_type, payload
|
|
return None
|
|
|
|
|
|
def _terminal_laminar_image_part_from_path(path: str, mime_type: str | None = None) -> dict[str, Any] | None:
|
|
image_path = Path(path)
|
|
if not image_path.exists() or not image_path.is_file():
|
|
return None
|
|
resolved_mime_type = mime_type or mimetypes.guess_type(str(image_path))[0] or 'image/png'
|
|
try:
|
|
data = base64.b64encode(image_path.read_bytes()).decode('ascii')
|
|
except OSError:
|
|
return None
|
|
return {'type': 'image_url', 'image_url': {'url': f'data:{resolved_mime_type};base64,{data}'}}
|
|
|
|
|
|
def _terminal_laminar_image_part_for_span(image: Any) -> dict[str, Any] | None:
|
|
if isinstance(image, str):
|
|
if image.startswith(('http://', 'https://', 'data:image/')):
|
|
return {'type': 'image_url', 'image_url': {'url': image}}
|
|
return _terminal_laminar_image_part_from_path(image)
|
|
if not isinstance(image, dict):
|
|
return None
|
|
mime_type = image.get('mime_type') or image.get('mimeType') or image.get('media_type') or image.get('mediaType')
|
|
mime_type = mime_type if isinstance(mime_type, str) and mime_type else None
|
|
data = image.get('data') or image.get('base64')
|
|
if isinstance(data, str) and data:
|
|
resolved_mime_type = mime_type or 'image/png'
|
|
return {'type': 'image_url', 'image_url': {'url': f'data:{resolved_mime_type};base64,{data}'}}
|
|
url = image.get('url') or image.get('image_url')
|
|
if isinstance(url, str) and url:
|
|
if url.startswith('file://'):
|
|
return _terminal_laminar_image_part_from_path(url.removeprefix('file://'), mime_type)
|
|
return {'type': 'image_url', 'image_url': {'url': url}}
|
|
path = image.get('path')
|
|
if isinstance(path, str) and path:
|
|
return _terminal_laminar_image_part_from_path(path, mime_type)
|
|
return None
|
|
|
|
|
|
def _terminal_laminar_image_parts_for_span(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
|
images = payload.get('images')
|
|
if not isinstance(images, list):
|
|
return []
|
|
parts: list[dict[str, Any]] = []
|
|
for image in images:
|
|
part = _terminal_laminar_image_part_for_span(image)
|
|
if part is not None:
|
|
parts.append(part)
|
|
return parts
|
|
|
|
|
|
def _terminal_laminar_tool_input_message(tool_call: dict[str, Any]) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
'role': 'assistant',
|
|
'tool_calls': [
|
|
{
|
|
'id': tool_call.get('tool_call_id'),
|
|
'name': tool_call.get('name'),
|
|
'arguments': tool_call.get('arguments'),
|
|
}
|
|
],
|
|
}
|
|
]
|
|
|
|
|
|
def _terminal_laminar_tool_output_message(event_type: str | None, payload: dict[str, Any]) -> list[dict[str, Any]]:
|
|
image_parts = _terminal_laminar_image_parts_for_span(payload)
|
|
content = payload.get('content')
|
|
if isinstance(content, list) and content:
|
|
parts = _terminal_laminar_content_parts_for_span(content)
|
|
else:
|
|
text = _tool_result_text(payload, include_completion_fallback=not bool(image_parts))
|
|
if not text and event_type == 'tool.finished':
|
|
text = _synthetic_tool_result_text(str(payload.get('name') or 'tool'))
|
|
parts = []
|
|
if text:
|
|
parts.append({'type': 'text', 'text': text})
|
|
parts.extend(image_parts)
|
|
if not parts:
|
|
parts = [{'type': 'text', 'text': ''}]
|
|
role = 'tool'
|
|
return [{'role': role, 'content': parts}]
|
|
|
|
|
|
def _record_laminar_terminal_tool_spans(events: list[dict[str, Any]], *, max_spans: int) -> None:
|
|
if not _laminar_ready() or max_spans <= 0:
|
|
return
|
|
tool_calls = _tool_started_calls(events)
|
|
for span_index, tool_call in enumerate(tool_calls[:max_spans], start=1):
|
|
tool_call_id = str(tool_call.get('tool_call_id') or '')
|
|
result = _terminal_laminar_tool_result_payload(events, tool_call_id)
|
|
event_type, payload = result if result is not None else (None, {})
|
|
input_messages = _terminal_laminar_tool_input_message(tool_call)
|
|
output_messages = _terminal_laminar_tool_output_message(event_type, payload)
|
|
with _laminar_start_span(f'rust_core.tool.{tool_call.get("name") or "tool"}', input=input_messages, span_type='TOOL'):
|
|
_laminar_set_span_attributes(
|
|
{
|
|
'runtime': 'browser_use.beta',
|
|
'tool_name': tool_call.get('name'),
|
|
'tool_call_id': tool_call_id,
|
|
'tool_index': span_index,
|
|
'event_type': event_type,
|
|
'ok': payload.get('ok') if payload else None,
|
|
'status': payload.get('status') if payload else None,
|
|
'error': payload.get('error') if payload else None,
|
|
'has_content': bool(payload.get('content')) if payload else False,
|
|
'has_images': bool(payload.get('images')) if payload else False,
|
|
'has_outputs': bool(payload.get('outputs')) if payload else False,
|
|
'has_summary': bool(payload.get('summary')) if payload else False,
|
|
}
|
|
)
|
|
_laminar_set_span_output(output_messages)
|
|
_laminar_force_flush()
|
|
if len(tool_calls) > max_spans:
|
|
_laminar_event('rust_core.tool_spans_truncated', {'recorded': max_spans, 'available': len(tool_calls)})
|
|
|
|
|
|
def _record_laminar_terminal_llm_spans(
|
|
events: list[dict[str, Any]],
|
|
*,
|
|
default_model: str,
|
|
default_provider: str,
|
|
) -> None:
|
|
if not _laminar_ready():
|
|
return
|
|
max_spans = _int_value(os.getenv('BROWSER_USE_RUST_LAMINAR_MAX_LLM_SPANS') or 80)
|
|
if max_spans <= 0:
|
|
return
|
|
ranges = _terminal_model_turn_event_ranges(events)
|
|
for span_index, (start, end) in enumerate(ranges[:max_spans], start=1):
|
|
turn_events = events[start:end]
|
|
if not turn_events:
|
|
continue
|
|
request_payload = _event_payload(turn_events[0])
|
|
span_input = _terminal_laminar_turn_input(
|
|
request_payload,
|
|
default_model=default_model,
|
|
default_provider=default_provider,
|
|
)
|
|
raw_usage = _terminal_turn_usage_payload(turn_events)
|
|
usage = _terminal_laminar_usage_summary(raw_usage)
|
|
cost = _terminal_laminar_usage_cost(str(span_input.get('model') or default_model), usage)
|
|
if cost and usage is not None:
|
|
usage.update(cost)
|
|
start_time = _event_time_seconds(turn_events[0], 0.0)
|
|
end_time = _event_time_seconds(turn_events[-1], start_time)
|
|
span_input_payload = _terminal_laminar_span_input_payload(span_input)
|
|
with _laminar_start_span('rust_core.llm', input=span_input_payload, span_type='LLM'):
|
|
span_output = _terminal_laminar_turn_output(turn_events, raw_usage)
|
|
span_output_messages = _terminal_laminar_span_output_messages(span_output)
|
|
_laminar_set_span_attributes(
|
|
{
|
|
'runtime': 'browser_use.beta',
|
|
'model': str(span_input.get('model') or default_model),
|
|
'provider': str(span_input.get('provider') or default_provider),
|
|
'turn_index': span_index,
|
|
'turn_idx': _int_value(span_input.get('turn_idx')),
|
|
'attempt': _int_value(span_input.get('attempt')),
|
|
'system_prompt_tokens': _int_value(span_input.get('system_prompt_tokens')),
|
|
'tools_count': _int_value(span_input.get('tools_count')),
|
|
'tool_names': _laminar_preview(span_input.get('tool_names') or [], limit=2000),
|
|
'message_count': _int_value(span_input.get('message_count')),
|
|
'omitted_earlier_messages': _int_value(span_input.get('omitted_earlier_messages')),
|
|
'truncated': bool(span_input.get('truncated')),
|
|
'assistant_output_preview': span_output.get('assistant_output_preview') or '',
|
|
'thinking_preview': span_output.get('thinking_preview') or '',
|
|
'tool_calls_count': _int_value(span_output.get('tool_calls_count')),
|
|
'tool_call_names': _laminar_preview(span_output.get('tool_call_names') or [], limit=2000),
|
|
'input_tokens': usage.get('input_tokens') if usage else None,
|
|
'cached_input_tokens': usage.get('cached_input_tokens') if usage else None,
|
|
'cache_creation_input_tokens': usage.get('cache_creation_input_tokens') if usage else None,
|
|
'output_tokens': usage.get('output_tokens') if usage else None,
|
|
'total_tokens': usage.get('total_tokens') if usage else None,
|
|
'input_cost_usd': usage.get('input_cost_usd') if usage else None,
|
|
'input_cached_cost_usd': usage.get('input_cached_cost_usd') if usage else None,
|
|
'input_cache_creation_cost_usd': usage.get('input_cache_creation_cost_usd') if usage else None,
|
|
'output_cost_usd': usage.get('output_cost_usd') if usage else None,
|
|
'cost_usd': usage.get('cost_usd') if usage else None,
|
|
'duration_seconds': max(0.0, end_time - start_time),
|
|
}
|
|
)
|
|
_laminar_set_span_attributes(_terminal_laminar_gen_ai_attributes(span_input, span_output, usage))
|
|
_laminar_set_span_output(span_output_messages)
|
|
_laminar_force_flush()
|
|
if len(ranges) > max_spans:
|
|
_laminar_event(
|
|
'rust_core.llm_spans_truncated',
|
|
{'recorded': max_spans, 'available': len(ranges)},
|
|
)
|
|
|
|
|
|
def _history_from_events(
|
|
events: list[dict[str, Any]],
|
|
*,
|
|
model: str,
|
|
started: float,
|
|
finished: float,
|
|
output_model_schema: type[AgentStructuredOutput] | None,
|
|
process_error: str | None,
|
|
) -> AgentHistoryList[AgentStructuredOutput]:
|
|
events = _events_after_terminal_rollbacks(_events_after_terminal_compaction(events))
|
|
final_result = _structured_result_text(_result_from_events(events), output_model_schema)
|
|
failure = process_error or _failure_from_events(events)
|
|
if final_result is None and failure is not None:
|
|
final_result = _structured_result_text(_last_streamed_assistant_text_from_events(events), output_model_schema)
|
|
if final_result is None and failure is None:
|
|
failure = _recoverable_failure_from_events(events)
|
|
if final_result is None and failure is None:
|
|
failure = 'Rust terminal session did not produce a final result.'
|
|
is_done = final_result is not None and failure is None
|
|
attachments = _attachments_from_events(events)
|
|
history_items = _history_items_from_terminal_turns(
|
|
events,
|
|
started=started,
|
|
finished=finished,
|
|
final_result=final_result,
|
|
attachments=attachments,
|
|
failure=failure,
|
|
is_done=is_done,
|
|
)
|
|
if history_items is not None:
|
|
history_list: AgentHistoryList[AgentStructuredOutput] = AgentHistoryList(
|
|
history=history_items,
|
|
usage=_usage_from_events(events, model),
|
|
)
|
|
history_list._output_model_schema = output_model_schema
|
|
return history_list
|
|
tool_calls = _tool_calls_with_final_done(
|
|
_tool_started_calls(events),
|
|
final_result=final_result,
|
|
attachments=attachments,
|
|
is_done=is_done,
|
|
)
|
|
history = AgentHistory(
|
|
model_output=_model_output_from_tool_calls(tool_calls, events),
|
|
result=_action_results_from_tool_calls(
|
|
tool_calls,
|
|
events,
|
|
final_result=final_result,
|
|
attachments=attachments,
|
|
failure=failure,
|
|
is_done=is_done,
|
|
),
|
|
state=_browser_state_from_events(events),
|
|
metadata=StepMetadata(step_start_time=started, step_end_time=finished, step_number=max(1, len(events))),
|
|
)
|
|
history_list: AgentHistoryList[AgentStructuredOutput] = AgentHistoryList(
|
|
history=[history],
|
|
usage=_usage_from_events(events, model),
|
|
)
|
|
history_list._output_model_schema = output_model_schema
|
|
return history_list
|
|
|
|
|
|
def _load_rust_history(file_path: str | Path) -> AgentHistoryList:
|
|
with open(file_path, encoding='utf-8') as history_file:
|
|
data = json.load(history_file)
|
|
if not isinstance(data, dict):
|
|
raise BetaAgentError(f'Invalid Browser Use history file: {file_path}')
|
|
for item in data.get('history', []):
|
|
if isinstance(item, dict):
|
|
item['model_output'] = None
|
|
state = item.get('state')
|
|
if isinstance(state, dict) and 'interacted_element' not in state:
|
|
state['interacted_element'] = None
|
|
return AgentHistoryList.model_validate(data)
|
|
|
|
|
|
def _default_browser_session(
|
|
agent_id: str,
|
|
browser_profile: BrowserProfile | None,
|
|
browser_session: Any | None,
|
|
browser: Any | None,
|
|
) -> tuple[Any | None, Any | None]:
|
|
if browser is not None:
|
|
return browser, getattr(browser, 'browser_profile', browser_profile)
|
|
if browser_session is not None:
|
|
return browser_session, getattr(browser_session, 'browser_profile', browser_profile)
|
|
if browser_profile is not None and not isinstance(browser_profile, BrowserProfile):
|
|
return None, browser_profile
|
|
resolved_profile = browser_profile or BrowserProfile()
|
|
session = BrowserSession(
|
|
browser_profile=resolved_profile,
|
|
id=uuid7str()[:-4] + agent_id[-4:],
|
|
)
|
|
return session, session.browser_profile
|
|
|
|
|
|
def _init_file_system(
|
|
state: AgentState,
|
|
agent_directory: Path,
|
|
file_system_path: str | None,
|
|
) -> tuple[FileSystem, str]:
|
|
if state.file_system_state and file_system_path:
|
|
raise ValueError(
|
|
'Cannot provide both file_system_state (from agent state) and file_system_path. '
|
|
'Either restore from existing state or create new file system at specified path, not both.'
|
|
)
|
|
if state.file_system_state:
|
|
file_system = FileSystem.from_state(state.file_system_state)
|
|
return file_system, str(file_system.base_dir)
|
|
if file_system_path:
|
|
file_system = FileSystem(file_system_path)
|
|
state.file_system_state = file_system.get_state()
|
|
return file_system, file_system_path
|
|
file_system = FileSystem(agent_directory)
|
|
state.file_system_state = file_system.get_state()
|
|
return file_system, str(agent_directory)
|
|
|
|
|
|
def _resolve_tools(
|
|
tools: Any | None,
|
|
controller: Any | None,
|
|
output_model_schema: type[BaseModel] | None,
|
|
use_vision: bool | Literal['auto'],
|
|
display_files_in_done_text: bool,
|
|
) -> Any:
|
|
resolved_tools = tools if tools is not None else controller
|
|
if resolved_tools is None:
|
|
exclude_actions = ['screenshot'] if use_vision is False else []
|
|
resolved_tools = Tools(exclude_actions=exclude_actions, display_files_in_done_text=display_files_in_done_text)
|
|
if output_model_schema is not None and hasattr(resolved_tools, 'use_structured_output_action'):
|
|
resolved_tools.use_structured_output_action(output_model_schema)
|
|
return resolved_tools
|
|
|
|
|
|
def _browser_use_version_and_source(source_override: str | None = None) -> tuple[str | None, str]:
|
|
version = get_browser_use_version()
|
|
try:
|
|
package_root = Path(__file__).parent.parent.parent
|
|
repo_files = ['.git', 'README.md', 'docs', 'examples']
|
|
source = 'git' if all((package_root / file).exists() for file in repo_files) else 'pip'
|
|
except Exception:
|
|
source = 'unknown'
|
|
if source_override is not None:
|
|
source = source_override
|
|
return version, source
|
|
|
|
|
|
def _register_llm_for_usage(token_cost_service: TokenCost, llm: Any | None) -> None:
|
|
if llm is None or not hasattr(llm, 'ainvoke'):
|
|
return
|
|
try:
|
|
token_cost_service.register_llm(llm)
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def _eventbus_name(agent_id: str, prefix: str = 'Agent') -> str:
|
|
suffix = re.sub(r'\W', '_', str(agent_id)[-4:])
|
|
return f'{prefix}_{suffix or "agent"}'
|
|
|
|
|
|
class _CloudEventLLMProxy:
|
|
def __init__(self, model_name: str):
|
|
self.model_name = model_name
|
|
|
|
|
|
class _CloudEventAgentProxy:
|
|
def __init__(self, agent: Agent):
|
|
self._agent = agent
|
|
self.llm = _CloudEventLLMProxy(getattr(agent, 'model', None) or _model_name(getattr(agent, 'llm', None)))
|
|
|
|
def __getattr__(self, name: str) -> Any:
|
|
return getattr(self._agent, name)
|
|
|
|
|
|
def _unique_eventbus_name(agent_id: str, prefix: str = 'Agent') -> str:
|
|
unique_suffix = re.sub(r'\W', '_', uuid7str()[-8:])
|
|
return f'{_eventbus_name(agent_id, prefix)}_{unique_suffix}'
|
|
|
|
|
|
def _action_payload(action: Any) -> dict[str, Any]:
|
|
"""Serialize a Browser Use action model into JSON-like data for Rust task context."""
|
|
if isinstance(action, dict):
|
|
return action
|
|
if hasattr(action, 'model_dump'):
|
|
try:
|
|
dumped = action.model_dump(exclude_unset=True, mode='json')
|
|
except TypeError:
|
|
dumped = action.model_dump(exclude_unset=True)
|
|
if isinstance(dumped, dict):
|
|
return dumped
|
|
return {'action': str(action)}
|
|
|
|
|
|
def _done_action_result(payload: dict[str, Any]) -> ActionResult | None:
|
|
done_payload = payload.get('done')
|
|
if done_payload is None:
|
|
return None
|
|
if hasattr(done_payload, 'model_dump'):
|
|
done_payload = done_payload.model_dump(exclude_unset=True, mode='json')
|
|
if not isinstance(done_payload, dict):
|
|
done_payload = {}
|
|
success = done_payload.get('success')
|
|
if not isinstance(success, bool):
|
|
success = True
|
|
text = done_payload.get('text')
|
|
if not isinstance(text, str) and 'data' in done_payload:
|
|
text = json.dumps(done_payload['data'], ensure_ascii=False, default=str)
|
|
if not isinstance(text, str):
|
|
text = ''
|
|
files = done_payload.get('files_to_display')
|
|
attachments = [str(item) for item in files if isinstance(item, (str, os.PathLike))] if isinstance(files, list) else None
|
|
return ActionResult(
|
|
is_done=True,
|
|
success=success,
|
|
extracted_content=text,
|
|
long_term_memory=f'Task completed. Success Status: {success}',
|
|
attachments=attachments or None,
|
|
)
|
|
|
|
|
|
def _actions_instruction(payloads: list[dict[str, Any]]) -> str:
|
|
actions_json = json.dumps(payloads, indent=2, ensure_ascii=False, default=str)
|
|
return (
|
|
'Execute these Browser Use action models in order using the current browser page/session. '
|
|
'Return a concise result for the executed actions.\n\n'
|
|
f'Actions:\n{actions_json}'
|
|
)
|
|
|
|
|
|
def _normalize_initial_action(action_name: str, params: Any) -> tuple[str, Any]:
|
|
if not isinstance(params, dict):
|
|
return action_name, params
|
|
if action_name in ('go_to_url', 'open_tab'):
|
|
normalized = dict(params)
|
|
if action_name == 'open_tab' and 'new_tab' not in normalized:
|
|
normalized['new_tab'] = True
|
|
return 'navigate', normalized
|
|
if action_name == 'click_element_by_index':
|
|
return 'click', params
|
|
if action_name == 'input_text':
|
|
return 'input', params
|
|
return action_name, params
|
|
|
|
|
|
class Agent(Generic[Context, AgentStructuredOutput]):
|
|
"""Browser Use-style Agent backed by the Rust browser-use-terminal core."""
|
|
|
|
def __init__(
|
|
self,
|
|
task: str,
|
|
llm: BaseChatModel | None = None,
|
|
browser_profile: BrowserProfile | None = None,
|
|
browser_session: BrowserSession | None = None,
|
|
browser: BrowserSession | None = None,
|
|
tools: Tools[Context] | None = None,
|
|
controller: Tools[Context] | None = None,
|
|
skill_ids: list[str | Literal['*']] | None = None,
|
|
skills: list[str | Literal['*']] | None = None,
|
|
skill_service: Any | None = None,
|
|
sensitive_data: dict[str, str | dict[str, str]] | None = None,
|
|
initial_actions: list[dict[str, dict[str, Any]]] | None = None,
|
|
register_new_step_callback: AgentNewStepCallback | None = None,
|
|
register_done_callback: AgentDoneCallback | None = None,
|
|
register_external_agent_status_raise_error_callback: Callable[[], Awaitable[bool]] | None = None,
|
|
register_should_stop_callback: Callable[[], Awaitable[bool]] | None = None,
|
|
output_model_schema: type[AgentStructuredOutput] | None = None,
|
|
extraction_schema: dict | None = None,
|
|
use_vision: bool | Literal['auto'] = True,
|
|
save_conversation_path: str | Path | None = None,
|
|
save_conversation_path_encoding: str | None = 'utf-8',
|
|
max_failures: int = 5,
|
|
override_system_message: str | None = None,
|
|
extend_system_message: str | None = None,
|
|
generate_gif: bool | str = False,
|
|
available_file_paths: list[str] | None = None,
|
|
include_attributes: list[str] | None = None,
|
|
max_actions_per_step: int = 5,
|
|
use_thinking: bool = True,
|
|
flash_mode: bool = False,
|
|
demo_mode: bool | None = None,
|
|
max_history_items: int | None = None,
|
|
page_extraction_llm: BaseChatModel | None = None,
|
|
fallback_llm: BaseChatModel | None = None,
|
|
use_judge: bool = True,
|
|
ground_truth: str | None = None,
|
|
judge_llm: BaseChatModel | None = None,
|
|
injected_agent_state: AgentState | None = None,
|
|
source: str | None = None,
|
|
file_system_path: str | None = None,
|
|
task_id: str | None = None,
|
|
calculate_cost: bool = False,
|
|
pricing_url: str | None = None,
|
|
display_files_in_done_text: bool = True,
|
|
include_tool_call_examples: bool = False,
|
|
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
|
|
llm_timeout: int | None = None,
|
|
step_timeout: int = 180,
|
|
directly_open_url: bool = True,
|
|
include_recent_events: bool = False,
|
|
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
|
|
final_response_after_failure: bool = True,
|
|
enable_planning: bool = True,
|
|
planning_replan_on_stall: int = 3,
|
|
planning_exploration_limit: int = 5,
|
|
loop_detection_window: int = 20,
|
|
loop_detection_enabled: bool = True,
|
|
llm_screenshot_size: tuple[int, int] | None = None,
|
|
message_compaction: MessageCompactionSettings | bool | None = True,
|
|
max_clickable_elements_length: int = 40000,
|
|
_url_shortening_limit: int = 25,
|
|
enable_signal_handler: bool = True,
|
|
**kwargs,
|
|
):
|
|
if llm_screenshot_size is not None:
|
|
if not isinstance(llm_screenshot_size, tuple) or len(llm_screenshot_size) != 2:
|
|
raise ValueError('llm_screenshot_size must be a tuple of (width, height)')
|
|
width, height = llm_screenshot_size
|
|
if not isinstance(width, int) or not isinstance(height, int):
|
|
raise ValueError('llm_screenshot_size dimensions must be integers')
|
|
if width < 100 or height < 100:
|
|
raise ValueError('llm_screenshot_size dimensions must be at least 100 pixels')
|
|
llm = _resolve_default_llm(llm)
|
|
use_vision = True
|
|
if browser and browser_session:
|
|
raise ValueError('Cannot specify both "browser" and "browser_session" parameters. Use "browser" for the cleaner API.')
|
|
if getattr(llm, 'provider', None) == 'browser-use':
|
|
flash_mode = True
|
|
if flash_mode:
|
|
enable_planning = False
|
|
if llm_screenshot_size is None:
|
|
model_name = getattr(llm, 'model', '')
|
|
# rsplit drops the provider prefix so gateway ids like 'anthropic/claude-sonnet-4-6'
|
|
# get the same screenshot auto-config as direct Claude Sonnet models.
|
|
if isinstance(model_name, str) and model_name.rsplit('/', 1)[-1].startswith('claude-sonnet'):
|
|
llm_screenshot_size = (1400, 850)
|
|
if page_extraction_llm is None:
|
|
page_extraction_llm = llm
|
|
if judge_llm is None:
|
|
judge_llm = llm
|
|
if llm_timeout is None:
|
|
llm_timeout = _llm_timeout_for_model(llm)
|
|
self.id = task_id or uuid7str()
|
|
self.task_id = self.id
|
|
self.llm = llm
|
|
self.judge_llm = judge_llm
|
|
self.browser_session, self._browser_profile = _default_browser_session(
|
|
self.id,
|
|
browser_profile,
|
|
browser_session,
|
|
browser,
|
|
)
|
|
if demo_mode is not None and self.browser_profile is not None:
|
|
profile_demo_mode = getattr(self.browser_profile, 'demo_mode', None)
|
|
if profile_demo_mode != demo_mode and hasattr(self.browser_profile, 'model_copy'):
|
|
updated_profile = self.browser_profile.model_copy(update={'demo_mode': demo_mode})
|
|
self._browser_profile = updated_profile
|
|
if self.browser_session is not None and hasattr(self.browser_session, 'browser_profile'):
|
|
self.browser_session.browser_profile = updated_profile
|
|
self.tools = _resolve_tools(
|
|
tools,
|
|
controller,
|
|
output_model_schema,
|
|
use_vision,
|
|
display_files_in_done_text,
|
|
)
|
|
if skills and skill_ids:
|
|
raise ValueError('Cannot specify both "skills" and "skill_ids" parameters. Use "skills" for the cleaner API.')
|
|
skill_ids = skills or skill_ids
|
|
self.skill_service = None
|
|
self._skills_registered = False
|
|
if skill_service is not None:
|
|
self.skill_service = skill_service
|
|
elif skill_ids:
|
|
from browser_use.skills import SkillService
|
|
|
|
self.skill_service = SkillService(skill_ids=skill_ids)
|
|
self.extraction_schema = extraction_schema
|
|
if self.extraction_schema is None and output_model_schema is not None:
|
|
self.extraction_schema = output_model_schema.model_json_schema()
|
|
self._fallback_llm = fallback_llm
|
|
self._using_fallback_llm = False
|
|
self._original_llm = llm
|
|
self.sensitive_data = sensitive_data
|
|
self.register_new_step_callback = register_new_step_callback
|
|
self.register_done_callback = register_done_callback
|
|
self.register_external_agent_status_raise_error_callback = register_external_agent_status_raise_error_callback
|
|
self.register_should_stop_callback = register_should_stop_callback
|
|
self.output_model_schema = output_model_schema
|
|
self._set_browser_use_version_and_source(source)
|
|
self.kwargs = kwargs
|
|
self.model = _model_name(llm)
|
|
if isinstance(message_compaction, bool):
|
|
message_compaction = MessageCompactionSettings(enabled=message_compaction)
|
|
self.state = injected_agent_state or AgentState()
|
|
self.state.loop_detector.window_size = loop_detection_window
|
|
timestamp = int(time.time())
|
|
self.agent_directory = Path(tempfile.gettempdir()) / f'browser_use_agent_{self.id}_{timestamp}'
|
|
self._set_file_system(file_system_path)
|
|
self._set_screenshot_service()
|
|
self.settings = AgentSettings(
|
|
use_vision=use_vision,
|
|
vision_detail_level=vision_detail_level,
|
|
save_conversation_path=save_conversation_path,
|
|
save_conversation_path_encoding=save_conversation_path_encoding,
|
|
max_failures=max_failures,
|
|
override_system_message=override_system_message,
|
|
extend_system_message=extend_system_message,
|
|
generate_gif=generate_gif,
|
|
include_attributes=include_attributes,
|
|
max_actions_per_step=max_actions_per_step,
|
|
use_thinking=use_thinking,
|
|
flash_mode=flash_mode,
|
|
max_history_items=max_history_items,
|
|
page_extraction_llm=page_extraction_llm,
|
|
calculate_cost=calculate_cost,
|
|
include_tool_call_examples=include_tool_call_examples,
|
|
llm_timeout=llm_timeout,
|
|
step_timeout=step_timeout,
|
|
final_response_after_failure=final_response_after_failure,
|
|
use_judge=use_judge,
|
|
ground_truth=ground_truth,
|
|
enable_planning=enable_planning,
|
|
planning_replan_on_stall=planning_replan_on_stall,
|
|
planning_exploration_limit=planning_exploration_limit,
|
|
loop_detection_window=loop_detection_window,
|
|
loop_detection_enabled=loop_detection_enabled,
|
|
message_compaction=message_compaction,
|
|
max_clickable_elements_length=max_clickable_elements_length,
|
|
)
|
|
if self.settings.save_conversation_path:
|
|
self.settings.save_conversation_path = Path(self.settings.save_conversation_path).expanduser().resolve()
|
|
self.logger.info(f'💬 Saving conversation to {_log_pretty_path(self.settings.save_conversation_path)}')
|
|
self._setup_action_models()
|
|
self._verify_and_setup_llm()
|
|
logger.debug(
|
|
f'{" +vision" if self.settings.use_vision else ""}'
|
|
f' extraction_model={getattr(self.settings.page_extraction_llm, "model", "Unknown") if self.settings.page_extraction_llm else "Unknown"}'
|
|
f'{" +file_system" if self.file_system else ""}'
|
|
)
|
|
self.token_cost_service = TokenCost(include_cost=calculate_cost, pricing_url=pricing_url)
|
|
_register_llm_for_usage(self.token_cost_service, llm)
|
|
_register_llm_for_usage(self.token_cost_service, page_extraction_llm)
|
|
_register_llm_for_usage(self.token_cost_service, judge_llm)
|
|
if self.settings.message_compaction and self.settings.message_compaction.compaction_llm:
|
|
_register_llm_for_usage(self.token_cost_service, self.settings.message_compaction.compaction_llm)
|
|
self.enable_signal_handler = enable_signal_handler
|
|
self.telemetry = ProductTelemetry()
|
|
self.eventbus = EventBus(name=_eventbus_name(self.id))
|
|
self._eventbus_stopped = False
|
|
self.available_file_paths = available_file_paths or []
|
|
self.allowed_domains = _extract_profile_domains(self.browser_session, self.browser_profile, 'allowed_domains')
|
|
self.prohibited_domains = _extract_profile_domains(self.browser_session, self.browser_profile, 'prohibited_domains')
|
|
self.managed_browser_args = _managed_browser_launch_args(self.browser_session, self.browser_profile)
|
|
self.managed_browser_profile_dir = _managed_browser_profile_dir(self.browser_session, self.browser_profile)
|
|
self.managed_browser_executable_path = _managed_browser_executable_path(self.browser_session, self.browser_profile)
|
|
self.managed_browser_env = _managed_browser_env(self.browser_session, self.browser_profile)
|
|
self.cdp_headers = _extract_cdp_headers(self.browser_session, self.browser_profile)
|
|
self.browser_user_agent = _extract_user_agent(self.browser_session, self.browser_profile)
|
|
self.highlight_enabled, self.highlight_color, self.highlight_duration_ms = _extract_highlight_settings(
|
|
self.browser_session, self.browser_profile
|
|
)
|
|
self.wait_timing_env = _extract_wait_timing_settings(self.browser_session, self.browser_profile)
|
|
self.block_ip_addresses = _extract_block_ip_addresses(self.browser_session, self.browser_profile)
|
|
self.browser_permissions = _extract_profile_permissions(self.browser_session, self.browser_profile)
|
|
self.browser_accept_downloads, self.browser_downloads_path = _extract_browser_downloads(
|
|
self.browser_session, self.browser_profile
|
|
)
|
|
self.browser_no_viewport, self.browser_viewport = _extract_browser_viewport(self.browser_session, self.browser_profile)
|
|
self.browser_window_size = _extract_browser_window_size(self.browser_session, self.browser_profile)
|
|
self.browser_storage_state = _extract_browser_storage_state(self.browser_session, self.browser_profile)
|
|
self.sensitive_data_context = _sensitive_data_context(sensitive_data)
|
|
_warn_sensitive_data_domain_constraints(self.logger, sensitive_data, self.allowed_domains)
|
|
self.display_files_in_done_text = display_files_in_done_text
|
|
self.has_downloads_path = getattr(self.browser_profile, 'downloads_path', None) is not None
|
|
self._last_known_downloads: list[str] = []
|
|
self.directly_open_url = directly_open_url
|
|
self.include_recent_events = include_recent_events
|
|
self.sample_images = sample_images
|
|
self._url_shortening_limit = _url_shortening_limit
|
|
self.initial_url = None
|
|
if self.directly_open_url and not self.state.follow_up_task and not initial_actions:
|
|
self.initial_url = _extract_start_url(task)
|
|
if self.initial_url:
|
|
self.logger.info(f'🔗 Found URL in task: {self.initial_url}, adding as initial action...')
|
|
initial_actions = [{'navigate': {'url': self.initial_url, 'new_tab': False}}]
|
|
self.initial_action_payloads = list(initial_actions or [])
|
|
self.initial_actions = (
|
|
self._convert_initial_actions(self.initial_action_payloads) if self.initial_action_payloads else None
|
|
)
|
|
self._initial_actions_executed = False
|
|
self._completed_initial_navigation_urls: list[str] = []
|
|
self._completed_initial_navigation_states: list[dict[str, Any]] = []
|
|
self._pending_history_prefix: list[AgentHistory] = []
|
|
self.task = _task_with_schema(
|
|
_task_with_available_files(
|
|
_task_with_sensitive_data_context(
|
|
_task_with_domain_constraints(
|
|
_task_with_initial_actions(task, self.initial_action_payloads),
|
|
self.allowed_domains,
|
|
self.prohibited_domains,
|
|
),
|
|
self.sensitive_data_context,
|
|
),
|
|
self.available_file_paths,
|
|
),
|
|
output_model_schema,
|
|
)
|
|
self._message_manager = MessageManager(
|
|
task=self.task,
|
|
system_message=SystemPrompt(
|
|
max_actions_per_step=self.settings.max_actions_per_step,
|
|
override_system_message=self.settings.override_system_message,
|
|
extend_system_message=self.settings.extend_system_message,
|
|
use_thinking=self.settings.use_thinking,
|
|
flash_mode=self.settings.flash_mode,
|
|
is_anthropic=str(getattr(self.llm, 'provider', '')).lower() == 'anthropic',
|
|
is_browser_use_model=str(getattr(self.llm, 'provider', '')).lower() == 'browser-use',
|
|
model_name=getattr(self.llm, 'model', self.model),
|
|
).get_system_message(),
|
|
file_system=self.file_system,
|
|
state=self.state.message_manager_state,
|
|
use_thinking=self.settings.use_thinking,
|
|
include_attributes=self.settings.include_attributes,
|
|
sensitive_data=sensitive_data,
|
|
max_history_items=self.settings.max_history_items,
|
|
vision_detail_level=self.settings.vision_detail_level,
|
|
include_tool_call_examples=self.settings.include_tool_call_examples,
|
|
include_recent_events=self.include_recent_events,
|
|
sample_images=self.sample_images,
|
|
llm_screenshot_size=llm_screenshot_size,
|
|
max_clickable_elements_length=self.settings.max_clickable_elements_length,
|
|
)
|
|
if self.browser_session is not None:
|
|
self.browser_session.llm_screenshot_size = llm_screenshot_size
|
|
self.session_id: str = uuid7str()
|
|
self.terminal_session_id: str | None = None
|
|
self._sdk_agent_id: str | None = None
|
|
self._sdk_browser_id: str | None = None
|
|
self._sdk_client: RustSdkClient | None = None
|
|
self._active_sdk_run_id: str | None = None
|
|
self.laminar_trace_id: str | None = None
|
|
self.history: AgentHistoryList[AgentStructuredOutput] = AgentHistoryList(history=[], usage=None)
|
|
self.result: AgentHistoryList[AgentStructuredOutput] | None = None
|
|
self.last_events: list[dict[str, Any]] = []
|
|
self.last_child_events: list[dict[str, Any]] = []
|
|
self.last_usage_events: list[dict[str, Any]] = []
|
|
self.last_observability_events: list[dict[str, Any]] = []
|
|
self.last_stdout = ''
|
|
self.last_stderr = ''
|
|
self._last_synced_history_id: int | None = None
|
|
self._last_step_callback_history_id: int | None = None
|
|
self._last_step_end_callback_history_id: int | None = None
|
|
self._external_pause_event = asyncio.Event()
|
|
self._external_pause_event.set()
|
|
|
|
def _set_file_system(self, file_system_path: str | None = None) -> None:
|
|
"""Initialize or restore Browser Use file-system state."""
|
|
self.file_system, self.file_system_path = _init_file_system(self.state, self.agent_directory, file_system_path)
|
|
|
|
def _set_screenshot_service(self) -> None:
|
|
"""Initialize Browser Use screenshot storage under the agent directory."""
|
|
self.screenshot_service = ScreenshotService(self.agent_directory)
|
|
|
|
def _set_browser_use_version_and_source(self, source_override: str | None = None) -> None:
|
|
"""Expose Browser Use version/source metadata on the Rust-backed wrapper."""
|
|
self.version, self.source = _browser_use_version_and_source(source_override)
|
|
|
|
def _verify_and_setup_llm(self):
|
|
"""Mirror Browser Use LLM verification state for callers that use the helper."""
|
|
if self.llm is None:
|
|
return True
|
|
try:
|
|
from browser_use.config import CONFIG
|
|
|
|
skip_verification = CONFIG.SKIP_LLM_API_KEY_VERIFICATION
|
|
except Exception:
|
|
skip_verification = False
|
|
if getattr(self.llm, '_verified_api_keys', None) is True or skip_verification:
|
|
setattr(self.llm, '_verified_api_keys', True)
|
|
return True
|
|
|
|
async def _log_agent_run(self) -> None:
|
|
"""Log Browser Use run metadata for the Rust-backed wrapper."""
|
|
self.logger.info(f'\033[34m🎯 Task: {self.task}\033[0m')
|
|
self.logger.debug(f'🤖 Browser-Use Library Version {self.version} ({self.source})')
|
|
latest_version = await check_latest_browser_use_version()
|
|
if latest_version and latest_version != self.version:
|
|
self.logger.info(
|
|
f'📦 Newer version available: {latest_version} (current: {self.version}). Upgrade with: uv add browser-use=={latest_version}'
|
|
)
|
|
|
|
def _log_agent_setup(self) -> None:
|
|
"""Log Browser Use run setup metadata for the Rust-backed wrapper."""
|
|
browser_session_id = getattr(self.browser_session, 'id', None) if self.browser_session else None
|
|
browser_session_suffix = str(browser_session_id)[-4:] if browser_session_id else 'None'
|
|
cdp_url = getattr(self.browser_session, 'cdp_url', None) if self.browser_session else None
|
|
connection_mode = '(connecting via CDP)' if cdp_url else '(launching local browser)'
|
|
self.logger.debug(
|
|
f'Agent setup: Agent Session ID {self.session_id[-4:]}, Task ID {self.task_id[-4:]}, Browser Session ID {browser_session_suffix} {connection_mode}'
|
|
)
|
|
|
|
def _log_first_step_startup(self) -> None:
|
|
"""Log the first-step startup line used by Browser Use callers."""
|
|
if len(self.history.history) != 0:
|
|
return
|
|
provider = getattr(self.llm, 'provider', None) or 'rust-terminal'
|
|
model = getattr(self.llm, 'model', None) or self.model
|
|
self.logger.info(f'Starting a browser-use agent with version {self.version}, with provider={provider} and model={model}')
|
|
|
|
def _log_main_execution_start(self, max_steps: int) -> None:
|
|
"""Log Browser Use's main execution-loop start for terminal-backed runs."""
|
|
self.logger.debug(f'Starting main execution loop with max {max_steps} steps...')
|
|
|
|
def _log_step_context(self, browser_state_summary: BrowserStateSummary) -> None:
|
|
"""Log current step and page context."""
|
|
url = getattr(browser_state_summary, 'url', '') if browser_state_summary else ''
|
|
url_short = url[:50] + '...' if len(url) > 50 else url
|
|
dom_state = getattr(browser_state_summary, 'dom_state', None)
|
|
selector_map = getattr(dom_state, 'selector_map', {}) if dom_state else {}
|
|
interactive_count = len(selector_map) if selector_map else 0
|
|
self.logger.info('')
|
|
self.logger.info(f'Step {self.state.n_steps}:')
|
|
self.logger.debug(f'Evaluating page with {interactive_count} interactive elements on: {url_short}')
|
|
|
|
def _log_next_action_summary(self, parsed: AgentOutput) -> None:
|
|
"""Log a concise summary of the next action list."""
|
|
actions = getattr(parsed, 'action', None)
|
|
if not (self.logger.isEnabledFor(logging.DEBUG) and actions):
|
|
return
|
|
action_details = []
|
|
for action in actions:
|
|
action_data = action.model_dump(exclude_unset=True)
|
|
action_name = next(iter(action_data.keys())) if action_data else 'unknown'
|
|
action_params = action_data.get(action_name, {}) if action_data else {}
|
|
param_summary = []
|
|
if isinstance(action_params, dict):
|
|
for key, value in action_params.items():
|
|
if key == 'index':
|
|
param_summary.append(f'#{value}')
|
|
elif key == 'text' and isinstance(value, str):
|
|
text_preview = value[:30] + '...' if len(value) > 30 else value
|
|
param_summary.append(f'text="{text_preview}"')
|
|
elif key == 'url':
|
|
param_summary.append(f'url="{value}"')
|
|
elif key == 'success':
|
|
param_summary.append(f'success={value}')
|
|
elif isinstance(value, (str, int, bool)):
|
|
value_str = str(value)
|
|
value_preview = value_str[:30] + '...' if len(value_str) > 30 else value_str
|
|
param_summary.append(f'{key}={value_preview}')
|
|
param_text = f'({", ".join(param_summary)})' if param_summary else ''
|
|
action_details.append(f'{action_name}{param_text}')
|
|
self.logger.debug(f'Next actions: {", ".join(action_details)}')
|
|
|
|
def _log_step_completion_summary(self, step_start_time: float, result: list[ActionResult]) -> None:
|
|
"""Log action count, timing, and success/failure counts for a completed step."""
|
|
if not result:
|
|
return
|
|
step_duration = time.time() - step_start_time
|
|
action_count = len(result)
|
|
success_count = sum(1 for item in result if not item.error)
|
|
failure_count = action_count - success_count
|
|
status_parts = []
|
|
if success_count > 0:
|
|
status_parts.append(f'success={success_count}')
|
|
if failure_count > 0:
|
|
status_parts.append(f'failed={failure_count}')
|
|
status_text = ' | '.join(status_parts) if status_parts else 'success=0'
|
|
self.logger.debug(
|
|
f'Step {self.state.n_steps}: Ran {action_count} action{"" if action_count == 1 else "s"} in {step_duration:.2f}s: {status_text}'
|
|
)
|
|
|
|
def _log_final_outcome_messages(self) -> None:
|
|
"""Log Browser Use-style guidance for failed runs."""
|
|
is_successful = self.history.is_successful()
|
|
if is_successful is not False and is_successful is not None:
|
|
return
|
|
final_result = self.history.final_result()
|
|
final_result_str = str(final_result).lower() if final_result else ''
|
|
captcha_keywords = ['captcha', 'cloudflare', 'recaptcha', 'challenge', 'bot detection', 'access denied']
|
|
if any(keyword in final_result_str for keyword in captcha_keywords):
|
|
task_preview = self.task[:10] if len(self.task) > 10 else self.task
|
|
self.logger.info('')
|
|
self.logger.info('Failed because of CAPTCHA? For better browser stealth, try:')
|
|
self.logger.info(f' agent = Agent(task="{task_preview}...", browser=Browser(use_cloud=True))')
|
|
self.logger.info('')
|
|
self.logger.info('Did the Agent not work as expected? Let us fix this!')
|
|
self.logger.info(' Please open a short issue here: https://github.com/browser-use/browser-use/issues')
|
|
|
|
@observe(ignore_input=True, ignore_output=False)
|
|
async def _judge_trace(self) -> JudgementResult | None:
|
|
"""Judge the reconstructed Rust terminal trace with the configured judge LLM."""
|
|
input_messages = construct_judge_messages(
|
|
task=self.task,
|
|
final_result=self.history.final_result() or '',
|
|
agent_steps=self.history.agent_steps(),
|
|
screenshot_paths=[path for path in self.history.screenshot_paths() if path is not None],
|
|
max_images=10,
|
|
ground_truth=self.settings.ground_truth,
|
|
use_vision=self.settings.use_vision,
|
|
)
|
|
|
|
kwargs: dict[str, Any] = {'output_format': JudgementResult}
|
|
if getattr(self.judge_llm, 'provider', None) == 'browser-use':
|
|
kwargs['request_type'] = 'judge'
|
|
kwargs['session_id'] = self.session_id
|
|
|
|
try:
|
|
response = await self.judge_llm.ainvoke(input_messages, **kwargs)
|
|
return response.completion # type: ignore[return-value]
|
|
except Exception as exc:
|
|
self.logger.error(f'Judge trace failed: {exc}')
|
|
return None
|
|
|
|
async def _judge_and_log(self) -> None:
|
|
"""Run judge evaluation and attach the verdict to the final action result."""
|
|
judgement = await self._judge_trace()
|
|
if not self.history.history:
|
|
return
|
|
last_step = self.history.history[-1]
|
|
if not last_step.result:
|
|
return
|
|
last_result = last_step.result[-1]
|
|
if not last_result.is_done:
|
|
return
|
|
|
|
last_result.judgement = judgement
|
|
self_reported_success = last_result.success
|
|
if not judgement:
|
|
return
|
|
if self_reported_success is True and judgement.verdict is True:
|
|
return
|
|
|
|
judge_log = '\n'
|
|
if self_reported_success is True and judgement.verdict is False:
|
|
judge_log += '⚠️ \033[33mAgent reported success but judge thinks task failed\033[0m\n'
|
|
|
|
verdict_color = '\033[32m' if judgement.verdict else '\033[31m'
|
|
verdict_text = '✅ PASS' if judgement.verdict else '❌ FAIL'
|
|
judge_log += f'⚖️ {verdict_color}Judge Verdict: {verdict_text}\033[0m\n'
|
|
if judgement.failure_reason:
|
|
judge_log += f' Failure Reason: {judgement.failure_reason}\n'
|
|
if judgement.reached_captcha:
|
|
self.logger.warning(
|
|
'Agent was blocked by a captcha. Cloud browsers include stealth fingerprinting and proxy rotation to avoid this.\n'
|
|
' Try: Browser(use_cloud=True) | Get an API key: https://cloud.browser-use.com?utm_source=oss&utm_medium=captcha_nudge'
|
|
)
|
|
judge_log += f' {judgement.reasoning}\n'
|
|
self.logger.info(judge_log)
|
|
|
|
def _log_agent_event(self, max_steps: int, agent_run_error: str | None = None) -> None:
|
|
"""Emit Browser Use telemetry for a Rust-backed run."""
|
|
usage = self.history.usage
|
|
if usage is None:
|
|
total_input_tokens = 0
|
|
total_output_tokens = 0
|
|
prompt_cached_tokens = 0
|
|
total_tokens = 0
|
|
else:
|
|
total_input_tokens = usage.total_prompt_tokens
|
|
total_output_tokens = usage.total_completion_tokens
|
|
prompt_cached_tokens = usage.total_prompt_cached_tokens
|
|
total_tokens = usage.total_tokens
|
|
|
|
action_history_data = []
|
|
for item in self.history.history:
|
|
if item.model_output and item.model_output.action:
|
|
action_history_data.append(
|
|
[action.model_dump(exclude_unset=True) for action in item.model_output.action if action]
|
|
)
|
|
else:
|
|
action_history_data.append(None)
|
|
|
|
final_result = self.history.final_result()
|
|
final_result_str = json.dumps(final_result) if final_result is not None else None
|
|
cdp_url = getattr(self.browser_session, 'cdp_url', None) if self.browser_session else None
|
|
model = getattr(self.llm, 'model', None) or self.model
|
|
provider = getattr(self.llm, 'provider', None) or 'rust-terminal'
|
|
|
|
self.telemetry.capture(
|
|
AgentTelemetryEvent(
|
|
task=self.task,
|
|
model=model,
|
|
model_provider=provider,
|
|
max_steps=max_steps,
|
|
max_actions_per_step=self.settings.max_actions_per_step,
|
|
use_vision=self.settings.use_vision,
|
|
version=self.version or '',
|
|
source=self.source,
|
|
cdp_url=urlparse(cdp_url).hostname if cdp_url else None,
|
|
agent_type='rust_core',
|
|
action_errors=self.history.errors(),
|
|
action_history=action_history_data,
|
|
urls_visited=[url for url in self.history.urls() if url],
|
|
steps=self.history.number_of_steps(),
|
|
total_input_tokens=total_input_tokens,
|
|
total_output_tokens=total_output_tokens,
|
|
prompt_cached_tokens=prompt_cached_tokens,
|
|
total_tokens=total_tokens,
|
|
total_duration_seconds=self.history.total_duration_seconds(),
|
|
success=self.history.is_successful(),
|
|
final_result_response=final_result_str,
|
|
error_message=agent_run_error,
|
|
)
|
|
)
|
|
|
|
def _record_run_telemetry(self, max_steps: int, agent_run_error: str | None = None) -> None:
|
|
"""Record Browser Use run telemetry without allowing telemetry failures to break the run."""
|
|
if getattr(self, '_force_exit_telemetry_logged', False):
|
|
self.logger.debug('Telemetry for force exit (SIGINT) was logged by custom exit callback.')
|
|
return
|
|
try:
|
|
self._log_agent_event(max_steps=max_steps, agent_run_error=agent_run_error)
|
|
except Exception as exc:
|
|
self.logger.error(f'Failed to log telemetry event: {exc}', exc_info=True)
|
|
|
|
def _record_laminar_run_observability(
|
|
self,
|
|
*,
|
|
max_steps: int,
|
|
duration_seconds: float | None,
|
|
process_error: str | None = None,
|
|
) -> None:
|
|
"""Populate the current Laminar agent.run span for terminal-backed runs."""
|
|
if not _laminar_ready():
|
|
return
|
|
errors = self.history.errors()
|
|
usage = self.history.usage
|
|
final_result = self.history.final_result()
|
|
model = getattr(self.llm, 'model', None) or self.model
|
|
provider = getattr(self.llm, 'provider', None) or 'rust-terminal'
|
|
summary = {
|
|
'runtime': 'browser_use.beta',
|
|
'model': model,
|
|
'provider': provider,
|
|
'max_steps': max_steps,
|
|
'steps': self.history.number_of_steps(),
|
|
'is_done': self.history.is_done(),
|
|
'is_successful': self.history.is_successful(),
|
|
'errors_count': len(errors or []),
|
|
'errors_preview': _laminar_preview(errors, limit=1500),
|
|
'final_result_preview': _laminar_preview(final_result, limit=2000),
|
|
'terminal_session_id': self.terminal_session_id,
|
|
'browser_use_session_id': self.session_id,
|
|
'terminal_events_count': len(self.last_observability_events or self.last_events or []),
|
|
'duration_seconds': duration_seconds,
|
|
'process_error': process_error,
|
|
}
|
|
if usage is not None:
|
|
summary.update(
|
|
{
|
|
'usage_total_tokens': usage.total_tokens,
|
|
'usage_prompt_tokens': usage.total_prompt_tokens,
|
|
'usage_prompt_cost': usage.total_prompt_cost,
|
|
'usage_prompt_cached_tokens': usage.total_prompt_cached_tokens,
|
|
'usage_prompt_cached_cost': usage.total_prompt_cached_cost,
|
|
'usage_prompt_cache_creation_tokens': usage.total_prompt_cache_creation_tokens,
|
|
'usage_prompt_cache_creation_cost': usage.total_prompt_cache_creation_cost,
|
|
'usage_completion_tokens': usage.total_completion_tokens,
|
|
'usage_completion_cost': usage.total_completion_cost,
|
|
'usage_total_cost': usage.total_cost,
|
|
}
|
|
)
|
|
observability_events = self.last_observability_events or self.last_events or []
|
|
_record_laminar_terminal_llm_spans(
|
|
observability_events,
|
|
default_model=str(model),
|
|
default_provider=str(provider),
|
|
)
|
|
_record_laminar_terminal_tool_spans(
|
|
observability_events,
|
|
max_spans=_int_value(os.getenv('BROWSER_USE_RUST_LAMINAR_MAX_TOOL_SPANS') or 160),
|
|
)
|
|
_laminar_set_span_attributes(
|
|
{
|
|
'runtime': summary['runtime'],
|
|
'model': model,
|
|
'provider': provider,
|
|
'max_steps': max_steps,
|
|
'steps': summary['steps'],
|
|
'is_done': summary['is_done'],
|
|
'is_successful': summary['is_successful'],
|
|
'terminal_session_id': self.terminal_session_id,
|
|
'terminal_events_count': summary['terminal_events_count'],
|
|
'duration_seconds': duration_seconds,
|
|
'process_error': process_error,
|
|
'usage_total_cost': usage.total_cost if usage is not None else None,
|
|
'usage_prompt_cached_tokens': usage.total_prompt_cached_tokens if usage is not None else None,
|
|
'usage_prompt_cache_creation_tokens': usage.total_prompt_cache_creation_tokens if usage is not None else None,
|
|
}
|
|
)
|
|
_laminar_set_span_output(summary)
|
|
_laminar_event(
|
|
'agent.run.terminal_summary',
|
|
{
|
|
'runtime': summary['runtime'],
|
|
'steps': summary['steps'],
|
|
'is_successful': summary['is_successful'],
|
|
'duration_seconds': duration_seconds,
|
|
'terminal_events_count': summary['terminal_events_count'],
|
|
},
|
|
)
|
|
|
|
async def _log_run_usage_summary(self) -> None:
|
|
"""Log Browser Use token usage summary for a completed Rust-backed run."""
|
|
await self.token_cost_service.log_usage_summary()
|
|
|
|
async def _apply_terminal_usage_costs(self, events: list[dict[str, Any]]) -> None:
|
|
"""Populate cost fields for terminal-reconstructed usage."""
|
|
if self.history is None:
|
|
return
|
|
try:
|
|
self.history.usage = await _usage_from_events_with_costs(events, self.model, self.token_cost_service)
|
|
except Exception as exc:
|
|
self.logger.debug(f'Failed to price Rust terminal usage: {exc}', exc_info=True)
|
|
|
|
def _cloud_event_agent(self):
|
|
llm_model_name = getattr(getattr(self, 'llm', None), 'model_name', None)
|
|
if isinstance(llm_model_name, str) and llm_model_name:
|
|
return self
|
|
return _CloudEventAgentProxy(self)
|
|
|
|
def _ensure_eventbus(self) -> None:
|
|
if getattr(self, '_eventbus_stopped', False):
|
|
self.eventbus = EventBus(name=_unique_eventbus_name(self.id))
|
|
self._eventbus_stopped = False
|
|
|
|
def _register_run_signal_handler(self, max_steps: int) -> SignalHandler:
|
|
"""Register Browser Use SIGINT/SIGTERM handling for a Rust-backed run."""
|
|
self._unregister_run_signal_handler()
|
|
self._force_exit_telemetry_logged = False
|
|
|
|
def on_force_exit_log_telemetry() -> None:
|
|
self._record_run_telemetry(max_steps=max_steps, agent_run_error='SIGINT: Cancelled by user')
|
|
if hasattr(self, 'telemetry') and self.telemetry:
|
|
self.telemetry.flush()
|
|
self._force_exit_telemetry_logged = True
|
|
|
|
signal_handler = SignalHandler(
|
|
loop=asyncio.get_event_loop(),
|
|
pause_callback=self.pause,
|
|
resume_callback=self.resume,
|
|
custom_exit_callback=on_force_exit_log_telemetry,
|
|
exit_on_second_int=True,
|
|
)
|
|
signal_handler.register()
|
|
self._run_signal_handler = signal_handler
|
|
return signal_handler
|
|
|
|
def _unregister_run_signal_handler(self) -> None:
|
|
signal_handler = getattr(self, '_run_signal_handler', None)
|
|
if signal_handler is None:
|
|
return
|
|
try:
|
|
signal_handler.unregister()
|
|
finally:
|
|
self._run_signal_handler = None
|
|
|
|
def _dispatch_run_start_events(self) -> None:
|
|
"""Emit Browser Use cloud lifecycle create events for a Rust-backed run."""
|
|
self._ensure_eventbus()
|
|
event_agent = self._cloud_event_agent()
|
|
if not self.state.session_initialized:
|
|
self.logger.debug('Dispatching CreateAgentSessionEvent...')
|
|
self.eventbus.dispatch(CreateAgentSessionEvent.from_agent(event_agent))
|
|
self.state.session_initialized = True
|
|
self.logger.debug('Dispatching CreateAgentTaskEvent...')
|
|
self.eventbus.dispatch(CreateAgentTaskEvent.from_agent(event_agent))
|
|
|
|
def _dispatch_run_update_event(self) -> None:
|
|
"""Emit Browser Use cloud lifecycle update event for a completed Rust-backed run."""
|
|
self.eventbus.dispatch(UpdateAgentTaskEvent.from_agent(self._cloud_event_agent()))
|
|
|
|
async def _stop_eventbus_after_run(self) -> None:
|
|
"""Stop the eventbus after run-level events have been dispatched."""
|
|
stop = getattr(self.eventbus, 'stop', None)
|
|
if not callable(stop):
|
|
return
|
|
handlers = getattr(self.eventbus, 'handlers', None)
|
|
has_handlers = any(handlers.values()) if isinstance(handlers, dict) else True
|
|
try:
|
|
result = stop(timeout=3.0, clear=not has_handlers)
|
|
except TypeError:
|
|
result = stop(timeout=3.0)
|
|
if inspect.isawaitable(result):
|
|
await result
|
|
self._eventbus_stopped = True
|
|
|
|
async def _finalize_run_cleanup(self) -> None:
|
|
"""Mirror Browser Use run cleanup ordering."""
|
|
self._unregister_run_signal_handler()
|
|
await self._stop_eventbus_after_run()
|
|
await self._close_browser_resources()
|
|
await self._close_sdk_client_if_not_keep_alive()
|
|
|
|
async def _finalize_exceptional_run(self, max_steps: int, agent_run_error: str) -> None:
|
|
"""Mirror Browser Use run finalization for exceptions that escape Rust execution."""
|
|
if not hasattr(self, '_task_start_time') or not hasattr(self, '_session_start_time'):
|
|
self._initialize_run_lifecycle_state()
|
|
await self._log_run_usage_summary()
|
|
self._record_laminar_run_observability(
|
|
max_steps=max_steps,
|
|
duration_seconds=None,
|
|
process_error=agent_run_error,
|
|
)
|
|
self._record_run_telemetry(max_steps=max_steps, agent_run_error=agent_run_error)
|
|
self._dispatch_run_update_event()
|
|
self._log_final_outcome_messages()
|
|
await self._finalize_run_cleanup()
|
|
|
|
def _initialize_run_lifecycle_state(self) -> None:
|
|
"""Initialize Browser Use run timing and session state."""
|
|
self._session_start_time = time.time()
|
|
self._task_start_time = self._session_start_time
|
|
self._dispatch_run_start_events()
|
|
|
|
def _log_action(self, action, action_name: str, action_num: int, total_actions: int) -> None:
|
|
"""Log an action before execution with Browser Use-style structure."""
|
|
action_header = f'[{action_num}/{total_actions}] {action_name}:' if total_actions > 1 else f'{action_name}:'
|
|
action_data = action.model_dump(exclude_unset=True)
|
|
params = action_data.get(action_name, {}) if isinstance(action_data, dict) else {}
|
|
param_parts = []
|
|
if isinstance(params, dict):
|
|
for param_name, value in params.items():
|
|
if isinstance(value, str) and len(value) > 150:
|
|
display_value = value[:150] + '...'
|
|
elif isinstance(value, list) and len(str(value)) > 200:
|
|
display_value = str(value)[:200] + '...'
|
|
else:
|
|
display_value = value
|
|
param_parts.append(f'{param_name}: {display_value}')
|
|
if param_parts:
|
|
self.logger.info(f' {action_header} {", ".join(param_parts)}')
|
|
else:
|
|
self.logger.info(f' {action_header}')
|
|
|
|
@observe(name='agent.run', ignore_input=True, ignore_output=True)
|
|
async def run(
|
|
self,
|
|
max_steps: int = 100,
|
|
on_step_start: AgentHookFunc | None = None,
|
|
on_step_end: AgentHookFunc | None = None,
|
|
) -> AgentHistoryList[AgentStructuredOutput]:
|
|
self.laminar_trace_id = _laminar_current_trace_id()
|
|
self._register_run_signal_handler(max_steps)
|
|
try:
|
|
return await self._run_terminal(max_steps=max_steps, on_step_start=on_step_start, on_step_end=on_step_end)
|
|
except asyncio.CancelledError:
|
|
await self._finalize_exceptional_run(max_steps=max_steps, agent_run_error='CancelledError')
|
|
raise
|
|
except KeyboardInterrupt:
|
|
self.logger.debug('Got KeyboardInterrupt during execution, returning current history')
|
|
await self._finalize_exceptional_run(max_steps=max_steps, agent_run_error='KeyboardInterrupt')
|
|
return self.history
|
|
except Exception as exc:
|
|
self.logger.error(f'Agent run failed with exception: {exc}', exc_info=True)
|
|
await self._finalize_exceptional_run(max_steps=max_steps, agent_run_error=str(exc))
|
|
raise
|
|
|
|
async def _run_terminal(
|
|
self,
|
|
max_steps: int,
|
|
on_step_start: AgentHookFunc | None,
|
|
on_step_end: AgentHookFunc | None,
|
|
) -> AgentHistoryList[AgentStructuredOutput]:
|
|
await self._log_agent_run()
|
|
self._log_agent_setup()
|
|
self._initialize_run_lifecycle_state()
|
|
self._log_first_step_startup()
|
|
started = time.time()
|
|
if await self._should_stop_before_run():
|
|
finished = time.time()
|
|
self.result = _history_from_events(
|
|
[],
|
|
model=self.model,
|
|
started=started,
|
|
finished=finished,
|
|
output_model_schema=self.output_model_schema,
|
|
process_error='Beta agent stopped before terminal run.',
|
|
)
|
|
self.history = self.result
|
|
await self._apply_terminal_usage_costs([])
|
|
await self._log_run_usage_summary()
|
|
self._record_laminar_run_observability(
|
|
max_steps=max_steps,
|
|
duration_seconds=finished - started,
|
|
process_error='Beta agent stopped before terminal run.',
|
|
)
|
|
self._record_run_telemetry(max_steps=max_steps, agent_run_error='Beta agent stopped before terminal run.')
|
|
self._dispatch_run_update_event()
|
|
self._log_final_outcome_messages()
|
|
await self._finalize_run_cleanup()
|
|
return self.history
|
|
if self.state.paused:
|
|
self.logger.debug('Agent paused before Rust terminal run, waiting to resume...')
|
|
await self._external_pause_event.wait()
|
|
signal_handler = getattr(self, '_run_signal_handler', None)
|
|
reset = getattr(signal_handler, 'reset', None)
|
|
if callable(reset):
|
|
reset()
|
|
if await self._should_stop_before_run():
|
|
finished = time.time()
|
|
self.result = _history_from_events(
|
|
[],
|
|
model=self.model,
|
|
started=started,
|
|
finished=finished,
|
|
output_model_schema=self.output_model_schema,
|
|
process_error='Beta agent stopped before terminal run.',
|
|
)
|
|
self.history = self.result
|
|
await self._apply_terminal_usage_costs([])
|
|
await self._log_run_usage_summary()
|
|
self._record_laminar_run_observability(
|
|
max_steps=max_steps,
|
|
duration_seconds=finished - started,
|
|
process_error='Beta agent stopped before terminal run.',
|
|
)
|
|
self._record_run_telemetry(max_steps=max_steps, agent_run_error='Beta agent stopped before terminal run.')
|
|
self._dispatch_run_update_event()
|
|
self._log_final_outcome_messages()
|
|
await self._finalize_run_cleanup()
|
|
return self.history
|
|
|
|
prefix_start = len(self.history.history)
|
|
await self._execute_initial_actions(allow_terminal_run=False)
|
|
if len(self.history.history) > prefix_start:
|
|
self._pending_history_prefix = list(self.history.history[prefix_start:])
|
|
await self._call_callback(on_step_start, self)
|
|
self._log_main_execution_start(max_steps)
|
|
followup = bool(self.state.follow_up_task and self._sdk_agent_id)
|
|
task = self.task
|
|
if not followup:
|
|
task = _task_with_completed_initial_navigation_context(
|
|
task,
|
|
self._completed_initial_navigation_urls,
|
|
self.initial_action_payloads,
|
|
self._completed_initial_navigation_states,
|
|
)
|
|
self.state.follow_up_task = False
|
|
return await self._run_sdk_agent(
|
|
task=task,
|
|
max_steps=max_steps,
|
|
started=started,
|
|
on_step_end=on_step_end,
|
|
source='follow_up' if followup else 'run',
|
|
followups=[task] if followup else None,
|
|
)
|
|
|
|
async def follow_up(
|
|
self,
|
|
task: str,
|
|
max_steps: int | None = None,
|
|
*,
|
|
step_timeout: int | None = None,
|
|
enqueue_timeout: int | None = None,
|
|
) -> AgentHistoryList[AgentStructuredOutput]:
|
|
if not self.terminal_session_id or not self._sdk_agent_id:
|
|
raise BetaAgentError('No active Rust session. Call run() before follow_up().')
|
|
resolved_max_steps = max_steps if max_steps is not None else self.kwargs.get('max_steps', 100)
|
|
self.add_new_task(task)
|
|
self.state.follow_up_task = False
|
|
self._register_run_signal_handler(resolved_max_steps)
|
|
try:
|
|
return await self._follow_up_terminal(
|
|
self.task,
|
|
max_steps=max_steps,
|
|
resolved_max_steps=resolved_max_steps,
|
|
step_timeout=step_timeout,
|
|
enqueue_timeout=enqueue_timeout,
|
|
)
|
|
except asyncio.CancelledError:
|
|
await self._finalize_exceptional_run(max_steps=resolved_max_steps, agent_run_error='CancelledError')
|
|
raise
|
|
except KeyboardInterrupt:
|
|
self.logger.debug('Got KeyboardInterrupt during execution, returning current history')
|
|
await self._finalize_exceptional_run(max_steps=resolved_max_steps, agent_run_error='KeyboardInterrupt')
|
|
return self.history
|
|
except Exception as exc:
|
|
self.logger.error(f'Agent follow-up failed with exception: {exc}', exc_info=True)
|
|
await self._finalize_exceptional_run(max_steps=resolved_max_steps, agent_run_error=str(exc))
|
|
raise
|
|
|
|
async def _follow_up_terminal(
|
|
self,
|
|
task: str,
|
|
max_steps: int | None,
|
|
resolved_max_steps: int,
|
|
initialize_lifecycle: bool = True,
|
|
on_step_end: AgentHookFunc | None = None,
|
|
step_timeout: int | None = None,
|
|
enqueue_timeout: int | None = None,
|
|
) -> AgentHistoryList[AgentStructuredOutput]:
|
|
if initialize_lifecycle:
|
|
self._initialize_run_lifecycle_state()
|
|
started = time.time()
|
|
return await self._run_sdk_agent(
|
|
task=task,
|
|
max_steps=resolved_max_steps,
|
|
started=started,
|
|
on_step_end=on_step_end,
|
|
source='follow_up',
|
|
followups=[task],
|
|
step_timeout=step_timeout,
|
|
enqueue_timeout=enqueue_timeout,
|
|
)
|
|
|
|
async def _run_sdk_agent(
|
|
self,
|
|
*,
|
|
task: str,
|
|
max_steps: int,
|
|
started: float,
|
|
on_step_end: AgentHookFunc | None,
|
|
source: str,
|
|
followups: list[str] | None = None,
|
|
step_timeout: int | None = None,
|
|
enqueue_timeout: int | None = None,
|
|
) -> AgentHistoryList[AgentStructuredOutput]:
|
|
events: list[dict[str, Any]] = []
|
|
process_error: str | None = None
|
|
result: Any = None
|
|
sdk = await self._ensure_sdk_client()
|
|
_ = step_timeout, enqueue_timeout
|
|
method = 'agent.run' if self._sdk_agent_id or followups else 'agent.run_task'
|
|
params = self._sdk_run_params(max_steps=max_steps, task=task, followups=followups)
|
|
self._active_sdk_run_id = self.terminal_session_id or self._sdk_agent_id
|
|
progress_task: asyncio.Task[Any] | None = None
|
|
self.logger.info(
|
|
'Rust SDK %s starting: max_steps=%s browser_mode=%s browser_id=%s llm_timeout=%s',
|
|
method,
|
|
params.get('max_steps'),
|
|
params.get('browser_mode'),
|
|
params.get('browser_id') or '<new>',
|
|
params.get('llm', {}).get('timeout'),
|
|
)
|
|
if hasattr(sdk, 'notification_queue'):
|
|
progress_task = asyncio.create_task(self._log_sdk_progress(sdk))
|
|
try:
|
|
result = await sdk.call(method, params)
|
|
except asyncio.CancelledError:
|
|
await self._preserve_sdk_notification_history(
|
|
sdk,
|
|
started=started,
|
|
process_error='CancelledError',
|
|
)
|
|
await self._cancel_active_sdk_run()
|
|
raise
|
|
except Exception as exc:
|
|
process_error = str(exc) or exc.__class__.__name__
|
|
finally:
|
|
if progress_task is not None:
|
|
progress_task.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await progress_task
|
|
self._active_sdk_run_id = None
|
|
finished = time.time()
|
|
self.last_stdout = str(getattr(sdk, 'stdout_text', '') or '')
|
|
self.last_stderr = str(getattr(sdk, 'stderr_text', '') or '\n'.join(line for line in sdk.stderr_lines if line))
|
|
notification_events = _sdk_notification_events(sdk)
|
|
child_events: list[dict[str, Any]] = []
|
|
usage_events: list[dict[str, Any]] = []
|
|
response_usage_event: dict[str, Any] | None = None
|
|
if isinstance(result, dict):
|
|
agent_id = result.get('agent_id')
|
|
session_id = result.get('session_id')
|
|
browser_id = result.get('browser_id')
|
|
if isinstance(agent_id, str) and agent_id:
|
|
self._sdk_agent_id = agent_id
|
|
if isinstance(session_id, str) and session_id:
|
|
self.terminal_session_id = session_id
|
|
if isinstance(browser_id, str) and browser_id:
|
|
self._sdk_browser_id = browser_id
|
|
history_payload = result.get('history')
|
|
if isinstance(history_payload, dict):
|
|
raw_events = history_payload.get('events')
|
|
if isinstance(raw_events, list):
|
|
events = _dedupe_sdk_events([event for event in raw_events if isinstance(event, dict)])
|
|
errors = history_payload.get('errors')
|
|
if process_error is None and history_payload.get('success') is False and isinstance(errors, list) and errors:
|
|
process_error = '\n'.join(str(error) for error in errors if error)
|
|
raw_child_events = history_payload.get('child_events')
|
|
if isinstance(raw_child_events, list):
|
|
child_events = _dedupe_sdk_events([event for event in raw_child_events if isinstance(event, dict)])
|
|
raw_usage_events = history_payload.get('usage_events')
|
|
if isinstance(raw_usage_events, list):
|
|
usage_events = _dedupe_sdk_events([event for event in raw_usage_events if isinstance(event, dict)])
|
|
response_usage_event = _usage_event_from_sdk_history_usage(history_payload.get('usage'))
|
|
elif process_error is None:
|
|
process_error = 'Rust SDK server returned an invalid response.'
|
|
events_result = _result_from_events(events)
|
|
notification_result = _result_from_events(notification_events)
|
|
used_notification_events = False
|
|
if notification_events and (
|
|
not events
|
|
or _sdk_events_truncated_for_transport(events)
|
|
or len(notification_events) > len(events)
|
|
or (notification_result is not None and events_result is None)
|
|
):
|
|
events = notification_events
|
|
child_events = []
|
|
usage_events = notification_events
|
|
events_result = notification_result
|
|
used_notification_events = True
|
|
if not usage_events:
|
|
usage_events = [*events, *child_events]
|
|
if response_usage_event is not None:
|
|
current_usage = _usage_from_events(usage_events, self.model)
|
|
response_usage = _usage_from_events([response_usage_event], self.model)
|
|
if _usage_tokens(response_usage) > _usage_tokens(current_usage):
|
|
usage_events = [response_usage_event]
|
|
if self.logger.isEnabledFor(logging.INFO):
|
|
self.logger.info(
|
|
'Rust SDK reconstructed history: response_events=%s notification_events=%s usage_events=%s final_from=%s usage_tokens=%s',
|
|
len(events),
|
|
len(notification_events),
|
|
len(usage_events),
|
|
'events' if events_result is not None else 'none',
|
|
_usage_tokens(_usage_from_events(usage_events, self.model)),
|
|
)
|
|
if events_result is not None and (
|
|
process_error == 'CancelledError' or _sdk_transport_error_after_final_result(process_error)
|
|
):
|
|
process_error = None
|
|
if used_notification_events and events_result is not None:
|
|
process_error = None
|
|
self.last_events = events
|
|
self.last_child_events = child_events
|
|
self.last_usage_events = usage_events
|
|
self.last_observability_events = usage_events
|
|
self.result = _history_from_events(
|
|
self.last_events,
|
|
model=self.model,
|
|
started=started,
|
|
finished=finished,
|
|
output_model_schema=self.output_model_schema,
|
|
process_error=process_error,
|
|
)
|
|
if self._pending_history_prefix:
|
|
self.result.history = [*self._pending_history_prefix, *self.result.history]
|
|
self._pending_history_prefix = []
|
|
self.history = self.result
|
|
await self._apply_terminal_usage_costs(self.last_usage_events)
|
|
self._sync_state_from_history()
|
|
await self._log_run_usage_summary()
|
|
self._record_laminar_run_observability(
|
|
max_steps=max_steps,
|
|
duration_seconds=finished - started,
|
|
process_error=process_error,
|
|
)
|
|
self._record_run_telemetry(max_steps=max_steps, agent_run_error=process_error)
|
|
self._dispatch_run_update_event()
|
|
await self._check_and_update_downloads(source)
|
|
await self._save_conversation_if_requested()
|
|
await self._call_new_step_callback()
|
|
await self._call_step_end_callbacks(on_step_end)
|
|
await self._call_done_callback()
|
|
await self._generate_gif_if_requested()
|
|
self._log_final_outcome_messages()
|
|
await self._finalize_run_cleanup()
|
|
return self.history
|
|
|
|
async def _preserve_sdk_notification_history(
|
|
self,
|
|
sdk: Any,
|
|
*,
|
|
started: float,
|
|
process_error: str,
|
|
) -> None:
|
|
events = _sdk_notification_events(sdk)
|
|
if not events:
|
|
return
|
|
finished = time.time()
|
|
self.last_stdout = str(getattr(sdk, 'stdout_text', '') or '')
|
|
self.last_stderr = str(
|
|
getattr(sdk, 'stderr_text', '') or '\n'.join(line for line in getattr(sdk, 'stderr_lines', []) if line)
|
|
)
|
|
effective_error = process_error
|
|
if _result_from_events(events) is not None and (
|
|
process_error == 'CancelledError' or _sdk_transport_error_after_final_result(process_error)
|
|
):
|
|
effective_error = None
|
|
self.last_events = events
|
|
self.last_child_events = []
|
|
self.last_usage_events = events
|
|
self.last_observability_events = events
|
|
self.result = _history_from_events(
|
|
self.last_events,
|
|
model=self.model,
|
|
started=started,
|
|
finished=finished,
|
|
output_model_schema=self.output_model_schema,
|
|
process_error=effective_error,
|
|
)
|
|
if self._pending_history_prefix:
|
|
self.result.history = [*self._pending_history_prefix, *self.result.history]
|
|
self._pending_history_prefix = []
|
|
self.history = self.result
|
|
await self._apply_terminal_usage_costs(self.last_usage_events)
|
|
self._sync_state_from_history()
|
|
|
|
async def _log_sdk_progress(self, sdk: RustSdkClient) -> None:
|
|
logged = 0
|
|
last_summary: str | None = None
|
|
last_logged_at = 0.0
|
|
while True:
|
|
notification = await sdk.notification_queue.get()
|
|
summary = _sdk_notification_summary(notification)
|
|
if not summary:
|
|
continue
|
|
now = time.time()
|
|
if summary == last_summary and now - last_logged_at < 30:
|
|
continue
|
|
last_summary = summary
|
|
last_logged_at = now
|
|
logged += 1
|
|
self.logger.info('Rust SDK event %s: %s', logged, summary)
|
|
|
|
follow_up_task = follow_up
|
|
|
|
@property
|
|
def usage(self) -> UsageSummary | None:
|
|
return self.history.usage
|
|
|
|
@property
|
|
def logger(self) -> logging.Logger:
|
|
browser_session_id = getattr(self.browser_session, 'id', '----') or '----'
|
|
target_id = '--'
|
|
agent_focus = getattr(self.browser_session, 'agent_focus', None)
|
|
focus_target_id = getattr(agent_focus, 'target_id', None)
|
|
if isinstance(focus_target_id, str) and focus_target_id:
|
|
target_id = focus_target_id[-2:]
|
|
return logging.getLogger(f'browser_use.Agent🅰 {self.task_id[-4:]} ⇢ 🅑 {str(browser_session_id)[-4:]} 🅣 {target_id}')
|
|
|
|
@property
|
|
def browser_profile(self) -> Any:
|
|
session_profile = getattr(self.browser_session, 'browser_profile', None)
|
|
return session_profile if session_profile is not None else self._browser_profile
|
|
|
|
@property
|
|
def message_manager(self) -> MessageManager:
|
|
return self._message_manager
|
|
|
|
def _enhance_task_with_schema(self, task: str, output_model_schema: type[AgentStructuredOutput] | None) -> str:
|
|
"""Enhance task description with Browser Use-style output schema information."""
|
|
if output_model_schema is None:
|
|
return task
|
|
try:
|
|
schema_json = json.dumps(output_model_schema.model_json_schema(), indent=2)
|
|
return f'{task}\nExpected output format: {output_model_schema.__name__}\n{schema_json}'
|
|
except Exception as exc:
|
|
self.logger.debug(f'Could not parse output schema: {exc}')
|
|
return task
|
|
|
|
def _extract_start_url(self, task: str) -> str | None:
|
|
"""Extract Browser Use-style direct startup URL from a task string."""
|
|
return _extract_start_url(task)
|
|
|
|
def _remove_think_tags(self, text: str) -> str:
|
|
think_tags = re.compile(r'<think>.*?</think>', re.DOTALL)
|
|
stray_close_tag = re.compile(r'.*?</think>', re.DOTALL)
|
|
text = re.sub(think_tags, '', text)
|
|
text = re.sub(stray_close_tag, '', text)
|
|
return text.strip()
|
|
|
|
def _replace_urls_in_text(self, text: str) -> tuple[str, dict[str, str]]:
|
|
"""Replace long URL query/fragment tails with shorter reversible forms."""
|
|
replaced_urls: dict[str, str] = {}
|
|
|
|
def replace_url(match: re.Match) -> str:
|
|
original_url = match.group(0)
|
|
query_start = original_url.find('?')
|
|
fragment_start = original_url.find('#')
|
|
after_path_start = len(original_url)
|
|
if query_start != -1:
|
|
after_path_start = min(after_path_start, query_start)
|
|
if fragment_start != -1:
|
|
after_path_start = min(after_path_start, fragment_start)
|
|
|
|
base_url = original_url[:after_path_start]
|
|
after_path = original_url[after_path_start:]
|
|
if len(after_path) <= self._url_shortening_limit:
|
|
return original_url
|
|
if not after_path:
|
|
return original_url
|
|
|
|
truncated = after_path[: self._url_shortening_limit]
|
|
short_hash = hashlib.md5(after_path.encode('utf-8')).hexdigest()[:7]
|
|
shortened = f'{base_url}{truncated}...{short_hash}'
|
|
if len(shortened) < len(original_url):
|
|
replaced_urls[shortened] = original_url
|
|
return shortened
|
|
return original_url
|
|
|
|
return URL_PATTERN.sub(replace_url, text), replaced_urls
|
|
|
|
def _process_messsages_and_replace_long_urls_shorter_ones(self, input_messages: list[BaseMessage]) -> dict[str, str]:
|
|
"""Replace long URLs in Browser Use LLM messages in place."""
|
|
from browser_use.llm.messages import AssistantMessage, ContentPartTextParam, UserMessage
|
|
|
|
urls_replaced: dict[str, str] = {}
|
|
for message in input_messages:
|
|
if not isinstance(message, (UserMessage, AssistantMessage)):
|
|
continue
|
|
if isinstance(message.content, str):
|
|
message.content, replaced_urls = self._replace_urls_in_text(message.content)
|
|
urls_replaced.update(replaced_urls)
|
|
elif isinstance(message.content, list):
|
|
for part in message.content:
|
|
if isinstance(part, ContentPartTextParam):
|
|
part.text, replaced_urls = self._replace_urls_in_text(part.text)
|
|
urls_replaced.update(replaced_urls)
|
|
return urls_replaced
|
|
|
|
@staticmethod
|
|
def _recursive_process_all_strings_inside_pydantic_model(model: BaseModel, url_replacements: dict[str, str]) -> None:
|
|
"""Replace shortened URLs with originals inside a Pydantic model in place."""
|
|
for field_name, field_value in model.__dict__.items():
|
|
if isinstance(field_value, str):
|
|
setattr(model, field_name, Agent._replace_shortened_urls_in_string(field_value, url_replacements))
|
|
elif isinstance(field_value, BaseModel):
|
|
Agent._recursive_process_all_strings_inside_pydantic_model(field_value, url_replacements)
|
|
elif isinstance(field_value, dict):
|
|
Agent._recursive_process_dict(field_value, url_replacements)
|
|
elif isinstance(field_value, (list, tuple)):
|
|
setattr(model, field_name, Agent._recursive_process_list_or_tuple(field_value, url_replacements))
|
|
|
|
@staticmethod
|
|
def _recursive_process_dict(dictionary: dict, url_replacements: dict[str, str]) -> None:
|
|
for key, value in dictionary.items():
|
|
if isinstance(value, str):
|
|
dictionary[key] = Agent._replace_shortened_urls_in_string(value, url_replacements)
|
|
elif isinstance(value, BaseModel):
|
|
Agent._recursive_process_all_strings_inside_pydantic_model(value, url_replacements)
|
|
elif isinstance(value, dict):
|
|
Agent._recursive_process_dict(value, url_replacements)
|
|
elif isinstance(value, (list, tuple)):
|
|
dictionary[key] = Agent._recursive_process_list_or_tuple(value, url_replacements)
|
|
|
|
@staticmethod
|
|
def _recursive_process_list_or_tuple(container: list | tuple, url_replacements: dict[str, str]) -> list | tuple:
|
|
if isinstance(container, tuple):
|
|
processed_items = []
|
|
for item in container:
|
|
if isinstance(item, str):
|
|
processed_items.append(Agent._replace_shortened_urls_in_string(item, url_replacements))
|
|
elif isinstance(item, BaseModel):
|
|
Agent._recursive_process_all_strings_inside_pydantic_model(item, url_replacements)
|
|
processed_items.append(item)
|
|
elif isinstance(item, dict):
|
|
Agent._recursive_process_dict(item, url_replacements)
|
|
processed_items.append(item)
|
|
elif isinstance(item, (list, tuple)):
|
|
processed_items.append(Agent._recursive_process_list_or_tuple(item, url_replacements))
|
|
else:
|
|
processed_items.append(item)
|
|
return tuple(processed_items)
|
|
for index, item in enumerate(container):
|
|
if isinstance(item, str):
|
|
container[index] = Agent._replace_shortened_urls_in_string(item, url_replacements)
|
|
elif isinstance(item, BaseModel):
|
|
Agent._recursive_process_all_strings_inside_pydantic_model(item, url_replacements)
|
|
elif isinstance(item, dict):
|
|
Agent._recursive_process_dict(item, url_replacements)
|
|
elif isinstance(item, (list, tuple)):
|
|
container[index] = Agent._recursive_process_list_or_tuple(item, url_replacements)
|
|
return container
|
|
|
|
@staticmethod
|
|
def _replace_shortened_urls_in_string(text: str, url_replacements: dict[str, str]) -> str:
|
|
result = text
|
|
for shortened_url, original_url in url_replacements.items():
|
|
result = result.replace(shortened_url, original_url)
|
|
return result
|
|
|
|
def _setup_action_models(self) -> None:
|
|
"""Expose Browser Use-style action model classes from the configured tools."""
|
|
self._setup_action_models_for_page(page_url=None)
|
|
|
|
def _setup_action_models_for_page(self, page_url: str | None) -> None:
|
|
"""Create action model classes, optionally filtered for a specific page."""
|
|
registry = getattr(self.tools, 'registry', None)
|
|
create_action_model = getattr(registry, 'create_action_model', None)
|
|
if not callable(create_action_model):
|
|
self.ActionModel = None
|
|
self.DoneActionModel = None
|
|
self.AgentOutput = AgentOutput
|
|
return
|
|
self.ActionModel = create_action_model(page_url=page_url)
|
|
if self.settings.flash_mode:
|
|
self.AgentOutput = AgentOutput.type_with_custom_actions_flash_mode(self.ActionModel)
|
|
elif self.settings.use_thinking:
|
|
self.AgentOutput = AgentOutput.type_with_custom_actions(self.ActionModel)
|
|
else:
|
|
self.AgentOutput = AgentOutput.type_with_custom_actions_no_thinking(self.ActionModel)
|
|
self.DoneActionModel = create_action_model(include_actions=['done'], page_url=page_url)
|
|
if self.settings.flash_mode:
|
|
self.DoneAgentOutput = AgentOutput.type_with_custom_actions_flash_mode(self.DoneActionModel)
|
|
elif self.settings.use_thinking:
|
|
self.DoneAgentOutput = AgentOutput.type_with_custom_actions(self.DoneActionModel)
|
|
else:
|
|
self.DoneAgentOutput = AgentOutput.type_with_custom_actions_no_thinking(self.DoneActionModel)
|
|
|
|
async def _update_action_models_for_page(self, page_url: str) -> None:
|
|
"""Update Browser Use-style action model classes for page-filtered tools."""
|
|
self._setup_action_models_for_page(page_url)
|
|
|
|
def _convert_initial_actions(self, actions: list[dict[str, dict[str, Any]]]) -> list[ActionModel]:
|
|
"""Convert dictionary initial actions to Browser Use action model instances when possible."""
|
|
converted_actions: list[Any] = []
|
|
registry = getattr(getattr(self.tools, 'registry', None), 'registry', None)
|
|
registry_actions = getattr(registry, 'actions', {})
|
|
action_model = getattr(self, 'ActionModel', None)
|
|
if action_model is None:
|
|
return list(actions)
|
|
for action_dict in actions:
|
|
if not isinstance(action_dict, dict) or not action_dict:
|
|
converted_actions.append(action_dict)
|
|
continue
|
|
action_name = next(iter(action_dict))
|
|
params = action_dict[action_name]
|
|
normalized_name, normalized_params = _normalize_initial_action(action_name, params)
|
|
action_info = registry_actions.get(normalized_name)
|
|
if action_info is None:
|
|
converted_actions.append(action_dict)
|
|
continue
|
|
try:
|
|
validated_params = action_info.param_model(**(normalized_params if isinstance(normalized_params, dict) else {}))
|
|
converted_actions.append(action_model(**{normalized_name: validated_params}))
|
|
except (TypeError, ValueError, ValidationError):
|
|
converted_actions.append(action_dict)
|
|
return converted_actions
|
|
|
|
async def _check_and_update_downloads(self, context: str = '') -> None:
|
|
"""Mirror Browser Use's downloaded-file tracking for supplied sessions."""
|
|
if not self.has_downloads_path or self.browser_session is None:
|
|
return
|
|
try:
|
|
current_downloads = getattr(self.browser_session, 'downloaded_files', None)
|
|
if callable(current_downloads):
|
|
current_downloads = current_downloads()
|
|
if inspect.isawaitable(current_downloads):
|
|
current_downloads = await current_downloads
|
|
if not isinstance(current_downloads, list):
|
|
return
|
|
if current_downloads != self._last_known_downloads:
|
|
self._update_available_file_paths(current_downloads)
|
|
self._last_known_downloads = list(current_downloads)
|
|
except Exception:
|
|
_ = context
|
|
|
|
def _update_available_file_paths(self, downloads: list[str]) -> None:
|
|
"""Update available_file_paths with downloaded files, preserving caller order."""
|
|
if not self.has_downloads_path:
|
|
return
|
|
current_files = list(self.available_file_paths or [])
|
|
seen = set(current_files)
|
|
for file_path in downloads:
|
|
if not isinstance(file_path, str) or not file_path or file_path in seen:
|
|
continue
|
|
current_files.append(file_path)
|
|
seen.add(file_path)
|
|
self.available_file_paths = current_files
|
|
|
|
def save_file_system_state(self) -> None:
|
|
"""Save current Browser Use file system state back onto AgentState."""
|
|
self.state.file_system_state = self.file_system.get_state()
|
|
|
|
async def _prepare_context(self, step_info: AgentStepInfo | None = None) -> BrowserStateSummary:
|
|
"""Prepare Browser Use step context from the configured browser session."""
|
|
if self.browser_session is None:
|
|
raise AssertionError('BrowserSession is not set up')
|
|
get_state = getattr(self.browser_session, 'get_browser_state_summary', None)
|
|
if not callable(get_state):
|
|
raise ValueError('BrowserSession does not expose get_browser_state_summary')
|
|
|
|
browser_state_summary = await get_state(
|
|
include_screenshot=True,
|
|
include_recent_events=self.include_recent_events,
|
|
)
|
|
await self._check_and_update_downloads(f'Step {self.state.n_steps}: after getting browser state')
|
|
self._log_step_context(browser_state_summary)
|
|
await self._check_stop_or_pause()
|
|
await self._update_action_models_for_page(browser_state_summary.url)
|
|
|
|
page_filtered_actions = None
|
|
registry = getattr(getattr(self.tools, 'registry', None), 'get_prompt_description', None)
|
|
if callable(registry):
|
|
page_filtered_actions = registry(browser_state_summary.url)
|
|
|
|
self._message_manager.create_state_messages(
|
|
browser_state_summary=browser_state_summary,
|
|
model_output=self.state.last_model_output,
|
|
result=self.state.last_result,
|
|
step_info=step_info,
|
|
use_vision=self.settings.use_vision,
|
|
page_filtered_actions=page_filtered_actions if page_filtered_actions else None,
|
|
sensitive_data=self.sensitive_data,
|
|
available_file_paths=self.available_file_paths,
|
|
)
|
|
|
|
await self._force_done_after_last_step(step_info)
|
|
await self._force_done_after_failure()
|
|
return browser_state_summary
|
|
|
|
async def get_model_output(self, input_messages: list[BaseMessage]) -> AgentOutput:
|
|
"""Get next Browser Use action output from the configured Python LLM."""
|
|
if self.llm is None or not hasattr(self.llm, 'ainvoke'):
|
|
raise ValueError('A Browser Use-compatible llm with ainvoke(...) is required for get_model_output().')
|
|
|
|
urls_replaced = self._process_messsages_and_replace_long_urls_shorter_ones(input_messages)
|
|
response = await self.llm.ainvoke(input_messages, output_format=self.AgentOutput)
|
|
parsed: AgentOutput = getattr(response, 'completion', response)
|
|
|
|
if urls_replaced:
|
|
self._recursive_process_all_strings_inside_pydantic_model(parsed, urls_replaced)
|
|
|
|
actions = getattr(parsed, 'action', None)
|
|
if actions and len(actions) > self.settings.max_actions_per_step:
|
|
parsed.action = actions[: self.settings.max_actions_per_step]
|
|
|
|
if not (hasattr(self.state, 'paused') and (self.state.paused or self.state.stopped)):
|
|
from browser_use.agent.service import log_response
|
|
|
|
registry = getattr(getattr(self.tools, 'registry', None), 'registry', None)
|
|
log_response(parsed, registry, self.logger)
|
|
|
|
self._log_next_action_summary(parsed)
|
|
return parsed
|
|
|
|
async def _get_model_output_with_retry(self, input_messages: list[BaseMessage]) -> AgentOutput:
|
|
"""Get model output, retrying once when the model returns no usable action."""
|
|
model_output = await self.get_model_output(input_messages)
|
|
action_count = len(model_output.action) if getattr(model_output, 'action', None) else 0
|
|
self.logger.debug(f'✅ Step {self.state.n_steps}: Got LLM response with {action_count} actions')
|
|
|
|
def has_empty_actions(output: AgentOutput) -> bool:
|
|
actions = getattr(output, 'action', None)
|
|
if not actions or not isinstance(actions, list):
|
|
return True
|
|
return all(getattr(action, 'model_dump', lambda **kwargs: {})() == {} for action in actions)
|
|
|
|
if has_empty_actions(model_output):
|
|
from browser_use.llm.messages import UserMessage
|
|
|
|
self.logger.warning('Model returned empty action. Retrying...')
|
|
clarification_message = UserMessage(
|
|
content='You forgot to return an action. Please respond with a valid JSON action according to the expected schema with your assessment and next actions.'
|
|
)
|
|
model_output = await self.get_model_output(input_messages + [clarification_message])
|
|
if has_empty_actions(model_output):
|
|
self.logger.warning('Model still returned empty after retry. Inserting safe noop action.')
|
|
try:
|
|
done_action = self.DoneActionModel(done={'success': False, 'text': 'No next action returned by LLM!'})
|
|
except Exception:
|
|
done_action = self.ActionModel()
|
|
setattr(done_action, 'done', {'success': False, 'text': 'No next action returned by LLM!'})
|
|
model_output.action = [done_action]
|
|
|
|
return model_output
|
|
|
|
async def _handle_post_llm_processing(
|
|
self,
|
|
browser_state_summary: BrowserStateSummary,
|
|
input_messages: list[BaseMessage],
|
|
) -> None:
|
|
"""Handle Browser Use callbacks and conversation saving after an LLM response."""
|
|
if self.register_new_step_callback and self.state.last_model_output:
|
|
if inspect.iscoroutinefunction(self.register_new_step_callback):
|
|
await self.register_new_step_callback(
|
|
browser_state_summary,
|
|
self.state.last_model_output,
|
|
self.state.n_steps,
|
|
)
|
|
else:
|
|
self.register_new_step_callback(
|
|
browser_state_summary,
|
|
self.state.last_model_output,
|
|
self.state.n_steps,
|
|
)
|
|
|
|
if self.settings.save_conversation_path and self.state.last_model_output:
|
|
conversation_dir = Path(self.settings.save_conversation_path)
|
|
conversation_filename = f'conversation_{self.id}_{self.state.n_steps}.txt'
|
|
target = conversation_dir / conversation_filename
|
|
await save_conversation(
|
|
input_messages,
|
|
self.state.last_model_output,
|
|
target,
|
|
self.settings.save_conversation_path_encoding,
|
|
)
|
|
|
|
async def _get_next_action(self, browser_state_summary: BrowserStateSummary) -> None:
|
|
"""Fetch the next model output and run Browser Use post-LLM hooks."""
|
|
input_messages = self._message_manager.get_messages()
|
|
try:
|
|
model_output = await asyncio.wait_for(
|
|
self._get_model_output_with_retry(input_messages), timeout=self.settings.llm_timeout
|
|
)
|
|
except TimeoutError:
|
|
raise TimeoutError(
|
|
f'LLM call timed out after {self.settings.llm_timeout} seconds. Keep your thinking and output short.'
|
|
)
|
|
|
|
self.state.last_model_output = model_output
|
|
await self._check_stop_or_pause()
|
|
await self._handle_post_llm_processing(browser_state_summary, input_messages)
|
|
await self._check_stop_or_pause()
|
|
|
|
async def _execute_actions(self) -> None:
|
|
"""Execute actions from the last model output through the Rust-backed action path."""
|
|
if self.state.last_model_output is None:
|
|
raise ValueError('No model output to execute actions from')
|
|
self.state.last_result = await self.multi_act(self.state.last_model_output.action)
|
|
|
|
async def _make_history_item(
|
|
self,
|
|
model_output: AgentOutput | None,
|
|
browser_state_summary: BrowserStateSummary,
|
|
result: list[ActionResult],
|
|
metadata: StepMetadata | None = None,
|
|
state_message: str | None = None,
|
|
) -> None:
|
|
"""Create and store a Browser Use history item from a browser-state summary."""
|
|
if model_output:
|
|
selector_map = getattr(getattr(browser_state_summary, 'dom_state', None), 'selector_map', {})
|
|
interacted_elements = AgentHistory.get_interacted_element(model_output, selector_map)
|
|
else:
|
|
interacted_elements = [None]
|
|
|
|
screenshot_path = None
|
|
screenshot = getattr(browser_state_summary, 'screenshot', None)
|
|
if screenshot:
|
|
screenshot_path = await self.screenshot_service.store_screenshot(screenshot, self.state.n_steps)
|
|
|
|
state_history = BrowserStateHistory(
|
|
url=getattr(browser_state_summary, 'url', ''),
|
|
title=getattr(browser_state_summary, 'title', ''),
|
|
tabs=getattr(browser_state_summary, 'tabs', []),
|
|
interacted_element=interacted_elements,
|
|
screenshot_path=screenshot_path,
|
|
)
|
|
self.history.add_item(
|
|
AgentHistory(
|
|
model_output=model_output,
|
|
result=result,
|
|
state=state_history,
|
|
metadata=metadata,
|
|
state_message=state_message,
|
|
)
|
|
)
|
|
|
|
async def _post_process(self) -> None:
|
|
"""Handle Browser Use-style post-action bookkeeping."""
|
|
await self._check_and_update_downloads('after executing actions')
|
|
if self.state.last_result and len(self.state.last_result) == 1 and self.state.last_result[-1].error:
|
|
self.state.consecutive_failures += 1
|
|
self.logger.debug(f'🔄 Step {self.state.n_steps}: Consecutive failures: {self.state.consecutive_failures}')
|
|
return
|
|
if self.state.consecutive_failures > 0:
|
|
self.state.consecutive_failures = 0
|
|
self.logger.debug(f'🔄 Step {self.state.n_steps}: Consecutive failures reset to: {self.state.consecutive_failures}')
|
|
if self.state.last_result and self.state.last_result[-1].is_done:
|
|
success = self.state.last_result[-1].success
|
|
if success:
|
|
self.logger.info(f'\n📄 \033[32m Final Result:\033[0m \n{self.state.last_result[-1].extracted_content}\n\n')
|
|
else:
|
|
self.logger.info(f'\n📄 \033[31m Final Result:\033[0m \n{self.state.last_result[-1].extracted_content}\n\n')
|
|
if self.state.last_result[-1].attachments:
|
|
total_attachments = len(self.state.last_result[-1].attachments)
|
|
for index, file_path in enumerate(self.state.last_result[-1].attachments):
|
|
self.logger.info(f'👉 Attachment {index + 1 if total_attachments > 1 else ""}: {file_path}')
|
|
|
|
async def _handle_step_error(self, error: Exception) -> None:
|
|
"""Convert a step exception into Browser Use-style state.last_result."""
|
|
if isinstance(error, InterruptedError):
|
|
self.logger.error('The agent was interrupted mid-step' + (f' - {error}' if str(error) else ''))
|
|
return
|
|
include_trace = self.logger.isEnabledFor(logging.DEBUG)
|
|
error_msg = AgentError.format_error(error, include_trace=include_trace)
|
|
prefix = f'❌ Result failed {self.state.consecutive_failures + 1}/{self.settings.max_failures + int(self.settings.final_response_after_failure)} times:\n '
|
|
self.state.consecutive_failures += 1
|
|
if 'Could not parse response' in error_msg or 'tool_use_failed' in error_msg:
|
|
logger.error(f'Model: {getattr(self.llm, "model", None)} failed')
|
|
logger.error(f'{prefix}{error_msg}')
|
|
else:
|
|
self.logger.error(f'{prefix}{error_msg}')
|
|
self.state.last_result = [ActionResult(error=error_msg)]
|
|
|
|
async def _finalize(self, browser_state_summary: BrowserStateSummary | None) -> None:
|
|
"""Finalize one Browser Use-style step after Rust-backed helper execution."""
|
|
step_end_time = time.time()
|
|
if not self.state.last_result:
|
|
return
|
|
step_start_time = getattr(self, 'step_start_time', step_end_time)
|
|
step_event = None
|
|
if browser_state_summary is not None:
|
|
metadata = StepMetadata(
|
|
step_number=self.state.n_steps,
|
|
step_start_time=step_start_time,
|
|
step_end_time=step_end_time,
|
|
)
|
|
state_message = getattr(self._message_manager, 'last_state_message_text', None)
|
|
await self._make_history_item(
|
|
self.state.last_model_output,
|
|
browser_state_summary,
|
|
self.state.last_result,
|
|
metadata,
|
|
state_message=state_message,
|
|
)
|
|
if self.state.last_model_output:
|
|
actions_data = []
|
|
for action in self.state.last_model_output.action or []:
|
|
action_dict = action.model_dump() if hasattr(action, 'model_dump') else {}
|
|
actions_data.append(action_dict)
|
|
step_event = CreateAgentStepEvent.from_agent_step(
|
|
self,
|
|
self.state.last_model_output,
|
|
self.state.last_result,
|
|
actions_data,
|
|
browser_state_summary,
|
|
)
|
|
self._log_step_completion_summary(step_start_time, self.state.last_result)
|
|
self.save_file_system_state()
|
|
if step_event is not None:
|
|
self.eventbus.dispatch(step_event)
|
|
self.state.n_steps += 1
|
|
|
|
async def _force_done_after_last_step(self, step_info: AgentStepInfo | None = None) -> None:
|
|
"""Switch to done-only output on the last configured step."""
|
|
if not (step_info and step_info.is_last_step()):
|
|
return
|
|
from browser_use.llm.messages import UserMessage
|
|
|
|
msg = 'You reached max_steps - this is your last step. Your only tool available is the "done" tool. No other tool is available. All other tools which you see in history or examples are not available.'
|
|
msg += '\nIf the task is not yet fully finished as requested by the user, set success in "done" to false! E.g. if not all steps are fully completed. Else success to true.'
|
|
msg += '\nInclude everything you found out for the ultimate task in the done text.'
|
|
self.logger.debug('Last step finishing up')
|
|
self._message_manager._add_context_message(UserMessage(content=msg))
|
|
self.AgentOutput = self.DoneAgentOutput
|
|
|
|
async def _force_done_after_failure(self) -> None:
|
|
"""Switch to done-only output after max failures when final response is enabled."""
|
|
if self.state.consecutive_failures < self.settings.max_failures or not self.settings.final_response_after_failure:
|
|
return
|
|
from browser_use.llm.messages import UserMessage
|
|
|
|
msg = f'You failed {self.settings.max_failures} times. Therefore we terminate the agent.'
|
|
msg += '\nYour only tool available is the "done" tool. No other tool is available. All other tools which you see in history or examples are not available.'
|
|
msg += '\nIf the task is not yet fully finished as requested by the user, set success in "done" to false! E.g. if not all steps are fully completed. Else success to true.'
|
|
msg += '\nInclude everything you found out for the ultimate task in the done text.'
|
|
self.logger.debug('Force done action, because we reached max_failures.')
|
|
self._message_manager._add_context_message(UserMessage(content=msg))
|
|
self.AgentOutput = self.DoneAgentOutput
|
|
|
|
async def step(self, step_info: AgentStepInfo | None = None) -> None:
|
|
"""Execute one Browser Use-style step through the Rust terminal core."""
|
|
await self.take_step(step_info)
|
|
|
|
async def take_step(self, step_info: AgentStepInfo | None = None) -> tuple[bool, bool]:
|
|
"""Take one Rust terminal turn and return Browser Use-style step status."""
|
|
if step_info is not None:
|
|
self.state.n_steps = max(self.state.n_steps, step_info.step_number)
|
|
if step_info.step_number == 0:
|
|
try:
|
|
await self._execute_initial_actions()
|
|
except InterruptedError:
|
|
pass
|
|
history = await self.run(max_steps=1)
|
|
if history.is_done():
|
|
return True, True
|
|
return False, False
|
|
|
|
async def multi_act(self, actions: list[ActionModel]) -> list[ActionResult]:
|
|
"""Execute Browser Use action models through the Rust-backed session.
|
|
|
|
The Rust terminal owns browser actions, so non-`done` action batches are
|
|
serialized as a follow-up instruction for the active Rust session. A
|
|
standalone `done` action preserves Browser Use's local completion semantics.
|
|
"""
|
|
payloads = []
|
|
total_actions = len(actions)
|
|
for index, action in enumerate(actions):
|
|
payload = _action_payload(action)
|
|
if index > 0 and payload.get('done') is not None:
|
|
self.logger.debug(
|
|
f'Done action is allowed only as a single action - stopped after action {index} / {total_actions}.'
|
|
)
|
|
break
|
|
payloads.append(payload)
|
|
if payload.get('done') is not None:
|
|
break
|
|
if not payloads:
|
|
return []
|
|
if len(payloads) == 1:
|
|
done_result = _done_action_result(payloads[0])
|
|
if done_result is not None:
|
|
return [done_result]
|
|
instruction = _actions_instruction(payloads)
|
|
max_steps = max(1, len(payloads))
|
|
if self.terminal_session_id:
|
|
history = await self.follow_up(instruction, max_steps=max_steps)
|
|
return history.action_results()
|
|
original_task = self.task
|
|
self.task = f'{self.task}\n\n{instruction}'
|
|
try:
|
|
history = await self.run(max_steps=max_steps)
|
|
finally:
|
|
self.task = original_task
|
|
return history.action_results()
|
|
|
|
async def _execute_initial_actions(self, *, allow_terminal_run: bool = True) -> None:
|
|
"""Execute configured Browser Use initial actions through the Rust-backed action path."""
|
|
if not self.initial_actions or self.state.follow_up_task or self._initial_actions_executed:
|
|
return
|
|
self.logger.debug(f'⚡ Executing {len(self.initial_actions)} initial actions...')
|
|
result = await self._execute_direct_initial_navigation_actions()
|
|
if result is None:
|
|
if not allow_terminal_run:
|
|
self.logger.debug('Initial actions left for Rust task context; direct CDP navigation was not available')
|
|
return
|
|
result = await self.multi_act(self.initial_actions)
|
|
self._initial_actions_executed = True
|
|
if result and self.initial_url and result[0].long_term_memory:
|
|
result[0].long_term_memory = f'Found initial url and automatically loaded it. {result[0].long_term_memory}'
|
|
self.state.last_result = result
|
|
|
|
if self.settings.flash_mode:
|
|
model_output = self.AgentOutput(
|
|
evaluation_previous_goal=None,
|
|
memory='Initial navigation',
|
|
next_goal=None,
|
|
action=self.initial_actions,
|
|
)
|
|
else:
|
|
model_output = self.AgentOutput(
|
|
evaluation_previous_goal='Start',
|
|
memory=None,
|
|
next_goal='Initial navigation',
|
|
action=self.initial_actions,
|
|
)
|
|
|
|
metadata = StepMetadata(
|
|
step_number=0,
|
|
step_start_time=time.time(),
|
|
step_end_time=time.time(),
|
|
)
|
|
initial_state_context = self._completed_initial_navigation_states[-1] if self._completed_initial_navigation_states else {}
|
|
initial_state_url = str(initial_state_context.get('url') or self.initial_url or '')
|
|
initial_state_title = str(initial_state_context.get('title') or 'Initial Actions')
|
|
initial_state_tabs = []
|
|
for index, tab in enumerate(initial_state_context.get('tabs') or []):
|
|
if not isinstance(tab, dict):
|
|
continue
|
|
tab_url = str(tab.get('url') or '')
|
|
if not tab_url:
|
|
continue
|
|
initial_state_tabs.append(TabInfo(url=tab_url, title=str(tab.get('title') or ''), target_id=f'initial-tab-{index}'))
|
|
state_history = BrowserStateHistory(
|
|
url=initial_state_url,
|
|
title=initial_state_title,
|
|
tabs=initial_state_tabs,
|
|
interacted_element=[None] * len(self.initial_actions),
|
|
screenshot_path=None,
|
|
)
|
|
self.history.add_item(
|
|
AgentHistory(
|
|
model_output=model_output,
|
|
result=result,
|
|
state=state_history,
|
|
metadata=metadata,
|
|
)
|
|
)
|
|
self.logger.debug('📝 Saved initial actions to history as step 0')
|
|
self.logger.debug('Initial actions completed')
|
|
|
|
async def _execute_direct_initial_navigation_actions(self) -> list[ActionResult] | None:
|
|
"""Pre-navigate existing CDP-backed browser sessions before the Beta agent starts."""
|
|
self._completed_initial_navigation_urls = []
|
|
self._completed_initial_navigation_states = []
|
|
if not _direct_initial_navigation_enabled():
|
|
return None
|
|
if self.browser_session is None:
|
|
return None
|
|
if not (_extract_cdp_url(self.browser_session) or _extract_profile_cdp_url(self.browser_profile)):
|
|
return None
|
|
navigate_to = getattr(self.browser_session, 'navigate_to', None)
|
|
if not callable(navigate_to):
|
|
return None
|
|
payloads = [_action_payload(action) for action in self.initial_actions or []]
|
|
nav_actions = [_initial_navigation_params_from_action(payload) for payload in payloads]
|
|
if not nav_actions or any(action is None for action in nav_actions):
|
|
return None
|
|
results: list[ActionResult] = []
|
|
for url, new_tab in nav_actions:
|
|
try:
|
|
await navigate_to(url, new_tab=new_tab)
|
|
except Exception as exc:
|
|
message = f'Initial navigation to {url} failed: {exc}'
|
|
self.logger.warning(message)
|
|
results.append(ActionResult(error=message))
|
|
continue
|
|
state = await self._capture_direct_initial_navigation_state(url)
|
|
if not self._direct_initial_navigation_state_matches(url, getattr(state, 'url', '') if state else None):
|
|
observed_url = getattr(state, 'url', None) if state else None
|
|
self.logger.warning(
|
|
'Initial navigation to %s was not confirmed by browser state; observed %s. '
|
|
'Leaving navigation in the Rust task context.',
|
|
url,
|
|
observed_url or '<unknown>',
|
|
)
|
|
self._completed_initial_navigation_urls = []
|
|
self._completed_initial_navigation_states = []
|
|
return None
|
|
state_context = self._direct_initial_navigation_state_context(url, state)
|
|
text = f'Navigated to {url}'
|
|
current_url = state_context.get('url')
|
|
title = state_context.get('title')
|
|
if current_url or title:
|
|
text += f'. Current page: {current_url or url}'
|
|
if title:
|
|
text += f' ({title})'
|
|
self._completed_initial_navigation_urls.append(url)
|
|
self._completed_initial_navigation_states.append(state_context)
|
|
results.append(ActionResult(extracted_content=text, long_term_memory=text))
|
|
return results
|
|
|
|
async def _capture_direct_initial_navigation_state(
|
|
self, requested_url: str, *, timeout_seconds: float = 7.0
|
|
) -> BrowserStateSummary | None:
|
|
if self.browser_session is None:
|
|
return None
|
|
get_state = getattr(self.browser_session, 'get_browser_state_summary', None)
|
|
if not callable(get_state):
|
|
return None
|
|
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
|
last_state: BrowserStateSummary | None = None
|
|
while True:
|
|
try:
|
|
state = await get_state(include_screenshot=False)
|
|
except Exception as exc:
|
|
self.logger.debug(f'Initial navigation state probe failed: {exc}')
|
|
state = None
|
|
if state is not None:
|
|
last_state = state
|
|
if self._direct_initial_navigation_state_matches(requested_url, getattr(state, 'url', '')):
|
|
return state
|
|
if time.monotonic() >= deadline:
|
|
return last_state
|
|
await asyncio.sleep(0.25)
|
|
|
|
@staticmethod
|
|
def _direct_initial_navigation_state_matches(requested_url: str, current_url: str | None) -> bool:
|
|
current = str(current_url or '')
|
|
requested = str(requested_url or '')
|
|
if not current:
|
|
return False
|
|
if current == requested:
|
|
return True
|
|
try:
|
|
current_parts = urlparse(current)
|
|
requested_parts = urlparse(requested)
|
|
if not requested_parts.netloc or current_parts.netloc != requested_parts.netloc:
|
|
return False
|
|
requested_path = (requested_parts.path or '/').rstrip('/') or '/'
|
|
current_path = (current_parts.path or '/').rstrip('/') or '/'
|
|
if requested_path != '/' and current_path != requested_path:
|
|
return False
|
|
if requested_parts.query and current_parts.query != requested_parts.query:
|
|
return False
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
@staticmethod
|
|
def _direct_initial_navigation_state_context(requested_url: str, state: BrowserStateSummary | None) -> dict[str, Any]:
|
|
context: dict[str, Any] = {'requested_url': requested_url}
|
|
if state is None:
|
|
return context
|
|
url = getattr(state, 'url', None)
|
|
title = getattr(state, 'title', None)
|
|
if isinstance(url, str) and url:
|
|
context['url'] = url
|
|
if isinstance(title, str) and title:
|
|
context['title'] = title
|
|
tabs = getattr(state, 'tabs', None)
|
|
if isinstance(tabs, list) and tabs:
|
|
tab_summaries = []
|
|
for tab in tabs[:5]:
|
|
tab_url = getattr(tab, 'url', None)
|
|
tab_title = getattr(tab, 'title', None)
|
|
if isinstance(tab_url, str) and tab_url:
|
|
tab_summaries.append({'url': tab_url, 'title': tab_title if isinstance(tab_title, str) else ''})
|
|
if tab_summaries:
|
|
context['tabs'] = tab_summaries
|
|
return context
|
|
|
|
def add_new_task(self, new_task: str) -> None:
|
|
"""Add a follow-up task while keeping the same Browser Use-style agent object."""
|
|
self.task = new_task
|
|
self._message_manager.add_new_task(new_task)
|
|
self.state.follow_up_task = True
|
|
self.state.stopped = False
|
|
self.state.paused = False
|
|
self.eventbus = EventBus(name=_unique_eventbus_name(self.id))
|
|
self._eventbus_stopped = False
|
|
self._external_pause_event.set()
|
|
|
|
def save_history(self, file_path: str | Path | None = None) -> None:
|
|
"""Save the current Browser Use history to disk."""
|
|
if not file_path:
|
|
file_path = 'AgentHistory.json'
|
|
self.history.save_to_file(file_path, sensitive_data=self.sensitive_data)
|
|
|
|
async def rerun_history(
|
|
self,
|
|
history: AgentHistoryList,
|
|
max_retries: int = 3,
|
|
skip_failures: bool = True,
|
|
delay_between_actions: float = 2.0,
|
|
) -> list[ActionResult]:
|
|
"""Rerun through the Rust core and return Browser Use-style action results.
|
|
|
|
The Python Agent replays serialized Python action models. Rust terminal
|
|
histories do not expose those action models, so this compatibility path
|
|
reruns the current task through the Rust core while preserving the public
|
|
retry and skip-failure controls from Browser Use's rerun API.
|
|
"""
|
|
_ = history
|
|
max_retries = max(1, max_retries)
|
|
last_results: list[ActionResult] = []
|
|
last_error: str | None = None
|
|
for attempt in range(1, max_retries + 1):
|
|
try:
|
|
result = await self.run(max_steps=self.kwargs.get('max_steps', 100))
|
|
last_results = result.action_results()
|
|
errors = [error for error in result.errors() if error]
|
|
if not errors:
|
|
return last_results
|
|
last_error = errors[0]
|
|
except Exception as error:
|
|
last_results = []
|
|
last_error = str(error)
|
|
|
|
if attempt < max_retries:
|
|
self.logger.warning(f'Rerun failed (attempt {attempt}/{max_retries}), retrying...')
|
|
await asyncio.sleep(delay_between_actions)
|
|
|
|
error_msg = f'Rerun failed after {max_retries} attempts: {last_error or "unknown error"}'
|
|
self.logger.error(error_msg)
|
|
if not skip_failures:
|
|
raise RuntimeError(error_msg)
|
|
return last_results or [ActionResult(error=error_msg)]
|
|
|
|
async def load_and_rerun(self, history_file: str | Path | None = None, **kwargs) -> list[ActionResult]:
|
|
"""Load a saved Rust-backed Browser Use history and rerun the task."""
|
|
if not history_file:
|
|
history_file = 'AgentHistory.json'
|
|
history = _load_rust_history(history_file)
|
|
return await self.rerun_history(history, **kwargs)
|
|
|
|
def pause(self) -> None:
|
|
"""Pause the Rust-backed agent before the next terminal run."""
|
|
print('\n\n⏸️ Paused the agent and left the browser open.\n\tPress [Enter] to resume or [Ctrl+C] again to quit.')
|
|
self.state.paused = True
|
|
self._external_pause_event.clear()
|
|
|
|
def resume(self) -> None:
|
|
"""Resume a paused Rust-backed agent."""
|
|
print('----------------------------------------------------------------------')
|
|
print('▶️ Resuming agent execution where it left off...\n')
|
|
self.state.paused = False
|
|
self._external_pause_event.set()
|
|
|
|
def stop(self) -> None:
|
|
"""Stop the Rust-backed agent before the next terminal run."""
|
|
self.logger.info('⏹️ Agent stopping')
|
|
self.state.stopped = True
|
|
self._external_pause_event.set()
|
|
if self._sdk_client is not None:
|
|
try:
|
|
loop = asyncio.get_running_loop()
|
|
except RuntimeError:
|
|
return
|
|
loop.create_task(self._cancel_active_sdk_run())
|
|
|
|
async def close(self):
|
|
"""Close Browser Use session resources when the caller did not request keep-alive."""
|
|
await self._close_browser_resources()
|
|
await self._close_sdk_client_if_not_keep_alive()
|
|
return None
|
|
|
|
def _should_keep_browser_alive(self) -> bool:
|
|
profile = getattr(self.browser_session, 'browser_profile', None) or self.browser_profile
|
|
return bool(getattr(profile, 'keep_alive', False))
|
|
|
|
async def _close_sdk_browser_resources(self) -> None:
|
|
"""Close Rust-owned SDK agent/browser resources when this run is not keep-alive."""
|
|
if self._should_keep_browser_alive() or self._sdk_client is None:
|
|
return
|
|
if isinstance(self._sdk_client, RustSdkClient) and self._sdk_client.process is None:
|
|
return
|
|
client = self._sdk_client
|
|
agent_id = self._sdk_agent_id
|
|
browser_id = self._sdk_browser_id
|
|
if agent_id:
|
|
with suppress(Exception):
|
|
await client.call('agent.close', {'agent_id': agent_id})
|
|
if browser_id:
|
|
with suppress(Exception):
|
|
await client.call('browser.close', {'browser_id': browser_id})
|
|
self._sdk_agent_id = None
|
|
self._sdk_browser_id = None
|
|
self.terminal_session_id = None
|
|
|
|
async def _close_sdk_client_if_not_keep_alive(self) -> None:
|
|
if self._should_keep_browser_alive() or self._sdk_client is None:
|
|
return
|
|
try:
|
|
await self._sdk_client.close()
|
|
except Exception as exc:
|
|
self.logger.error(f'Error closing Rust SDK client: {exc}')
|
|
else:
|
|
self._sdk_client = None
|
|
|
|
async def _close_browser_resources(self):
|
|
"""Close Browser Use session resources without tearing down the Rust SDK process."""
|
|
try:
|
|
await self._close_sdk_browser_resources()
|
|
if self.browser_session is not None:
|
|
if not self._should_keep_browser_alive():
|
|
kill = getattr(self.browser_session, 'kill', None)
|
|
if callable(kill):
|
|
result = kill()
|
|
if inspect.isawaitable(result):
|
|
await result
|
|
import gc
|
|
|
|
gc.collect()
|
|
except Exception as exc:
|
|
self.logger.error(f'Error during cleanup: {exc}')
|
|
return None
|
|
|
|
async def log_completion(self) -> None:
|
|
"""Log Browser Use-style task completion."""
|
|
if self.history.is_successful():
|
|
self.logger.info('✅ Task completed successfully')
|
|
|
|
def run_sync(
|
|
self,
|
|
max_steps: int = 100,
|
|
on_step_start: AgentHookFunc | None = None,
|
|
on_step_end: AgentHookFunc | None = None,
|
|
) -> AgentHistoryList[AgentStructuredOutput]:
|
|
"""Synchronous wrapper around the async run method."""
|
|
return asyncio.run(self.run(max_steps=max_steps, on_step_start=on_step_start, on_step_end=on_step_end))
|
|
|
|
async def authenticate_cloud_sync(self, show_instructions: bool = True) -> bool:
|
|
"""Browser Use-compatible cloud-sync hook.
|
|
|
|
The upstream Python Agent currently reports cloud sync as unavailable.
|
|
The Rust wrapper mirrors that contract.
|
|
"""
|
|
_ = show_instructions
|
|
self.logger.warning('Cloud sync has been removed and is no longer available')
|
|
return False
|
|
|
|
def get_trace_object(self) -> dict[str, Any]:
|
|
"""Get Browser Use-style trace and trace_details data for the Rust-backed run."""
|
|
|
|
def extract_task_website(task_text: str) -> str | None:
|
|
match = re.search(
|
|
r'https?://[^\s<>"\']+|www\.[^\s<>"\']+|[^\s<>"\']+\.[a-z]{2,}(?:/[^\s<>"\']*)?', task_text, re.IGNORECASE
|
|
)
|
|
return match.group(0) if match else None
|
|
|
|
def json_default(value: Any) -> str:
|
|
return str(value)
|
|
|
|
def complete_history_without_screenshots() -> str:
|
|
history_data = self.history.model_dump(sensitive_data=self.sensitive_data)
|
|
for item in history_data.get('history', []):
|
|
state = item.get('state')
|
|
if isinstance(state, dict) and 'screenshot' in state:
|
|
state['screenshot'] = None
|
|
return json.dumps(history_data, default=json_default)
|
|
|
|
trace_id = uuid7str()
|
|
timestamp = datetime.now().isoformat()
|
|
git_info = get_git_info()
|
|
structured_output = self.history.structured_output
|
|
structured_output_json = json.dumps(structured_output.model_dump(), default=json_default) if structured_output else None
|
|
final_result = self.history.final_result()
|
|
action_history = self.history.action_history()
|
|
action_errors = self.history.errors()
|
|
urls = self.history.urls()
|
|
usage = self.history.usage
|
|
|
|
return {
|
|
'trace': {
|
|
'trace_id': trace_id,
|
|
'timestamp': timestamp,
|
|
'browser_use_version': get_browser_use_version(),
|
|
'git_info': json.dumps(git_info, default=json_default) if git_info else None,
|
|
'model': self.model,
|
|
'settings': json.dumps(self.settings.model_dump(), default=json_default) if self.settings else None,
|
|
'task_id': self.task_id,
|
|
'task_truncated': self.task[:20000] if len(self.task) > 20000 else self.task,
|
|
'task_website': extract_task_website(self.task),
|
|
'structured_output_truncated': (
|
|
structured_output_json[:20000]
|
|
if structured_output_json and len(structured_output_json) > 20000
|
|
else structured_output_json
|
|
),
|
|
'action_history_truncated': json.dumps(action_history, default=json_default) if action_history else None,
|
|
'action_errors': json.dumps(action_errors, default=json_default) if action_errors else None,
|
|
'urls': json.dumps(urls, default=json_default) if urls else None,
|
|
'final_result_response_truncated': final_result[:20000]
|
|
if final_result and len(final_result) > 20000
|
|
else final_result,
|
|
'self_report_completed': 1 if self.history.is_done() else 0,
|
|
'self_report_success': 1 if self.history.is_successful() else 0,
|
|
'duration': self.history.total_duration_seconds(),
|
|
'steps_taken': self.history.number_of_steps(),
|
|
'usage': json.dumps(usage.model_dump(), default=json_default) if usage else None,
|
|
},
|
|
'trace_details': {
|
|
'trace_id': trace_id,
|
|
'timestamp': timestamp,
|
|
'task': self.task,
|
|
'structured_output': structured_output_json,
|
|
'final_result_response': final_result,
|
|
'complete_history': complete_history_without_screenshots(),
|
|
},
|
|
}
|
|
|
|
def _sdk_server_argv(self) -> list[str]:
|
|
explicit = os.environ.get('BROWSER_USE_SDK_SERVER')
|
|
command = [explicit] if explicit else [find_browser_use_terminal_binary()]
|
|
return [
|
|
*command,
|
|
*self._state_dir_args(),
|
|
'sdk-server',
|
|
'--transport',
|
|
'stdio',
|
|
]
|
|
|
|
async def _ensure_sdk_client(self) -> RustSdkClient:
|
|
if self._sdk_client is not None and self._sdk_client.process is not None and self._sdk_client.process.returncode is None:
|
|
return self._sdk_client
|
|
if self._sdk_client is not None:
|
|
self._sdk_agent_id = None
|
|
self._sdk_browser_id = None
|
|
self.terminal_session_id = None
|
|
self._sdk_client = RustSdkClient(self._sdk_server_argv(), self._run_env())
|
|
try:
|
|
ping = await self._sdk_client.call('runtime.ping')
|
|
protocol_version = ping.get('sdk_protocol_version') if isinstance(ping, dict) else None
|
|
if protocol_version == 1:
|
|
return self._sdk_client
|
|
except Exception as exc:
|
|
await self._sdk_client.close()
|
|
self._sdk_client = None
|
|
raise BetaAgentError(
|
|
'Failed to negotiate browser-use-terminal SDK protocol via runtime.ping. '
|
|
'Install a compatible browser-use-core package or set BROWSER_USE_TERMINAL_BINARY to a compatible binary.'
|
|
) from exc
|
|
await self._sdk_client.close()
|
|
self._sdk_client = None
|
|
raise BetaAgentError(
|
|
f'Unsupported browser-use-terminal SDK protocol {protocol_version!r}; expected 1. '
|
|
'Install a compatible browser-use-core package or set BROWSER_USE_TERMINAL_BINARY to a compatible binary.'
|
|
)
|
|
|
|
def _sdk_llm_payload(self) -> dict[str, Any]:
|
|
provider = _llm_provider_name(self.llm) or 'browser-use'
|
|
payload: dict[str, Any] = {
|
|
'provider': provider,
|
|
'model': self.model,
|
|
}
|
|
if self.settings.llm_timeout is not None:
|
|
payload['timeout'] = int(self.settings.llm_timeout)
|
|
return payload
|
|
|
|
def _sdk_browser_payload(self) -> dict[str, Any]:
|
|
payload: dict[str, Any] = {}
|
|
|
|
def put(key: str, value: Any) -> None:
|
|
if value is None:
|
|
return
|
|
if isinstance(value, str) and not value:
|
|
return
|
|
payload[key] = value
|
|
|
|
put('cdp_url', _extract_cdp_url(self.browser_session) or _extract_profile_cdp_url(self.browser_profile))
|
|
put('cdp_headers', self.cdp_headers or None)
|
|
put('user_agent', self.browser_user_agent)
|
|
put('viewport', self.browser_viewport)
|
|
put('window_size', self.browser_window_size)
|
|
put('storage_state', self.browser_storage_state)
|
|
put('downloads_path', self.browser_downloads_path)
|
|
put('allowed_domains', self.allowed_domains or None)
|
|
put('blocked_domains', self.prohibited_domains or None)
|
|
put('state_dir', self._sdk_browser_state_dir())
|
|
put('no_viewport', self.browser_no_viewport)
|
|
put('accept_downloads', self.browser_accept_downloads)
|
|
put('headless', _extract_headless_preference(self.browser_session, self.browser_profile))
|
|
put('keep_alive', getattr(self.browser_profile, 'keep_alive', None))
|
|
profile_id = self._sdk_profile_id()
|
|
put('profile_id', profile_id)
|
|
proxy_country_code = self._sdk_proxy_country_code()
|
|
put('proxy_country_code', proxy_country_code)
|
|
return payload
|
|
|
|
def _sdk_profile_id(self) -> str | None:
|
|
session_profile = getattr(self.browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, self.browser_profile, self.browser_session):
|
|
for attr in ('profile_id', 'cloud_profile_id'):
|
|
value = getattr(profile, attr, None)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return None
|
|
|
|
def _sdk_proxy_country_code(self) -> str | None:
|
|
session_profile = getattr(self.browser_session, 'browser_profile', None)
|
|
for profile in (session_profile, self.browser_profile, self.browser_session):
|
|
for attr in ('cloud_proxy_country_code', 'proxy_country_code'):
|
|
value = getattr(profile, attr, None)
|
|
if isinstance(value, str) and value:
|
|
return value
|
|
return None
|
|
|
|
def _sdk_browser_state_dir(self) -> str | None:
|
|
value = getattr(self.browser_profile, 'state_dir', None)
|
|
if isinstance(value, (str, os.PathLike)) and str(value):
|
|
return str(Path(value).expanduser())
|
|
return None
|
|
|
|
def _sdk_run_params(self, *, max_steps: int, task: str, followups: list[str] | None = None) -> dict[str, Any]:
|
|
params: dict[str, Any] = {
|
|
'task': task,
|
|
'cwd': os.getcwd(),
|
|
'llm': self._sdk_llm_payload(),
|
|
'max_steps': int(max_steps),
|
|
'browser_mode': self._browser_mode(),
|
|
'browser': self._sdk_browser_payload(),
|
|
'calculate_cost': bool(self.settings.calculate_cost),
|
|
'use_vision': self.settings.use_vision,
|
|
'max_actions_per_step': int(self.settings.max_actions_per_step),
|
|
'config_overrides': {'full_llm_input_events': True},
|
|
}
|
|
if self._sdk_agent_id:
|
|
params['agent_id'] = self._sdk_agent_id
|
|
if self._sdk_browser_id:
|
|
params['browser_id'] = self._sdk_browser_id
|
|
if followups:
|
|
params['followups'] = list(followups)
|
|
if self.extraction_schema:
|
|
params['output_schema'] = self.extraction_schema
|
|
return params
|
|
|
|
async def _cancel_active_sdk_run(self) -> None:
|
|
client = self._sdk_client
|
|
if client is None:
|
|
return
|
|
await client.close()
|
|
|
|
async def _call_callback(self, callback: AgentHookFunc | None, *args: Any) -> None:
|
|
if callback is None:
|
|
return
|
|
result = callback(*args)
|
|
if inspect.isawaitable(result):
|
|
await result
|
|
|
|
async def _call_done_callback(self) -> None:
|
|
if not self.history.is_done():
|
|
return
|
|
await self.log_completion()
|
|
if self.register_done_callback is None:
|
|
return
|
|
result = self.register_done_callback(self.history)
|
|
if inspect.isawaitable(result):
|
|
await result
|
|
|
|
async def _call_new_step_callback(self) -> None:
|
|
if self.register_new_step_callback is None or not self.history.history:
|
|
return
|
|
history_id = id(self.history)
|
|
if history_id == self._last_step_callback_history_id:
|
|
return
|
|
start_step = max(1, self.state.n_steps - len(self.history.history) + 1)
|
|
for offset, history_item in enumerate(self.history.history):
|
|
result = self.register_new_step_callback(history_item.state, None, start_step + offset)
|
|
if inspect.isawaitable(result):
|
|
await result
|
|
self._last_step_callback_history_id = history_id
|
|
|
|
async def _call_step_end_callbacks(self, callback: AgentHookFunc | None) -> None:
|
|
if callback is None or not self.history.history:
|
|
return
|
|
history_id = id(self.history)
|
|
if history_id == self._last_step_end_callback_history_id:
|
|
return
|
|
for _history_item in self.history.history:
|
|
await self._call_callback(callback, self)
|
|
self._last_step_end_callback_history_id = history_id
|
|
|
|
def _sync_state_from_history(self) -> None:
|
|
if not self.history.history:
|
|
return
|
|
history_id = id(self.history)
|
|
if history_id != self._last_synced_history_id:
|
|
self.state.n_steps += max(1, len(self.history.history))
|
|
self._last_synced_history_id = history_id
|
|
action_results = self.history.action_results()
|
|
if action_results:
|
|
self.state.last_result = action_results
|
|
|
|
async def _save_conversation_if_requested(self) -> None:
|
|
if not self.settings.save_conversation_path:
|
|
return
|
|
conversation_dir = Path(self.settings.save_conversation_path).expanduser()
|
|
conversation_dir.mkdir(parents=True, exist_ok=True)
|
|
target = conversation_dir / f'conversation_{self.id}_{self.state.n_steps}.json'
|
|
target.write_text(
|
|
json.dumps(self._conversation_snapshot(), indent=2, default=str),
|
|
encoding=self.settings.save_conversation_path_encoding or 'utf-8',
|
|
)
|
|
|
|
async def _generate_gif_if_requested(self) -> None:
|
|
if not self.settings.generate_gif:
|
|
return
|
|
output_path = 'agent_history.gif'
|
|
if isinstance(self.settings.generate_gif, str):
|
|
output_path = self.settings.generate_gif
|
|
|
|
from browser_use.agent.gif import create_history_gif
|
|
|
|
create_history_gif(task=self.task, history=self.history, output_path=output_path)
|
|
if Path(output_path).exists():
|
|
output_event = await CreateAgentOutputFileEvent.from_agent_and_file(self, output_path)
|
|
self.eventbus.dispatch(output_event)
|
|
|
|
def _conversation_snapshot(self) -> dict[str, Any]:
|
|
return {
|
|
'agent': 'browser_use.beta.Agent',
|
|
'task_id': self.task_id,
|
|
'session_id': self.session_id,
|
|
'terminal_session_id': self.terminal_session_id,
|
|
'model': self.model,
|
|
'browser_mode': self._browser_mode(),
|
|
'task': self.task,
|
|
'final_result': self.history.final_result(),
|
|
'is_done': self.history.is_done(),
|
|
'is_successful': self.history.is_successful(),
|
|
'errors': self.history.errors(),
|
|
'urls': self.history.urls(),
|
|
'usage': self.history.usage.model_dump() if self.history.usage else None,
|
|
'events': self.last_events,
|
|
'child_events': self.last_child_events,
|
|
'usage_events': self.last_usage_events,
|
|
'stdout': self.last_stdout,
|
|
'stderr': self.last_stderr,
|
|
}
|
|
|
|
async def _check_stop_or_pause(self) -> None:
|
|
"""Check Browser Use-style stop/pause controls and raise when interrupted."""
|
|
if self.register_should_stop_callback is not None:
|
|
should_stop = self.register_should_stop_callback()
|
|
if inspect.isawaitable(should_stop):
|
|
should_stop = await should_stop
|
|
if should_stop:
|
|
self.logger.info('External callback requested stop')
|
|
self.state.stopped = True
|
|
raise InterruptedError
|
|
|
|
if self.register_external_agent_status_raise_error_callback is not None:
|
|
should_stop = self.register_external_agent_status_raise_error_callback()
|
|
if inspect.isawaitable(should_stop):
|
|
should_stop = await should_stop
|
|
if should_stop:
|
|
raise InterruptedError
|
|
|
|
if self.state.stopped:
|
|
raise InterruptedError
|
|
|
|
if self.state.paused:
|
|
raise InterruptedError
|
|
|
|
async def _should_stop_before_run(self) -> bool:
|
|
if self.state.stopped:
|
|
return True
|
|
if self.register_should_stop_callback is not None:
|
|
should_stop = self.register_should_stop_callback()
|
|
if inspect.isawaitable(should_stop):
|
|
should_stop = await should_stop
|
|
if should_stop:
|
|
self.state.stopped = True
|
|
return True
|
|
if self.register_external_agent_status_raise_error_callback is not None:
|
|
should_stop = self.register_external_agent_status_raise_error_callback()
|
|
if inspect.isawaitable(should_stop):
|
|
should_stop = await should_stop
|
|
if should_stop:
|
|
self.state.stopped = True
|
|
return True
|
|
return False
|
|
|
|
def _state_dir_args(self) -> list[str]:
|
|
state_dir = os.environ.get('BROWSER_USE_RUST_STATE_DIR')
|
|
return ['--state-dir', state_dir] if state_dir else []
|
|
|
|
def _browser_mode(self) -> str:
|
|
if _extract_cdp_url(self.browser_session) or _extract_profile_cdp_url(self.browser_profile):
|
|
return 'remote-cdp'
|
|
value = os.environ.get('BROWSER_USE_RUST_BROWSER_MODE')
|
|
if value:
|
|
return value
|
|
value = os.environ.get('BROWSER_USE_BROWSER_MODE')
|
|
if value:
|
|
return value
|
|
if _extract_cloud_preference(self.browser_session, self.browser_profile):
|
|
return 'cloud'
|
|
headless = _extract_headless_preference(self.browser_session, self.browser_profile)
|
|
if headless is False:
|
|
return 'managed-headed'
|
|
if headless is True:
|
|
return 'managed-headless'
|
|
return 'managed-headless'
|
|
|
|
def _run_env(self) -> dict[str, str]:
|
|
env = os.environ.copy()
|
|
env.update(_llm_env_overrides(self.llm))
|
|
if self.settings.calculate_cost:
|
|
env['BU_USE_CALCULATE_COST'] = 'true'
|
|
if _llm_provider_name(self.llm) in {'openrouter', 'deepseek'}:
|
|
env['LLM_BROWSER_OPENAI_COMPAT_INCLUDE_USAGE'] = 'true'
|
|
browser_mode = self._browser_mode()
|
|
env['LLM_BROWSER_BROWSER_MODE'] = browser_mode
|
|
env.setdefault('BROWSER_USE_PYTHON', sys.executable)
|
|
_apply_agent_tools_env(env)
|
|
cdp_url = _extract_cdp_url(self.browser_session) or _extract_profile_cdp_url(self.browser_profile)
|
|
if cdp_url:
|
|
env['BU_CDP_URL'] = cdp_url
|
|
if self.cdp_headers:
|
|
env['BU_CDP_HEADERS'] = json.dumps(self.cdp_headers)
|
|
if self.browser_user_agent:
|
|
env['BU_BROWSER_USER_AGENT'] = self.browser_user_agent
|
|
if self.highlight_enabled is not None:
|
|
env['BROWSER_USE_TERMINAL_AUTO_HIGHLIGHT'] = 'true' if self.highlight_enabled else 'false'
|
|
if self.highlight_color:
|
|
env['BROWSER_USE_TERMINAL_HIGHLIGHT_COLOR'] = self.highlight_color
|
|
if self.highlight_duration_ms is not None:
|
|
env['BROWSER_USE_TERMINAL_HIGHLIGHT_DURATION_MS'] = str(self.highlight_duration_ms)
|
|
env.update(self.wait_timing_env)
|
|
if self.block_ip_addresses is not None:
|
|
env['BU_BROWSER_BLOCK_IP_ADDRESSES'] = 'true' if self.block_ip_addresses else 'false'
|
|
if self.allowed_domains:
|
|
env['BU_BROWSER_ALLOWED_DOMAINS'] = json.dumps(self.allowed_domains)
|
|
if self.prohibited_domains:
|
|
env['BU_BROWSER_PROHIBITED_DOMAINS'] = json.dumps(self.prohibited_domains)
|
|
if self.browser_permissions:
|
|
env['BU_BROWSER_PERMISSIONS'] = json.dumps(self.browser_permissions)
|
|
if self.browser_accept_downloads is not None:
|
|
env['BU_BROWSER_ACCEPT_DOWNLOADS'] = 'true' if self.browser_accept_downloads else 'false'
|
|
if self.browser_downloads_path:
|
|
env['BU_BROWSER_DOWNLOADS_PATH'] = self.browser_downloads_path
|
|
if self.browser_no_viewport is not None:
|
|
env['BU_BROWSER_NO_VIEWPORT'] = 'true' if self.browser_no_viewport else 'false'
|
|
if self.browser_viewport:
|
|
env['BU_BROWSER_VIEWPORT'] = json.dumps(self.browser_viewport)
|
|
if self.browser_storage_state:
|
|
env['BU_BROWSER_STORAGE_STATE'] = json.dumps(self.browser_storage_state)
|
|
if self.managed_browser_env and _is_managed_browser_mode(browser_mode):
|
|
env.update(self.managed_browser_env)
|
|
if self.managed_browser_args and _is_managed_browser_mode(browser_mode):
|
|
env['BU_MANAGED_BROWSER_ARGS'] = json.dumps(self.managed_browser_args)
|
|
if self.managed_browser_profile_dir and _is_managed_browser_mode(browser_mode):
|
|
env['BU_MANAGED_BROWSER_PROFILE'] = self.managed_browser_profile_dir
|
|
if self.managed_browser_executable_path and _is_managed_browser_mode(browser_mode):
|
|
env['CHROME_PATH'] = self.managed_browser_executable_path
|
|
return env
|
|
|
|
|
|
Agent.__module__ = 'browser_use.agent.service'
|
|
Agent.__doc__ = None
|
|
|
|
|
|
def _align_browser_use_agent_signatures() -> None:
|
|
from browser_use.agent.service import _PythonAgent
|
|
|
|
for name, browser_use_method in vars(_PythonAgent).items():
|
|
if name.startswith('__') and name != '__init__':
|
|
continue
|
|
beta_method = getattr(Agent, name, None)
|
|
if beta_method is None or not callable(browser_use_method) or not callable(beta_method):
|
|
continue
|
|
try:
|
|
beta_method.__signature__ = inspect.signature(browser_use_method)
|
|
beta_method.__annotations__ = dict(getattr(browser_use_method, '__annotations__', {}))
|
|
except (TypeError, ValueError, AttributeError):
|
|
continue
|
|
try:
|
|
Agent.__signature__ = inspect.signature(_PythonAgent)
|
|
except (TypeError, ValueError, AttributeError):
|
|
pass
|
|
agent_hook_func = Callable[[_PythonAgent], Awaitable[None]]
|
|
for name in ('run', 'run_sync'):
|
|
beta_method = getattr(Agent, name, None)
|
|
if beta_method is None:
|
|
continue
|
|
try:
|
|
beta_method.__annotations__['on_step_start'] = agent_hook_func | None
|
|
beta_method.__annotations__['on_step_end'] = agent_hook_func | None
|
|
except (TypeError, AttributeError, KeyError):
|
|
continue
|
|
|
|
|
|
_align_browser_use_agent_signatures()
|