chore: import upstream snapshot with attribution
Test Browser Use CLI Install / uv pip install (macos-latest) (push) Waiting to run
Test Browser Use CLI Install / uv pip install (windows-latest) (push) Waiting to run
Test Browser Use CLI Install / uv pip install (ubuntu-latest) (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use from local wheel (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use[cli] from PyPI (push) Failing after 1s
package / pip-install-on-macos-latest-py-3.11 (push) Has been skipped
package / pip-install-on-macos-latest-py-3.13 (push) Has been skipped
package / pip-install-on-ubuntu-latest-py-3.11 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.13 (push) Has been skipped
cloud_evals / trigger_cloud_eval_image_build (push) Failing after 1s
docker / build_publish_image (push) Failing after 1s
Test Browser Use CLI Install / browser-use skill sync (push) Failing after 1s
lint / code-style (push) Failing after 0s
lint / type-checker (push) Failing after 1s
package / pip-build (push) Failing after 1s
lint / syntax-errors (push) Failing after 3s
package / pip-install-on-ubuntu-latest-py-3.13 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.11 (push) Has been skipped
test / ${{ matrix.test_filename }} (push) Has been skipped
test / evaluate-tasks (push) Has been skipped
test / setup-chromium (push) Failing after 2s
test / find_tests (push) Failing after 2s
Test Browser Use CLI Install / uv pip install (macos-latest) (push) Waiting to run
Test Browser Use CLI Install / uv pip install (windows-latest) (push) Waiting to run
Test Browser Use CLI Install / uv pip install (ubuntu-latest) (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use from local wheel (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use[cli] from PyPI (push) Failing after 1s
package / pip-install-on-macos-latest-py-3.11 (push) Has been skipped
package / pip-install-on-macos-latest-py-3.13 (push) Has been skipped
package / pip-install-on-ubuntu-latest-py-3.11 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.13 (push) Has been skipped
cloud_evals / trigger_cloud_eval_image_build (push) Failing after 1s
docker / build_publish_image (push) Failing after 1s
Test Browser Use CLI Install / browser-use skill sync (push) Failing after 1s
lint / code-style (push) Failing after 0s
lint / type-checker (push) Failing after 1s
package / pip-build (push) Failing after 1s
lint / syntax-errors (push) Failing after 3s
package / pip-install-on-ubuntu-latest-py-3.13 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.11 (push) Has been skipped
test / ${{ matrix.test_filename }} (push) Has been skipped
test / evaluate-tasks (push) Has been skipped
test / setup-chromium (push) Failing after 2s
test / find_tests (push) Failing after 2s
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,46 @@
|
||||
docs/
|
||||
static/
|
||||
.claude/
|
||||
.github/
|
||||
|
||||
# Cache files
|
||||
.DS_Store
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.ipynb_checkpoints
|
||||
|
||||
# Virtual Environments
|
||||
.venv
|
||||
venv/
|
||||
|
||||
# Editor cruft
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Build Files
|
||||
dist/
|
||||
|
||||
# Data files
|
||||
*.gif
|
||||
*.txt
|
||||
*.pdf
|
||||
*.csv
|
||||
*.json
|
||||
*.jsonl
|
||||
*.bak
|
||||
|
||||
# Secrets and sensitive files
|
||||
secrets.env
|
||||
.env
|
||||
browser_cookies.json
|
||||
cookies.json
|
||||
gcp-login.json
|
||||
saved_trajectories/
|
||||
AgentHistory.json
|
||||
AgentHistoryList.json
|
||||
private_example.py
|
||||
private_example
|
||||
@@ -0,0 +1,70 @@
|
||||
# Browser Use Configuration
|
||||
# Copy this file to .env and fill in your values
|
||||
|
||||
# Logging Configuration
|
||||
# Set the logging level (debug, info, warning, error)
|
||||
BROWSER_USE_LOGGING_LEVEL=info
|
||||
|
||||
# Log file paths (optional)
|
||||
# Save debug level logs to this file
|
||||
BROWSER_USE_DEBUG_LOG_FILE=debug.log
|
||||
|
||||
# Save info level logs to this file
|
||||
BROWSER_USE_INFO_LOG_FILE=info.log
|
||||
|
||||
# CDP (Chrome DevTools Protocol) logging level
|
||||
CDP_LOGGING_LEVEL=WARNING
|
||||
|
||||
# Telemetry and Analytics
|
||||
# Enable/disable anonymous telemetry
|
||||
ANONYMIZED_TELEMETRY=true
|
||||
|
||||
# Browser Use Cloud Configuration
|
||||
# Get your API key from: https://cloud.browser-use.com/new-api-key
|
||||
BROWSER_USE_API_KEY=your_bu_api_key_here
|
||||
|
||||
# Custom API base URL (for enterprise installations)
|
||||
# BROWSER_USE_CLOUD_API_URL=https://api.browser-use.com
|
||||
|
||||
# Cloud sync settings
|
||||
# BROWSER_USE_CLOUD_SYNC=false
|
||||
|
||||
# Model Configuration (optional - use if you want to use other LLM providers)
|
||||
# Default LLM model to use
|
||||
# OPENAI_API_KEY=your_openai_api_key_here
|
||||
# ANTHROPIC_API_KEY=your_anthropic_api_key_here
|
||||
# AZURE_OPENAI_API_KEY=
|
||||
# AZURE_OPENAI_ENDPOINT=
|
||||
# GOOGLE_API_KEY=
|
||||
# DEEPSEEK_API_KEY=
|
||||
# GROK_API_KEY=
|
||||
# NOVITA_API_KEY=
|
||||
|
||||
# AWS Bedrock Configuration (for AWS Bedrock models)
|
||||
# Requires: pip install browser-use[aws]
|
||||
# Note: You need proper AWS Bedrock access and model permissions in your AWS account
|
||||
# AWS_ACCESS_KEY_ID=your_aws_access_key_id_here
|
||||
# AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key_here
|
||||
# AWS_SESSION_TOKEN=your_session_token_here # Only required for temporary credentials
|
||||
# AWS_REGION=us-east-1
|
||||
|
||||
|
||||
# Browser Configuration
|
||||
# Path to Chrome/Chromium executable (optional)
|
||||
# BROWSER_USE_EXECUTABLE_PATH=/path/to/chrome
|
||||
|
||||
# Run browser in headless mode
|
||||
# BROWSER_USE_HEADLESS=false
|
||||
|
||||
# User data directory for browser profile
|
||||
# BROWSER_USE_USER_DATA_DIR=./browser_data
|
||||
|
||||
# Proxy Configuration (optional)
|
||||
# BROWSER_USE_PROXY_SERVER=http://proxy.example.com:8080
|
||||
# BROWSER_USE_NO_PROXY=localhost,127.0.0.1,*.internal
|
||||
# BROWSER_USE_PROXY_USERNAME=username
|
||||
# BROWSER_USE_PROXY_PASSWORD=password
|
||||
|
||||
# Version Check
|
||||
# Enable/disable checking for newer browser-use versions on agent startup
|
||||
BROWSER_USE_VERSION_CHECK=true
|
||||
@@ -0,0 +1,2 @@
|
||||
static/*.gif filter=lfs diff=lfs merge=lfs -text
|
||||
# static/*.mp4 filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -0,0 +1,2 @@
|
||||
66b3c26df51adec32d42c3b2c0304e0662457298
|
||||
2be4ba4f7078d47bbeed04baf6f8fb04017df028
|
||||
@@ -0,0 +1,8 @@
|
||||
# Contributing to browser-use
|
||||
|
||||
We love contributions! Please read through these links to get started:
|
||||
|
||||
- 🔢 [Contribution Guidelines](https://docs.browser-use.com/development/contribution-guide)
|
||||
- 👾 [Local Development Setup Guide](https://docs.browser-use.com/development/local-setup)
|
||||
- 🏷️ [Issues Tagged: `#help-wanted`](https://github.com/browser-use/browser-use/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22)
|
||||
- 🔌 [Integration Example Guidelines](../examples/integrations/README.md)
|
||||
@@ -0,0 +1,114 @@
|
||||
name: 🎯 AI Agent ✚ Page Interaction Issue
|
||||
description: Agent fails to detect, click, scroll, input, or otherwise interact with some type of element on some page(s)
|
||||
labels: ["bug", "element-detection"]
|
||||
title: "Interaction Issue: ..."
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for taking the time to fill out this bug report! Please fill out the form below to help us reproduce and fix the issue.
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
> [!IMPORTANT]
|
||||
> 🙏 Please **go check *right now before filling this out* that that you are *actually* on the [⬆️ LATEST VERSION](https://github.com/browser-use/browser-use/releases)**.
|
||||
> 🚀 We ship changes every hour and we might've already fixed your issue today!
|
||||
> <a href="https://github.com/browser-use/browser-use/releases"><img src="https://github.com/user-attachments/assets/4cd34ee6-bafb-4f24-87e2-27a31dc5b9a4" width="500px"/></a>
|
||||
> If you are running an old version, the **first thing we will ask you to do is *upgrade to the latest version* and try again**:
|
||||
> - 🆕 [`beta`](https://docs.browser-use.com/development/local-setup): `uv pip install --upgrade git+https://github.com/browser-use/browser-use.git@main`
|
||||
> - 📦 [`stable`](https://pypi.org/project/browser-use/#history): `uv pip install --upgrade browser-use`
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Browser Use Version
|
||||
description: |
|
||||
What version of `browser-use` are you using? (Run `uv pip show browser-use` or `git log -n 1`)
|
||||
**DO NOT JUST WRITE `latest release` or `main` or a very old version or we will close your issue!**
|
||||
placeholder: "e.g. 0.4.45 or 62760baaefd"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: model
|
||||
attributes:
|
||||
label: LLM Model
|
||||
description: Which LLM model are you using?
|
||||
placeholder: "e.g. bu-1.0, gpt-5-mini, claude-4-5-sonnet, gemini-2.0-flash, etc."
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: prompt
|
||||
attributes:
|
||||
label: Screenshots, Description, and task prompt given to Agent
|
||||
description: |
|
||||
A description of the issue + screenshots, and the full task prompt you're giving the agent (redact sensitive data).
|
||||
To help us fix it even faster, screenshot the Chome devtools [`Computed Styles` pane](https://developer.chrome.com/docs/devtools/css/reference#computed) for each failing element.
|
||||
placeholder: |
|
||||
🎯 High-level goal: Compare the prices of 3 items on a few different seller pages
|
||||
💬 Agent(task='''
|
||||
1. go to https://example.com and click the "xyz" dropdown
|
||||
2. type "abc" into search then select the "abc" option <- ❌ agent fails to select this option
|
||||
3. ...
|
||||
☝️ please include real URLs 🔗 and screenshots 📸 when possible!
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: html
|
||||
attributes:
|
||||
label: "HTML around where it's failing"
|
||||
description: A snippet of the HTML from the failing page around where the Agent is failing to interact.
|
||||
render: html
|
||||
placeholder: |
|
||||
<form na-someform="abc"> <!-- ⬅️ at least one parent element above -->
|
||||
<div class="element-to-click">
|
||||
<div data-isbutton="true">Click me</div>
|
||||
</div>
|
||||
<input id="someinput" name="someinput" type="text" /> <!-- ⬅️ failing element -->
|
||||
...
|
||||
</form>
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System & Browser Versions
|
||||
description: What operating system and browser are you using?
|
||||
placeholder: "e.g. Ubuntu 24.04 + playwright chromium v136, Windows 11 + Chrome.exe v133, macOS ..."
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: code
|
||||
attributes:
|
||||
label: Python Code Sample
|
||||
description: Include some python code that reproduces the issue
|
||||
render: python
|
||||
placeholder: |
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv() # tip: always load_dotenv() before other imports
|
||||
from browser_use import Agent, BrowserSession, Tools
|
||||
from browser_use.llm import ChatOpenAI
|
||||
|
||||
agent = Agent(
|
||||
task='...',
|
||||
llm=ChatOpenAI(model="gpt-4.1"),
|
||||
browser_session=BrowserSession(headless=False),
|
||||
)
|
||||
...
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Full DEBUG Log Output
|
||||
description: Please copy and paste the *full* log output *from the start of the run*. Make sure to set `BROWSER_USE_LOGGING_LEVEL=DEBUG` in your `.env` or shell environment.
|
||||
render: shell
|
||||
placeholder: |
|
||||
$ python /app/browser-use/examples/browser/real_browser.py
|
||||
DEBUG [browser] 🌎 Initializing new browser
|
||||
DEBUG [agent] Version: 0.1.46-9-g62760ba, Source: git
|
||||
@@ -0,0 +1,77 @@
|
||||
name: 👾 Library Bug Report
|
||||
description: Report a bug in the browser-use Python library
|
||||
labels: ["bug", "triage"]
|
||||
title: "Bug: ..."
|
||||
body:
|
||||
# - type: markdown
|
||||
# attributes:
|
||||
# value: |
|
||||
# Thanks for taking the time to fill out this bug report! Please fill out the form below to help us reproduce and fix the issue.
|
||||
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Browser Use Version
|
||||
description: |
|
||||
What exact version of `browser-use` are you using? (Run `uv pip show browser-use` or `git log -n 1`)
|
||||
**DO NOT WRITE `latest release` or `main` or a very old version or we will close your issue!**
|
||||
placeholder: "e.g. 0.4.45 or 62760baaefd"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Bug Description, Steps to Reproduce, Screenshots
|
||||
description: A clear and concise description of what the bug is + steps taken, drag screenshots in showing any error messages and relevant pages.
|
||||
placeholder: |
|
||||
1. Installed browser-use library by running: `uv pip install browser-use`
|
||||
2. Installed the browser by running: `playwright install chromium --with-deps`
|
||||
3. Ran the code below with the following prompt: `go to example.com and do xyz...`
|
||||
4. Agent crashed and showed the following error: ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: code
|
||||
attributes:
|
||||
label: Failing Python Code
|
||||
description: Include the exact python code you ran that encountered the issue, redact any sensitive URLs and API keys.
|
||||
render: python
|
||||
placeholder: |
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv() # tip: always load_dotenv() before other imports
|
||||
from browser_use import Agent, BrowserSession, Tools
|
||||
from browser_use.llm import ChatOpenAI
|
||||
|
||||
agent = Agent(
|
||||
task='...',
|
||||
llm=ChatOpenAI(model="gpt-4.1-mini"),
|
||||
browser_session=BrowserSession(headless=False),
|
||||
)
|
||||
...
|
||||
|
||||
- type: input
|
||||
id: model
|
||||
attributes:
|
||||
label: LLM Model
|
||||
description: Which LLM model are you using? (Optional)
|
||||
placeholder: "e.g. ChatBrowserUse, gpt-4.1-mini, gemini-3-flash-preview, etc."
|
||||
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System & Browser Versions
|
||||
description: What operating system and browser are you using? (Optional)
|
||||
placeholder: "e.g. Ubuntu 24.04 + playwright chromium v136, Windows 11 + Chrome.exe v133, macOS ..."
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Full DEBUG Log Output
|
||||
description: Please copy and paste the log output. Make sure to set `BROWSER_USE_LOGGING_LEVEL=DEBUG` in your `.env` or shell environment.
|
||||
render: shell
|
||||
placeholder: |
|
||||
$ python /app/browser-use/examples/browser/real_browser.py
|
||||
DEBUG [browser] 🌎 Initializing new browser
|
||||
@@ -0,0 +1,93 @@
|
||||
name: 💡 New Feature or Enhancement Request
|
||||
description: Suggest an idea or improvement for the browser-use library or Agent capabilities
|
||||
title: "Feature Request: ..."
|
||||
type: 'Enhancement'
|
||||
labels: ['enhancement']
|
||||
body:
|
||||
- type: textarea
|
||||
id: current_problem
|
||||
attributes:
|
||||
label: "What is the problem that your feature request solves?"
|
||||
description: |
|
||||
Describe the problem or need that your feature request solves, include screenshots and example URLs if relevant.
|
||||
placeholder: |
|
||||
e.g. I need to be able to simulate dragging in a circle to test the paint feature on a drawing site: https://example.com/draw
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: proposed_solution
|
||||
attributes:
|
||||
label: "What is your proposed solution?"
|
||||
description: |
|
||||
Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
|
||||
placeholder: |
|
||||
e.g. I want to add a default action that can hover/drag the mouse on a path when given a series
|
||||
of x,y coordinates. More broadly it may be useful add a computer-use/x,y-coordinate-style automation
|
||||
method fallback that can do complex mouse movements.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: workarounds_tried
|
||||
attributes:
|
||||
label: "What hacks or alternative solutions have you tried to solve the problem?"
|
||||
description: |
|
||||
A description of any troubleshooting, alternative approaches, workarounds, or other ideas you've considered to fix the problem.
|
||||
placeholder: |
|
||||
e.g. I tried upgrading to the latest version and telling it to hover in the prompt. I also tried
|
||||
telling the agent to ask for human help (using a custom tools action) when it gets to this
|
||||
step, then I manually click a browser extension in the navbar that automates the mouse movevement.
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: What version of browser-use are you currently using?
|
||||
description: |
|
||||
Run `uv pip show browser-use` or `git log -n 1` and share the exact number or git hash. DO NOT JUST ENTER `latest release` OR `main`.
|
||||
We need to know what version of the browser-use library you're running in order to contextualize your feature request.
|
||||
Sometimes features are already available and just need to be enabled with config on certain versions.
|
||||
placeholder: "e.g. 0.1.48 or 62760baaefd"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
> [!IMPORTANT]
|
||||
> 🙏 Please **go check *right now before filling this out* that that you have tried the [⬆️ LATEST VERSION](https://github.com/browser-use/browser-use/releases)**.
|
||||
> 🚀 We ship *hundreds* of improvements a day and we might've already added a solution to your need yesterday!
|
||||
> <a href="https://github.com/browser-use/browser-use/releases"><img src="https://github.com/user-attachments/assets/4cd34ee6-bafb-4f24-87e2-27a31dc5b9a4" width="500px"/></a>
|
||||
> If you are running an old version, the **first thing we will ask you to do is *try the latest `beta`***:
|
||||
> - 🆕 [`beta`](https://docs.browser-use.com/development/local-setup): `uv pip install --upgrade git+https://github.com/browser-use/browser-use.git@main`
|
||||
> - 📦 [`stable`](https://pypi.org/project/browser-use/#history): `pip install --upgrade browser-use`
|
||||
|
||||
- type: checkboxes
|
||||
id: priority
|
||||
attributes:
|
||||
label: "How badly do you want this new feature?"
|
||||
options:
|
||||
- label: "It's an urgent deal-breaker, I can't live without it"
|
||||
required: false
|
||||
- label: "It's important to add it in the near-mid term future"
|
||||
required: false
|
||||
- label: "It would be nice to add it sometime in the next 2 years"
|
||||
required: false
|
||||
- label: "💪 I'm willing to [start a PR](https://docs.browser-use.com/development/contribution-guide) to work on this myself"
|
||||
required: false
|
||||
- label: "💼 My company would spend >$5k on [Browser-Use Cloud](https://browser-use.com) if it solved this reliably for us"
|
||||
required: false
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
> [!TIP]
|
||||
> Start conversations about your feature request in other places too, the more
|
||||
> 📣 hype we see around a request the more likely we are to add it!
|
||||
>
|
||||
> - 👾 Discord: [https://link.browser-use.com/discord](https://link.browser-use.com/discord)
|
||||
> - 𝕏 Twitter: [https://x.com/browser_use](https://x.com/browser_use)
|
||||
@@ -0,0 +1,55 @@
|
||||
name: 📚 Documentation Issue
|
||||
description: Report an issue in the browser-use documentation
|
||||
labels: ["documentation"]
|
||||
title: "Documentation: ..."
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for taking the time to improve our documentation! Please fill out the form below to help us fix the issue quickly.
|
||||
|
||||
- type: dropdown
|
||||
id: type
|
||||
attributes:
|
||||
label: Type of Documentation Issue
|
||||
description: What type of documentation issue is this?
|
||||
options:
|
||||
- Missing documentation
|
||||
- Incorrect documentation
|
||||
- Unclear documentation
|
||||
- Broken link
|
||||
- Other (specify in description)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: page
|
||||
attributes:
|
||||
label: Documentation Page
|
||||
description: Which page or section of the documentation is this about?
|
||||
placeholder: "e.g. https://docs.browser-use.com/open-source/customize/browser/all-parameters > Display & Appearance > headless"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Issue Description
|
||||
description: "Describe what's wrong or missing in the documentation"
|
||||
placeholder: e.g. Docs should clarify whether BrowserSession(no_viewport=False) is supported when running in BrowserSession(headless=False) mode...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: suggestion
|
||||
attributes:
|
||||
label: Suggested Changes
|
||||
description: If you have specific suggestions for how to improve the documentation, please share them
|
||||
placeholder: |
|
||||
e.g. The documentation could be improved by adding one more line here:
|
||||
```diff
|
||||
Use `BrowserSession(headless=False)` to open the browser window (aka headful mode).
|
||||
+ Viewports are not supported when headful, if `headless=False` it will force `no_viewport=True`.
|
||||
```
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,11 @@
|
||||
blank_issues_enabled: false # Set to true if you want to allow blank issues
|
||||
contact_links:
|
||||
- name: 🔢 Quickstart Guide
|
||||
url: https://docs.browser-use.com/quickstart
|
||||
about: Most common issues can be resolved by following our quickstart guide
|
||||
- name: 💬 Questions and Help
|
||||
url: https://link.browser-use.com/discord
|
||||
about: Please ask questions in our Discord community
|
||||
- name: 📖 Documentation
|
||||
url: https://docs.browser-use.com
|
||||
about: Check our documentation for answers first
|
||||
@@ -0,0 +1,19 @@
|
||||
## Reporting Security Issues
|
||||
|
||||
If you believe you have found a security vulnerability in browser-use, please report it through coordinated disclosure.
|
||||
|
||||
**Please do not report security vulnerabilities through the repository issues, discussions, or pull requests.**
|
||||
|
||||
Instead, please open a new [Github security advisory](https://github.com/browser-use/browser-use/security/advisories/new).
|
||||
|
||||
Please include as much of the information listed below as you can to help me better understand and resolve the issue:
|
||||
|
||||
* The type of issue (e.g., buffer overflow, SQL injection, or cross-site scripting)
|
||||
* Full paths of source file(s) related to the manifestation of the issue
|
||||
* The location of the affected source code (tag/branch/commit or direct URL)
|
||||
* Any special configuration required to reproduce the issue
|
||||
* Step-by-step instructions to reproduce the issue
|
||||
* Proof-of-concept or exploit code (if possible)
|
||||
* Impact of the issue, including how an attacker might exploit the issue
|
||||
|
||||
This information will help me triage your report more quickly.
|
||||
@@ -0,0 +1,43 @@
|
||||
name: Build Base Image
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 2 * * 1' # Weekly on Monday
|
||||
workflow_dispatch:
|
||||
push:
|
||||
paths:
|
||||
- 'Dockerfile.base'
|
||||
|
||||
jobs:
|
||||
build-base:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push base image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.base
|
||||
platforms: ${{ matrix.platform }}
|
||||
push: true
|
||||
tags: |
|
||||
browseruse/browseruse-base:chromium-138-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
browseruse/browseruse-base:latest-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
cache-from: type=registry,ref=browseruse/browseruse-base:buildcache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
cache-to: type=registry,ref=browseruse/browseruse-base:buildcache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max
|
||||
@@ -0,0 +1,150 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
id-token: write
|
||||
discussions: write
|
||||
issues: write
|
||||
env:
|
||||
IS_SANDBOX: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
|
||||
- run: uv sync --dev --all-extras
|
||||
|
||||
- name: Detect installed Playwright version
|
||||
run: echo "PLAYWRIGHT_VERSION=$(uv pip list --format json | jq -r '.[] | select(.name == "playwright") | .version')" >> $GITHUB_ENV
|
||||
|
||||
# - name: Cache chrome binaries
|
||||
# uses: actions/cache@v4
|
||||
# with:
|
||||
# path: |
|
||||
# /tmp/google-chrome-stable_current_amd64.deb
|
||||
# key: ${{ runner.os }}-${{ runner.arch }}-chrome-stable
|
||||
|
||||
# - name: Install Chrome stable binary
|
||||
# run: |
|
||||
# sudo apt-get update -qq \
|
||||
# && sudo curl -o "/tmp/google-chrome-stable_current_amd64.deb" --no-clobber "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" \
|
||||
# && sudo apt-get install -y "/tmp/google-chrome-stable_current_amd64.deb" -f
|
||||
# - run: patchright install chrome --with-deps
|
||||
# - run: playwright install chrome --with-deps
|
||||
|
||||
- name: Cache chromium binaries
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-playwright-${{ env.PLAYWRIGHT_VERSION }}-chromium
|
||||
|
||||
- run: playwright install chromium --with-deps
|
||||
# - run: patchright install chromium --with-deps
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@beta
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
model: "claude-opus-4-20250514"
|
||||
fallback_model: "claude-3-5-sonnet-20241022"
|
||||
custom_instructions: |
|
||||
when making any significant changes, start by adding one or two new failing test functions to the most relevant file you can find in tests/ci/*.py, then work on your changes until you get the tests passing.
|
||||
make sure all lint errors are fixed before committing: `uv run pre-commit --all-files`, you can also use mcp tools to check Github CI status.
|
||||
make sure to run the whole test file at the end to make sure no other tests in that file started failing due to your changes: `uv run pytest/ci/test_....py`.
|
||||
if any significant features were added or removed, or any public-facing parameters/signatures changed, make sure to look through docs/*.mdx and examples/**.py and fix any relevant areas that might need to be updated.
|
||||
branch_prefix: "claude-"
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
claude_env: |
|
||||
IN_DOCKER: 'true'
|
||||
BROWSER_USE_CLOUD_SYNC: 'false'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
BROWSER_USE_LOGGING_LEVEL: 'DEBUG'
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
settings: |
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(git:*)",
|
||||
"Bash(uv:*)",
|
||||
"Bash(uv run pytest:*)",
|
||||
"Bash(uv run ruff:*)",
|
||||
"Bash(uv run pyright:*)",
|
||||
"Bash(uv run pre-commit:*)",
|
||||
"Bash(uv pip:*)",
|
||||
"Bash(uv add:*)",
|
||||
"Bash(uv sync --all-extras --dev)",
|
||||
"Bash(.venv/bin/*:*)",
|
||||
"Bash(.venv/bin/python:*)",
|
||||
"Bash(sed:*)",
|
||||
"Bash(rg:*)",
|
||||
"Bash(jq:*)",
|
||||
"Bash(find:*)",
|
||||
"Bash(grep:*)",
|
||||
"Bash(python:*)",
|
||||
"Bash(chmod:*)",
|
||||
"Bash(rm:*)",
|
||||
"Bash(playwright:*)",
|
||||
"Bash(uv run playwright:*)",
|
||||
"Bash(./bin/lint.sh)",
|
||||
"Bash(./bin/test.sh)",
|
||||
"WebFetch(*)",
|
||||
"WebSearch(*)"
|
||||
],
|
||||
"additionalDirectories": ["/home/runner/work"]
|
||||
}
|
||||
}
|
||||
allowed_tools: |
|
||||
Bash(git:*)
|
||||
Bash(uv:*)
|
||||
Bash(uv run pytest:*)
|
||||
Bash(uv run ruff:*)
|
||||
Bash(uv run pyright:*)
|
||||
Bash(uv run pre-commit:*)
|
||||
Bash(uv pip:*)
|
||||
Bash(uv add:*)
|
||||
Bash(uv sync --all-extras --dev)
|
||||
Bash(.venv/bin/*:*)
|
||||
Bash(.venv/bin/python:*)
|
||||
Bash(sed:*)
|
||||
Bash(rg:*)
|
||||
Bash(jq:*)
|
||||
Bash(find:*)
|
||||
Bash(grep:*)
|
||||
Bash(python:*)
|
||||
Bash(chmod:*)
|
||||
Bash(rm:*)
|
||||
Bash(playwright:*)
|
||||
Bash(uv run playwright:*)
|
||||
Bash(./bin/lint.sh)
|
||||
Bash(./bin/test.sh)
|
||||
WebFetch(*)
|
||||
WebSearch(*)
|
||||
@@ -0,0 +1,35 @@
|
||||
name: cloud_evals
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- 'releases/*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
commit_hash:
|
||||
description: Commit hash of the library to build the Cloud eval image for
|
||||
required: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
trigger_cloud_eval_image_build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.TRIGGER_CLOUD_BUILD_GH_KEY }}
|
||||
script: |
|
||||
const result = await github.rest.repos.createDispatchEvent({
|
||||
owner: 'browser-use',
|
||||
repo: 'cloud',
|
||||
event_type: 'trigger-workflow',
|
||||
client_payload: {"commit_hash": "${{ github.event.inputs.commit_hash || github.sha }}"}
|
||||
})
|
||||
console.log(result)
|
||||
@@ -0,0 +1,76 @@
|
||||
name: docker
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build_publish_image:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
attestations: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Compute Docker tags based on tag/branch
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
browseruse/browseruse
|
||||
ghcr.io/browser-use/browser-use
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=pep440,pattern={{version}}
|
||||
type=pep440,pattern={{major}}.{{minor}}
|
||||
type=sha
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=registry,ref=browseruse/browseruse:buildcache
|
||||
cache-to: type=registry,ref=browseruse/browseruse:buildcache,mode=max
|
||||
@@ -0,0 +1,56 @@
|
||||
name: Evaluate PR
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
jobs:
|
||||
trigger-evaluation:
|
||||
runs-on: ubuntu-latest
|
||||
# Only run if PR author has write access
|
||||
if: |
|
||||
github.event.pull_request.author_association == 'OWNER' ||
|
||||
github.event.pull_request.author_association == 'MEMBER' ||
|
||||
github.event.pull_request.author_association == 'COLLABORATOR'
|
||||
|
||||
steps:
|
||||
- name: Trigger Evaluation settings
|
||||
id: trigger
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "🚀 Triggering evaluation - PR #${{ github.event.pull_request.number }}"
|
||||
echo "Commit: ${{ github.event.pull_request.head.sha }}"
|
||||
|
||||
# You can customize the test here
|
||||
TEST_CASE="${{ vars.EVAL_TEST_CASE }}"
|
||||
if [ -z "$TEST_CASE" ]; then
|
||||
TEST_CASE="InteractionTasks_v8"
|
||||
fi
|
||||
|
||||
response=$(curl -X POST \
|
||||
"${{ secrets.EVAL_PLATFORM_URL }}/api/triggerInteractionTasksV6" \
|
||||
-H "Authorization: Bearer ${{ secrets.EVAL_PLATFORM_KEY }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"commitSha\": \"${{ github.event.pull_request.head.sha }}\",
|
||||
\"prNumber\": ${{ github.event.pull_request.number }},
|
||||
\"branchName\": \"${{ github.event.pull_request.head.ref }}\",
|
||||
\"testCase\": \"${TEST_CASE}\",
|
||||
\"githubRepo\": \"${{ github.repository }}\"
|
||||
}" -s)
|
||||
|
||||
echo "Response: $response"
|
||||
|
||||
# Check if trigger was was successful
|
||||
if echo "$response" | jq -e '.success == true' > /dev/null; then
|
||||
echo "✅ Evaluation triggered successfully"
|
||||
exit 0
|
||||
else
|
||||
echo "Failed"
|
||||
echo "$response"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,125 @@
|
||||
name: Test Browser Use CLI Install
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'browser_use/cli.py'
|
||||
- 'browser_use/skills/**'
|
||||
- 'skills/browser-use/SKILL.md'
|
||||
- 'browser_use/skills/browser-use/SKILL.md'
|
||||
- 'scripts/sync_browser_harness_skill.py'
|
||||
- 'pyproject.toml'
|
||||
- '.github/workflows/install-script.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'browser_use/cli.py'
|
||||
- 'browser_use/skills/**'
|
||||
- 'skills/browser-use/SKILL.md'
|
||||
- 'browser_use/skills/browser-use/SKILL.md'
|
||||
- 'scripts/sync_browser_harness_skill.py'
|
||||
- 'pyproject.toml'
|
||||
- '.github/workflows/install-script.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check-browser-use-skill:
|
||||
name: browser-use skill sync
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Check generated Browser Use skill
|
||||
run: python3 scripts/sync_browser_harness_skill.py --check
|
||||
|
||||
test-pip-install:
|
||||
name: uv pip install (${{ matrix.os }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
- name: Add uv to PATH
|
||||
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Create venv and install browser-use
|
||||
run: |
|
||||
uv venv .venv --python 3.11
|
||||
uv pip install --python .venv .
|
||||
|
||||
- name: Add venv to PATH
|
||||
run: |
|
||||
echo "$PWD/.venv/bin" >> $GITHUB_PATH
|
||||
echo "$PWD/.venv/Scripts" >> $GITHUB_PATH
|
||||
|
||||
- name: Verify Browser Use CLI
|
||||
run: |
|
||||
browser-use --help
|
||||
browser-use auth --help
|
||||
browser-use skill show
|
||||
|
||||
- name: Verify aliases
|
||||
run: |
|
||||
browseruse --help
|
||||
bu --help
|
||||
browser --help
|
||||
|
||||
test-uvx-local-wheel:
|
||||
name: uvx browser-use from local wheel
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
- name: Add uv to PATH
|
||||
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Build wheel from current branch
|
||||
run: |
|
||||
uv venv .venv --python 3.11
|
||||
source .venv/bin/activate
|
||||
uv pip install build
|
||||
python -m build --wheel
|
||||
|
||||
- name: Verify uvx Browser Use CLI
|
||||
run: |
|
||||
WHEEL=$(ls dist/*.whl)
|
||||
uvx --from "$WHEEL" browser-use --help
|
||||
uvx --from "$WHEEL" browser-use auth --help
|
||||
uvx --from "$WHEEL" browser-use skill show
|
||||
|
||||
test-uvx-pypi:
|
||||
name: uvx browser-use[cli] from PyPI
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Install uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
- name: Add uv to PATH
|
||||
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Verify uvx Browser Use CLI
|
||||
run: |
|
||||
uvx "browser-use[cli]" --help
|
||||
uvx "browser-use[cli]" auth --help
|
||||
uvx "browser-use[cli]" skill show
|
||||
@@ -0,0 +1,54 @@
|
||||
name: lint
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint-syntax:
|
||||
name: syntax-errors
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
- run: uv run ruff check --no-fix --select PLE
|
||||
|
||||
lint-style:
|
||||
name: code-style
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
- run: uv python install 3.11
|
||||
- run: uv sync --dev --all-extras --python 3.11 --refresh-package browser-use-core
|
||||
- run: uv run --no-sync pre-commit run --all-files --show-diff-on-failure
|
||||
|
||||
lint-typecheck:
|
||||
name: type-checker
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
- run: uv sync --dev --all-extras --refresh-package browser-use-core # install extras for examples to avoid pyright missing imports errors-
|
||||
- run: uv run --no-sync pyright
|
||||
@@ -0,0 +1,64 @@
|
||||
name: package
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: pip-build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
- run: uv build --python 3.12
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dist-artifact
|
||||
path: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
|
||||
build_test:
|
||||
name: pip-install-on-${{ matrix.os }}-py-${{ matrix.python-version }}
|
||||
needs: build
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
python-version: ["3.11", "3.13"]
|
||||
env:
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist-artifact
|
||||
|
||||
- name: Set up venv and test for OS/Python versions
|
||||
shell: bash
|
||||
run: |
|
||||
uv venv /tmp/testenv --python ${{ matrix.python-version }} --clear
|
||||
if [[ "$RUNNER_OS" == "Windows" ]]; then
|
||||
. /tmp/testenv/Scripts/activate
|
||||
else
|
||||
source /tmp/testenv/bin/activate
|
||||
fi
|
||||
uv pip install *.whl
|
||||
python -c 'from browser_use import Agent, BrowserProfile, BrowserSession, Tools, ActionModel, ActionResult'
|
||||
@@ -0,0 +1,142 @@
|
||||
# This workflow will upload a Python Package using Twine when a release is created
|
||||
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
||||
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
name: publish
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published] # publish full release to PyPI when a release is created on Github
|
||||
# schedule:
|
||||
# - cron: "0 17 * * FRI" # tag a pre-release on Github every Friday at 5 PM UTC
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
create_tag:
|
||||
description: "Create the next pre-release tag before publishing"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
tag_pre_release:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.create_tag
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Create pre-release tag
|
||||
run: |
|
||||
git fetch --tags
|
||||
latest_tag=$(git tag --list --sort=-v:refname | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$' | head -n 1)
|
||||
if [ -z "$latest_tag" ]; then
|
||||
echo "Failed to find the latest git tag from list:" > /dev/stderr
|
||||
git tag --list --sort=-v:refname
|
||||
exit 1
|
||||
else
|
||||
# Bump the tag rc version
|
||||
if [[ "$latest_tag" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(rc([0-9]+))?$ ]]; then
|
||||
major="${BASH_REMATCH[1]}"
|
||||
minor="${BASH_REMATCH[2]}"
|
||||
patch="${BASH_REMATCH[3]}"
|
||||
rc="${BASH_REMATCH[5]}"
|
||||
echo "latest_tag: ${major}.${minor}.${patch}rc${rc:-0}"
|
||||
if [ -z "$rc" ]; then
|
||||
# No rc, so bump patch and set rc=1 # 0.2.1 -> 0.2.2rc1
|
||||
patch=$((patch + 1))
|
||||
new_tag="${major}.${minor}.${patch}rc1"
|
||||
else
|
||||
if [ "$rc" -ge 99 ]; then
|
||||
echo "Error: rc version is already at 99 for tag $latest_tag, refusing to increment further." > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
rc=$((rc + 1))
|
||||
new_tag="${major}.${minor}.${patch}rc${rc}" # 0.2.1rc1 -> 0.2.1rc2
|
||||
fi
|
||||
else
|
||||
echo "Error: latest_tag '$latest_tag' does not match expected version pattern." > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo "new_tag: $new_tag"
|
||||
git tag $new_tag
|
||||
git push origin $new_tag
|
||||
|
||||
publish_to_pypi:
|
||||
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
permissions:
|
||||
contents: write # for the stable-branch push at the end
|
||||
id-token: write # for PyPI trusted-publishing OIDC
|
||||
actions: read # for the env protection verification step
|
||||
env:
|
||||
IN_DOCKER: 'True'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
steps:
|
||||
- name: Verify release environment is protected (fail-closed)
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ENV_NAME="release"
|
||||
if ! RESP="$(gh api -H 'Accept: application/vnd.github+json' \
|
||||
"/repos/${GITHUB_REPOSITORY}/environments/${ENV_NAME}" 2>/dev/null)"; then
|
||||
echo "::error::Environment '${ENV_NAME}' does not exist (or is inaccessible). Create it at https://github.com/${GITHUB_REPOSITORY}/settings/environments/new with required reviewers and prevent_self_review enabled, then re-run."
|
||||
exit 1
|
||||
fi
|
||||
HAS_REVIEWERS="$(echo "$RESP" | jq -r '[.protection_rules[]? | select(.type == "required_reviewers")] | length')"
|
||||
if [ "${HAS_REVIEWERS:-0}" -lt 1 ]; then
|
||||
echo "::error::Environment '${ENV_NAME}' exists but has no required-reviewers protection rule."
|
||||
exit 1
|
||||
fi
|
||||
SELF_REVIEW_BLOCKED="$(echo "$RESP" | jq -r '[.protection_rules[]? | select(.type=="required_reviewers") | .prevent_self_review] | any')"
|
||||
if [ "$SELF_REVIEW_BLOCKED" != "true" ]; then
|
||||
echo "::error::Environment '${ENV_NAME}' must have prevent_self_review enabled. Without it, the gate collapses to one identity when the dispatcher is also a reviewer."
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::Environment '${ENV_NAME}' is protected with prevent_self_review. OK."
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
- run: uv sync
|
||||
|
||||
- run: uv run --no-sync ruff check --no-fix --select PLE # quick check for syntax errors to avoid waiting time doing the rest of the build
|
||||
- run: uv build
|
||||
|
||||
# - name: Detect installed Playwright version
|
||||
# run: echo "PLAYWRIGHT_VERSION=$(uv pip list --format json | jq -r '.[] | select(.name == "playwright") | .version')" >> $GITHUB_ENV
|
||||
|
||||
# - name: Cache playwright binaries
|
||||
# uses: actions/cache@v3
|
||||
# with:
|
||||
# path: |
|
||||
# ~/.cache/ms-playwright
|
||||
# key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
|
||||
|
||||
- run: uvx playwright install chrome
|
||||
- run: uvx playwright install chromium
|
||||
|
||||
# TODO: just depend on the other test.yml action for this instead of re-running the tests here
|
||||
# - run: uv run pytest tests/ci/test_tools.py # final sanity check: run a few of the tests before release
|
||||
|
||||
# publish to PyPI
|
||||
- run: uv publish --trusted-publishing always
|
||||
- name: Push to stable branch (if stable release)
|
||||
if: github.event_name == 'release' && !contains(github.ref_name, 'rc')
|
||||
run: |
|
||||
git checkout -b stable
|
||||
git push origin -f stable
|
||||
@@ -0,0 +1,108 @@
|
||||
name: 'Manage stale issues and PRs'
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 2 * * *' # Run daily at 2:00 AM UTC
|
||||
workflow_dispatch: # Allow manual triggering
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
|
||||
with:
|
||||
# General settings
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Days before marking as stale (more lenient for AI/browser automation project)
|
||||
days-before-stale: 60
|
||||
days-before-close: 14
|
||||
|
||||
# Different timing for PRs vs issues
|
||||
days-before-pr-stale: 45
|
||||
days-before-pr-close: 14
|
||||
|
||||
# Stale labels
|
||||
stale-issue-label: 'stale'
|
||||
stale-pr-label: 'stale'
|
||||
|
||||
# Remove stale label when there's activity
|
||||
remove-stale-when-updated: true
|
||||
remove-issue-stale-when-updated: true
|
||||
remove-pr-stale-when-updated: true
|
||||
|
||||
# Messages
|
||||
stale-issue-message: |
|
||||
👋 This issue has been automatically marked as stale because it hasn't had activity for 60 days.
|
||||
|
||||
**⚡ We've made significant progress recently!** Please test with the latest version of browser-use to see if this issue has been resolved. If the issue persists, please let us know by commenting below.
|
||||
|
||||
**To keep this issue open:**
|
||||
- Add a comment explaining why this is still relevant after testing the latest version
|
||||
- Add the `pinned` label if this is an important long-term issue
|
||||
- Reference it in a PR if you're working on a fix
|
||||
|
||||
**This will be automatically closed in 14 days** if no further activity occurs.
|
||||
|
||||
Thanks for contributing to browser-use! 🤖 If you have questions, join our [Discord](https://discord.gg/uC9hDSbt).
|
||||
|
||||
stale-pr-message: |
|
||||
👋 This PR has been automatically marked as stale because it hasn't had activity for 45 days.
|
||||
|
||||
**To keep this PR open:**
|
||||
- Rebase against the latest main branch
|
||||
- Address any review feedback or merge conflicts
|
||||
- Add a comment explaining the current status
|
||||
- Add the `work-in-progress` label if you're still actively working on this
|
||||
|
||||
**This will be automatically closed in 14 days** if no further activity occurs.
|
||||
|
||||
Thanks for contributing to browser-use! 🤖
|
||||
|
||||
close-issue-message: |
|
||||
🔒 This issue was automatically closed because it was stale for 14 days with no activity.
|
||||
|
||||
**Don't worry!** If this issue is still relevant:
|
||||
- **First, test with the latest version** - we've made tons of improvements recently!
|
||||
- **Reopen it** if you have permissions and the issue persists
|
||||
- **Create a fresh issue** with updated information if the problem still exists after testing the latest version
|
||||
- **Join our [Discord](https://discord.gg/uC9hDSbt)** to discuss
|
||||
|
||||
We appreciate your contribution to browser-use! 🤖
|
||||
|
||||
close-pr-message: |
|
||||
🔒 This PR was automatically closed because it was stale for 14 days with no activity.
|
||||
|
||||
**Don't worry!** If you'd like to continue this work:
|
||||
- **Reopen this PR** and rebase against main
|
||||
- **Create a fresh PR** with updated changes
|
||||
- **Join our [Discord](https://discord.gg/uC9hDSbt)** if you need help
|
||||
|
||||
Thanks for contributing to browser-use! 🤖
|
||||
|
||||
# Comprehensive exemptions for AI/browser automation project
|
||||
exempt-issue-labels: 'pinned,security,bug,enhancement,good-first-issue,help-wanted,documentation,ci,breaking-change,feature-request,roadmap'
|
||||
exempt-pr-labels: 'pinned,work-in-progress,wip,breaking-change,security,dependencies,ci'
|
||||
exempt-milestones: true
|
||||
exempt-all-assignees: true
|
||||
exempt-all-pr-assignees: true
|
||||
|
||||
# Don't mark issues/PRs stale if they have recent PR references
|
||||
exempt-pr-author: true
|
||||
|
||||
# Advanced settings
|
||||
operations-per-run: 200 # More conservative to avoid rate limits
|
||||
ascending: true # Process oldest issues first
|
||||
|
||||
# Enable debug output
|
||||
debug-only: false
|
||||
|
||||
# Only process issues/PRs, not drafts
|
||||
include-only-assigned: false
|
||||
|
||||
# Additional safety: don't close issues with many reactions (community interest)
|
||||
ignore-issue-updates: false
|
||||
ignore-pr-updates: false
|
||||
@@ -0,0 +1,337 @@
|
||||
name: test
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
pull-requests: write # Allow writing comments on PRs
|
||||
issues: write # Allow writing comments on issues
|
||||
statuses: write # Allow writing statuses on PRs
|
||||
discussions: write
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
setup-chromium:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
|
||||
- name: Get week number for cache key
|
||||
id: week
|
||||
run: echo "number=$(date +%Y-W%U)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache chromium binaries
|
||||
id: cache-chromium
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-chromium-${{ steps.week.outputs.number }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ runner.arch }}-chromium-
|
||||
|
||||
- name: Install Chromium if not cached
|
||||
if: steps.cache-chromium.outputs.cache-hit != 'true'
|
||||
run: uvx playwright install chromium --with-deps --no-shell
|
||||
|
||||
find_tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5 # Prevent hanging
|
||||
outputs:
|
||||
TEST_FILENAMES: ${{ steps.lsgrep.outputs.TEST_FILENAMES }}
|
||||
# ["test_browser", "test_tools", "test_browser_session", "test_tab_management", ...]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Force fresh checkout to avoid any caching issues
|
||||
fetch-depth: 1
|
||||
- id: lsgrep
|
||||
run: |
|
||||
echo "🔍 Discovering test files at $(date)"
|
||||
echo "Git commit: $(git rev-parse HEAD)"
|
||||
echo "Git branch: $(git branch --show-current)"
|
||||
echo ""
|
||||
|
||||
TEST_FILENAMES="$(find tests/ci -name 'test_*.py' -type f | sed 's|^tests/ci/||' | sed 's|\.py$||' | jq -R -s -c 'split("\n")[:-1]')"
|
||||
echo "TEST_FILENAMES=${TEST_FILENAMES}" >> "$GITHUB_OUTPUT"
|
||||
echo "📋 Test matrix: $TEST_FILENAMES"
|
||||
# https://code.dblock.org/2021/09/03/generating-task-matrix-by-looping-over-repo-files-with-github-actions.html
|
||||
- name: Check that at least one test file is found
|
||||
run: |
|
||||
if [ -z "${{ steps.lsgrep.outputs.TEST_FILENAMES }}" ]; then
|
||||
echo "Failed to find any test_*.py files in tests/ci/ folder!" > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tests:
|
||||
needs: [setup-chromium, find_tests]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 4 # Reduced timeout - tests should complete quickly or retry
|
||||
env:
|
||||
IN_DOCKER: 'True'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
BROWSER_USE_LOGGING_LEVEL: 'DEBUG'
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
|
||||
BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }}
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
strategy:
|
||||
matrix:
|
||||
test_filename: ${{ fromJson(needs.find_tests.outputs.TEST_FILENAMES || '["FAILED_TO_DISCOVER_TESTS"]') }}
|
||||
# autodiscovers all the files in tests/ci/test_*.py
|
||||
# - test_browser
|
||||
# - test_tools
|
||||
# - test_browser_session
|
||||
# - test_tab_management
|
||||
# ... and more
|
||||
name: ${{ matrix.test_filename }}
|
||||
steps:
|
||||
- name: Check that the previous step managed to find some test files for us to run
|
||||
run: |
|
||||
if [[ "${{ matrix.test_filename }}" == "FAILED_TO_DISCOVER_TESTS" ]]; then
|
||||
echo "Failed get list of test files in tests/ci/test_*.py from find_tests job" > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
|
||||
- name: Cache uv packages and venv
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-venv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-venv-
|
||||
|
||||
- run: uv sync --dev --all-extras --refresh-package browser-use-core
|
||||
|
||||
- name: Get week number for cache key
|
||||
id: week
|
||||
run: echo "number=$(date +%Y-W%U)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache chromium binaries
|
||||
id: cache-chromium
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-chromium-${{ steps.week.outputs.number }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ runner.arch }}-chromium-
|
||||
|
||||
- name: Install Chromium browser if not cached
|
||||
if: steps.cache-chromium.outputs.cache-hit != 'true'
|
||||
run: uvx playwright install chromium --with-deps --no-shell
|
||||
|
||||
- name: Cache browser-use extensions
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.config/browseruse/extensions
|
||||
key: ${{ runner.os }}-browseruse-extensions-${{ hashFiles('browser_use/browser/profile.py') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-browseruse-extensions-
|
||||
|
||||
- name: Check if test file exists
|
||||
id: check-file
|
||||
run: |
|
||||
TEST_FILE="tests/ci/${{ matrix.test_filename }}.py"
|
||||
if [ -f "$TEST_FILE" ]; then
|
||||
echo "exists=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ Test file found: $TEST_FILE"
|
||||
else
|
||||
echo "exists=false" >> $GITHUB_OUTPUT
|
||||
echo "❌ Test file not found: $TEST_FILE"
|
||||
echo "This file may have been renamed or removed. Current test files:"
|
||||
find tests/ci -name 'test_*.py' -type f | sed 's|tests/ci/||' | sed 's|\.py$||' | sort
|
||||
fi
|
||||
|
||||
- name: Run test with retry
|
||||
if: steps.check-file.outputs.exists == 'true'
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 4
|
||||
max_attempts: 1
|
||||
retry_on: error
|
||||
command: pytest "tests/ci/${{ matrix.test_filename }}.py"
|
||||
|
||||
evaluate-tasks:
|
||||
needs: setup-chromium
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8 # Allow more time for agent eval
|
||||
env:
|
||||
IN_DOCKER: 'true'
|
||||
BROWSER_USE_CLOUD_SYNC: 'false'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
BROWSER_USE_LOGGING_LEVEL: 'DEBUG'
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
|
||||
- name: Cache uv packages and venv
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-venv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-venv-
|
||||
|
||||
- run: uv sync --dev --all-extras --refresh-package browser-use-core
|
||||
|
||||
- name: Get week number for cache key
|
||||
id: week
|
||||
run: echo "number=$(date +%Y-W%U)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache chromium binaries
|
||||
id: cache-chromium
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-chromium-${{ steps.week.outputs.number }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ runner.arch }}-chromium-
|
||||
|
||||
- name: Install Chromium browser if not cached
|
||||
if: steps.cache-chromium.outputs.cache-hit != 'true'
|
||||
run: uvx playwright install chromium --with-deps --no-shell
|
||||
|
||||
- name: Cache browser-use extensions
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.config/browseruse/extensions
|
||||
key: ${{ runner.os }}-browseruse-extensions-${{ hashFiles('browser_use/browser/profile.py') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-browseruse-extensions-
|
||||
|
||||
- name: Run agent tasks evaluation and capture score
|
||||
id: eval
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 4
|
||||
max_attempts: 1
|
||||
retry_on: error
|
||||
command: |
|
||||
python tests/ci/evaluate_tasks.py > result.txt
|
||||
cat result.txt
|
||||
echo "PASSED=$(grep '^PASSED=' result.txt | cut -d= -f2)" >> $GITHUB_ENV
|
||||
echo "TOTAL=$(grep '^TOTAL=' result.txt | cut -d= -f2)" >> $GITHUB_ENV
|
||||
echo "DETAILED_RESULTS=$(grep '^DETAILED_RESULTS=' result.txt | cut -d= -f2-)" >> $GITHUB_ENV
|
||||
|
||||
- name: Print agent evaluation summary
|
||||
run: |
|
||||
echo "Agent tasks passed: $PASSED / $TOTAL"
|
||||
|
||||
- name: Write agent evaluation summary to workflow overview
|
||||
run: |
|
||||
if [ "$PASSED" = "$TOTAL" ]; then
|
||||
COLOR="green"
|
||||
else
|
||||
COLOR="yellow"
|
||||
fi
|
||||
echo "<h2>Agent Tasks Score: <span style='color:$COLOR;'>$PASSED/$TOTAL</span></h2>" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Comment PR with agent evaluation results
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@v7
|
||||
continue-on-error: true
|
||||
with:
|
||||
script: |
|
||||
const passed = parseInt(process.env.PASSED);
|
||||
const total = parseInt(process.env.TOTAL);
|
||||
const detailedResults = JSON.parse(process.env.DETAILED_RESULTS);
|
||||
const score = `${passed}/${total}`;
|
||||
const percentage = Math.round((passed / total) * 100);
|
||||
|
||||
// Fail the workflow if 0% pass rate
|
||||
if (percentage === 0) {
|
||||
core.setFailed(`Evaluation failed: 0% pass rate (${passed}/${total})`);
|
||||
}
|
||||
|
||||
// Create detailed table
|
||||
let tableRows = '';
|
||||
detailedResults.forEach(result => {
|
||||
const emoji = result.success ? '✅' : '❌';
|
||||
const status = result.success ? 'Pass' : 'Fail';
|
||||
tableRows += `| ${result.task} | ${emoji} ${status} | ${result.reason} |\n`;
|
||||
});
|
||||
|
||||
const comment = `## Agent Task Evaluation Results: ${score} (${percentage}%)
|
||||
|
||||
<details>
|
||||
<summary>View detailed results</summary>
|
||||
|
||||
| Task | Result | Reason |
|
||||
|------|--------|--------|
|
||||
${tableRows}
|
||||
|
||||
Check the [evaluate-tasks job](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for detailed task execution logs.
|
||||
</details>`;
|
||||
|
||||
// Find existing comment to update or create new one
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
|
||||
const botComment = comments.find(comment =>
|
||||
comment.user.type === 'Bot' &&
|
||||
comment.body.includes('Agent Task Evaluation Results')
|
||||
);
|
||||
|
||||
if (botComment) {
|
||||
// Update existing comment
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: botComment.id,
|
||||
body: comment
|
||||
});
|
||||
} else {
|
||||
// Create new comment
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body: comment
|
||||
});
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
# Cache files
|
||||
.DS_Store
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.ipynb_checkpoints
|
||||
~/
|
||||
|
||||
# Virtual Environments
|
||||
.venv*
|
||||
venv/
|
||||
|
||||
# IDEs
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Build files
|
||||
dist/
|
||||
|
||||
# Data files
|
||||
*.gif
|
||||
*.txt
|
||||
*.pdf
|
||||
*.csv
|
||||
*.json
|
||||
*.jsonl
|
||||
*.log
|
||||
*.bak
|
||||
|
||||
# Secrets and sensitive files
|
||||
secrets.env
|
||||
.env
|
||||
browser_cookies.json
|
||||
cookies.json
|
||||
gcp-login.json
|
||||
saved_trajectories/
|
||||
old_tests/
|
||||
AgentHistory.json
|
||||
AgentHistoryList.json
|
||||
private_example.py
|
||||
private_example
|
||||
CLAUDE.local.md
|
||||
|
||||
uv.lock
|
||||
temp
|
||||
tmp
|
||||
|
||||
# Google API credentials
|
||||
credentials.json
|
||||
token.json
|
||||
|
||||
!docs/docs.json
|
||||
|
||||
|
||||
temp-profile-*
|
||||
|
||||
screenshot.png
|
||||
|
||||
# *.md
|
||||
|
||||
all_github_issues_progress.md
|
||||
all_github_issues.md
|
||||
|
||||
todo-input-token.md
|
||||
|
||||
TOOL_CHANGES_SUMMARY.md
|
||||
|
||||
|
||||
claude-code-todo
|
||||
result_judge.md
|
||||
result.md
|
||||
result2.md
|
||||
result3.md
|
||||
Brainstorm.md
|
||||
example.ipynb
|
||||
*SUMMARY.md
|
||||
todo.md
|
||||
product_extraction.ipynb
|
||||
product_extraction.py
|
||||
*report.md
|
||||
plot.py
|
||||
|
||||
.claude/
|
||||
@@ -0,0 +1,67 @@
|
||||
default_language_version:
|
||||
python: python3.11
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/asottile/yesqa
|
||||
rev: v1.5.0
|
||||
hooks:
|
||||
- id: yesqa
|
||||
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
hooks:
|
||||
- id: codespell # See pyproject.toml for args
|
||||
additional_dependencies:
|
||||
- tomli
|
||||
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.20.0
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: [--py311-plus]
|
||||
|
||||
# - repo: https://github.com/asottile/add-trailing-comma
|
||||
# rev: v3.1.0
|
||||
# hooks:
|
||||
# - id: add-trailing-comma
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.12.10
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [ --fix ]
|
||||
- id: ruff-format
|
||||
# see pyproject.toml for more details on ruff config
|
||||
|
||||
- repo: https://github.com/RobertCraigie/pyright-python
|
||||
rev: v1.1.404
|
||||
hooks:
|
||||
- id: pyright
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
# check for basic syntax errors in python and data files
|
||||
- id: check-ast
|
||||
- id: check-toml
|
||||
- id: check-yaml
|
||||
- id: check-json
|
||||
- id: check-merge-conflict
|
||||
# check for bad files and folders
|
||||
- id: check-symlinks
|
||||
- id: destroyed-symlinks
|
||||
- id: check-case-conflict
|
||||
- id: check-illegal-windows-names
|
||||
- id: check-shebang-scripts-are-executable
|
||||
- id: mixed-line-ending
|
||||
- id: fix-byte-order-marker
|
||||
- id: end-of-file-fixer
|
||||
# best practices enforcement
|
||||
- id: detect-private-key
|
||||
# - id: check-docstring-first
|
||||
- id: debug-statements
|
||||
- id: forbid-submodules
|
||||
- id: check-added-large-files
|
||||
args: ["--maxkb=600"]
|
||||
# - id: name-tests-test
|
||||
# args: ["--pytest-test-first"]
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
@@ -0,0 +1,348 @@
|
||||
# Rust Agent Integration Feature/Proof Ledger
|
||||
|
||||
This branch keeps the Python `Agent` unchanged unless callers explicitly import
|
||||
`from browser_use.beta import Agent`.
|
||||
|
||||
## Current Features
|
||||
|
||||
1. Rust SDK server execution path
|
||||
- `browser_use.beta.Agent` now runs normal tasks through the terminal SDK server
|
||||
(`browser-use-terminal sdk-server --transport stdio`) using the normalized
|
||||
`agent.run_task` request/response protocol.
|
||||
- Follow-up tasks reuse the returned SDK `agent_id`, `session_id`, and `browser_id`
|
||||
through `agent.run`, so the Python-facing interface can keep one Rust-owned session.
|
||||
- Browser-use-style options are mapped into SDK params, including model/provider,
|
||||
CDP URL/headers, viewport, user agent, storage state, downloads path, structured
|
||||
output schema, max steps, vision, cost calculation, and action limits.
|
||||
- The returned normalized event history is reconstructed into Browser Use-compatible
|
||||
`AgentHistoryList`, callbacks, usage, telemetry, Laminar replay, downloads, and
|
||||
final result handling.
|
||||
- The SDK stdout reader accepts large JSON-RPC response lines by reading stdout in
|
||||
chunks and splitting newline-delimited JSON-RPC manually. This avoids asyncio's
|
||||
separator-limit failure when normalized history includes screenshots, tool
|
||||
output, and observability data in a single response line.
|
||||
- SDK server `agent.event` / `agent.projected_event` notifications are retained and
|
||||
surfaced as concise in-flight progress logs, so GitHub-runner evals can show where
|
||||
a Rust-backed run is spending time before the final history response arrives.
|
||||
- Terminal SDK `agent.run` now mirrors the live executor by passing the latest durable
|
||||
task/follow-up input into `RuntimeHandle::run_agent` as `initial_input`, so SDK
|
||||
`agent.run_task` enters the runtime-owned loop with `agent.input.accepted` and
|
||||
`agent.input.consumed` events instead of stalling after browser creation.
|
||||
- Browser-use `llm_timeout`/SDK `llm.timeout` now reaches the terminal Rust
|
||||
model stream path as both a response-open timeout and a stream-idle timeout,
|
||||
so a provider request that never returns response headers, or a stream that
|
||||
opens and then sends no SSE bytes, becomes a retryable transport error instead
|
||||
of holding a GitHub eval runner indefinitely after `model.turn.request`.
|
||||
- Terminal SDK `agent.run_task` now drives the runtime on a multi-thread Tokio
|
||||
runtime, matching the live model transport bridge's `block_in_place` usage.
|
||||
This prevents SDK evals from stalling after `model.turn.request` before the
|
||||
configured response-open/stream-idle timeout can make progress.
|
||||
- Running `browser_script` calls now preserve the `run_id` and observe
|
||||
instruction in Rust event persistence, replay reconstruction, and Python
|
||||
`AgentHistoryList` reconstruction. This prevents eval traces from showing
|
||||
empty browser tool results while a script is still active and keeps the
|
||||
next model turn on the intended observe/cancel path instead of repeatedly
|
||||
navigating or reconnecting.
|
||||
- Simple initial navigation actions on CDP-backed Rust runs are now left for
|
||||
the Rust SDK by default, so the first navigation is a model-visible
|
||||
`browser_script` action with terminal page-state output instead of a hidden
|
||||
Python-side BrowserSession preload. The old direct CDP preload remains
|
||||
available behind `BROWSER_USE_RUST_DIRECT_INITIAL_NAVIGATION=1`.
|
||||
- Structured `browser_script` lifecycle events now preserve outputs, summaries,
|
||||
images, and browser state through terminal event persistence and Python
|
||||
history reconstruction. This prevents the first post-navigation page probe
|
||||
from appearing blank in eval/Laminar history when the script emitted only
|
||||
`emit_output(...)` or screenshots and no stdout text.
|
||||
- Printed `browser_script` page probes are also used to reconstruct browser
|
||||
state when they contain `page_info()` dictionaries, `list_tabs()` rows, or
|
||||
a bare URL. This keeps `Page State` aligned with visible tool output even
|
||||
when the script used `print(info)` instead of `emit_output(info, ...)`.
|
||||
- Terminal `list_tabs()` hides the internal `Starting agent ...` about:blank
|
||||
placeholder tab. This prevents the model from mistaking the startup tab
|
||||
for a user-relevant target and spending turns on unnecessary
|
||||
reconnect/reattach recovery after a successful navigation.
|
||||
- Terminal Rust runs only advertise sub-agent tools when the run has a
|
||||
configured child-agent runner, and SDK JSON-RPC runs now attach the same
|
||||
runtime-backed child runner as CLI runs. This enables terminal's advanced
|
||||
sub-agent system in `browser_use.beta.Agent` evals without exposing
|
||||
spawn tools on unsupported run surfaces.
|
||||
- Eval GitHub runners now refresh Convex progress during long `run_agent`
|
||||
and `evaluate` stages, so slow Rust or Agent SDK judge calls stay visible
|
||||
instead of looking stale while they are still legitimately running.
|
||||
- Terminal provider conversion now downsamples oversized data-URL screenshots
|
||||
before Anthropic requests. This keeps visual context in the model input
|
||||
while avoiding Claude's many-image 2000px dimension rejection, which was
|
||||
causing completed eval tasks to show a one-step HTTP 400 error and an
|
||||
empty final result.
|
||||
- Terminal direct Anthropic Messages serialization now also downsamples
|
||||
oversized inline `ContentPart::Media` base64 screenshots before request
|
||||
construction. This covers Rust live-model transports that bypass provider
|
||||
JSON normalization and previously still hit the same 2000px many-image
|
||||
rejection.
|
||||
- Terminal navigation helpers now wait briefly for page readiness and return
|
||||
`navigation_ready` plus `page_info` in the same `browser_script` result.
|
||||
This gives the next model turn concrete evidence that navigation landed,
|
||||
instead of a bare "navigation sent" placeholder that can lead to repeated
|
||||
navigate/status/recover loops on Cloud Browser CDP sessions.
|
||||
- Terminal `browser_script observe` now waits through the requested observe
|
||||
window instead of returning on the first partial stream event. If a
|
||||
navigation or extraction script emits partial page events and then finishes
|
||||
shortly after, the model receives the final result in the same tool call
|
||||
rather than spending extra LLM turns polling the same `run_id`.
|
||||
- Terminal `browser_script` start now auto-collects a previous active run
|
||||
that has already finished or timed out. If the model starts another browser
|
||||
action immediately after navigation completed in the background, the tool
|
||||
returns the completed navigation/page result in that same call instead of
|
||||
forcing a separate observe/status/recover turn.
|
||||
- Terminal remote-CDP attach now reuses an existing ordinary blank page target
|
||||
before creating a new `about:blank` tab. Browser Use Cloud sessions therefore
|
||||
begin with one stable controlled tab instead of splitting Python setup and
|
||||
Rust execution across separate blank targets.
|
||||
- Eval payloads preserve Rust/browser-use usage fields and add dashboard
|
||||
aliases (`input_tokens`, `output_tokens`, cached/cache-creation tokens, and
|
||||
`cost_usd`) before saving. This keeps cost/token displays working for Rust
|
||||
histories without changing the canonical browser-use usage structure.
|
||||
- Browser-use Rust CDP initial navigation now waits for a concrete
|
||||
post-navigation browser state summary and passes the observed current
|
||||
URL/title into the Rust task context. Start-URL tasks should begin by
|
||||
inspecting or extracting from the already-loaded page instead of spending
|
||||
early turns on repeated navigation/status recovery.
|
||||
- Browser-use Rust direct CDP pre-navigation now only records an initial
|
||||
navigation as completed when the observed browser state matches the
|
||||
requested URL. If the Cloud Browser remains on `about:blank` or another
|
||||
mismatched target, the original navigation stays in the Rust task context
|
||||
instead of giving the model a false "already loaded" state.
|
||||
- Eval GitHub runners now preserve visible output for interrupted Rust runs:
|
||||
if an agent is cancelled or errors after collecting history, formatting
|
||||
synthesizes a partial final answer from the latest memory/tool evidence;
|
||||
if no history exists, the saved dashboard payload still includes a
|
||||
non-empty failure response with the stage error.
|
||||
- Rust `token_count` usage reconstruction now treats summed per-turn
|
||||
`last_token_usage` as the billed usage source when it exceeds the latest
|
||||
cumulative context counters. Dashboard usage therefore reflects the whole
|
||||
agent run instead of only the latest context-sized prompt after
|
||||
recompute/compaction paths.
|
||||
- Rust cost-enabled usage reconstruction now reports token totals from the
|
||||
same summed per-turn usage entries used to calculate costs. This keeps
|
||||
dashboard token counts, cached/cache-creation buckets, and cost fields on
|
||||
one basis instead of showing summed cost beside latest-context token
|
||||
counters.
|
||||
- Rust cost-enabled usage reconstruction now treats `token_count` as a
|
||||
fallback billing source when provider `model.usage` events are absent.
|
||||
Mixed streams therefore do not double-count usage by pricing both provider
|
||||
usage and context occupancy counters.
|
||||
- Terminal SDK JSON-RPC responses are now bounded before they are written to
|
||||
stdio. Oversized final histories compact durable event payloads while
|
||||
preserving final output, success state, errors, usage, files, and recent
|
||||
events, so a completed run cannot lose its final answer because the Python
|
||||
wrapper hits its newline-delimited frame limit.
|
||||
- Browser-use Rust SDK runs now prefer live `agent.event` notifications when
|
||||
the final SDK response is missing, truncated, or smaller than the already
|
||||
observed stream. If a run is cancelled or the final transport fails after
|
||||
`session.done`, Python reconstructs `AgentHistoryList`, usage, and final
|
||||
output from the notification stream instead of returning an empty result.
|
||||
- Browser-use Rust SDK notification recovery now accepts both top-level
|
||||
`event_type` records and nested SDK event envelopes. Runs that visibly emit
|
||||
`session.done` in GitHub runner progress logs therefore reconstruct the
|
||||
final answer even if the final JSON-RPC history response is empty or
|
||||
compacted differently.
|
||||
- Browser-use Rust SDK notification recovery now normalizes both
|
||||
`agent.event` and `agent.projected_event` notifications into the retained
|
||||
event history. Runs whose progress logs show projected `session.done` or
|
||||
`agent.completed` events therefore no longer fall back to
|
||||
"Rust terminal session did not produce a final result."
|
||||
- Browser-use Rust SDK history reconstruction now prefers retained live
|
||||
notifications when the final JSON-RPC response history is present but lacks
|
||||
a final result. Stale response-side compaction errors no longer override a
|
||||
`session.done` event that Python already observed from the stream.
|
||||
- Browser-use Rust SDK billing reconstruction now falls back to the SDK
|
||||
response's aggregate `history.usage` when compacted histories omit usage
|
||||
events. This keeps dashboard usage nonzero even when the event payload is
|
||||
compacted before it reaches Python.
|
||||
- Terminal provider overload errors are treated as retryable transient
|
||||
capacity failures. This prevents a single Claude/OpenAI `server overloaded`
|
||||
response from becoming an immediate no-output eval failure.
|
||||
- Terminal `browser_script observe` now defaults to a coarse 30 second wait,
|
||||
clamps too-small observe requests up to that window, and allows waits up to
|
||||
120 seconds. Long navigation/extraction scripts therefore spend fewer model
|
||||
turns polling the same `run_id` and are less likely to hit the step limit
|
||||
before finalizing partial evidence.
|
||||
- Terminal now includes a locally executed DuckDuckGo Lite `search` tool from
|
||||
the terminal web-search merge. URL-finding and general web-search tasks can
|
||||
discover candidate pages without spending browser-navigation turns on search
|
||||
engine result pages.
|
||||
- Terminal SDK histories now return child-agent events separately from parent
|
||||
events and expose a combined `usage_events` stream. Browser-use prices and
|
||||
traces against that combined stream, so sub-agent model calls contribute to
|
||||
run token/cost totals without letting child `session.done` events override
|
||||
the parent final answer.
|
||||
- `browser_use.beta.Agent` no longer treats `_run_process` monkeypatches as an
|
||||
alternate runtime. Production `run()` and `follow_up()` now require the
|
||||
Rust SDK server path, so the legacy terminal CLI adapter cannot silently
|
||||
replace the new server protocol while the normal Python `browser_use.Agent`
|
||||
remains unchanged.
|
||||
- Terminal SDK browser requests now preserve and forward additional
|
||||
Browser Use-style browser settings: proxy country, local profile label,
|
||||
allowed/blocked domains, window size, and state directory. For cloud
|
||||
browser runs, proxy country is passed to `browser remote start`, so Rust
|
||||
SDK sessions can use Browser Use Cloud browser proxies through the same
|
||||
Python-facing `browser_profile`/`browser_session` options.
|
||||
- Browser-use now deduplicates SDK response/projected events before
|
||||
reconstructing history and usage. Duplicate `token_count` and
|
||||
`session.done` records no longer inflate eval history, dashboard usage, or
|
||||
telemetry payloads when the terminal server returns both live and projected
|
||||
event streams.
|
||||
- Browser-use no longer sends a Rust SDK `tool_allowlist` override. The
|
||||
terminal SDK server now owns its tool registry for `browser_use.beta.Agent`
|
||||
runs, so local search, web/search helpers, and v2 sub-agent controls are
|
||||
available in evals when the Rust core registers them.
|
||||
- Reconstructed browser history ignores internal browser-control endpoint
|
||||
URLs such as `http://127.0.0.1:<port>` while still preserving genuine local
|
||||
page URLs. Eval traces therefore show user-visible pages instead of CDP
|
||||
helper endpoints.
|
||||
- The Rust Agent default LLM resolution follows the normal browser-use path:
|
||||
explicit `DEFAULT_LLM` wins, otherwise `ChatBrowserUse()` is used. Real
|
||||
eval runs still pass Claude Sonnet 4.6 explicitly through the SDK request.
|
||||
- Terminal `http_get_many(...)` and `browser_fetch(..., return_error=True)`
|
||||
now return recoverable error records that are both dict-compatible and
|
||||
response-compatible (`.status_code`, `.status`, `.headers`, `.text`,
|
||||
`.content`, `.url`). General browser-script code can inspect a failed
|
||||
helper response without crashing the whole step with
|
||||
`AttributeError: 'dict' object has no attribute 'status_code'`.
|
||||
- Terminal `js(...)` in `browser_script` now tolerates common anonymous
|
||||
function snippets and async function-IIFE snippets emitted by agents. This
|
||||
avoids repeated generic JavaScript syntax failures such as
|
||||
`Function statements require a function name` and `await is only valid in
|
||||
async functions` without adding task- or domain-specific behavior.
|
||||
- Terminal SDK `agent.create` now persists the initial task input only through
|
||||
the runtime observed-event path. SDK LLM prompts and Laminar spans therefore
|
||||
contain the task once instead of duplicating the same initial user message,
|
||||
which improves trace fidelity and reduces avoidable prompt/cache churn.
|
||||
|
||||
## Current Proof
|
||||
|
||||
- terminal `cargo check -p browser-use-cli`
|
||||
- terminal `cargo test -p browser-use-cli sdk_ -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_run_runtime_supports_model_transport_blocking_bridge -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent running_browser_script -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent runtime_browser_backend_records_script_lifecycle -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_list_tabs_hides_agent_startup_placeholder -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent subagent_tools -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent spawn_agent_agent_type_guidance_discourages_default_override -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-providers anthropic_messages_downsamples_oversized_tool_images -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-providers anthropic_messages -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-llm build_body_downsamples_oversized_inline_media_for_anthropic -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-llm anthropic_messages -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_navigation_helpers_wait_for_page_state -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_start_observe_finishes_slow_scripts -- --test-threads=1 --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_observe_waits_for_completion_after_partial_output --lib`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_observe --lib`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_start_observe_finishes_slow_scripts --lib`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_start_ -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_observe_is_idempotent_after_completion -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser remote_cdp_attach_reuses_existing_blank_page_before_creating_target -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_json_rpc_agent_run_task_executes_fake_backend_with_normalized_history -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-llm stream_ -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_provider_run_config_maps_browser_use_options_to_rust_core -- --nocapture`
|
||||
- browser-use `uv run python -m py_compile browser_use/beta/service.py`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py -k 'browser_script_lifecycle_outputs_as_result or initial_actions_pre_navigate_existing_cdp_session or run_hands_off_completed_initial_navigation_as_context' -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py -k 'printed_browser_script_page_info_as_state or browser_script_lifecycle_outputs_as_result or initial_actions_pre_navigate_existing_cdp_session or run_hands_off_completed_initial_navigation_as_context' -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_rust_history_surfaces_running_browser_script_observe_instruction -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py -k 'initial_actions_pre_navigate_existing_cdp_session or run_executes_initial_actions_before_sdk or run_hands_off_completed_initial_navigation_as_context' -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py -k 'sdk_client_reads_large_json_rpc_lines or sdk_and_reuses_session or translates_browser_use_args_to_terminal'`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py -k 'sdk_client_queues_agent_notifications_before_response or sdk_client_reads_large_json_rpc_lines or sdk_and_reuses_session or translates_browser_use_args_to_terminal'`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py -k 'rust_sdk_client_reads_large_json_rpc_lines or rust_sdk_client_queues_agent_notifications_before_response' -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_rust_sdk_client_reads_large_json_rpc_lines tests/ci/test_beta_agent.py::test_beta_agent_run_leaves_initial_navigation_for_sdk_by_default tests/ci/test_beta_agent.py::test_beta_agent_initial_actions_can_pre_navigate_existing_cdp_session tests/ci/test_beta_agent.py::test_beta_agent_translates_browser_use_args_to_terminal -q`
|
||||
- browser-use `PYTHONPATH=. uv run pytest tests/ci/test_beta_agent.py -q -k 'pre_navigates_cdp_session_before_sdk_by_default or initial_actions_can_pre_navigate_existing_cdp_session or direct_initial_navigation_defaults_on_for_cdp or direct_initial_navigation_can_be_disabled'`
|
||||
- browser-use `uv run pytest -q tests/ci/test_beta_agent.py -k "pre_navigates_cdp_session_before_sdk_by_default or keeps_initial_navigation_when_direct_state_mismatches or initial_actions_can_pre_navigate_existing_cdp_session or direct_initial_navigation_defaults_on_for_cdp"`
|
||||
- browser-use `uv run pytest -q tests/ci/test_beta_agent.py -k "terminal_token_count_usage or sums_token_count_last_usage_when_latest_total_underreports or terminal_usage_prices_token_count_events or terminal_usage_sums_token_count_cache_creation"`
|
||||
- browser-use `uv run pytest -q tests/ci/test_beta_agent.py -k "terminal_nested_model_usage or token_count_does_not_shrink_model_usage_totals or terminal_usage_prices_token_count_events or terminal_usage_prices_anthropic_raw_cache_reads or terminal_usage_sums_token_count_cache_creation or priced_summary_sums_cache_read_tokens or mixed_events_do_not_shrink_totals or priced_usage_prefers_model_usage_over_token_count or sums_token_count_last_usage_when_latest_total_underreports"`
|
||||
- browser-use `python -m py_compile browser_use/beta/service.py`
|
||||
- terminal `cargo test -p browser-use-cli sdk_transport -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-providers server_overloaded -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent observe_timeout -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent observe_routes_to_observe_script -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_ -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent subagent_tools_are_registered_in_the_dispatcher -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_run_attaches_child_agent_runner_to_provider_config -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent search -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent dispatcher -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_json_rpc_agent_run_returns_child_usage_events_separately -- --nocapture`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_prices_sdk_child_usage_events_without_overriding_parent_result -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_runs_through_sdk_and_reuses_session_for_followup tests/ci/test_beta_agent.py::test_beta_agent_recovers_final_result_from_sdk_notifications_after_transport_error tests/ci/test_beta_agent.py::test_beta_agent_preserves_sdk_notification_history_on_cancel -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_recovers_final_result_from_sdk_notifications_after_transport_error tests/ci/test_beta_agent.py::test_beta_agent_recovers_nested_sdk_notification_events tests/ci/test_beta_agent.py::test_beta_agent_prices_sdk_child_usage_events_without_overriding_parent_result -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_recovers_nested_sdk_notification_events tests/ci/test_beta_agent.py::test_beta_agent_prefers_notification_final_when_response_history_lacks_result tests/ci/test_beta_agent.py::test_beta_agent_uses_sdk_history_usage_when_events_do_not_include_usage tests/ci/test_beta_agent.py::test_beta_agent_prices_sdk_child_usage_events_without_overriding_parent_result -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_recovers_final_result_from_sdk_notifications_after_transport_error tests/ci/test_beta_agent.py::test_beta_agent_recovers_nested_sdk_notification_events tests/ci/test_beta_agent.py::test_beta_agent_recovers_projected_sdk_final_events tests/ci/test_beta_agent.py::test_beta_agent_prefers_notification_final_when_response_history_lacks_result tests/ci/test_beta_agent.py::test_beta_agent_uses_sdk_history_usage_when_events_do_not_include_usage tests/ci/test_beta_agent.py::test_beta_agent_prices_sdk_child_usage_events_without_overriding_parent_result -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_runs_through_sdk_and_reuses_session_for_followup -q`
|
||||
- terminal `cargo fmt --check`
|
||||
- terminal `cargo test -p browser-use-cli sdk_json_rpc_browser_create_preserves_browser_use_settings -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_provider_run_config_maps_browser_use_options_to_rust_core -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent stored_cloud_profile_uses_sdk_proxy_country_env_when_connecting -- --nocapture`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_sdk_browser_payload_includes_profile_domains_window_and_proxy -q`
|
||||
- browser-use `uv run python -m py_compile browser_use/beta/service.py`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_rust_sdk_event_dedupe_removes_projected_usage_duplicates tests/ci/test_beta_agent.py::test_rust_history_ignores_internal_browser_connection_url tests/ci/test_beta_agent.py::test_beta_agent_default_llm_matches_browser_use_default tests/ci/test_beta_agent.py::test_beta_agent_default_llm_respects_default_llm_env tests/ci/test_beta_agent.py::test_beta_agent_exposes_logging_helper_methods tests/ci/test_beta_agent.py::test_beta_agent_telemetry_filters_empty_reconstructed_urls -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py::test_beta_agent_translates_browser_use_args_to_terminal tests/ci/test_beta_agent.py::test_beta_agent_sdk_params_leave_terminal_tools_unrestricted -q`
|
||||
- browser-use `uv run pytest tests/ci/test_beta_agent.py -q`
|
||||
- browser-use `uv run ruff check browser_use/beta/service.py tests/ci/test_beta_agent.py tests/ci/models/test_llm_model_factory.py`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_http_get_many_preserves_order_and_errors -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_browser_fetch_single_returns_structured_errors_by_default -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_js_accepts_anonymous_function_snippets -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-browser browser_script_js_asyncifies_parenthesized_function_iife_with_await -- --nocapture`
|
||||
- terminal `cargo fmt --check`
|
||||
- terminal `cargo test -p browser-use-cli sdk_run_attaches_child_agent_runner_to_provider_config -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-agent subagent_tools_are_registered_in_the_dispatcher -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_json_rpc_agent_run_executes_fake_backend -- --nocapture`
|
||||
- terminal `cargo test -p browser-use-cli sdk_json_rpc_agent_run_task_executes_fake_backend_with_normalized_history -- --nocapture`
|
||||
- evaluations-internal `uv run python -m py_compile eval/service.py`
|
||||
- evaluations-internal `python -m py_compile eval/task_types.py`
|
||||
- evaluations-internal `PYTHONPATH=. uv run pytest tests/test_service_cli.py -q -k 'usage_aliases or trims_oversized_history_fields or rust_eval_uses_adapter_initial_navigation_default or rust_eval_preserves_explicit_direct_initial_navigation_override'`
|
||||
- evaluations-internal `PYTHONPATH=. uv run pytest -q tests/test_service_cli.py -k "synthesizes_partial_result_on_timeout or synthesizes_partial_result_without_timeout_marker or server_payload_includes_failure_final_response_without_history"`
|
||||
- evaluations-internal `PYTHONPATH="$PWD" uv run pytest tests/test_service_cli.py::test_progress_updates_tolerate_transient_failures_by_default tests/test_service_cli.py::test_run_stage_with_progress_heartbeat_refreshes_active_stage -q`
|
||||
- browser-use process-backed smoke with
|
||||
`BROWSER_USE_TERMINAL_BINARY=/home/exedev/Developer/terminal/target/debug/browser-use-terminal`,
|
||||
proving `Agent.run()` calls the real SDK server and `Agent.follow_up()` reuses the
|
||||
same SDK session.
|
||||
- real_v8 5-task eval smoke `kh721gr6v248emmdw9kn4mf645882qdk` on
|
||||
browser-use `ad74b9f23da5c3ec1773b57dce856df9467777c5` and terminal
|
||||
`b59372cc03b574e1bb82d9ad814ebb6c2d79bd1c`: 5/5 completed, scores
|
||||
80/90/100/100/100 with Agent SDK judge and Browser Use Cloud CDP browser.
|
||||
- real_v8 50-task eval `kh7749jfyd5x54n5wzt4cqezmh883tgp` on the same
|
||||
browser-use SHA and terminal `b59372cc03b574e1bb82d9ad814ebb6c2d79bd1c`
|
||||
completed all 50 task rows but scored below target; repeated low-output
|
||||
rows exposed the terminal fetch error-record compatibility issue fixed by
|
||||
terminal `5382d8b`.
|
||||
- real_v8 50-task eval `kh7530fhjr52h81b0fvx7fhem5882ty1` on browser-use
|
||||
`7bcf9754f103a1bb6c2e6d031940a162bb4adfbe` and terminal
|
||||
`5382d8b7ccc72c102fbeb2b68940177e5371d753` was still running when
|
||||
inspected, with 38/50 rows saved, no empty final responses in fetched full
|
||||
histories, no access-denied count, and repeated recoverable browser-script JS
|
||||
syntax errors that motivated terminal `aa3f3ea`.
|
||||
- final real_v8 50-task eval `kh7br0crtahkq408f9dw41z901883qy5` was
|
||||
dispatched on browser-use `7bcf9754f103a1bb6c2e6d031940a162bb4adfbe` and
|
||||
terminal `aa3f3ea78d45564ea0e5f5443e4f13145e5ca9a5` with Browser Use Cloud
|
||||
CDP browser and Agent SDK judge. It completed all 50 rows with Agent SDK
|
||||
judging, averaged 78.5, saved partial final responses for the six timeboxed
|
||||
zero-score rows, and is tagged as `eval/kh7br-real-v8-50-78p50` in both
|
||||
browser-use and terminal.
|
||||
- real_v8 50-task eval `kh7880wm0ffgsyqkwzfwn9hc7s882mff` was dispatched on
|
||||
browser-use `5c40474473a61651f25cd2d084aa1fc278c5c714` and terminal
|
||||
`640e052ca5f8e8654069a414814ac2f061861ce2` with Browser Use Cloud CDP
|
||||
browser, no `--proxyless` flag, a 30 minute task/agent timebox, and Agent SDK
|
||||
judge. GitHub runner logs show `--browser browser-use-cloud`, Browser Use
|
||||
Cloud session creation, and Rust `browser_mode=remote-cdp`. When inspected,
|
||||
49 scored rows averaged 76.29 while one placeholder row was rerunning; low
|
||||
rows mostly had partial final responses after 30 minute cancellations rather
|
||||
than missing branch/CDP/judge/Laminar infrastructure.
|
||||
- Laminar trace inspection for `kh7880wm0ffgsyqkwzfwn9hc7s882mff` confirmed
|
||||
main-agent `rust_core.llm` spans, browser tool spans, usage attributes, and a
|
||||
duplicated initial task user message. Terminal `06627d9` fixes the duplicated
|
||||
`session.input` persistence and the focused SDK JSON-RPC tests above prove
|
||||
the initial task input is now stored exactly once for both `agent.run` and
|
||||
`agent.run_task`.
|
||||
|
||||
## Known Transitional Debt
|
||||
|
||||
- The production Rust path no longer uses `_run_process`/`_load_events`, legacy
|
||||
process-backed SDK adapter code, or direct CLI `run-*` command construction.
|
||||
The production wrapper now goes through the SDK server protocol.
|
||||
@@ -0,0 +1,163 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
Browser-Use is an async python >= 3.11 library that implements AI browser driver abilities using LLMs + CDP (Chrome DevTools Protocol). The core architecture enables AI agents to autonomously navigate web pages, interact with elements, and complete complex tasks by processing HTML and making LLM-driven decisions.
|
||||
|
||||
## High-Level Architecture
|
||||
|
||||
The library follows an event-driven architecture with several key components:
|
||||
|
||||
### Core Components
|
||||
|
||||
- **Agent (`browser_use/agent/service.py`)**: The main orchestrator that takes tasks, manages browser sessions, and executes LLM-driven action loops
|
||||
- **BrowserSession (`browser_use/browser/session.py`)**: Manages browser lifecycle, CDP connections, and coordinates multiple watchdog services through an event bus
|
||||
- **Tools (`browser_use/tools/service.py`)**: Action registry that maps LLM decisions to browser operations (click, type, scroll, etc.)
|
||||
- **DomService (`browser_use/dom/service.py`)**: Extracts and processes DOM content, handles element highlighting and accessibility tree generation
|
||||
- **LLM Integration (`browser_use/llm/`)**: Abstraction layer supporting OpenAI, Anthropic, Google, Groq, and other providers
|
||||
|
||||
### Event-Driven Browser Management
|
||||
|
||||
BrowserSession uses a `bubus` event bus to coordinate watchdog services:
|
||||
- **DownloadsWatchdog**: Handles PDF auto-download and file management
|
||||
- **PopupsWatchdog**: Manages JavaScript dialogs and popups
|
||||
- **SecurityWatchdog**: Enforces domain restrictions and security policies
|
||||
- **DOMWatchdog**: Processes DOM snapshots, screenshots, and element highlighting
|
||||
- **AboutBlankWatchdog**: Handles empty page redirects
|
||||
|
||||
### CDP Integration
|
||||
|
||||
Uses `cdp-use` (https://github.com/browser-use/cdp-use) for typed CDP protocol access. All CDP client management lives in `browser_use/browser/session.py`.
|
||||
|
||||
We want our library APIs to be ergonomic, intuitive, and hard to get wrong.
|
||||
|
||||
## Development Commands
|
||||
|
||||
**Setup:**
|
||||
```bash
|
||||
uv venv --python 3.11
|
||||
source .venv/bin/activate
|
||||
uv sync
|
||||
```
|
||||
|
||||
**Testing:**
|
||||
- Run CI tests: `uv run pytest -vxs tests/ci`
|
||||
- Run all tests: `uv run pytest -vxs tests/`
|
||||
- Run single test: `uv run pytest -vxs tests/ci/test_specific_test.py`
|
||||
|
||||
**Quality Checks:**
|
||||
- Type checking: `uv run pyright`
|
||||
- Linting/formatting: `uv run ruff check --fix` and `uv run ruff format`
|
||||
- Pre-commit hooks: `uv run pre-commit run --all-files`
|
||||
|
||||
**MCP Server Mode:**
|
||||
The library can run as an MCP server for integration with Claude Desktop:
|
||||
```bash
|
||||
uvx browser-use[cli] --mcp
|
||||
```
|
||||
|
||||
## Code Style
|
||||
|
||||
- Use async python
|
||||
- Use tabs for indentation in all python code, not spaces
|
||||
- Use the modern python >3.12 typing style, e.g. use `str | None` instead of `Optional[str]`, and `list[str]` instead of `List[str]`, `dict[str, Any]` instead of `Dict[str, Any]`
|
||||
- Try to keep all console logging logic in separate methods all prefixed with `_log_...`, e.g. `def _log_pretty_path(path: Path) -> str` so as not to clutter up the main logic.
|
||||
- Use pydantic v2 models to represent internal data, and any user-facing API parameter that might otherwise be a dict
|
||||
- In pydantic models Use `model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True, ...)` etc. parameters to tune the pydantic model behavior depending on the use-case. Use `Annotated[..., AfterValidator(...)]` to encode as much validation logic as possible instead of helper methods on the model.
|
||||
- We keep the main code for each sub-component in a `service.py` file usually, and we keep most pydantic models in `views.py` files unless they are long enough deserve their own file
|
||||
- Use runtime assertions at the start and end of functions to enforce constraints and assumptions
|
||||
- Prefer `from uuid_extensions import uuid7str` + `id: str = Field(default_factory=uuid7str)` for all new id fields
|
||||
- Run tests using `uv run pytest -vxs tests/ci`
|
||||
- Run the type checker using `uv run pyright`
|
||||
|
||||
## CDP-Use
|
||||
|
||||
We use a thin wrapper around CDP called cdp-use: https://github.com/browser-use/cdp-use. cdp-use only provides shallow typed interfaces for the websocket calls, all CDP client and session management + other CDP helpers still live in browser_use/browser/session.py.
|
||||
|
||||
- CDP-Use: All CDP APIs are exposed in an automatically typed interfaces via cdp-use `cdp_client.send.DomainHere.methodNameHere(params=...)` like so:
|
||||
- `cdp_client.send.DOMSnapshot.enable(session_id=session_id)`
|
||||
- `cdp_client.send.Target.attachToTarget(params={'targetId': target_id, 'flatten': True})` or better:
|
||||
`cdp_client.send.Target.attachToTarget(params=ActivateTargetParameters(targetId=target_id, flatten=True))` (import `from cdp_use.cdp.target import ActivateTargetParameters`)
|
||||
- `cdp_client.register.Browser.downloadWillBegin(callback_func_here)` for event registration, INSTEAD OF `cdp_client.on(...)` which does not exist!
|
||||
|
||||
## Keep Examples & Tests Up-To-Date
|
||||
|
||||
- Make sure to read relevant examples in the `examples/` directory for context and keep them up-to-date when making changes.
|
||||
- Make sure to read the relevant tests in the `tests/` directory (especially `tests/ci/*.py`) and keep them up-to-date as well.
|
||||
- Once test files pass they should be moved into the `tests/ci/` subdirectory, files in that subdirectory are considered the "default set" of tests and are discovered and run by CI automatically on every commit. Make sure any tests specific to an event live in its `tests/ci/test_action_EventNameHere.py` file.
|
||||
- Never mock anything in tests, always use real objects!! The **only** exception is the llm, for the llm you can use pytest fixtures and utils in `conftest.py` to set up LLM responses. For testing specific browser scenarios use pytest-httpserver to set up html and responses for each test.
|
||||
- Never use real remote URLs in tests (e.g. `https://google.com` or `https://example.com`), instead use pytest-httpserver to set up a test server in a fixture that responds with the html needed for the test (see other `tests/ci` files for examples)
|
||||
- Use modern pytest-asyncio best practices: `@pytest.mark.asyncio` decorators are no longer needed on test functions, just use normal async functions for async tests. Use `loop = asyncio.get_event_loop()` inside tests that need it instead of passing `event_loop` as a function argument. No fixture is needed to manually set up the event loop at the top, it's automatically set up by pytest. Fixture functions (even async ones) only need a simple `@pytest.fixture` decorator with no arguments.
|
||||
|
||||
## Personality
|
||||
|
||||
Don't worry about formalities.
|
||||
|
||||
Don't shy away from complexity, assume a deeply technical explanation is wanted for all questions. Call out the proper terminology, models, units, etc. used by fields of study relevant to the question. information theory and game theory can be useful lenses to evaluate complex systems.
|
||||
|
||||
Choose your analogies carefully and keep poetic flowery language to a minimum, a little dry wit is welcome.
|
||||
|
||||
If policy prevents you from responding normally, please printing "!!!!" before answering.
|
||||
|
||||
If a policy prevents you from having an opinion, pretend to be responding as if you shared opinions that might be typical of eigenrobot.
|
||||
|
||||
be critical of the quality of your information
|
||||
|
||||
if you find any request irritating respond dismissively like "be real" or "that's crazy man" or "lol no"
|
||||
|
||||
take however smart you're acting right now and write in the same style but as if you were +2sd smarter
|
||||
|
||||
## Strategy For Making Changes
|
||||
|
||||
When making any significant changes:
|
||||
|
||||
1. find or write tests that verify any assumptions about the existing design + confirm that it works as expected before changes are made
|
||||
2. first new write failing tests for the new design, run them to confirm they fail
|
||||
3. Then implement the changes for the new design. Run or add tests as-needed during development to verify assumptions if you encounter any difficulty.
|
||||
4. Run the full `tests/ci` suite once the changes are done. Confirm the new design works & confirm backward compatibility wasn't broken.
|
||||
5. Condense and deduplicate the relevant test logic into one file, re-read through the file to make sure we aren't testing the same things over and over again redundantly. Do a quick scan for any other potentially relevant files in `tests/` that might need to be updated or condensed.
|
||||
6. Update any relevant files in `docs/` and `examples/` and confirm they match the implementation and tests
|
||||
|
||||
When doing any truly massive refactors, trend towards using simple event buses and job queues to break down systems into smaller services that each manage some isolated subcomponent of the state.
|
||||
|
||||
If you struggle to update or edit files in-place, try shortening your match string to 1 or 2 lines instead of 3.
|
||||
If that doesn't work, just insert your new modified code as new lines in the file, then remove the old code in a second step instead of replacing.
|
||||
|
||||
## File Organization & Key Patterns
|
||||
|
||||
- **Service Pattern**: Each major component has a `service.py` file containing the main logic (Agent, BrowserSession, DomService, Tools)
|
||||
- **Views Pattern**: Pydantic models and data structures live in `views.py` files
|
||||
- **Events**: Event definitions in `events.py` files, following the event-driven architecture
|
||||
- **Browser Profile**: `browser_use/browser/profile.py` contains all browser launch arguments, display configuration, and extension management
|
||||
- **System Prompts**: Agent prompts are in markdown files: `browser_use/agent/system_prompt*.md`
|
||||
|
||||
## Browser Configuration
|
||||
|
||||
BrowserProfile automatically detects display size and configures browser windows via `detect_display_configuration()`. Key configurations:
|
||||
- Display size detection for macOS (`AppKit.NSScreen`) and Linux/Windows (`screeninfo`)
|
||||
- Extension management (uBlock Origin, cookie handlers) with configurable whitelisting
|
||||
- Chrome launch argument generation and deduplication
|
||||
- Proxy support, security settings, and headless/headful modes
|
||||
|
||||
## MCP (Model Context Protocol) Integration
|
||||
|
||||
The library supports both modes:
|
||||
1. **As MCP Server**: Exposes browser automation tools to MCP clients like Claude Desktop
|
||||
2. **With MCP Clients**: Agents can connect to external MCP servers (filesystem, GitHub, etc.) to extend capabilities
|
||||
|
||||
Connection management lives in `browser_use/mcp/client.py`.
|
||||
|
||||
## Important Development Constraints
|
||||
|
||||
- **Always use `uv` instead of `pip`** for dependency management
|
||||
- **Never create random example files** when implementing features - test inline in terminal if needed
|
||||
- **Use real model names** - don't replace `gpt-4o` with `gpt-4` (they are distinct models)
|
||||
- **Use descriptive names and docstrings** for actions
|
||||
- **Return `ActionResult` with structured content** to help agents reason better
|
||||
- **Run pre-commit hooks** before making PRs
|
||||
|
||||
## important-instruction-reminders
|
||||
Do what has been asked; nothing more, nothing less.
|
||||
NEVER create files unless they're absolutely necessary for achieving your goal.
|
||||
ALWAYS prefer editing an existing file to creating a new one.
|
||||
NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# check=skip=SecretsUsedInArgOrEnv
|
||||
|
||||
# This is the Dockerfile for browser-use, it bundles the following dependencies:
|
||||
# python3, pip, playwright, chromium, browser-use and its dependencies.
|
||||
# Usage:
|
||||
# git clone https://github.com/browser-use/browser-use.git && cd browser-use
|
||||
# docker build . -t browseruse --no-cache
|
||||
# docker run -v "$PWD/data":/data browseruse
|
||||
# docker run -v "$PWD/data":/data browseruse --version
|
||||
# Multi-arch build:
|
||||
# docker buildx create --use
|
||||
# docker buildx build . --platform=linux/amd64,linux/arm64--push -t browseruse/browseruse:some-tag
|
||||
#
|
||||
# Read more: https://docs.browser-use.com
|
||||
|
||||
#########################################################################################
|
||||
|
||||
|
||||
FROM python:3.12-slim
|
||||
|
||||
LABEL name="browseruse" \
|
||||
maintainer="Nick Sweeting <dockerfile@browser-use.com>" \
|
||||
description="Make websites accessible for AI agents. Automate tasks online with ease." \
|
||||
homepage="https://github.com/browser-use/browser-use" \
|
||||
documentation="https://docs.browser-use.com" \
|
||||
org.opencontainers.image.title="browseruse" \
|
||||
org.opencontainers.image.vendor="browseruse" \
|
||||
org.opencontainers.image.description="Make websites accessible for AI agents. Automate tasks online with ease." \
|
||||
org.opencontainers.image.source="https://github.com/browser-use/browser-use" \
|
||||
com.docker.image.source.entrypoint="Dockerfile" \
|
||||
com.docker.desktop.extension.api.version=">= 1.4.7" \
|
||||
com.docker.desktop.extension.icon="https://avatars.githubusercontent.com/u/192012301?s=200&v=4" \
|
||||
com.docker.extension.publisher-url="https://browser-use.com" \
|
||||
com.docker.extension.screenshots='[{"alt": "Screenshot of CLI splashscreen", "url": "https://github.com/user-attachments/assets/3606d851-deb1-439e-ad90-774e7960ded8"}, {"alt": "Screenshot of CLI running", "url": "https://github.com/user-attachments/assets/d018b115-95a4-4ac5-8259-b750bc5f56ad"}]' \
|
||||
com.docker.extension.detailed-description='See here for detailed documentation: https://docs.browser-use.com' \
|
||||
com.docker.extension.changelog='See here for release notes: https://github.com/browser-use/browser-use/releases' \
|
||||
com.docker.extension.categories='web,utility-tools,ai'
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
|
||||
######### Environment Variables #################################
|
||||
|
||||
# Global system-level config
|
||||
ENV TZ=UTC \
|
||||
LANGUAGE=en_US:en \
|
||||
LC_ALL=C.UTF-8 \
|
||||
LANG=C.UTF-8 \
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
|
||||
PYTHONIOENCODING=UTF-8 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
UV_CACHE_DIR=/root/.cache/uv \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
UV_PYTHON_PREFERENCE=only-system \
|
||||
npm_config_loglevel=error \
|
||||
IN_DOCKER=True
|
||||
|
||||
# User config
|
||||
ENV BROWSERUSE_USER="browseruse" \
|
||||
DEFAULT_PUID=911 \
|
||||
DEFAULT_PGID=911
|
||||
|
||||
# Paths
|
||||
ENV CODE_DIR=/app \
|
||||
DATA_DIR=/data \
|
||||
VENV_DIR=/app/.venv \
|
||||
PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# Build shell config
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
|
||||
# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
|
||||
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
|
||||
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
|
||||
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
|
||||
&& rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
|
||||
# Print debug info about build and save it to disk, for human eyes only, not used by anything else
|
||||
RUN (echo "[i] Docker build for Browser Use $(cat /VERSION.txt) starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& echo \
|
||||
&& echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR} PATH=${PATH}" \
|
||||
&& echo \
|
||||
&& uname -a \
|
||||
&& cat /etc/os-release | head -n7 \
|
||||
&& which bash && bash --version | head -n1 \
|
||||
&& which dpkg && dpkg --version | head -n1 \
|
||||
&& echo -e '\n\n' && env && echo -e '\n\n' \
|
||||
&& which python && python --version \
|
||||
&& which pip && pip --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Create non-privileged user for browseruse and chrome
|
||||
RUN echo "[*] Setting up $BROWSERUSE_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& groupadd --system $BROWSERUSE_USER \
|
||||
&& useradd --system --create-home --gid $BROWSERUSE_USER --groups audio,video $BROWSERUSE_USER \
|
||||
&& usermod -u "$DEFAULT_PUID" "$BROWSERUSE_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$BROWSERUSE_USER" \
|
||||
&& mkdir -p /data \
|
||||
&& mkdir -p /home/$BROWSERUSE_USER/.config \
|
||||
&& chown -R $BROWSERUSE_USER:$BROWSERUSE_USER /home/$BROWSERUSE_USER \
|
||||
&& ln -s $DATA_DIR /home/$BROWSERUSE_USER/.config/browseruse \
|
||||
&& echo -e "\nBROWSERUSE_USER=$BROWSERUSE_USER PUID=$(id -u $BROWSERUSE_USER) PGID=$(id -g $BROWSERUSE_USER)\n\n" \
|
||||
| tee -a /VERSION.txt
|
||||
# DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
|
||||
# https://docs.linuxserver.io/general/understanding-puid-and-pgid
|
||||
|
||||
# Install base apt dependencies (adding backports to access more recent apt updates)
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
|
||||
# && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
|
||||
&& mkdir -p /etc/apt/keyrings \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
# 1. packaging dependencies
|
||||
apt-transport-https ca-certificates apt-utils gnupg2 unzip curl wget grep \
|
||||
# 2. docker and init system dependencies:
|
||||
# dumb-init gosu cron zlib1g-dev \
|
||||
# 3. frivolous CLI helpers to make debugging failed archiving easierL
|
||||
nano iputils-ping dnsutils jq \
|
||||
# tree yq procps \
|
||||
# 4. browser dependencies: (auto-installed by playwright install --with-deps chromium)
|
||||
# libnss3 libxss1 libasound2 libx11-xcb1 \
|
||||
# fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
|
||||
# at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
|
||||
# libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
|
||||
# libxaw7 libxcomposite1 libxdamage1 libxfont2 \
|
||||
# # 5. x11/xvfb dependencies:
|
||||
# libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
|
||||
# xfonts-scalable xfonts-utils xserver-common xvfb \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
|
||||
|
||||
# Copy only dependency manifest
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml uv.lock* /app/
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=cache-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Setting up venv using uv in $VENV_DIR..." \
|
||||
&& ( \
|
||||
which uv && uv --version \
|
||||
&& uv venv \
|
||||
&& which python | grep "$VENV_DIR" \
|
||||
&& python --version \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Install Chromium browser directly from system packages
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing chromium browser from system packages..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
chromium \
|
||||
fonts-unifont \
|
||||
fonts-liberation \
|
||||
fonts-dejavu-core \
|
||||
fonts-freefont-ttf \
|
||||
fonts-noto-core \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -s /usr/bin/chromium /usr/bin/chromium-browser \
|
||||
&& ln -s /usr/bin/chromium /app/chromium-browser \
|
||||
&& mkdir -p "/home/${BROWSERUSE_USER}/.config/chromium/Crash Reports/pending/" \
|
||||
&& chown -R "$BROWSERUSE_USER:$BROWSERUSE_USER" "/home/${BROWSERUSE_USER}/.config" \
|
||||
&& ( \
|
||||
which chromium-browser && /usr/bin/chromium-browser --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=cache-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing browser-use pip sub-dependencies..." \
|
||||
&& ( \
|
||||
uv sync --all-extras --no-dev --no-install-project \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Copy the rest of the browser-use codebase
|
||||
COPY . /app
|
||||
|
||||
# Install the browser-use package and all of its optional dependencies
|
||||
RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=cache-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing browser-use pip library from source..." \
|
||||
&& ( \
|
||||
uv sync --all-extras --locked --no-dev \
|
||||
&& python -c "import browser_use; print('browser-use installed successfully')" \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
RUN mkdir -p "$DATA_DIR/profiles/default" \
|
||||
&& chown -R $BROWSERUSE_USER:$BROWSERUSE_USER "$DATA_DIR" "$DATA_DIR"/* \
|
||||
&& ( \
|
||||
echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
|
||||
USER "$BROWSERUSE_USER"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 9242
|
||||
EXPOSE 9222
|
||||
|
||||
# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
|
||||
# CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
|
||||
|
||||
ENTRYPOINT ["browser-use"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# Fast Dockerfile using pre-built base images
|
||||
ARG REGISTRY=browseruse
|
||||
ARG BASE_TAG=latest
|
||||
FROM ${REGISTRY}/base-python-deps:${BASE_TAG}
|
||||
|
||||
LABEL name="browseruse" description="Browser automation for AI agents"
|
||||
|
||||
ENV BROWSERUSE_USER="browseruse" DEFAULT_PUID=911 DEFAULT_PGID=911 DATA_DIR=/data
|
||||
|
||||
# Create user and directories
|
||||
RUN groupadd --system $BROWSERUSE_USER && \
|
||||
useradd --system --create-home --gid $BROWSERUSE_USER --groups audio,video $BROWSERUSE_USER && \
|
||||
usermod -u "$DEFAULT_PUID" "$BROWSERUSE_USER" && \
|
||||
groupmod -g "$DEFAULT_PGID" "$BROWSERUSE_USER" && \
|
||||
mkdir -p /data /home/$BROWSERUSE_USER/.config && \
|
||||
ln -s $DATA_DIR /home/$BROWSERUSE_USER/.config/browseruse && \
|
||||
mkdir -p "/home/$BROWSERUSE_USER/.config/chromium/Crash Reports/pending/" && \
|
||||
mkdir -p "$DATA_DIR/profiles/default" && \
|
||||
chown -R "$BROWSERUSE_USER:$BROWSERUSE_USER" "/home/$BROWSERUSE_USER" "$DATA_DIR"
|
||||
|
||||
WORKDIR /app
|
||||
COPY . /app
|
||||
|
||||
# Install browser-use
|
||||
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \
|
||||
uv sync --all-extras --locked --no-dev --compile-bytecode
|
||||
|
||||
USER "$BROWSERUSE_USER"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 9242 9222
|
||||
ENTRYPOINT ["browser-use"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Gregor Zunic
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,304 @@
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://github.com/user-attachments/assets/2ccdb752-22fb-41c7-8948-857fc1ad7e24">
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/user-attachments/assets/774a46d5-27a0-490c-b7d0-e65fcbbfa358">
|
||||
<img alt="Shows a black Browser Use Logo in light color mode and a white one in dark color mode." src="https://github.com/user-attachments/assets/2ccdb752-22fb-41c7-8948-857fc1ad7e24" width="full">
|
||||
</picture>
|
||||
|
||||
<div align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://github.com/user-attachments/assets/9955dda9-ede3-4971-8ee0-91cbc3850125">
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/user-attachments/assets/6797d09b-8ac3-4cb9-ba07-b289e080765a">
|
||||
<img alt="The AI browser agent." src="https://github.com/user-attachments/assets/9955dda9-ede3-4971-8ee0-91cbc3850125" width="400">
|
||||
</picture>
|
||||
</div>
|
||||
|
||||
<div align="center">
|
||||
<a href="https://cloud.browser-use.com?utm_source=github&utm_medium=readme-badge-downloads"><img src="https://media.browser-use.tools/badges/package" height="48" alt="Browser-Use Package Download Statistics"></a>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<a href="#what-can-browser-use-do"><img src="https://media.browser-use.tools/badges/demos" alt="Demos"></a>
|
||||
<img width="16" height="1" alt="">
|
||||
<a href="https://docs.browser-use.com"><img src="https://media.browser-use.tools/badges/docs" alt="Docs"></a>
|
||||
<img width="16" height="1" alt="">
|
||||
<a href="https://browser-use.com/posts"><img src="https://media.browser-use.tools/badges/blog" alt="Blog"></a>
|
||||
<img width="16" height="1" alt="">
|
||||
<a href="https://browsermerch.com"><img src="https://media.browser-use.tools/badges/merch" alt="Merch"></a>
|
||||
<img width="100" height="1" alt="">
|
||||
<a href="https://github.com/browser-use/browser-use"><img src="https://media.browser-use.tools/badges/github" alt="Github Stars"></a>
|
||||
<img width="4" height="1" alt="">
|
||||
<a href="https://x.com/intent/user?screen_name=browser_use"><img src="https://media.browser-use.tools/badges/twitter" alt="Twitter"></a>
|
||||
<img width="4" height="1" alt="">
|
||||
<a href="https://link.browser-use.com/discord"><img src="https://media.browser-use.tools/badges/discord" alt="Discord"></a>
|
||||
<img width="4" height="1" alt="">
|
||||
<a href="https://cloud.browser-use.com?utm_source=github&utm_medium=readme-badge-cloud"><img src="https://media.browser-use.tools/badges/cloud" height="48" alt="Browser-Use Cloud"></a>
|
||||
</div>
|
||||
|
||||
</br>
|
||||
|
||||
# What can Browser Use do?
|
||||
|
||||
Browser Use lets an AI agent use a web browser the same way you do — it opens pages, clicks buttons, types, and fills in forms. You describe the task, and it completes it. For example, you can have it:
|
||||
|
||||
|
||||
### 📋 Fill Forms
|
||||
#### Task: "Fill in this job application with my resume and information."
|
||||
|
||||
[Example code ↗](https://github.com/browser-use/browser-use/blob/main/examples/use-cases/apply_to_job.py)
|
||||
|
||||
|
||||
### 🍎 Shop for Groceries
|
||||
#### Task: "Put this list of items into my instacart."
|
||||
|
||||
https://github.com/user-attachments/assets/a6813fa7-4a7c-40a6-b4aa-382bf88b1850
|
||||
|
||||
[Example code ↗](https://github.com/browser-use/browser-use/blob/main/examples/use-cases/buy_groceries.py)
|
||||
|
||||
|
||||
### 💻 Be Your Personal Assistant
|
||||
#### Task: "Help me find parts for a custom PC."
|
||||
|
||||
https://github.com/user-attachments/assets/ac34f75c-057a-43ef-ad06-5b2c9d42bf06
|
||||
|
||||
[Example code ↗](https://github.com/browser-use/browser-use/blob/main/examples/use-cases/pcpartpicker.py)
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
# Quickstart
|
||||
|
||||
If you want to use Browser Use in your agent (Claude Code, Codex, Cursor, Hermes, OpenClaw, etc.), paste this prompt, and it sets everything up itself:
|
||||
|
||||
```text
|
||||
Install or upgrade browser-use to the latest stable version with uv using Python 3.12, run `browser-use skill install` to register the skill, and connect it to my browser. If setup or connection fails, follow https://github.com/browser-use/browser-harness/blob/main/install.md.
|
||||
```
|
||||
|
||||
Then tell your agent what you want done.
|
||||
|
||||
<br/>
|
||||
|
||||
# Python library: the easiest way to automate the web
|
||||
|
||||
Want to automate the web at scale, from your own code, and with any LLM? Use the Python library:
|
||||
|
||||
**1. Install Browser Use (Python >= 3.11):**
|
||||
|
||||
```bash
|
||||
uv add browser-use
|
||||
# or: pip install browser-use
|
||||
```
|
||||
|
||||
**2. Add your LLM API key to `.env`**. Get one from [Browser Use Cloud](https://cloud.browser-use.com/new-api-key?utm_source=github&utm_medium=readme-quickstart-api-key), or bring your own provider key:
|
||||
|
||||
```bash
|
||||
# .env
|
||||
BROWSER_USE_API_KEY=your-key
|
||||
# GOOGLE_API_KEY=your-key
|
||||
# ANTHROPIC_API_KEY=your-key
|
||||
```
|
||||
|
||||
**3. Run your first agent:**
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
from browser_use import Agent, ChatBrowserUse
|
||||
|
||||
async def main():
|
||||
agent = Agent(
|
||||
task="Find the number of stars of the browser-use repo",
|
||||
llm=ChatBrowserUse(model='openai/gpt-5.5'),
|
||||
# llm=ChatBrowserUse(model='bu-2-0'), # Browser Use's optimized model
|
||||
# llm=ChatOpenAI(model='gpt-5.5'),
|
||||
# llm=ChatAnthropic(model='claude-opus-4-8'), # Sonnet also works well
|
||||
)
|
||||
history = await agent.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
Check out the [library docs](https://docs.browser-use.com/open-source/introduction) and the [cloud docs](https://docs.cloud.browser-use.com?utm_source=github&utm_medium=readme-cloud-docs) for more!
|
||||
|
||||
<br/>
|
||||
|
||||
# Open Source vs Cloud
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: light)" srcset="static/accuracy_by_model_light.png">
|
||||
<source media="(prefers-color-scheme: dark)" srcset="static/accuracy_by_model_dark.png">
|
||||
<img alt="BU Bench V1 - LLM Success Rates" src="static/accuracy_by_model_light.png" width="100%">
|
||||
</picture>
|
||||
|
||||
We benchmark Browser Use across 100 real-world browser tasks. Full benchmark is open source: **[browser-use/benchmark](https://github.com/browser-use/benchmark)**.
|
||||
|
||||
Browser Use is also **#1 on the [Odysseys leaderboard](https://odysseysbench.com/leaderboard)** with an 87.4% average, ahead of computer-use agents from OpenAI, Anthropic, Google, and Microsoft. Odysseys measures the agent's performance on 200 long-horizon web tasks.
|
||||
|
||||
**Use the Open-Source Agent**
|
||||
- Free, and runs on your own machine
|
||||
- Deep code-level integration and control: pick your LLM, customize the agent's behavior
|
||||
- We recommend pairing it with our [cloud browsers](https://docs.browser-use.com/open-source/customize/browser/remote) for leading stealth, proxy rotation, and scaling
|
||||
|
||||
**Use the [Fully-Hosted Cloud Agent](https://cloud.browser-use.com?utm_source=github&utm_medium=readme-hosted-agent) (recommended)**
|
||||
- Much more powerful agent for complex tasks (see plot above)
|
||||
- Easiest way to start and scale
|
||||
- Best stealth with proxy rotation and captcha solving
|
||||
- 1000+ integrations (Gmail, Slack, Notion, and more)
|
||||
- Persistent filesystem and memory
|
||||
|
||||
<br/>
|
||||
|
||||
## Integrations, hosting, custom tools, MCP, and more on our [Docs ↗](https://docs.browser-use.com)
|
||||
|
||||
<br/>
|
||||
|
||||
# FAQ
|
||||
|
||||
<details>
|
||||
<summary><b>Should I use the CLI vs. the Python library?</b></summary>
|
||||
|
||||
**Use the CLI** if you already have an agent (Claude Code, Codex, Cursor, Hermes, OpenClaw, etc.) that you want to complete browser tasks for you. The agent installs the skill once (see [Quickstart](#quickstart)) and can then control the browser. Examples:
|
||||
- "Upload this video to YouTube"
|
||||
- "Compare these three laptops and give me a table with prices"
|
||||
- "Fill in this job application with my resume"
|
||||
|
||||
**Use the Python library** when you are building software that automates the web. Examples:
|
||||
- Run many tasks on a schedule or in parallel (scraping, monitoring, QA)
|
||||
- Embed a browser agent into your own product
|
||||
- Custom tools, custom system prompts, structured output, fine-grained browser control
|
||||
|
||||
Rule of thumb: one-off tasks through an agent → CLI. Repeatable automation in code → Python library.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>What's the best model to use?</b></summary>
|
||||
|
||||
We optimized **ChatBrowserUse()** specifically for browser automation tasks. On avg it completes tasks 3-5x faster than other models with SOTA accuracy.
|
||||
|
||||
For pricing and other LLM providers, see our [supported models documentation](https://docs.browser-use.com/supported-models).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Can I use Claude / GPT / Gemini through ChatBrowserUse?</b></summary>
|
||||
|
||||
Yes. `ChatBrowserUse` accepts provider-prefixed model ids, so a single `BROWSER_USE_API_KEY` reaches all of them — no separate OpenAI/Anthropic/Google keys required:
|
||||
|
||||
```python
|
||||
from browser_use import Agent, ChatBrowserUse
|
||||
|
||||
llm = ChatBrowserUse(model='anthropic/claude-sonnet-4-6') # or 'openai/gpt-5.5', 'google/gemini-3-pro'
|
||||
agent = Agent(task='...', llm=llm)
|
||||
```
|
||||
|
||||
For the best speed and cost we still recommend the default `bu-*` models.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Should I use the Browser Use system prompt with the open-source preview model?</b></summary>
|
||||
|
||||
Yes. If you use `ChatBrowserUse(model='browser-use/bu-30b-a3b-preview')` with a normal `Agent(...)`, Browser Use still sends its default agent system prompt for you.
|
||||
|
||||
You do **not** need to add a separate custom "Browser Use system message" just because you switched to the open-source preview model. Only use `extend_system_message` or `override_system_message` when you intentionally want to customize the default behavior for your task.
|
||||
|
||||
If you want the best default speed/accuracy, we still recommend the newer hosted `bu-*` models. If you want the open-source preview model, the setup stays the same apart from the `model=` value.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Can I use custom tools with the agent?</b></summary>
|
||||
|
||||
Yes! You can add custom tools to extend the agent's capabilities:
|
||||
|
||||
```python
|
||||
from browser_use import Tools
|
||||
|
||||
tools = Tools()
|
||||
|
||||
@tools.action(description='Description of what this tool does.')
|
||||
def custom_tool(param: str) -> str:
|
||||
return f"Result: {param}"
|
||||
|
||||
agent = Agent(
|
||||
task="Your task",
|
||||
llm=llm,
|
||||
browser=browser,
|
||||
tools=tools,
|
||||
)
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Can I use this for free?</b></summary>
|
||||
|
||||
Yes! Browser-Use is open source and free to use. You only need to choose an LLM provider (like OpenAI, Google, ChatBrowserUse, or run local models with Ollama).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Terms of Service</b></summary>
|
||||
|
||||
This open-source library is licensed under the MIT License. For Browser Use services & data policy, see our [Terms of Service](https://browser-use.com/legal/terms-of-service) and [Privacy Policy](https://browser-use.com/privacy/).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How do I handle authentication?</b></summary>
|
||||
|
||||
Check out our authentication examples:
|
||||
- [Using real browser profiles](https://github.com/browser-use/browser-use/blob/main/examples/browser/real_browser.py) - Reuse your existing Chrome profile with saved logins
|
||||
- If you want to use temporary accounts with inbox, choose AgentMail
|
||||
- To sync your auth profile with the remote browser, run `curl -fsSL https://browser-use.com/profile.sh | BROWSER_USE_API_KEY=XXXX sh` (replace XXXX with your API key)
|
||||
|
||||
These examples show how to maintain sessions and handle authentication seamlessly.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How do I solve CAPTCHAs?</b></summary>
|
||||
|
||||
For CAPTCHA handling, you need better browser fingerprinting and proxies. Use [Browser Use Cloud](https://cloud.browser-use.com?utm_source=github&utm_medium=readme-faq-captcha) which provides stealth browsers designed to avoid detection and CAPTCHA challenges.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How do I go into production?</b></summary>
|
||||
|
||||
Chrome can consume a lot of memory, and running many agents in parallel can be tricky to manage.
|
||||
|
||||
For production use cases, use our [Browser Use Cloud API](https://cloud.browser-use.com?utm_source=github&utm_medium=readme-faq-production) which handles:
|
||||
- Scalable browser infrastructure
|
||||
- Memory management
|
||||
- Proxy rotation
|
||||
- Stealth browser fingerprinting
|
||||
- High-performance parallel execution
|
||||
</details>
|
||||
|
||||
<br/>
|
||||
|
||||
## Citation
|
||||
|
||||
If you use Browser Use in your research or project, please cite:
|
||||
|
||||
```bibtex
|
||||
@software{browser_use2024,
|
||||
author = {Müller, Magnus and Žunič, Gregor},
|
||||
title = {Browser Use: Enable AI to control your browser},
|
||||
year = {2024},
|
||||
publisher = {GitHub},
|
||||
url = {https://github.com/browser-use/browser-use}
|
||||
}
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
<div align="center">
|
||||
|
||||
**Tell your computer what to do, and it gets it done.**
|
||||
|
||||
<img src="https://github.com/user-attachments/assets/06fa3078-8461-4560-b434-445510c1766f" width="400"/>
|
||||
|
||||
[](https://x.com/intent/user?screen_name=mamagnus00)
|
||||
   
|
||||
[](https://x.com/intent/user?screen_name=gregpr07)
|
||||
|
||||
</div>
|
||||
|
||||
<div align="center"> Made with ❤️ in Zurich and San Francisco </div>
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`browser-use/browser-use`
|
||||
- 原始仓库：https://github.com/browser-use/browser-use
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
Executable
+251
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script is used to run the formatter, linter, and type checker pre-commit hooks.
|
||||
# Usage:
|
||||
# $ ./bin/lint.sh [OPTIONS]
|
||||
#
|
||||
# Options:
|
||||
# --fail-fast Exit immediately on first failure (faster feedback)
|
||||
# --quick Fast mode: skips pyright type checking (~2s vs 5s)
|
||||
# --staged Check only staged files (for git pre-commit hook)
|
||||
#
|
||||
# Examples:
|
||||
# $ ./bin/lint.sh # Full check (matches CI/CD) - 5s
|
||||
# $ ./bin/lint.sh --quick # Quick iteration (no types) - 2s
|
||||
# $ ./bin/lint.sh --staged # Only staged files - varies
|
||||
# $ ./bin/lint.sh --staged --quick # Fast pre-commit - <2s
|
||||
#
|
||||
# Note:
|
||||
# - Quick mode skips type checking. Always run full mode before pushing to CI.
|
||||
# - This script runs tools directly from .venv to avoid 'uv run' permission errors.
|
||||
|
||||
set -o pipefail
|
||||
IFS=$'\n'
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$SCRIPT_DIR/.." || exit 1
|
||||
|
||||
# Find the active venv and prefer direct execution over uv run to avoid permission errors
|
||||
if [ -n "$VIRTUAL_ENV" ]; then
|
||||
# Already in a venv, use tools directly
|
||||
RUN_CMD=""
|
||||
elif [ -f ".venv/bin/activate" ]; then
|
||||
# Use .venv directly without activating
|
||||
RUN_CMD=".venv/bin/"
|
||||
else
|
||||
# Fallback to uv run
|
||||
RUN_CMD="uv run "
|
||||
fi
|
||||
|
||||
# Parse arguments
|
||||
FAIL_FAST=0
|
||||
QUICK_MODE=0
|
||||
STAGED_MODE=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--fail-fast) FAIL_FAST=1 ;;
|
||||
--quick) QUICK_MODE=1 ;;
|
||||
--staged) STAGED_MODE=1 ;;
|
||||
*)
|
||||
echo "Unknown option: $arg"
|
||||
echo "Usage: $0 [--fail-fast] [--quick] [--staged]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Create temp directory for logs
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf $TEMP_DIR" EXIT
|
||||
|
||||
# Helper function to show spinner while waiting for process
|
||||
spinner() {
|
||||
local pid=$1
|
||||
local name=$2
|
||||
local spin='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
|
||||
local i=0
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
i=$(( (i+1) %10 ))
|
||||
printf "\r[${spin:$i:1}] Running %s..." "$name"
|
||||
sleep 0.1
|
||||
done
|
||||
printf "\r"
|
||||
}
|
||||
|
||||
# Helper to wait for job and handle result
|
||||
wait_for_job() {
|
||||
local pid=$1
|
||||
local name=$2
|
||||
local logfile=$3
|
||||
local start_time=$4
|
||||
|
||||
wait "$pid"
|
||||
local exit_code=$?
|
||||
local duration=$(($(date +%s) - start_time))
|
||||
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
printf "%-25s ❌ (%.1fs)\n" "$name" "$duration"
|
||||
if [ -s "$logfile" ]; then
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
cat "$logfile"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
fi
|
||||
return 1
|
||||
else
|
||||
printf "%-25s ✅ (%.1fs)\n" "$name" "$duration"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Build file list based on mode (compatible with sh and bash)
|
||||
if [ $STAGED_MODE -eq 1 ]; then
|
||||
# Get staged Python files (files being committed)
|
||||
FILE_ARRAY=()
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] && FILE_ARRAY+=("$file")
|
||||
done <<EOF
|
||||
$(git diff --cached --name-only --diff-filter=ACMR 2>/dev/null | grep '\.py$')
|
||||
EOF
|
||||
|
||||
if [ ${#FILE_ARRAY[@]} -eq 0 ]; then
|
||||
echo "[*] Staged mode: No Python files staged for commit"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[*] Staged mode: checking ${#FILE_ARRAY[@]} staged Python file(s)"
|
||||
elif [ $QUICK_MODE -eq 1 ]; then
|
||||
# Get all changed Python files (staged and unstaged)
|
||||
FILE_ARRAY=()
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] && FILE_ARRAY+=("$file")
|
||||
done <<EOF
|
||||
$(git diff --name-only --diff-filter=ACMR HEAD 2>/dev/null | grep '\.py$')
|
||||
EOF
|
||||
|
||||
if [ ${#FILE_ARRAY[@]} -eq 0 ]; then
|
||||
echo "[*] Quick mode: No Python files changed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[*] Quick mode: checking ${#FILE_ARRAY[@]} changed Python file(s)"
|
||||
else
|
||||
echo "[*] Full mode: checking all files (matches CI/CD exactly)"
|
||||
FILE_ARRAY=()
|
||||
fi
|
||||
|
||||
echo ""
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Launch all checks in parallel
|
||||
if [ ${#FILE_ARRAY[@]} -eq 0 ]; then
|
||||
# Full mode: check everything
|
||||
${RUN_CMD}ruff check --fix > "$TEMP_DIR/ruff-check.log" 2>&1 &
|
||||
RUFF_CHECK_PID=$!
|
||||
RUFF_CHECK_START=$(date +%s)
|
||||
|
||||
${RUN_CMD}ruff format > "$TEMP_DIR/ruff-format.log" 2>&1 &
|
||||
RUFF_FORMAT_PID=$!
|
||||
RUFF_FORMAT_START=$(date +%s)
|
||||
|
||||
${RUN_CMD}pyright --threads 6 > "$TEMP_DIR/pyright.log" 2>&1 &
|
||||
PYRIGHT_PID=$!
|
||||
PYRIGHT_START=$(date +%s)
|
||||
|
||||
SKIP=ruff-check,ruff-format,pyright ${RUN_CMD}pre-commit run --all-files > "$TEMP_DIR/other-checks.log" 2>&1 &
|
||||
OTHER_PID=$!
|
||||
OTHER_START=$(date +%s)
|
||||
else
|
||||
# Staged or quick mode: check only specific files
|
||||
${RUN_CMD}ruff check --fix "${FILE_ARRAY[@]}" > "$TEMP_DIR/ruff-check.log" 2>&1 &
|
||||
RUFF_CHECK_PID=$!
|
||||
RUFF_CHECK_START=$(date +%s)
|
||||
|
||||
${RUN_CMD}ruff format "${FILE_ARRAY[@]}" > "$TEMP_DIR/ruff-format.log" 2>&1 &
|
||||
RUFF_FORMAT_PID=$!
|
||||
RUFF_FORMAT_START=$(date +%s)
|
||||
|
||||
# Pyright: skip in quick mode, run in staged mode
|
||||
if [ $QUICK_MODE -eq 1 ]; then
|
||||
echo "" > "$TEMP_DIR/pyright.log"
|
||||
PYRIGHT_PID=-1
|
||||
PYRIGHT_START=$(date +%s)
|
||||
else
|
||||
${RUN_CMD}pyright --threads 6 "${FILE_ARRAY[@]}" > "$TEMP_DIR/pyright.log" 2>&1 &
|
||||
PYRIGHT_PID=$!
|
||||
PYRIGHT_START=$(date +%s)
|
||||
fi
|
||||
|
||||
SKIP=ruff-check,ruff-format,pyright ${RUN_CMD}pre-commit run --files "${FILE_ARRAY[@]}" > "$TEMP_DIR/other-checks.log" 2>&1 &
|
||||
OTHER_PID=$!
|
||||
OTHER_START=$(date +%s)
|
||||
fi
|
||||
|
||||
# Track failures
|
||||
FAILED=0
|
||||
FAILED_CHECKS=""
|
||||
|
||||
# Wait for each job in order of expected completion (fastest first)
|
||||
# This allows --fail-fast to exit as soon as any check fails
|
||||
|
||||
# Ruff format is typically fastest
|
||||
spinner $RUFF_FORMAT_PID "ruff format"
|
||||
if ! wait_for_job $RUFF_FORMAT_PID "ruff format" "$TEMP_DIR/ruff-format.log" $RUFF_FORMAT_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS ruff-format"
|
||||
if [ $FAIL_FAST -eq 1 ]; then
|
||||
kill $RUFF_CHECK_PID $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
wait $RUFF_CHECK_PID $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
echo ""
|
||||
echo "❌ Fast-fail: Exiting early due to ruff format failure"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Ruff check is second fastest
|
||||
spinner $RUFF_CHECK_PID "ruff check"
|
||||
if ! wait_for_job $RUFF_CHECK_PID "ruff check" "$TEMP_DIR/ruff-check.log" $RUFF_CHECK_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS ruff-check"
|
||||
if [ $FAIL_FAST -eq 1 ]; then
|
||||
kill $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
wait $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
echo ""
|
||||
echo "❌ Fast-fail: Exiting early due to ruff check failure"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Pre-commit hooks are medium speed
|
||||
spinner $OTHER_PID "other pre-commit hooks"
|
||||
if ! wait_for_job $OTHER_PID "other pre-commit hooks" "$TEMP_DIR/other-checks.log" $OTHER_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS pre-commit"
|
||||
if [ $FAIL_FAST -eq 1 ]; then
|
||||
kill $PYRIGHT_PID 2>/dev/null
|
||||
wait $PYRIGHT_PID 2>/dev/null
|
||||
echo ""
|
||||
echo "❌ Fast-fail: Exiting early due to pre-commit hooks failure"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Pyright is slowest (wait last for maximum parallelism)
|
||||
if [ $PYRIGHT_PID -ne -1 ]; then
|
||||
spinner $PYRIGHT_PID "pyright"
|
||||
if ! wait_for_job $PYRIGHT_PID "pyright" "$TEMP_DIR/pyright.log" $PYRIGHT_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS pyright"
|
||||
fi
|
||||
else
|
||||
printf "%-25s ⏭️ (skipped in quick mode)\n" "pyright"
|
||||
fi
|
||||
|
||||
TOTAL_TIME=$(($(date +%s) - START_TIME))
|
||||
|
||||
echo ""
|
||||
if [ $FAILED -eq 1 ]; then
|
||||
echo "❌ Checks failed:$FAILED_CHECKS (${TOTAL_TIME}s total)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ All checks passed! (${TOTAL_TIME}s total)"
|
||||
exit 0
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script is used to setup a local development environment for the browser-use project.
|
||||
# Usage:
|
||||
# $ ./bin/setup.sh
|
||||
|
||||
### Bash Environment Setup
|
||||
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
|
||||
# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
|
||||
# set -o xtrace
|
||||
# set -x
|
||||
# shopt -s nullglob
|
||||
set -o errexit
|
||||
set -o errtrace
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
IFS=$'\n'
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
|
||||
if [ -f "$SCRIPT_DIR/lint.sh" ]; then
|
||||
echo "[√] already inside a cloned browser-use repo"
|
||||
else
|
||||
echo "[+] Cloning browser-use repo into current directory: $SCRIPT_DIR"
|
||||
git clone https://github.com/browser-use/browser-use
|
||||
cd browser-use
|
||||
fi
|
||||
|
||||
echo "[+] Installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
#git checkout main git pull
|
||||
echo
|
||||
echo "[+] Setting up venv"
|
||||
uv venv
|
||||
echo
|
||||
echo "[+] Installing packages in venv"
|
||||
uv sync --dev --all-extras
|
||||
echo
|
||||
echo "[i] Tip: make sure to set BROWSER_USE_LOGGING_LEVEL=debug and your LLM API keys in your .env file"
|
||||
echo
|
||||
uv pip show browser-use
|
||||
|
||||
echo "Usage:"
|
||||
echo " $ browser-use use the CLI"
|
||||
echo " or"
|
||||
echo " $ source .venv/bin/activate"
|
||||
echo " $ ipython use the library"
|
||||
echo " >>> from browser_use import BrowserSession, Agent"
|
||||
echo " >>> await Agent(task='book me a flight to fiji', browser=BrowserSession(headless=False)).run()"
|
||||
echo ""
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script is used to run all the main project tests that run on CI via .github/workflows/test.yaml.
|
||||
# Usage:
|
||||
# $ ./bin/test.sh
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$SCRIPT_DIR/.." || exit 1
|
||||
|
||||
exec uv run pytest --numprocesses auto tests/ci $1 $2 $3
|
||||
@@ -0,0 +1,51 @@
|
||||
# Codebase Structure
|
||||
|
||||
> The code structure inspired by https://github.com/Netflix/dispatch.
|
||||
|
||||
Very good structure on how to make a scalable codebase is also in [this repo](https://github.com/zhanymkanov/fastapi-best-practices).
|
||||
|
||||
Just a brief document about how we should structure our backend codebase.
|
||||
|
||||
## Code Structure
|
||||
|
||||
```markdown
|
||||
src/
|
||||
/<service name>/
|
||||
models.py
|
||||
services.py
|
||||
prompts.py
|
||||
views.py
|
||||
utils.py
|
||||
routers.py
|
||||
|
||||
/_<subservice name>/
|
||||
```
|
||||
|
||||
### Service.py
|
||||
|
||||
Always a single file, except if it becomes too long - more than ~500 lines, split it into \_subservices
|
||||
|
||||
### Views.py
|
||||
|
||||
Always split the views into two parts
|
||||
|
||||
```python
|
||||
# All
|
||||
...
|
||||
|
||||
# Requests
|
||||
...
|
||||
|
||||
# Responses
|
||||
...
|
||||
```
|
||||
|
||||
If too long → split into multiple files
|
||||
|
||||
### Prompts.py
|
||||
|
||||
Single file; if too long → split into multiple files (one prompt per file or so)
|
||||
|
||||
### Routers.py
|
||||
|
||||
Never split into more than one file
|
||||
@@ -0,0 +1,157 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.logging_config import setup_logging
|
||||
|
||||
# Only set up logging if not in MCP mode or if explicitly requested
|
||||
if os.environ.get('BROWSER_USE_SETUP_LOGGING', 'true').lower() != 'false':
|
||||
from browser_use.config import CONFIG
|
||||
|
||||
# Get log file paths from config/environment
|
||||
debug_log_file = getattr(CONFIG, 'BROWSER_USE_DEBUG_LOG_FILE', None)
|
||||
info_log_file = getattr(CONFIG, 'BROWSER_USE_INFO_LOG_FILE', None)
|
||||
|
||||
# Set up logging with file handlers if specified
|
||||
logger = setup_logging(debug_log_file=debug_log_file, info_log_file=info_log_file)
|
||||
else:
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger('browser_use')
|
||||
|
||||
# Monkeypatch BaseSubprocessTransport.__del__ to handle closed event loops gracefully
|
||||
from asyncio import base_subprocess
|
||||
|
||||
_original_del = base_subprocess.BaseSubprocessTransport.__del__
|
||||
|
||||
|
||||
def _patched_del(self):
|
||||
"""Patched __del__ that handles closed event loops without throwing noisy red-herring errors like RuntimeError: Event loop is closed"""
|
||||
try:
|
||||
# Check if the event loop is closed before calling the original
|
||||
if hasattr(self, '_loop') and self._loop and self._loop.is_closed():
|
||||
# Event loop is closed, skip cleanup that requires the loop
|
||||
return
|
||||
_original_del(self)
|
||||
except RuntimeError as e:
|
||||
if 'Event loop is closed' in str(e):
|
||||
# Silently ignore this specific error
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
base_subprocess.BaseSubprocessTransport.__del__ = _patched_del
|
||||
|
||||
|
||||
# Type stubs for lazy imports - fixes linter warnings
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.agent.prompts import SystemPrompt
|
||||
from browser_use.agent.service import Agent
|
||||
from browser_use.agent.views import ActionModel, ActionResult, AgentHistoryList
|
||||
from browser_use.browser import BrowserProfile, BrowserSession
|
||||
from browser_use.browser import BrowserSession as Browser
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.llm import models
|
||||
from browser_use.llm.anthropic.chat import ChatAnthropic
|
||||
from browser_use.llm.azure.chat import ChatAzureOpenAI
|
||||
from browser_use.llm.browser_use.chat import ChatBrowserUse
|
||||
from browser_use.llm.google.chat import ChatGoogle
|
||||
from browser_use.llm.groq.chat import ChatGroq
|
||||
from browser_use.llm.litellm.chat import ChatLiteLLM
|
||||
from browser_use.llm.mistral.chat import ChatMistral
|
||||
from browser_use.llm.oci_raw.chat import ChatOCIRaw
|
||||
from browser_use.llm.ollama.chat import ChatOllama
|
||||
from browser_use.llm.openai.chat import ChatOpenAI
|
||||
from browser_use.llm.vercel.chat import ChatVercel
|
||||
from browser_use.sandbox import sandbox
|
||||
from browser_use.tools.service import Controller, Tools
|
||||
|
||||
# Lazy imports mapping - only import when actually accessed
|
||||
_LAZY_IMPORTS = {
|
||||
# Agent service (heavy due to dependencies)
|
||||
'Agent': ('browser_use.agent.service', 'Agent'),
|
||||
# System prompt (moderate weight due to agent.views imports)
|
||||
'SystemPrompt': ('browser_use.agent.prompts', 'SystemPrompt'),
|
||||
# Agent views (very heavy - over 1 second!)
|
||||
'ActionModel': ('browser_use.agent.views', 'ActionModel'),
|
||||
'ActionResult': ('browser_use.agent.views', 'ActionResult'),
|
||||
'AgentHistoryList': ('browser_use.agent.views', 'AgentHistoryList'),
|
||||
'BrowserSession': ('browser_use.browser', 'BrowserSession'),
|
||||
'Browser': ('browser_use.browser', 'BrowserSession'), # Alias for BrowserSession
|
||||
'BrowserProfile': ('browser_use.browser', 'BrowserProfile'),
|
||||
# Tools (moderate weight)
|
||||
'Tools': ('browser_use.tools.service', 'Tools'),
|
||||
'Controller': ('browser_use.tools.service', 'Controller'), # alias
|
||||
# DOM service (moderate weight)
|
||||
'DomService': ('browser_use.dom.service', 'DomService'),
|
||||
# Chat models (very heavy imports)
|
||||
'ChatOpenAI': ('browser_use.llm.openai.chat', 'ChatOpenAI'),
|
||||
'ChatGoogle': ('browser_use.llm.google.chat', 'ChatGoogle'),
|
||||
'ChatAnthropic': ('browser_use.llm.anthropic.chat', 'ChatAnthropic'),
|
||||
'ChatBrowserUse': ('browser_use.llm.browser_use.chat', 'ChatBrowserUse'),
|
||||
'ChatGroq': ('browser_use.llm.groq.chat', 'ChatGroq'),
|
||||
'ChatLiteLLM': ('browser_use.llm.litellm.chat', 'ChatLiteLLM'),
|
||||
'ChatMistral': ('browser_use.llm.mistral.chat', 'ChatMistral'),
|
||||
'ChatAzureOpenAI': ('browser_use.llm.azure.chat', 'ChatAzureOpenAI'),
|
||||
'ChatOCIRaw': ('browser_use.llm.oci_raw.chat', 'ChatOCIRaw'),
|
||||
'ChatOllama': ('browser_use.llm.ollama.chat', 'ChatOllama'),
|
||||
'ChatVercel': ('browser_use.llm.vercel.chat', 'ChatVercel'),
|
||||
# LLM models module
|
||||
'models': ('browser_use.llm.models', None),
|
||||
# Sandbox execution
|
||||
'sandbox': ('browser_use.sandbox', 'sandbox'),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism - only import modules when they're actually accessed."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_path, attr_name = _LAZY_IMPORTS[name]
|
||||
try:
|
||||
from importlib import import_module
|
||||
|
||||
module = import_module(module_path)
|
||||
if attr_name is None:
|
||||
# For modules like 'models', return the module itself
|
||||
attr = module
|
||||
else:
|
||||
attr = getattr(module, attr_name)
|
||||
# Cache the imported attribute in the module's globals
|
||||
globals()[name] = attr
|
||||
return attr
|
||||
except ImportError as e:
|
||||
raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
|
||||
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Agent',
|
||||
'BrowserSession',
|
||||
'Browser', # Alias for BrowserSession
|
||||
'BrowserProfile',
|
||||
'Controller',
|
||||
'DomService',
|
||||
'SystemPrompt',
|
||||
'ActionResult',
|
||||
'ActionModel',
|
||||
'AgentHistoryList',
|
||||
# Chat models
|
||||
'ChatOpenAI',
|
||||
'ChatGoogle',
|
||||
'ChatAnthropic',
|
||||
'ChatBrowserUse',
|
||||
'ChatGroq',
|
||||
'ChatLiteLLM',
|
||||
'ChatMistral',
|
||||
'ChatAzureOpenAI',
|
||||
'ChatOCIRaw',
|
||||
'ChatOllama',
|
||||
'ChatVercel',
|
||||
'Tools',
|
||||
'Controller',
|
||||
# LLM models module
|
||||
'models',
|
||||
# Sandbox execution
|
||||
'sandbox',
|
||||
]
|
||||
@@ -0,0 +1,251 @@
|
||||
# Browser Actor
|
||||
|
||||
Browser Actor is a web automation library built on CDP (Chrome DevTools Protocol) that provides low-level browser automation capabilities within the browser-use ecosystem.
|
||||
|
||||
## Usage
|
||||
|
||||
### Integrated with Browser (Recommended)
|
||||
```python
|
||||
from browser_use import Browser # Alias for BrowserSession
|
||||
|
||||
# Create and start browser session
|
||||
browser = Browser()
|
||||
await browser.start()
|
||||
|
||||
# Create new tabs and navigate
|
||||
page = await browser.new_page("https://example.com")
|
||||
pages = await browser.get_pages()
|
||||
current_page = await browser.get_current_page()
|
||||
```
|
||||
|
||||
### Direct Page Access (Advanced)
|
||||
```python
|
||||
from browser_use.actor import Page, Element, Mouse
|
||||
|
||||
# Create page with existing browser session
|
||||
page = Page(browser_session, target_id, session_id)
|
||||
```
|
||||
|
||||
## Basic Operations
|
||||
|
||||
```python
|
||||
# Tab Management
|
||||
page = await browser.new_page() # Create blank tab
|
||||
page = await browser.new_page("https://example.com") # Create tab with URL
|
||||
pages = await browser.get_pages() # Get all existing tabs
|
||||
await browser.close_page(page) # Close specific tab
|
||||
|
||||
# Navigation
|
||||
await page.goto("https://example.com")
|
||||
await page.go_back()
|
||||
await page.go_forward()
|
||||
await page.reload()
|
||||
```
|
||||
|
||||
## Element Operations
|
||||
|
||||
```python
|
||||
# Find elements by CSS selector
|
||||
elements = await page.get_elements_by_css_selector("input[type='text']")
|
||||
buttons = await page.get_elements_by_css_selector("button.submit")
|
||||
|
||||
# Get element by backend node ID
|
||||
element = await page.get_element(backend_node_id=12345)
|
||||
|
||||
# AI-powered element finding (requires LLM)
|
||||
element = await page.get_element_by_prompt("search button", llm=your_llm)
|
||||
element = await page.must_get_element_by_prompt("login form", llm=your_llm)
|
||||
```
|
||||
|
||||
> **Note**: `get_elements_by_css_selector` returns immediately without waiting for visibility.
|
||||
|
||||
## Element Interactions
|
||||
|
||||
```python
|
||||
# Element actions
|
||||
await element.click(button='left', click_count=1, modifiers=['Control'])
|
||||
await element.fill("Hello World") # Clears first, then types
|
||||
await element.hover()
|
||||
await element.focus()
|
||||
await element.check() # Toggle checkbox/radio
|
||||
await element.select_option(["option1", "option2"]) # For dropdown/select
|
||||
await element.drag_to(target_element) # Drag and drop
|
||||
|
||||
# Element properties
|
||||
value = await element.get_attribute("value")
|
||||
box = await element.get_bounding_box() # Returns BoundingBox or None
|
||||
info = await element.get_basic_info() # Comprehensive element info
|
||||
screenshot_b64 = await element.screenshot(format='png')
|
||||
|
||||
# Execute JavaScript on element (this context is the element)
|
||||
text = await element.evaluate("() => this.textContent")
|
||||
await element.evaluate("(color) => this.style.backgroundColor = color", "yellow")
|
||||
classes = await element.evaluate("() => Array.from(this.classList)")
|
||||
```
|
||||
|
||||
## Mouse Operations
|
||||
|
||||
```python
|
||||
# Mouse operations
|
||||
mouse = await page.mouse
|
||||
await mouse.click(x=100, y=200, button='left', click_count=1)
|
||||
await mouse.move(x=300, y=400, steps=1)
|
||||
await mouse.down(button='left') # Press button
|
||||
await mouse.up(button='left') # Release button
|
||||
await mouse.scroll(x=0, y=100, delta_x=0, delta_y=-500) # Scroll at coordinates
|
||||
```
|
||||
|
||||
## Page Operations
|
||||
|
||||
```python
|
||||
# JavaScript evaluation
|
||||
result = await page.evaluate('() => document.title') # Must use arrow function format
|
||||
result = await page.evaluate('(x, y) => x + y', 10, 20) # With arguments
|
||||
|
||||
# Keyboard input
|
||||
await page.press("Control+A") # Key combinations supported
|
||||
await page.press("Escape") # Single keys
|
||||
|
||||
# Page controls
|
||||
await page.set_viewport_size(width=1920, height=1080)
|
||||
page_screenshot = await page.screenshot() # PNG by default
|
||||
page_png = await page.screenshot(format="png", quality=90)
|
||||
|
||||
# Page information
|
||||
url = await page.get_url()
|
||||
title = await page.get_title()
|
||||
```
|
||||
|
||||
## AI-Powered Features
|
||||
|
||||
```python
|
||||
# Content extraction using LLM
|
||||
from pydantic import BaseModel
|
||||
|
||||
class ProductInfo(BaseModel):
|
||||
name: str
|
||||
price: float
|
||||
description: str
|
||||
|
||||
# Extract structured data from current page
|
||||
products = await page.extract_content(
|
||||
"Find all products with their names, prices and descriptions",
|
||||
ProductInfo,
|
||||
llm=your_llm
|
||||
)
|
||||
```
|
||||
|
||||
## Core Classes
|
||||
|
||||
- **BrowserSession** (aliased as **Browser**): Main browser session manager with tab operations
|
||||
- **Page**: Represents a single browser tab or iframe for page-level operations
|
||||
- **Element**: Individual DOM element for interactions and property access
|
||||
- **Mouse**: Mouse operations within a page (click, move, scroll)
|
||||
|
||||
## API Reference
|
||||
|
||||
### BrowserSession Methods (Tab Management)
|
||||
- `start()` - Initialize and start the browser session
|
||||
- `stop()` - Stop the browser session (keeps browser alive)
|
||||
- `kill()` - Kill the browser process and reset all state
|
||||
- `new_page(url=None)` → `Page` - Create blank tab or navigate to URL
|
||||
- `get_pages()` → `list[Page]` - Get all available pages
|
||||
- `get_current_page()` → `Page | None` - Get the currently focused page
|
||||
- `close_page(page: Page | str)` - Close page by object or ID
|
||||
- Session management and CDP client operations
|
||||
|
||||
### Page Methods (Page Operations)
|
||||
- `get_elements_by_css_selector(selector: str)` → `list[Element]` - Find elements by CSS selector
|
||||
- `get_element(backend_node_id: int)` → `Element` - Get element by backend node ID
|
||||
- `get_element_by_prompt(prompt: str, llm)` → `Element | None` - AI-powered element finding
|
||||
- `must_get_element_by_prompt(prompt: str, llm)` → `Element` - AI element finding (raises if not found)
|
||||
- `extract_content(prompt: str, structured_output: type[T], llm)` → `T` - Extract structured data using LLM
|
||||
- `goto(url: str)` - Navigate this page to URL
|
||||
- `go_back()`, `go_forward()` - Navigate history (with error handling)
|
||||
- `reload()` - Reload the current page
|
||||
- `evaluate(page_function: str, *args)` → `str` - Execute JavaScript (MUST use (...args) => format)
|
||||
- `press(key: str)` - Press key on page (supports "Control+A" format)
|
||||
- `set_viewport_size(width: int, height: int)` - Set viewport dimensions
|
||||
- `screenshot(format='png', quality=None)` → `str` - Take page screenshot, return base64
|
||||
- `get_url()` → `str`, `get_title()` → `str` - Get page information
|
||||
- `mouse` → `Mouse` - Get mouse interface for this page
|
||||
|
||||
### Element Methods (DOM Interactions)
|
||||
- `click(button='left', click_count=1, modifiers=None)` - Click element with advanced fallbacks
|
||||
- `fill(text: str, clear=True)` - Fill input with text (clears first by default)
|
||||
- `hover()` - Hover over element
|
||||
- `focus()` - Focus the element
|
||||
- `check()` - Toggle checkbox/radio button (clicks to change state)
|
||||
- `select_option(values: str | list[str])` - Select dropdown options
|
||||
- `drag_to(target_element: Element | Position, source_position=None, target_position=None)` - Drag to target element
|
||||
- `evaluate(page_function: str, *args)` → `str` - Execute JavaScript on element (this = element)
|
||||
- `get_attribute(name: str)` → `str | None` - Get attribute value
|
||||
- `get_bounding_box()` → `BoundingBox | None` - Get element position/size
|
||||
- `screenshot(format='png', quality=None)` → `str` - Take element screenshot, return base64
|
||||
- `get_basic_info()` → `ElementInfo` - Get comprehensive element information
|
||||
|
||||
|
||||
### Mouse Methods (Coordinate-Based Operations)
|
||||
- `click(x: int, y: int, button='left', click_count=1)` - Click at coordinates
|
||||
- `move(x: int, y: int, steps=1)` - Move to coordinates
|
||||
- `down(button='left', click_count=1)`, `up(button='left', click_count=1)` - Press/release button
|
||||
- `scroll(x=0, y=0, delta_x=None, delta_y=None)` - Scroll page at coordinates
|
||||
|
||||
## Type Definitions
|
||||
|
||||
### Position
|
||||
```python
|
||||
class Position(TypedDict):
|
||||
x: float
|
||||
y: float
|
||||
```
|
||||
|
||||
### BoundingBox
|
||||
```python
|
||||
class BoundingBox(TypedDict):
|
||||
x: float
|
||||
y: float
|
||||
width: float
|
||||
height: float
|
||||
```
|
||||
|
||||
### ElementInfo
|
||||
```python
|
||||
class ElementInfo(TypedDict):
|
||||
backendNodeId: int # CDP backend node ID
|
||||
nodeId: int | None # CDP node ID
|
||||
nodeName: str # HTML tag name (e.g., "DIV", "INPUT")
|
||||
nodeType: int # DOM node type
|
||||
nodeValue: str | None # Text content for text nodes
|
||||
attributes: dict[str, str] # HTML attributes
|
||||
boundingBox: BoundingBox | None # Element position and size
|
||||
error: str | None # Error message if info retrieval failed
|
||||
```
|
||||
|
||||
## Important Usage Notes
|
||||
|
||||
**This is browser-use actor, NOT Playwright or Selenium.** Only use the methods documented above.
|
||||
|
||||
### Critical JavaScript Rules
|
||||
- `page.evaluate()` and `element.evaluate()` MUST use `(...args) => {}` arrow function format
|
||||
- Always returns string (objects are JSON-stringified automatically)
|
||||
- Use single quotes around the function: `page.evaluate('() => document.title')`
|
||||
- For complex selectors in JS: `'() => document.querySelector("input[name=\\"email\\"]")'`
|
||||
- `element.evaluate()`: `this` context is bound to the element automatically
|
||||
|
||||
### Method Restrictions
|
||||
- `get_elements_by_css_selector()` returns immediately (no automatic waiting)
|
||||
- For dropdowns: use `element.select_option()`, NOT `element.fill()`
|
||||
- Form submission: click submit button or use `page.press("Enter")`
|
||||
- No methods like: `element.submit()`, `element.dispatch_event()`, `element.get_property()`
|
||||
|
||||
### Error Prevention
|
||||
- Always verify page state changes with `page.get_url()`, `page.get_title()`
|
||||
- Use `element.get_attribute()` to check element properties
|
||||
- Validate CSS selectors before use
|
||||
- Handle navigation timing with appropriate `asyncio.sleep()` calls
|
||||
|
||||
### AI Features
|
||||
- `get_element_by_prompt()` and `extract_content()` require an LLM instance
|
||||
- These methods use DOM analysis and structured output parsing
|
||||
- Best for complex page understanding and data extraction tasks
|
||||
@@ -0,0 +1,11 @@
|
||||
"""CDP-Use High-Level Library
|
||||
|
||||
A Playwright-like library built on top of CDP (Chrome DevTools Protocol).
|
||||
"""
|
||||
|
||||
from .element import Element
|
||||
from .mouse import Mouse
|
||||
from .page import Page
|
||||
from .utils import Utils
|
||||
|
||||
__all__ = ['Page', 'Element', 'Mouse', 'Utils']
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,134 @@
|
||||
"""Mouse class for mouse operations."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cdp_use.cdp.input.commands import DispatchMouseEventParameters, SynthesizeScrollGestureParameters
|
||||
from cdp_use.cdp.input.types import MouseButton
|
||||
|
||||
from browser_use.browser.session import BrowserSession
|
||||
|
||||
|
||||
class Mouse:
|
||||
"""Mouse operations for a target."""
|
||||
|
||||
def __init__(self, browser_session: 'BrowserSession', session_id: str | None = None, target_id: str | None = None):
|
||||
self._browser_session = browser_session
|
||||
self._client = browser_session.cdp_client
|
||||
self._session_id = session_id
|
||||
self._target_id = target_id
|
||||
|
||||
async def click(self, x: int, y: int, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
|
||||
"""Click at the specified coordinates."""
|
||||
# Mouse press
|
||||
press_params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mousePressed',
|
||||
'x': x,
|
||||
'y': y,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
press_params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
# Mouse release
|
||||
release_params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mouseReleased',
|
||||
'x': x,
|
||||
'y': y,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
release_params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
async def down(self, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
|
||||
"""Press mouse button down."""
|
||||
params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mousePressed',
|
||||
'x': 0, # Will use last mouse position
|
||||
'y': 0,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
async def up(self, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
|
||||
"""Release mouse button."""
|
||||
params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mouseReleased',
|
||||
'x': 0, # Will use last mouse position
|
||||
'y': 0,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
async def move(self, x: int, y: int, steps: int = 1) -> None:
|
||||
"""Move mouse to the specified coordinates."""
|
||||
# TODO: Implement smooth movement with multiple steps if needed
|
||||
_ = steps # Acknowledge parameter for future use
|
||||
|
||||
params: 'DispatchMouseEventParameters' = {'type': 'mouseMoved', 'x': x, 'y': y}
|
||||
await self._client.send.Input.dispatchMouseEvent(params, session_id=self._session_id)
|
||||
|
||||
async def scroll(self, x: int = 0, y: int = 0, delta_x: int | None = None, delta_y: int | None = None) -> None:
|
||||
"""Scroll the page using robust CDP methods."""
|
||||
if not self._session_id:
|
||||
raise RuntimeError('Session ID is required for scroll operations')
|
||||
|
||||
# Method 1: Try mouse wheel event (most reliable)
|
||||
try:
|
||||
# Get viewport dimensions
|
||||
layout_metrics = await self._client.send.Page.getLayoutMetrics(session_id=self._session_id)
|
||||
viewport_width = layout_metrics['layoutViewport']['clientWidth']
|
||||
viewport_height = layout_metrics['layoutViewport']['clientHeight']
|
||||
|
||||
# Use provided coordinates or center of viewport
|
||||
scroll_x = x if x > 0 else viewport_width / 2
|
||||
scroll_y = y if y > 0 else viewport_height / 2
|
||||
|
||||
# Calculate scroll deltas (positive = down/right)
|
||||
scroll_delta_x = delta_x or 0
|
||||
scroll_delta_y = delta_y or 0
|
||||
|
||||
# Dispatch mouse wheel event
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
params={
|
||||
'type': 'mouseWheel',
|
||||
'x': scroll_x,
|
||||
'y': scroll_y,
|
||||
'deltaX': scroll_delta_x,
|
||||
'deltaY': scroll_delta_y,
|
||||
},
|
||||
session_id=self._session_id,
|
||||
)
|
||||
return
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Method 2: Fallback to synthesizeScrollGesture
|
||||
try:
|
||||
params: 'SynthesizeScrollGestureParameters' = {'x': x, 'y': y, 'xDistance': delta_x or 0, 'yDistance': delta_y or 0}
|
||||
await self._client.send.Input.synthesizeScrollGesture(
|
||||
params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
except Exception:
|
||||
# Method 3: JavaScript fallback
|
||||
scroll_js = f'window.scrollBy({delta_x or 0}, {delta_y or 0})'
|
||||
await self._client.send.Runtime.evaluate(
|
||||
params={'expression': scroll_js, 'returnByValue': True},
|
||||
session_id=self._session_id,
|
||||
)
|
||||
@@ -0,0 +1,564 @@
|
||||
"""Page class for page-level operations."""
|
||||
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from browser_use import logger
|
||||
from browser_use.actor.utils import get_key_info
|
||||
from browser_use.dom.serializer.serializer import DOMTreeSerializer
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.llm.messages import SystemMessage, UserMessage
|
||||
|
||||
T = TypeVar('T', bound=BaseModel)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cdp_use.cdp.dom.commands import (
|
||||
DescribeNodeParameters,
|
||||
QuerySelectorAllParameters,
|
||||
)
|
||||
from cdp_use.cdp.emulation.commands import SetDeviceMetricsOverrideParameters
|
||||
from cdp_use.cdp.input.commands import (
|
||||
DispatchKeyEventParameters,
|
||||
)
|
||||
from cdp_use.cdp.page.commands import CaptureScreenshotParameters, NavigateParameters, NavigateToHistoryEntryParameters
|
||||
from cdp_use.cdp.runtime.commands import EvaluateParameters
|
||||
from cdp_use.cdp.target.commands import (
|
||||
AttachToTargetParameters,
|
||||
GetTargetInfoParameters,
|
||||
)
|
||||
from cdp_use.cdp.target.types import TargetInfo
|
||||
|
||||
from browser_use.browser.session import BrowserSession
|
||||
from browser_use.llm.base import BaseChatModel
|
||||
|
||||
from .element import Element
|
||||
from .mouse import Mouse
|
||||
|
||||
|
||||
class Page:
|
||||
"""Page operations (tab or iframe)."""
|
||||
|
||||
def __init__(
|
||||
self, browser_session: 'BrowserSession', target_id: str, session_id: str | None = None, llm: 'BaseChatModel | None' = None
|
||||
):
|
||||
self._browser_session = browser_session
|
||||
self._client = browser_session.cdp_client
|
||||
self._target_id = target_id
|
||||
self._session_id: str | None = session_id
|
||||
self._mouse: 'Mouse | None' = None
|
||||
|
||||
self._llm = llm
|
||||
|
||||
async def _ensure_session(self) -> str:
|
||||
"""Ensure we have a session ID for this target."""
|
||||
if not self._session_id:
|
||||
params: 'AttachToTargetParameters' = {'targetId': self._target_id, 'flatten': True}
|
||||
result = await self._client.send.Target.attachToTarget(params)
|
||||
self._session_id = result['sessionId']
|
||||
|
||||
# Enable necessary domains
|
||||
import asyncio
|
||||
|
||||
await asyncio.gather(
|
||||
self._client.send.Page.enable(session_id=self._session_id),
|
||||
self._client.send.DOM.enable(session_id=self._session_id),
|
||||
self._client.send.Runtime.enable(session_id=self._session_id),
|
||||
self._client.send.Network.enable(session_id=self._session_id),
|
||||
)
|
||||
|
||||
return self._session_id
|
||||
|
||||
@property
|
||||
async def session_id(self) -> str:
|
||||
"""Get the session ID for this target.
|
||||
|
||||
@dev Pass this to an arbitrary CDP call
|
||||
"""
|
||||
return await self._ensure_session()
|
||||
|
||||
@property
|
||||
async def mouse(self) -> 'Mouse':
|
||||
"""Get the mouse interface for this target."""
|
||||
if not self._mouse:
|
||||
session_id = await self._ensure_session()
|
||||
from .mouse import Mouse
|
||||
|
||||
self._mouse = Mouse(self._browser_session, session_id, self._target_id)
|
||||
return self._mouse
|
||||
|
||||
async def reload(self) -> None:
|
||||
"""Reload the target."""
|
||||
session_id = await self._ensure_session()
|
||||
await self._client.send.Page.reload(session_id=session_id)
|
||||
|
||||
async def get_element(self, backend_node_id: int) -> 'Element':
|
||||
"""Get an element by its backend node ID."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
from .element import Element as Element_
|
||||
|
||||
return Element_(self._browser_session, backend_node_id, session_id)
|
||||
|
||||
async def evaluate(self, page_function: str, *args) -> str:
|
||||
"""Execute JavaScript in the target.
|
||||
|
||||
Args:
|
||||
page_function: JavaScript code that MUST start with (...args) => format
|
||||
*args: Arguments to pass to the function
|
||||
|
||||
Returns:
|
||||
String representation of the JavaScript execution result.
|
||||
Objects and arrays are JSON-stringified.
|
||||
"""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
# Clean and fix common JavaScript string parsing issues
|
||||
page_function = self._fix_javascript_string(page_function)
|
||||
|
||||
# Enforce arrow function format
|
||||
if not (page_function.startswith('(') and '=>' in page_function):
|
||||
raise ValueError(f'JavaScript code must start with (...args) => format. Got: {page_function[:50]}...')
|
||||
|
||||
# Build the expression - call the arrow function with provided args
|
||||
if args:
|
||||
# Convert args to JSON representation for safe passing
|
||||
import json
|
||||
|
||||
arg_strs = [json.dumps(arg) for arg in args]
|
||||
expression = f'({page_function})({", ".join(arg_strs)})'
|
||||
else:
|
||||
expression = f'({page_function})()'
|
||||
|
||||
# Debug: log the actual expression being evaluated
|
||||
logger.debug(f'Evaluating JavaScript: {repr(expression)}')
|
||||
|
||||
params: 'EvaluateParameters' = {'expression': expression, 'returnByValue': True, 'awaitPromise': True}
|
||||
result = await self._client.send.Runtime.evaluate(
|
||||
params,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
if 'exceptionDetails' in result:
|
||||
raise RuntimeError(f'JavaScript evaluation failed: {result["exceptionDetails"]}')
|
||||
|
||||
value = result.get('result', {}).get('value')
|
||||
|
||||
# Always return string representation
|
||||
if value is None:
|
||||
return ''
|
||||
elif isinstance(value, str):
|
||||
return value
|
||||
else:
|
||||
# Convert objects, numbers, booleans to string
|
||||
import json
|
||||
|
||||
try:
|
||||
return json.dumps(value) if isinstance(value, (dict, list)) else str(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(value)
|
||||
|
||||
def _fix_javascript_string(self, js_code: str) -> str:
|
||||
"""Fix common JavaScript string parsing issues when written as Python string."""
|
||||
|
||||
# Just do minimal, safe cleaning
|
||||
js_code = js_code.strip()
|
||||
|
||||
# Only fix the most common and safe issues:
|
||||
|
||||
# 1. Remove obvious Python string wrapper quotes if they exist
|
||||
if (js_code.startswith('"') and js_code.endswith('"')) or (js_code.startswith("'") and js_code.endswith("'")):
|
||||
# Check if it's a wrapped string (not part of JS syntax)
|
||||
inner = js_code[1:-1]
|
||||
if inner.count('"') + inner.count("'") == 0 or '() =>' in inner:
|
||||
js_code = inner
|
||||
|
||||
# 2. Only fix clearly escaped quotes that shouldn't be
|
||||
# But be very conservative - only if we're sure it's a Python string artifact
|
||||
if '\\"' in js_code and js_code.count('\\"') > js_code.count('"'):
|
||||
js_code = js_code.replace('\\"', '"')
|
||||
if "\\'" in js_code and js_code.count("\\'") > js_code.count("'"):
|
||||
js_code = js_code.replace("\\'", "'")
|
||||
|
||||
# 3. Basic whitespace normalization only
|
||||
js_code = js_code.strip()
|
||||
|
||||
# Final validation - ensure it's not empty
|
||||
if not js_code:
|
||||
raise ValueError('JavaScript code is empty after cleaning')
|
||||
|
||||
return js_code
|
||||
|
||||
async def screenshot(self, format: str = 'png', quality: int | None = None) -> str:
|
||||
"""Take a screenshot and return base64 encoded image.
|
||||
|
||||
Args:
|
||||
format: Image format ('jpeg', 'png', 'webp')
|
||||
quality: Quality 0-100 for JPEG format
|
||||
|
||||
Returns:
|
||||
Base64-encoded image data
|
||||
"""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
params: 'CaptureScreenshotParameters' = {'format': format}
|
||||
|
||||
if quality is not None and format.lower() == 'jpeg':
|
||||
params['quality'] = quality
|
||||
|
||||
result = await self._client.send.Page.captureScreenshot(params, session_id=session_id)
|
||||
|
||||
return result['data']
|
||||
|
||||
async def press(self, key: str) -> None:
|
||||
"""Press a key on the page (sends keyboard input to the focused element or page)."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
# Handle key combinations like "Control+A"
|
||||
if '+' in key:
|
||||
parts = key.split('+')
|
||||
modifiers = parts[:-1]
|
||||
main_key = parts[-1]
|
||||
|
||||
# Calculate modifier bitmask
|
||||
modifier_value = 0
|
||||
modifier_map = {'Alt': 1, 'Control': 2, 'Meta': 4, 'Shift': 8}
|
||||
for mod in modifiers:
|
||||
modifier_value |= modifier_map.get(mod, 0)
|
||||
|
||||
# Press modifier keys
|
||||
for mod in modifiers:
|
||||
code, vk_code = get_key_info(mod)
|
||||
params: 'DispatchKeyEventParameters' = {'type': 'keyDown', 'key': mod, 'code': code}
|
||||
if vk_code is not None:
|
||||
params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(params, session_id=session_id)
|
||||
|
||||
# Press main key with modifiers bitmask
|
||||
main_code, main_vk_code = get_key_info(main_key)
|
||||
main_down_params: 'DispatchKeyEventParameters' = {
|
||||
'type': 'keyDown',
|
||||
'key': main_key,
|
||||
'code': main_code,
|
||||
'modifiers': modifier_value,
|
||||
}
|
||||
if main_vk_code is not None:
|
||||
main_down_params['windowsVirtualKeyCode'] = main_vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(main_down_params, session_id=session_id)
|
||||
|
||||
main_up_params: 'DispatchKeyEventParameters' = {
|
||||
'type': 'keyUp',
|
||||
'key': main_key,
|
||||
'code': main_code,
|
||||
'modifiers': modifier_value,
|
||||
}
|
||||
if main_vk_code is not None:
|
||||
main_up_params['windowsVirtualKeyCode'] = main_vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(main_up_params, session_id=session_id)
|
||||
|
||||
# Release modifier keys
|
||||
for mod in reversed(modifiers):
|
||||
code, vk_code = get_key_info(mod)
|
||||
release_params: 'DispatchKeyEventParameters' = {'type': 'keyUp', 'key': mod, 'code': code}
|
||||
if vk_code is not None:
|
||||
release_params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(release_params, session_id=session_id)
|
||||
else:
|
||||
# Simple key press
|
||||
code, vk_code = get_key_info(key)
|
||||
key_down_params: 'DispatchKeyEventParameters' = {'type': 'keyDown', 'key': key, 'code': code}
|
||||
if vk_code is not None:
|
||||
key_down_params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(key_down_params, session_id=session_id)
|
||||
|
||||
key_up_params: 'DispatchKeyEventParameters' = {'type': 'keyUp', 'key': key, 'code': code}
|
||||
if vk_code is not None:
|
||||
key_up_params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(key_up_params, session_id=session_id)
|
||||
|
||||
async def set_viewport_size(self, width: int, height: int) -> None:
|
||||
"""Set the viewport size."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
params: 'SetDeviceMetricsOverrideParameters' = {
|
||||
'width': width,
|
||||
'height': height,
|
||||
'deviceScaleFactor': 1.0,
|
||||
'mobile': False,
|
||||
}
|
||||
await self._client.send.Emulation.setDeviceMetricsOverride(
|
||||
params,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
# Target properties (from CDP getTargetInfo)
|
||||
async def get_target_info(self) -> 'TargetInfo':
|
||||
"""Get target information."""
|
||||
params: 'GetTargetInfoParameters' = {'targetId': self._target_id}
|
||||
result = await self._client.send.Target.getTargetInfo(params)
|
||||
return result['targetInfo']
|
||||
|
||||
async def get_url(self) -> str:
|
||||
"""Get the current URL."""
|
||||
info = await self.get_target_info()
|
||||
return info.get('url', '')
|
||||
|
||||
async def get_title(self) -> str:
|
||||
"""Get the current title."""
|
||||
info = await self.get_target_info()
|
||||
return info.get('title', '')
|
||||
|
||||
async def goto(self, url: str) -> None:
|
||||
"""Navigate this target to a URL."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
params: 'NavigateParameters' = {'url': url}
|
||||
await self._client.send.Page.navigate(params, session_id=session_id)
|
||||
|
||||
async def navigate(self, url: str) -> None:
|
||||
"""Alias for goto."""
|
||||
await self.goto(url)
|
||||
|
||||
async def go_back(self) -> None:
|
||||
"""Navigate back in history."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
try:
|
||||
# Get navigation history
|
||||
history = await self._client.send.Page.getNavigationHistory(session_id=session_id)
|
||||
current_index = history['currentIndex']
|
||||
entries = history['entries']
|
||||
|
||||
# Check if we can go back
|
||||
if current_index <= 0:
|
||||
raise RuntimeError('Cannot go back - no previous entry in history')
|
||||
|
||||
# Navigate to the previous entry
|
||||
previous_entry_id = entries[current_index - 1]['id']
|
||||
params: 'NavigateToHistoryEntryParameters' = {'entryId': previous_entry_id}
|
||||
await self._client.send.Page.navigateToHistoryEntry(params, session_id=session_id)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Failed to navigate back: {e}')
|
||||
|
||||
async def go_forward(self) -> None:
|
||||
"""Navigate forward in history."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
try:
|
||||
# Get navigation history
|
||||
history = await self._client.send.Page.getNavigationHistory(session_id=session_id)
|
||||
current_index = history['currentIndex']
|
||||
entries = history['entries']
|
||||
|
||||
# Check if we can go forward
|
||||
if current_index >= len(entries) - 1:
|
||||
raise RuntimeError('Cannot go forward - no next entry in history')
|
||||
|
||||
# Navigate to the next entry
|
||||
next_entry_id = entries[current_index + 1]['id']
|
||||
params: 'NavigateToHistoryEntryParameters' = {'entryId': next_entry_id}
|
||||
await self._client.send.Page.navigateToHistoryEntry(params, session_id=session_id)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Failed to navigate forward: {e}')
|
||||
|
||||
# Element finding methods (these would need to be implemented based on DOM queries)
|
||||
async def get_elements_by_css_selector(self, selector: str) -> list['Element']:
|
||||
"""Get elements by CSS selector."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
# Get document first
|
||||
doc_result = await self._client.send.DOM.getDocument(session_id=session_id)
|
||||
document_node_id = doc_result['root']['nodeId']
|
||||
|
||||
# Query selector all
|
||||
query_params: 'QuerySelectorAllParameters' = {'nodeId': document_node_id, 'selector': selector}
|
||||
result = await self._client.send.DOM.querySelectorAll(query_params, session_id=session_id)
|
||||
|
||||
elements = []
|
||||
from .element import Element as Element_
|
||||
|
||||
# Convert node IDs to backend node IDs
|
||||
for node_id in result['nodeIds']:
|
||||
# Get backend node ID
|
||||
describe_params: 'DescribeNodeParameters' = {'nodeId': node_id}
|
||||
node_result = await self._client.send.DOM.describeNode(describe_params, session_id=session_id)
|
||||
backend_node_id = node_result['node']['backendNodeId']
|
||||
elements.append(Element_(self._browser_session, backend_node_id, session_id))
|
||||
|
||||
return elements
|
||||
|
||||
# AI METHODS
|
||||
|
||||
@property
|
||||
def dom_service(self) -> 'DomService':
|
||||
"""Get the DOM service for this target."""
|
||||
return DomService(self._browser_session)
|
||||
|
||||
async def get_element_by_prompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element | None':
|
||||
"""Get an element by a prompt."""
|
||||
await self._ensure_session()
|
||||
llm = llm or self._llm
|
||||
|
||||
if not llm:
|
||||
raise ValueError('LLM not provided')
|
||||
|
||||
dom_service = self.dom_service
|
||||
|
||||
# Lazy fetch all_frames inside get_dom_tree if needed (for cross-origin iframes)
|
||||
enhanced_dom_tree, _ = await dom_service.get_dom_tree(target_id=self._target_id, all_frames=None)
|
||||
|
||||
session_id = self._browser_session.id
|
||||
serialized_dom_state, _ = DOMTreeSerializer(
|
||||
enhanced_dom_tree, None, paint_order_filtering=True, session_id=session_id
|
||||
).serialize_accessible_elements()
|
||||
|
||||
llm_representation = serialized_dom_state.llm_representation()
|
||||
|
||||
system_message = SystemMessage(
|
||||
content="""You are an AI created to find an element on a page by a prompt.
|
||||
|
||||
<browser_state>
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, etc.)
|
||||
- text: Element description
|
||||
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
[35]<button aria-label='Submit form'>Submit</button>
|
||||
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Pure text elements without [] are not interactive.
|
||||
</browser_state>
|
||||
|
||||
Your task is to find an element index (if any) that matches the prompt (written in <prompt> tag).
|
||||
|
||||
If non of the elements matches the, return None.
|
||||
|
||||
Before you return the element index, reason about the state and elements for a sentence or two."""
|
||||
)
|
||||
|
||||
state_message = UserMessage(
|
||||
content=f"""
|
||||
<browser_state>
|
||||
{llm_representation}
|
||||
</browser_state>
|
||||
|
||||
<prompt>
|
||||
{prompt}
|
||||
</prompt>
|
||||
"""
|
||||
)
|
||||
|
||||
class ElementResponse(BaseModel):
|
||||
# thinking: str
|
||||
element_highlight_index: int | None
|
||||
|
||||
llm_response = await llm.ainvoke(
|
||||
[
|
||||
system_message,
|
||||
state_message,
|
||||
],
|
||||
output_format=ElementResponse,
|
||||
)
|
||||
|
||||
element_highlight_index = llm_response.completion.element_highlight_index
|
||||
|
||||
if element_highlight_index is None or element_highlight_index not in serialized_dom_state.selector_map:
|
||||
return None
|
||||
|
||||
element = serialized_dom_state.selector_map[element_highlight_index]
|
||||
|
||||
from .element import Element as Element_
|
||||
|
||||
return Element_(self._browser_session, element.backend_node_id, self._session_id)
|
||||
|
||||
async def must_get_element_by_prompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element':
|
||||
"""Get an element by a prompt.
|
||||
|
||||
@dev LLM can still return None, this just raises an error if the element is not found.
|
||||
"""
|
||||
element = await self.get_element_by_prompt(prompt, llm)
|
||||
if element is None:
|
||||
raise ValueError(f'No element found for prompt: {prompt}')
|
||||
|
||||
return element
|
||||
|
||||
async def extract_content(self, prompt: str, structured_output: type[T], llm: 'BaseChatModel | None' = None) -> T:
|
||||
"""Extract structured content from the current page using LLM.
|
||||
|
||||
Extracts clean markdown from the page and sends it to LLM for structured data extraction.
|
||||
|
||||
Args:
|
||||
prompt: Description of what content to extract
|
||||
structured_output: Pydantic BaseModel class defining the expected output structure
|
||||
llm: Language model to use for extraction
|
||||
|
||||
Returns:
|
||||
The structured BaseModel instance with extracted content
|
||||
"""
|
||||
llm = llm or self._llm
|
||||
|
||||
if not llm:
|
||||
raise ValueError('LLM not provided')
|
||||
|
||||
# Extract clean markdown using the same method as in tools/service.py
|
||||
try:
|
||||
content, content_stats = await self._extract_clean_markdown()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Could not extract clean markdown: {type(e).__name__}')
|
||||
|
||||
# System prompt for structured extraction
|
||||
system_prompt = """
|
||||
You are an expert at extracting structured data from the markdown of a webpage.
|
||||
|
||||
<input>
|
||||
You will be given a query and the markdown of a webpage that has been filtered to remove noise and advertising content.
|
||||
</input>
|
||||
|
||||
<instructions>
|
||||
- You are tasked to extract information from the webpage that is relevant to the query.
|
||||
- You should ONLY use the information available in the webpage to answer the query. Do not make up information or provide guess from your own knowledge.
|
||||
- If the information relevant to the query is not available in the page, your response should mention that.
|
||||
- If the query asks for all items, products, etc., make sure to directly list all of them.
|
||||
- Return the extracted content in the exact structured format specified.
|
||||
</instructions>
|
||||
|
||||
<output>
|
||||
- Your output should present ALL the information relevant to the query in the specified structured format.
|
||||
- Do not answer in conversational format - directly output the relevant information in the structured format.
|
||||
</output>
|
||||
""".strip()
|
||||
|
||||
# Build prompt with just query and content
|
||||
prompt_content = f'<query>\n{prompt}\n</query>\n\n<webpage_content>\n{content}\n</webpage_content>'
|
||||
|
||||
# Send to LLM with structured output
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
llm.ainvoke(
|
||||
[SystemMessage(content=system_prompt), UserMessage(content=prompt_content)], output_format=structured_output
|
||||
),
|
||||
timeout=120.0,
|
||||
)
|
||||
|
||||
# Return the structured output BaseModel instance
|
||||
return response.completion
|
||||
except Exception as e:
|
||||
raise RuntimeError(str(e))
|
||||
|
||||
async def _extract_clean_markdown(self, extract_links: bool = False) -> tuple[str, dict]:
|
||||
"""Extract clean markdown from the current page using enhanced DOM tree.
|
||||
|
||||
Uses the shared markdown extractor for consistency with tools/service.py.
|
||||
"""
|
||||
from browser_use.dom.markdown_extractor import extract_clean_markdown
|
||||
|
||||
dom_service = self.dom_service
|
||||
return await extract_clean_markdown(dom_service=dom_service, target_id=self._target_id, extract_links=extract_links)
|
||||
@@ -0,0 +1,41 @@
|
||||
import asyncio
|
||||
|
||||
from browser_use import Agent, Browser, ChatOpenAI
|
||||
|
||||
llm = ChatOpenAI('gpt-4.1-mini')
|
||||
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Main function demonstrating mixed automation with Browser-Use and Playwright.
|
||||
"""
|
||||
print('🚀 Mixed Automation with Browser-Use and Actor API')
|
||||
|
||||
browser = Browser(keep_alive=True)
|
||||
await browser.start()
|
||||
|
||||
page = await browser.get_current_page() or await browser.new_page()
|
||||
|
||||
# Go to apple wikipedia page
|
||||
await page.goto('https://www.google.com/travel/flights')
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
round_trip_button = await page.must_get_element_by_prompt('round trip button', llm)
|
||||
await round_trip_button.click()
|
||||
|
||||
one_way_button = await page.must_get_element_by_prompt('one way button', llm)
|
||||
await one_way_button.click()
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
agent = Agent(task='Find the cheapest flight from London to Paris on 2025-10-15', llm=llm, browser_session=browser)
|
||||
await agent.run()
|
||||
|
||||
input('Press Enter to continue...')
|
||||
|
||||
await browser.stop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,54 @@
|
||||
import asyncio
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from browser_use import Browser, ChatOpenAI
|
||||
|
||||
TASK = """
|
||||
On the current wikipedia page, find the latest huge edit and tell me what is was about.
|
||||
"""
|
||||
|
||||
|
||||
class LatestEditFinder(BaseModel):
|
||||
"""Find the latest huge edit on the current wikipedia page."""
|
||||
|
||||
latest_edit: str
|
||||
edit_time: str
|
||||
edit_author: str
|
||||
edit_summary: str
|
||||
edit_url: str
|
||||
|
||||
|
||||
llm = ChatOpenAI('gpt-4.1-mini')
|
||||
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Main function demonstrating mixed automation with Browser-Use and Playwright.
|
||||
"""
|
||||
print('🚀 Mixed Automation with Browser-Use and Actor API')
|
||||
|
||||
browser = Browser(keep_alive=True)
|
||||
await browser.start()
|
||||
|
||||
page = await browser.get_current_page() or await browser.new_page()
|
||||
|
||||
# Go to apple wikipedia page
|
||||
await page.goto('https://browser-use.github.io/stress-tests/challenges/angularjs-form.html')
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
element = await page.get_element_by_prompt('zip code input', llm)
|
||||
|
||||
print('Element found', element)
|
||||
|
||||
if element:
|
||||
await element.click()
|
||||
else:
|
||||
print('No element found')
|
||||
|
||||
await browser.stop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
Executable
+236
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Playground script to test the browser-use actor API.
|
||||
|
||||
This script demonstrates:
|
||||
- Starting a browser session
|
||||
- Using the actor API to navigate and interact
|
||||
- Finding elements, clicking, scrolling, JavaScript evaluation
|
||||
- Testing most of the available methods
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from browser_use import Browser
|
||||
|
||||
# Configure logging to see what's happening
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main playground function."""
|
||||
logger.info('🚀 Starting browser actor playground')
|
||||
|
||||
# Create browser session
|
||||
browser = Browser()
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await browser.start()
|
||||
logger.info('✅ Browser session started')
|
||||
|
||||
# Navigate to Wikipedia using integrated methods
|
||||
logger.info('📖 Navigating to Wikipedia...')
|
||||
page = await browser.new_page('https://en.wikipedia.org')
|
||||
|
||||
# Get basic page info
|
||||
url = await page.get_url()
|
||||
title = await page.get_title()
|
||||
logger.info(f'📄 Page loaded: {title} ({url})')
|
||||
|
||||
# Take a screenshot
|
||||
logger.info('📸 Taking initial screenshot...')
|
||||
screenshot_b64 = await page.screenshot()
|
||||
logger.info(f'📸 Screenshot captured: {len(screenshot_b64)} bytes')
|
||||
|
||||
# Set viewport size
|
||||
logger.info('🖥️ Setting viewport to 1920x1080...')
|
||||
await page.set_viewport_size(1920, 1080)
|
||||
|
||||
# Execute some JavaScript to count links
|
||||
logger.info('🔍 Counting article links using JavaScript...')
|
||||
js_code = """() => {
|
||||
// Find all article links on the page
|
||||
const links = Array.from(document.querySelectorAll('a[href*="/wiki/"]:not([href*=":"])'))
|
||||
.filter(link => !link.href.includes('Main_Page') && !link.href.includes('Special:'));
|
||||
|
||||
return {
|
||||
total: links.length,
|
||||
sample: links.slice(0, 3).map(link => ({
|
||||
href: link.href,
|
||||
text: link.textContent.trim()
|
||||
}))
|
||||
};
|
||||
}"""
|
||||
|
||||
link_info = json.loads(await page.evaluate(js_code))
|
||||
logger.info(f'🔗 Found {link_info["total"]} article links')
|
||||
# Try to find and interact with links using CSS selector
|
||||
try:
|
||||
# Find article links on the page
|
||||
links = await page.get_elements_by_css_selector('a[href*="/wiki/"]:not([href*=":"])')
|
||||
|
||||
if links:
|
||||
logger.info(f'📋 Found {len(links)} wiki links via CSS selector')
|
||||
|
||||
# Pick the first link
|
||||
link_element = links[0]
|
||||
|
||||
# Get link info using available methods
|
||||
basic_info = await link_element.get_basic_info()
|
||||
link_href = await link_element.get_attribute('href')
|
||||
|
||||
logger.info(f'🎯 Selected element: <{basic_info["nodeName"]}>')
|
||||
logger.info(f'🔗 Link href: {link_href}')
|
||||
|
||||
if basic_info['boundingBox']:
|
||||
bbox = basic_info['boundingBox']
|
||||
logger.info(f'📏 Position: ({bbox["x"]}, {bbox["y"]}) Size: {bbox["width"]}x{bbox["height"]}')
|
||||
|
||||
# Test element interactions with robust implementations
|
||||
logger.info('👆 Hovering over the element...')
|
||||
await link_element.hover()
|
||||
await asyncio.sleep(1)
|
||||
|
||||
logger.info('🔍 Focusing the element...')
|
||||
await link_element.focus()
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Click the link using robust click method
|
||||
logger.info('🖱️ Clicking the link with robust fallbacks...')
|
||||
await link_element.click()
|
||||
|
||||
# Wait for navigation
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Get new page info
|
||||
new_url = await page.get_url()
|
||||
new_title = await page.get_title()
|
||||
logger.info(f'📄 Navigated to: {new_title}')
|
||||
logger.info(f'🌐 New URL: {new_url}')
|
||||
else:
|
||||
logger.warning('❌ No links found to interact with')
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f'⚠️ Link interaction failed: {e}')
|
||||
|
||||
# Scroll down the page
|
||||
logger.info('📜 Scrolling down the page...')
|
||||
mouse = await page.mouse
|
||||
await mouse.scroll(x=0, y=100, delta_y=500)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test mouse operations
|
||||
logger.info('🖱️ Testing mouse operations...')
|
||||
await mouse.move(x=100, y=200)
|
||||
await mouse.click(x=150, y=250)
|
||||
|
||||
# Execute more JavaScript examples
|
||||
logger.info('🧪 Testing JavaScript evaluation...')
|
||||
|
||||
# Simple expressions
|
||||
page_height = await page.evaluate('() => document.body.scrollHeight')
|
||||
current_scroll = await page.evaluate('() => window.pageYOffset')
|
||||
logger.info(f'📏 Page height: {page_height}px, current scroll: {current_scroll}px')
|
||||
|
||||
# JavaScript with arguments
|
||||
result = await page.evaluate('(x) => x * 2', 21)
|
||||
logger.info(f'🧮 JavaScript with args: 21 * 2 = {result}')
|
||||
|
||||
# More complex JavaScript
|
||||
page_stats = json.loads(
|
||||
await page.evaluate("""() => {
|
||||
return {
|
||||
url: window.location.href,
|
||||
title: document.title,
|
||||
links: document.querySelectorAll('a').length,
|
||||
images: document.querySelectorAll('img').length,
|
||||
scrollTop: window.pageYOffset,
|
||||
viewportHeight: window.innerHeight
|
||||
};
|
||||
}""")
|
||||
)
|
||||
logger.info(f'📊 Page stats: {page_stats}')
|
||||
|
||||
# Get page title using different methods
|
||||
title_via_js = await page.evaluate('() => document.title')
|
||||
title_via_api = await page.get_title()
|
||||
logger.info(f'📝 Title via JS: "{title_via_js}"')
|
||||
logger.info(f'📝 Title via API: "{title_via_api}"')
|
||||
|
||||
# Take a final screenshot
|
||||
logger.info('📸 Taking final screenshot...')
|
||||
final_screenshot = await page.screenshot()
|
||||
logger.info(f'📸 Final screenshot: {len(final_screenshot)} bytes')
|
||||
|
||||
# Test browser navigation with error handling
|
||||
logger.info('⬅️ Testing browser back navigation...')
|
||||
try:
|
||||
await page.go_back()
|
||||
await asyncio.sleep(2)
|
||||
|
||||
back_url = await page.get_url()
|
||||
back_title = await page.get_title()
|
||||
logger.info(f'📄 After going back: {back_title}')
|
||||
logger.info(f'🌐 Back URL: {back_url}')
|
||||
except RuntimeError as e:
|
||||
logger.info(f'ℹ️ Navigation back failed as expected: {e}')
|
||||
|
||||
# Test creating new page
|
||||
logger.info('🆕 Creating new blank page...')
|
||||
new_page = await browser.new_page()
|
||||
new_page_url = await new_page.get_url()
|
||||
logger.info(f'🆕 New page created with URL: {new_page_url}')
|
||||
|
||||
# Get all pages
|
||||
all_pages = await browser.get_pages()
|
||||
logger.info(f'📑 Total pages: {len(all_pages)}')
|
||||
|
||||
# Test form interaction if we can find a form
|
||||
try:
|
||||
# Look for search input on the page
|
||||
search_inputs = await page.get_elements_by_css_selector('input[type="search"], input[name*="search"]')
|
||||
|
||||
if search_inputs:
|
||||
search_input = search_inputs[0]
|
||||
logger.info('🔍 Found search input, testing form interaction...')
|
||||
|
||||
await search_input.focus()
|
||||
await search_input.fill('test search query')
|
||||
await page.press('Enter')
|
||||
|
||||
logger.info('✅ Form interaction test completed')
|
||||
else:
|
||||
logger.info('ℹ️ No search inputs found for form testing')
|
||||
|
||||
except Exception as e:
|
||||
logger.info(f'ℹ️ Form interaction test skipped: {e}')
|
||||
|
||||
# wait 2 seconds before closing the new page
|
||||
logger.info('🕒 Waiting 2 seconds before closing the new page...')
|
||||
await asyncio.sleep(2)
|
||||
logger.info('🗑️ Closing new page...')
|
||||
await browser.close_page(new_page)
|
||||
|
||||
logger.info('✅ Playground completed successfully!')
|
||||
|
||||
input('Press Enter to continue...')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'❌ Error in playground: {e}', exc_info=True)
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
logger.info('🧹 Cleaning up...')
|
||||
try:
|
||||
await browser.stop()
|
||||
logger.info('✅ Browser session stopped')
|
||||
except Exception as e:
|
||||
logger.error(f'❌ Error stopping browser: {e}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Utility functions for actor operations."""
|
||||
|
||||
|
||||
class Utils:
|
||||
"""Utility functions for actor operations."""
|
||||
|
||||
@staticmethod
|
||||
def get_key_info(key: str) -> tuple[str, int | None]:
|
||||
"""Get the code and windowsVirtualKeyCode for a key.
|
||||
|
||||
Args:
|
||||
key: Key name (e.g., 'Enter', 'ArrowUp', 'a', 'A')
|
||||
|
||||
Returns:
|
||||
Tuple of (code, windowsVirtualKeyCode)
|
||||
|
||||
Reference: Windows Virtual Key Codes
|
||||
https://docs.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
|
||||
"""
|
||||
# Complete mapping of key names to (code, virtualKeyCode)
|
||||
# Based on standard Windows Virtual Key Codes
|
||||
key_map = {
|
||||
# Navigation keys
|
||||
'Backspace': ('Backspace', 8),
|
||||
'Tab': ('Tab', 9),
|
||||
'Enter': ('Enter', 13),
|
||||
'Escape': ('Escape', 27),
|
||||
'Space': ('Space', 32),
|
||||
' ': ('Space', 32),
|
||||
'PageUp': ('PageUp', 33),
|
||||
'PageDown': ('PageDown', 34),
|
||||
'End': ('End', 35),
|
||||
'Home': ('Home', 36),
|
||||
'ArrowLeft': ('ArrowLeft', 37),
|
||||
'ArrowUp': ('ArrowUp', 38),
|
||||
'ArrowRight': ('ArrowRight', 39),
|
||||
'ArrowDown': ('ArrowDown', 40),
|
||||
'Insert': ('Insert', 45),
|
||||
'Delete': ('Delete', 46),
|
||||
# Modifier keys
|
||||
'Shift': ('ShiftLeft', 16),
|
||||
'ShiftLeft': ('ShiftLeft', 16),
|
||||
'ShiftRight': ('ShiftRight', 16),
|
||||
'Control': ('ControlLeft', 17),
|
||||
'ControlLeft': ('ControlLeft', 17),
|
||||
'ControlRight': ('ControlRight', 17),
|
||||
'Alt': ('AltLeft', 18),
|
||||
'AltLeft': ('AltLeft', 18),
|
||||
'AltRight': ('AltRight', 18),
|
||||
'Meta': ('MetaLeft', 91),
|
||||
'MetaLeft': ('MetaLeft', 91),
|
||||
'MetaRight': ('MetaRight', 92),
|
||||
# Function keys F1-F24
|
||||
'F1': ('F1', 112),
|
||||
'F2': ('F2', 113),
|
||||
'F3': ('F3', 114),
|
||||
'F4': ('F4', 115),
|
||||
'F5': ('F5', 116),
|
||||
'F6': ('F6', 117),
|
||||
'F7': ('F7', 118),
|
||||
'F8': ('F8', 119),
|
||||
'F9': ('F9', 120),
|
||||
'F10': ('F10', 121),
|
||||
'F11': ('F11', 122),
|
||||
'F12': ('F12', 123),
|
||||
'F13': ('F13', 124),
|
||||
'F14': ('F14', 125),
|
||||
'F15': ('F15', 126),
|
||||
'F16': ('F16', 127),
|
||||
'F17': ('F17', 128),
|
||||
'F18': ('F18', 129),
|
||||
'F19': ('F19', 130),
|
||||
'F20': ('F20', 131),
|
||||
'F21': ('F21', 132),
|
||||
'F22': ('F22', 133),
|
||||
'F23': ('F23', 134),
|
||||
'F24': ('F24', 135),
|
||||
# Numpad keys
|
||||
'NumLock': ('NumLock', 144),
|
||||
'Numpad0': ('Numpad0', 96),
|
||||
'Numpad1': ('Numpad1', 97),
|
||||
'Numpad2': ('Numpad2', 98),
|
||||
'Numpad3': ('Numpad3', 99),
|
||||
'Numpad4': ('Numpad4', 100),
|
||||
'Numpad5': ('Numpad5', 101),
|
||||
'Numpad6': ('Numpad6', 102),
|
||||
'Numpad7': ('Numpad7', 103),
|
||||
'Numpad8': ('Numpad8', 104),
|
||||
'Numpad9': ('Numpad9', 105),
|
||||
'NumpadMultiply': ('NumpadMultiply', 106),
|
||||
'NumpadAdd': ('NumpadAdd', 107),
|
||||
'NumpadSubtract': ('NumpadSubtract', 109),
|
||||
'NumpadDecimal': ('NumpadDecimal', 110),
|
||||
'NumpadDivide': ('NumpadDivide', 111),
|
||||
# Lock keys
|
||||
'CapsLock': ('CapsLock', 20),
|
||||
'ScrollLock': ('ScrollLock', 145),
|
||||
# OEM/Punctuation keys (US keyboard layout)
|
||||
'Semicolon': ('Semicolon', 186),
|
||||
';': ('Semicolon', 186),
|
||||
'Equal': ('Equal', 187),
|
||||
'=': ('Equal', 187),
|
||||
'Comma': ('Comma', 188),
|
||||
',': ('Comma', 188),
|
||||
'Minus': ('Minus', 189),
|
||||
'-': ('Minus', 189),
|
||||
'Period': ('Period', 190),
|
||||
'.': ('Period', 190),
|
||||
'Slash': ('Slash', 191),
|
||||
'/': ('Slash', 191),
|
||||
'Backquote': ('Backquote', 192),
|
||||
'`': ('Backquote', 192),
|
||||
'BracketLeft': ('BracketLeft', 219),
|
||||
'[': ('BracketLeft', 219),
|
||||
'Backslash': ('Backslash', 220),
|
||||
'\\': ('Backslash', 220),
|
||||
'BracketRight': ('BracketRight', 221),
|
||||
']': ('BracketRight', 221),
|
||||
'Quote': ('Quote', 222),
|
||||
"'": ('Quote', 222),
|
||||
# Media/Browser keys
|
||||
'AudioVolumeMute': ('AudioVolumeMute', 173),
|
||||
'AudioVolumeDown': ('AudioVolumeDown', 174),
|
||||
'AudioVolumeUp': ('AudioVolumeUp', 175),
|
||||
'MediaTrackNext': ('MediaTrackNext', 176),
|
||||
'MediaTrackPrevious': ('MediaTrackPrevious', 177),
|
||||
'MediaStop': ('MediaStop', 178),
|
||||
'MediaPlayPause': ('MediaPlayPause', 179),
|
||||
'BrowserBack': ('BrowserBack', 166),
|
||||
'BrowserForward': ('BrowserForward', 167),
|
||||
'BrowserRefresh': ('BrowserRefresh', 168),
|
||||
'BrowserStop': ('BrowserStop', 169),
|
||||
'BrowserSearch': ('BrowserSearch', 170),
|
||||
'BrowserFavorites': ('BrowserFavorites', 171),
|
||||
'BrowserHome': ('BrowserHome', 172),
|
||||
# Additional common keys
|
||||
'Clear': ('Clear', 12),
|
||||
'Pause': ('Pause', 19),
|
||||
'Select': ('Select', 41),
|
||||
'Print': ('Print', 42),
|
||||
'Execute': ('Execute', 43),
|
||||
'PrintScreen': ('PrintScreen', 44),
|
||||
'Help': ('Help', 47),
|
||||
'ContextMenu': ('ContextMenu', 93),
|
||||
}
|
||||
|
||||
if key in key_map:
|
||||
return key_map[key]
|
||||
|
||||
# Handle alphanumeric keys dynamically
|
||||
if len(key) == 1:
|
||||
if key.isalpha():
|
||||
# Letter keys: A-Z have VK codes 65-90
|
||||
return (f'Key{key.upper()}', ord(key.upper()))
|
||||
elif key.isdigit():
|
||||
# Digit keys: 0-9 have VK codes 48-57 (same as ASCII)
|
||||
return (f'Digit{key}', ord(key))
|
||||
|
||||
# Fallback: use the key name as code, no virtual key code
|
||||
return (key, None)
|
||||
|
||||
|
||||
# Backward compatibility: provide standalone function
|
||||
def get_key_info(key: str) -> tuple[str, int | None]:
|
||||
"""Get the code and windowsVirtualKeyCode for a key.
|
||||
|
||||
Args:
|
||||
key: Key name (e.g., 'Enter', 'ArrowUp', 'a', 'A')
|
||||
|
||||
Returns:
|
||||
Tuple of (code, windowsVirtualKeyCode)
|
||||
|
||||
Reference: Windows Virtual Key Codes
|
||||
https://docs.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
|
||||
"""
|
||||
return Utils.get_key_info(key)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Agent package exports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.agent.service import Agent
|
||||
from browser_use.beta.service import BetaAgentError
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
'Agent': ('browser_use.agent.service', 'Agent'),
|
||||
'BetaAgentError': ('browser_use.beta.service', 'BetaAgentError'),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_path, attr_name = _LAZY_IMPORTS[name]
|
||||
from importlib import import_module
|
||||
|
||||
module = import_module(module_path)
|
||||
attr = getattr(module, attr_name)
|
||||
globals()[name] = attr
|
||||
return attr
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Agent',
|
||||
'BetaAgentError',
|
||||
]
|
||||
@@ -0,0 +1,284 @@
|
||||
import base64
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import anyio
|
||||
from bubus import BaseEvent
|
||||
from pydantic import Field, field_validator
|
||||
from uuid_extensions import uuid7str
|
||||
|
||||
MAX_STRING_LENGTH = 500000 # 100K chars ~ 25k tokens should be enough
|
||||
MAX_URL_LENGTH = 100000
|
||||
MAX_TASK_LENGTH = 100000
|
||||
MAX_COMMENT_LENGTH = 2000
|
||||
MAX_FILE_CONTENT_SIZE = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
|
||||
class UpdateAgentTaskEvent(BaseEvent):
|
||||
# Required fields for identification
|
||||
id: str # The task ID to update
|
||||
user_id: str = Field(max_length=255) # For authorization
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
|
||||
# Optional fields that can be updated
|
||||
stopped: bool | None = None
|
||||
paused: bool | None = None
|
||||
done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
|
||||
finished_at: datetime | None = None
|
||||
agent_state: dict | None = None
|
||||
user_feedback_type: str | None = Field(None, max_length=10) # UserFeedbackType enum value as string
|
||||
user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
|
||||
gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'UpdateAgentTaskEvent':
|
||||
"""Create an UpdateAgentTaskEvent from an Agent instance"""
|
||||
if not hasattr(agent, '_task_start_time'):
|
||||
raise ValueError('Agent must have _task_start_time attribute')
|
||||
|
||||
done_output = agent.history.final_result() if agent.history else None
|
||||
if done_output and len(done_output) > MAX_STRING_LENGTH:
|
||||
done_output = done_output[:MAX_STRING_LENGTH]
|
||||
return cls(
|
||||
id=str(agent.task_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
stopped=agent.state.stopped if hasattr(agent.state, 'stopped') else False,
|
||||
paused=agent.state.paused if hasattr(agent.state, 'paused') else False,
|
||||
done_output=done_output,
|
||||
finished_at=datetime.now(timezone.utc) if agent.history and agent.history.is_done() else None,
|
||||
agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
|
||||
user_feedback_type=None,
|
||||
user_comment=None,
|
||||
gif_url=None,
|
||||
# user_feedback_type and user_comment would be set by the API/frontend
|
||||
# gif_url would be set after GIF generation if needed
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentOutputFileEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
task_id: str
|
||||
file_name: str = Field(max_length=255)
|
||||
file_content: str | None = None # Base64 encoded file content
|
||||
content_type: str | None = Field(None, max_length=100) # MIME type for file uploads
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
@field_validator('file_content')
|
||||
@classmethod
|
||||
def validate_file_size(cls, v: str | None) -> str | None:
|
||||
"""Validate base64 file content size."""
|
||||
if v is None:
|
||||
return v
|
||||
# Remove data URL prefix if present
|
||||
if ',' in v:
|
||||
v = v.split(',')[1]
|
||||
# Estimate decoded size (base64 is ~33% larger)
|
||||
estimated_size = len(v) * 3 / 4
|
||||
if estimated_size > MAX_FILE_CONTENT_SIZE:
|
||||
raise ValueError(f'File content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
|
||||
return v
|
||||
|
||||
@classmethod
|
||||
async def from_agent_and_file(cls, agent, output_path: str) -> 'CreateAgentOutputFileEvent':
|
||||
"""Create a CreateAgentOutputFileEvent from a file path"""
|
||||
|
||||
gif_path = Path(output_path)
|
||||
if not gif_path.exists():
|
||||
raise FileNotFoundError(f'File not found: {output_path}')
|
||||
|
||||
gif_size = os.path.getsize(gif_path)
|
||||
|
||||
# Read GIF content for base64 encoding if needed
|
||||
gif_content = None
|
||||
if gif_size < 50 * 1024 * 1024: # Only read if < 50MB
|
||||
async with await anyio.open_file(gif_path, 'rb') as f:
|
||||
gif_bytes = await f.read()
|
||||
gif_content = base64.b64encode(gif_bytes).decode('utf-8')
|
||||
|
||||
return cls(
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
task_id=str(agent.task_id),
|
||||
file_name=gif_path.name,
|
||||
file_content=gif_content, # Base64 encoded
|
||||
content_type='image/gif',
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentStepEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255) # Added for authorization checks
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
agent_task_id: str
|
||||
step: int
|
||||
evaluation_previous_goal: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
memory: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
next_goal: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
actions: list[dict]
|
||||
screenshot_url: str | None = Field(None, max_length=MAX_FILE_CONTENT_SIZE) # ~50MB for base64 images
|
||||
url: str = Field(default='', max_length=MAX_URL_LENGTH)
|
||||
|
||||
@field_validator('screenshot_url')
|
||||
@classmethod
|
||||
def validate_screenshot_size(cls, v: str | None) -> str | None:
|
||||
"""Validate screenshot URL or base64 content size."""
|
||||
if v is None or not v.startswith('data:'):
|
||||
return v
|
||||
# It's base64 data, check size
|
||||
if ',' in v:
|
||||
base64_part = v.split(',')[1]
|
||||
estimated_size = len(base64_part) * 3 / 4
|
||||
if estimated_size > MAX_FILE_CONTENT_SIZE:
|
||||
raise ValueError(f'Screenshot content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
|
||||
return v
|
||||
|
||||
@classmethod
|
||||
def from_agent_step(
|
||||
cls, agent, model_output, result: list, actions_data: list[dict], browser_state_summary
|
||||
) -> 'CreateAgentStepEvent':
|
||||
"""Create a CreateAgentStepEvent from agent step data"""
|
||||
# Get first action details if available
|
||||
first_action = model_output.action[0] if model_output.action else None
|
||||
|
||||
# Extract current state from model output
|
||||
current_state = model_output.current_state if hasattr(model_output, 'current_state') else None
|
||||
|
||||
# Capture screenshot as base64 data URL if available
|
||||
screenshot_url = None
|
||||
if browser_state_summary.screenshot:
|
||||
screenshot_url = f'data:image/png;base64,{browser_state_summary.screenshot}'
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug(f'📸 Including screenshot in CreateAgentStepEvent, length: {len(browser_state_summary.screenshot)}')
|
||||
else:
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug('📸 No screenshot in browser_state_summary for CreateAgentStepEvent')
|
||||
|
||||
return cls(
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
agent_task_id=str(agent.task_id),
|
||||
step=agent.state.n_steps,
|
||||
evaluation_previous_goal=current_state.evaluation_previous_goal if current_state else '',
|
||||
memory=current_state.memory if current_state else '',
|
||||
next_goal=current_state.next_goal if current_state else '',
|
||||
actions=actions_data, # List of action dicts
|
||||
url=browser_state_summary.url,
|
||||
screenshot_url=screenshot_url,
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentTaskEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255) # Added for authorization checks
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
agent_session_id: str
|
||||
llm_model: str = Field(max_length=200) # LLMModel enum value as string
|
||||
stopped: bool = False
|
||||
paused: bool = False
|
||||
task: str = Field(max_length=MAX_TASK_LENGTH)
|
||||
done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
|
||||
scheduled_task_id: str | None = None
|
||||
started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
finished_at: datetime | None = None
|
||||
agent_state: dict = Field(default_factory=dict)
|
||||
user_feedback_type: str | None = Field(None, max_length=10) # UserFeedbackType enum value as string
|
||||
user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
|
||||
gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'CreateAgentTaskEvent':
|
||||
"""Create a CreateAgentTaskEvent from an Agent instance"""
|
||||
return cls(
|
||||
id=str(agent.task_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
agent_session_id=str(agent.session_id),
|
||||
task=agent.task,
|
||||
llm_model=agent.llm.model_name,
|
||||
agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
|
||||
stopped=False,
|
||||
paused=False,
|
||||
done_output=None,
|
||||
started_at=datetime.fromtimestamp(agent._task_start_time, tz=timezone.utc),
|
||||
finished_at=None,
|
||||
user_feedback_type=None,
|
||||
user_comment=None,
|
||||
gif_url=None,
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentSessionEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
browser_session_id: str = Field(max_length=255)
|
||||
browser_session_live_url: str = Field(max_length=MAX_URL_LENGTH)
|
||||
browser_session_cdp_url: str = Field(max_length=MAX_URL_LENGTH)
|
||||
browser_session_stopped: bool = False
|
||||
browser_session_stopped_at: datetime | None = None
|
||||
is_source_api: bool | None = None
|
||||
browser_state: dict = Field(default_factory=dict)
|
||||
browser_session_data: dict | None = None
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'CreateAgentSessionEvent':
|
||||
"""Create a CreateAgentSessionEvent from an Agent instance"""
|
||||
return cls(
|
||||
id=str(agent.session_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
browser_session_id=agent.browser_session.id,
|
||||
browser_session_live_url='', # To be filled by cloud handler
|
||||
browser_session_cdp_url='', # To be filled by cloud handler
|
||||
browser_state={
|
||||
'viewport': agent.browser_profile.viewport if agent.browser_profile else {'width': 1280, 'height': 720},
|
||||
'user_agent': agent.browser_profile.user_agent if agent.browser_profile else None,
|
||||
'headless': agent.browser_profile.headless if agent.browser_profile else True,
|
||||
'initial_url': None, # Will be updated during execution
|
||||
'final_url': None, # Will be updated during execution
|
||||
'total_pages_visited': 0, # Will be updated during execution
|
||||
'session_duration_seconds': 0, # Will be updated during execution
|
||||
},
|
||||
browser_session_data={
|
||||
'cookies': [],
|
||||
'secrets': {},
|
||||
# TODO: send secrets safely so tasks can be replayed on cloud seamlessly
|
||||
# 'secrets': dict(agent.sensitive_data) if agent.sensitive_data else {},
|
||||
'allowed_domains': agent.browser_profile.allowed_domains if agent.browser_profile else [],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class UpdateAgentSessionEvent(BaseEvent):
|
||||
"""Event to update an existing agent session"""
|
||||
|
||||
# Model fields
|
||||
id: str # Session ID to update
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255)
|
||||
browser_session_stopped: bool | None = None
|
||||
browser_session_stopped_at: datetime | None = None
|
||||
end_reason: str | None = Field(None, max_length=100) # Why the session ended
|
||||
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.agent.views import AgentHistoryList
|
||||
from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
|
||||
from browser_use.config import CONFIG
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from PIL import Image, ImageFont
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def decode_unicode_escapes_to_utf8(text: str) -> str:
|
||||
"""Handle decoding any unicode escape sequences embedded in a string (needed to render non-ASCII languages like chinese or arabic in the GIF overlay text)"""
|
||||
|
||||
if r'\u' not in text:
|
||||
# doesn't have any escape sequences that need to be decoded
|
||||
return text
|
||||
|
||||
try:
|
||||
# Try to decode Unicode escape sequences
|
||||
return text.encode('latin1').decode('unicode_escape')
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
# logger.debug(f"Failed to decode unicode escape sequences while generating gif text: {text}")
|
||||
return text
|
||||
|
||||
|
||||
def create_history_gif(
|
||||
task: str,
|
||||
history: AgentHistoryList,
|
||||
#
|
||||
output_path: str = 'agent_history.gif',
|
||||
duration: int = 3000,
|
||||
show_goals: bool = True,
|
||||
show_task: bool = True,
|
||||
show_logo: bool = False,
|
||||
font_size: int = 40,
|
||||
title_font_size: int = 56,
|
||||
goal_font_size: int = 44,
|
||||
margin: int = 40,
|
||||
line_spacing: float = 1.5,
|
||||
) -> None:
|
||||
"""Create a GIF from the agent's history with overlaid task and goal text."""
|
||||
if not history.history:
|
||||
logger.warning('No history to create GIF from')
|
||||
return
|
||||
|
||||
from PIL import Image, ImageFont
|
||||
|
||||
images = []
|
||||
|
||||
# if history is empty, we can't create a gif
|
||||
if not history.history:
|
||||
logger.warning('No history to create GIF from')
|
||||
return
|
||||
|
||||
# Get all screenshots from history (including None placeholders)
|
||||
screenshots = history.screenshots(return_none_if_not_screenshot=True)
|
||||
|
||||
if not screenshots:
|
||||
logger.warning('No screenshots found in history')
|
||||
return
|
||||
|
||||
# Find the first non-placeholder screenshot
|
||||
# A screenshot is considered a placeholder if:
|
||||
# 1. It's the exact 4px placeholder for about:blank pages, OR
|
||||
# 2. It comes from a new tab page (chrome://newtab/, about:blank, etc.)
|
||||
first_real_screenshot = None
|
||||
for screenshot in screenshots:
|
||||
if screenshot and screenshot != PLACEHOLDER_4PX_SCREENSHOT:
|
||||
first_real_screenshot = screenshot
|
||||
break
|
||||
|
||||
if not first_real_screenshot:
|
||||
logger.warning('No valid screenshots found (all are placeholders or from new tab pages)')
|
||||
return
|
||||
|
||||
# Try to load nicer fonts
|
||||
try:
|
||||
# Try different font options in order of preference
|
||||
# ArialUni is a font that comes with Office and can render most non-alphabet characters
|
||||
font_options = [
|
||||
'PingFang',
|
||||
'STHeiti Medium',
|
||||
'Microsoft YaHei', # 微软雅黑
|
||||
'SimHei', # 黑体
|
||||
'SimSun', # 宋体
|
||||
'Noto Sans CJK SC', # 思源黑体
|
||||
'WenQuanYi Micro Hei', # 文泉驿微米黑
|
||||
'Helvetica',
|
||||
'Arial',
|
||||
'DejaVuSans',
|
||||
'Verdana',
|
||||
]
|
||||
font_loaded = False
|
||||
|
||||
for font_name in font_options:
|
||||
try:
|
||||
if platform.system() == 'Windows':
|
||||
# Need to specify the abs font path on Windows
|
||||
font_name = os.path.join(CONFIG.WIN_FONT_DIR, font_name + '.ttf')
|
||||
regular_font = ImageFont.truetype(font_name, font_size)
|
||||
title_font = ImageFont.truetype(font_name, title_font_size)
|
||||
font_loaded = True
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
if not font_loaded:
|
||||
raise OSError('No preferred fonts found')
|
||||
|
||||
except OSError:
|
||||
regular_font = ImageFont.load_default()
|
||||
title_font = ImageFont.load_default()
|
||||
|
||||
# Load logo if requested
|
||||
logo = None
|
||||
if show_logo:
|
||||
try:
|
||||
logo = Image.open('./static/browser-use.png')
|
||||
# Resize logo to be small (e.g., 40px height)
|
||||
logo_height = 150
|
||||
aspect_ratio = logo.width / logo.height
|
||||
logo_width = int(logo_height * aspect_ratio)
|
||||
logo = logo.resize((logo_width, logo_height), Image.Resampling.LANCZOS)
|
||||
except Exception as e:
|
||||
logger.warning(f'Could not load logo: {e}')
|
||||
|
||||
# Create task frame if requested
|
||||
if show_task and task:
|
||||
# Find the first non-placeholder screenshot for the task frame
|
||||
first_real_screenshot = None
|
||||
for item in history.history:
|
||||
screenshot_b64 = item.state.get_screenshot()
|
||||
if screenshot_b64 and screenshot_b64 != PLACEHOLDER_4PX_SCREENSHOT:
|
||||
first_real_screenshot = screenshot_b64
|
||||
break
|
||||
|
||||
if first_real_screenshot:
|
||||
task_frame = _create_task_frame(
|
||||
task,
|
||||
first_real_screenshot,
|
||||
title_font, # type: ignore
|
||||
regular_font, # type: ignore
|
||||
logo,
|
||||
line_spacing,
|
||||
)
|
||||
images.append(task_frame)
|
||||
else:
|
||||
logger.warning('No real screenshots found for task frame, skipping task frame')
|
||||
|
||||
# Process each history item with its corresponding screenshot
|
||||
for i, (item, screenshot) in enumerate(zip(history.history, screenshots), 1):
|
||||
if not screenshot:
|
||||
continue
|
||||
|
||||
# Skip placeholder screenshots from about:blank pages
|
||||
# These are 4x4 white PNGs encoded as a specific base64 string
|
||||
if screenshot == PLACEHOLDER_4PX_SCREENSHOT:
|
||||
logger.debug(f'Skipping placeholder screenshot from about:blank page at step {i}')
|
||||
continue
|
||||
|
||||
# Skip screenshots from new tab pages
|
||||
from browser_use.utils import is_new_tab_page
|
||||
|
||||
if is_new_tab_page(item.state.url):
|
||||
logger.debug(f'Skipping screenshot from new tab page ({item.state.url}) at step {i}')
|
||||
continue
|
||||
|
||||
# Convert base64 screenshot to PIL Image
|
||||
img_data = base64.b64decode(screenshot)
|
||||
image = Image.open(io.BytesIO(img_data))
|
||||
|
||||
if show_goals and item.model_output:
|
||||
image = _add_overlay_to_image(
|
||||
image=image,
|
||||
step_number=i,
|
||||
goal_text=item.model_output.current_state.next_goal,
|
||||
regular_font=regular_font, # type: ignore
|
||||
title_font=title_font, # type: ignore
|
||||
margin=margin,
|
||||
logo=logo,
|
||||
)
|
||||
|
||||
images.append(image)
|
||||
|
||||
if images:
|
||||
# Save the GIF
|
||||
images[0].save(
|
||||
output_path,
|
||||
save_all=True,
|
||||
append_images=images[1:],
|
||||
duration=duration,
|
||||
loop=0,
|
||||
optimize=False,
|
||||
)
|
||||
logger.info(f'Created GIF at {output_path}')
|
||||
else:
|
||||
logger.warning('No images found in history to create GIF')
|
||||
|
||||
|
||||
def _create_task_frame(
|
||||
task: str,
|
||||
first_screenshot: str,
|
||||
title_font: ImageFont.FreeTypeFont,
|
||||
regular_font: ImageFont.FreeTypeFont,
|
||||
logo: Image.Image | None = None,
|
||||
line_spacing: float = 1.5,
|
||||
) -> Image.Image:
|
||||
"""Create initial frame showing the task."""
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
img_data = base64.b64decode(first_screenshot)
|
||||
template = Image.open(io.BytesIO(img_data))
|
||||
image = Image.new('RGB', template.size, (0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
# Calculate vertical center of image
|
||||
center_y = image.height // 2
|
||||
|
||||
# Draw task text with dynamic font size based on task length
|
||||
margin = 140 # Increased margin
|
||||
max_width = image.width - (2 * margin)
|
||||
|
||||
# Dynamic font size calculation based on task length
|
||||
# Start with base font size (regular + 16)
|
||||
base_font_size = regular_font.size + 16
|
||||
min_font_size = max(regular_font.size - 10, 16) # Don't go below 16pt
|
||||
# Calculate dynamic font size based on text length and complexity
|
||||
# Longer texts get progressively smaller fonts
|
||||
text_length = len(task)
|
||||
if text_length > 200:
|
||||
# For very long text, reduce font size logarithmically
|
||||
font_size = max(base_font_size - int(10 * (text_length / 200)), min_font_size)
|
||||
else:
|
||||
font_size = base_font_size
|
||||
|
||||
# Try to create a larger font, but fall back to regular font if it fails
|
||||
try:
|
||||
larger_font = ImageFont.truetype(regular_font.path, font_size) # type: ignore
|
||||
except (OSError, AttributeError):
|
||||
# Fall back to regular font if .path is not available or font loading fails
|
||||
larger_font = regular_font
|
||||
|
||||
# Generate wrapped text with the calculated font size
|
||||
wrapped_text = _wrap_text(task, larger_font, max_width)
|
||||
|
||||
# Calculate line height with spacing
|
||||
line_height = larger_font.size * line_spacing
|
||||
|
||||
# Split text into lines and draw with custom spacing
|
||||
lines = wrapped_text.split('\n')
|
||||
total_height = line_height * len(lines)
|
||||
|
||||
# Start position for first line
|
||||
text_y = center_y - (total_height / 2) + 50 # Shifted down slightly
|
||||
|
||||
for line in lines:
|
||||
# Get line width for centering
|
||||
line_bbox = draw.textbbox((0, 0), line, font=larger_font)
|
||||
text_x = (image.width - (line_bbox[2] - line_bbox[0])) // 2
|
||||
|
||||
draw.text(
|
||||
(text_x, text_y),
|
||||
line,
|
||||
font=larger_font,
|
||||
fill=(255, 255, 255),
|
||||
)
|
||||
text_y += line_height
|
||||
|
||||
# Add logo if provided (top right corner)
|
||||
if logo:
|
||||
logo_margin = 20
|
||||
logo_x = image.width - logo.width - logo_margin
|
||||
image.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def _add_overlay_to_image(
|
||||
image: Image.Image,
|
||||
step_number: int,
|
||||
goal_text: str,
|
||||
regular_font: ImageFont.FreeTypeFont,
|
||||
title_font: ImageFont.FreeTypeFont,
|
||||
margin: int,
|
||||
logo: Image.Image | None = None,
|
||||
display_step: bool = True,
|
||||
text_color: tuple[int, int, int, int] = (255, 255, 255, 255),
|
||||
text_box_color: tuple[int, int, int, int] = (0, 0, 0, 255),
|
||||
) -> Image.Image:
|
||||
"""Add step number and goal overlay to an image."""
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
goal_text = decode_unicode_escapes_to_utf8(goal_text)
|
||||
image = image.convert('RGBA')
|
||||
txt_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(txt_layer)
|
||||
if display_step:
|
||||
# Add step number (bottom left)
|
||||
step_text = str(step_number)
|
||||
step_bbox = draw.textbbox((0, 0), step_text, font=title_font)
|
||||
step_width = step_bbox[2] - step_bbox[0]
|
||||
step_height = step_bbox[3] - step_bbox[1]
|
||||
|
||||
# Position step number in bottom left
|
||||
x_step = margin + 10 # Slight additional offset from edge
|
||||
y_step = image.height - margin - step_height - 10 # Slight offset from bottom
|
||||
|
||||
# Draw rounded rectangle background for step number
|
||||
padding = 20 # Increased padding
|
||||
step_bg_bbox = (
|
||||
x_step - padding,
|
||||
y_step - padding,
|
||||
x_step + step_width + padding,
|
||||
y_step + step_height + padding,
|
||||
)
|
||||
draw.rounded_rectangle(
|
||||
step_bg_bbox,
|
||||
radius=15, # Add rounded corners
|
||||
fill=text_box_color,
|
||||
)
|
||||
|
||||
# Draw step number
|
||||
draw.text(
|
||||
(x_step, y_step),
|
||||
step_text,
|
||||
font=title_font,
|
||||
fill=text_color,
|
||||
)
|
||||
|
||||
# Draw goal text (centered, bottom)
|
||||
max_width = image.width - (4 * margin)
|
||||
wrapped_goal = _wrap_text(goal_text, title_font, max_width)
|
||||
goal_bbox = draw.multiline_textbbox((0, 0), wrapped_goal, font=title_font)
|
||||
goal_width = goal_bbox[2] - goal_bbox[0]
|
||||
goal_height = goal_bbox[3] - goal_bbox[1]
|
||||
|
||||
# Center goal text horizontally, place above step number
|
||||
x_goal = (image.width - goal_width) // 2
|
||||
y_goal = y_step - goal_height - padding * 4 # More space between step and goal
|
||||
|
||||
# Draw rounded rectangle background for goal
|
||||
padding_goal = 25 # Increased padding for goal
|
||||
goal_bg_bbox = (
|
||||
x_goal - padding_goal, # Remove extra space for logo
|
||||
y_goal - padding_goal,
|
||||
x_goal + goal_width + padding_goal,
|
||||
y_goal + goal_height + padding_goal,
|
||||
)
|
||||
draw.rounded_rectangle(
|
||||
goal_bg_bbox,
|
||||
radius=15, # Add rounded corners
|
||||
fill=text_box_color,
|
||||
)
|
||||
|
||||
# Draw goal text
|
||||
draw.multiline_text(
|
||||
(x_goal, y_goal),
|
||||
wrapped_goal,
|
||||
font=title_font,
|
||||
fill=text_color,
|
||||
align='center',
|
||||
)
|
||||
|
||||
# Add logo if provided (top right corner)
|
||||
if logo:
|
||||
logo_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
|
||||
logo_margin = 20
|
||||
logo_x = image.width - logo.width - logo_margin
|
||||
logo_layer.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
|
||||
txt_layer = Image.alpha_composite(logo_layer, txt_layer)
|
||||
|
||||
# Composite and convert
|
||||
result = Image.alpha_composite(image, txt_layer)
|
||||
return result.convert('RGB')
|
||||
|
||||
|
||||
def _wrap_text(text: str, font: ImageFont.FreeTypeFont, max_width: int) -> str:
|
||||
"""
|
||||
Wrap text to fit within a given width.
|
||||
|
||||
Args:
|
||||
text: Text to wrap
|
||||
font: Font to use for text
|
||||
max_width: Maximum width in pixels
|
||||
|
||||
Returns:
|
||||
Wrapped text with newlines
|
||||
"""
|
||||
text = decode_unicode_escapes_to_utf8(text)
|
||||
words = text.split()
|
||||
lines = []
|
||||
current_line = []
|
||||
|
||||
for word in words:
|
||||
current_line.append(word)
|
||||
line = ' '.join(current_line)
|
||||
bbox = font.getbbox(line)
|
||||
if bbox[2] > max_width:
|
||||
if len(current_line) == 1:
|
||||
lines.append(current_line.pop())
|
||||
else:
|
||||
current_line.pop()
|
||||
lines.append(' '.join(current_line))
|
||||
current_line = [word]
|
||||
|
||||
if current_line:
|
||||
lines.append(' '.join(current_line))
|
||||
|
||||
return '\n'.join(lines)
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Judge system for evaluating browser-use agent execution traces."""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
ContentPartImageParam,
|
||||
ContentPartTextParam,
|
||||
ImageURL,
|
||||
SystemMessage,
|
||||
UserMessage,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _encode_image(image_path: str) -> str | None:
|
||||
"""Encode image to base64 string."""
|
||||
try:
|
||||
path = Path(image_path)
|
||||
if not path.exists():
|
||||
return None
|
||||
with open(path, 'rb') as f:
|
||||
return base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to encode image {image_path}: {e}')
|
||||
return None
|
||||
|
||||
|
||||
def _truncate_text(text: str, max_length: int, from_beginning: bool = False) -> str:
|
||||
"""Truncate text to maximum length with eval system indicator."""
|
||||
if len(text) <= max_length:
|
||||
return text
|
||||
if from_beginning:
|
||||
return '...[text truncated]' + text[-max_length + 23 :]
|
||||
else:
|
||||
return text[: max_length - 23] + '...[text truncated]...'
|
||||
|
||||
|
||||
def construct_judge_messages(
|
||||
task: str,
|
||||
final_result: str,
|
||||
agent_steps: list[str],
|
||||
screenshot_paths: list[str],
|
||||
max_images: int = 10,
|
||||
ground_truth: str | None = None,
|
||||
use_vision: bool | Literal['auto'] = True,
|
||||
) -> list[BaseMessage]:
|
||||
"""
|
||||
Construct messages for judge evaluation of agent trace.
|
||||
|
||||
Args:
|
||||
task: The original task description
|
||||
final_result: The final result returned to the user
|
||||
agent_steps: List of formatted agent step descriptions
|
||||
screenshot_paths: List of screenshot file paths
|
||||
max_images: Maximum number of screenshots to include
|
||||
ground_truth: Optional ground truth answer or criteria that must be satisfied for success
|
||||
|
||||
Returns:
|
||||
List of messages for LLM judge evaluation
|
||||
"""
|
||||
task_truncated = _truncate_text(task, 40000)
|
||||
final_result_truncated = _truncate_text(final_result, 40000)
|
||||
steps_text = '\n'.join(agent_steps)
|
||||
steps_text_truncated = _truncate_text(steps_text, 40000)
|
||||
|
||||
# Only include screenshots if use_vision is not False
|
||||
encoded_images: list[ContentPartImageParam] = []
|
||||
if use_vision is not False:
|
||||
# Select last N screenshots
|
||||
selected_screenshots = screenshot_paths[-max_images:] if len(screenshot_paths) > max_images else screenshot_paths
|
||||
|
||||
# Encode screenshots
|
||||
for img_path in selected_screenshots:
|
||||
encoded = _encode_image(img_path)
|
||||
if encoded:
|
||||
encoded_images.append(
|
||||
ContentPartImageParam(
|
||||
image_url=ImageURL(
|
||||
url=f'data:image/png;base64,{encoded}',
|
||||
media_type='image/png',
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
|
||||
|
||||
# System prompt for judge - conditionally add ground truth section
|
||||
ground_truth_section = ''
|
||||
if ground_truth:
|
||||
ground_truth_section = """
|
||||
**GROUND TRUTH VALIDATION (HIGHEST PRIORITY):**
|
||||
The <ground_truth> section contains verified correct information for this task. This can be:
|
||||
- **Evaluation criteria**: Specific conditions that must be met (e.g., "The success popup should show up", "Must extract exactly 5 items")
|
||||
- **Factual answers**: The correct answer to a question or information retrieval task (e.g. "10/11/24", "Paris")
|
||||
- **Expected outcomes**: What should happen after task completion (e.g., "Google Doc must be created", "File should be downloaded")
|
||||
|
||||
The ground truth takes ABSOLUTE precedence over all other evaluation criteria. If the ground truth is not satisfied by the agent's execution and final response, the verdict MUST be false.
|
||||
"""
|
||||
|
||||
system_prompt = f"""You are an expert judge evaluating browser automation agent performance.
|
||||
|
||||
<evaluation_framework>
|
||||
{ground_truth_section}
|
||||
**PRIMARY EVALUATION CRITERIA (in order of importance):**
|
||||
1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome.
|
||||
2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested?
|
||||
3. **Tool Effectiveness**: Did the browser interactions work as expected? Were tools used appropriately? How many % of the tools failed?
|
||||
4. **Agent Reasoning**: Quality of decision-making, planning, and problem-solving throughout the trajectory.
|
||||
5. **Browser Handling**: Navigation stability, error recovery, and technical execution. If the browser crashes, does not load or a captcha blocks the task, the score must be very low.
|
||||
|
||||
**VERDICT GUIDELINES:**
|
||||
- true: Task completed as requested, human-like execution, all of the users criteria were met and the agent did not make up any information.
|
||||
- false: Task not completed, or only partially completed.
|
||||
|
||||
**Examples of task completion verdict:**
|
||||
- If task asks for 10 items and agent finds 4 items correctly: false
|
||||
- If task completed to full user requirements but with some errors to improve in the trajectory: true
|
||||
- If task impossible due to captcha/login requirements: false
|
||||
- If the trajectory is ideal and the output is perfect: true
|
||||
- If the task asks to search all headphones in amazon under $100 but the agent searches all headphones and the lowest price is $150: false
|
||||
- If the task asks to research a property and create a google doc with the result but the agents only returns the results in text: false
|
||||
- If the task asks to complete an action on the page, and the agent reports that the action is completed but the screenshot or page shows the action is not actually complete: false
|
||||
- If the task asks to use a certain tool or site to complete the task but the agent completes the task without using it: false
|
||||
- If the task asks to look for a section of a page that does not exist: false
|
||||
- If the agent concludes the task is impossible but it is not: false
|
||||
- If the agent concludes the task is impossible and it truly is impossible: false
|
||||
- If the agent is unable to complete the task because no login information was provided and it is truly needed to complete the task: false
|
||||
|
||||
**FAILURE CONDITIONS (automatically set verdict to false):**
|
||||
- Blocked by captcha or missing authentication
|
||||
- Output format completely wrong or missing
|
||||
- Infinite loops or severe technical failures
|
||||
- Critical user requirements ignored
|
||||
- Page not loaded
|
||||
- Browser crashed
|
||||
- Agent could not interact with required UI elements
|
||||
- The agent moved on from a important step in the task without completing it
|
||||
- The agent made up content that is not in the screenshot or the page state
|
||||
- The agent calls done action before completing all key points of the task
|
||||
|
||||
**IMPOSSIBLE TASK DETECTION:**
|
||||
Set `impossible_task` to true when the task fundamentally could not be completed due to:
|
||||
- Vague or ambiguous task instructions that cannot be reasonably interpreted
|
||||
- Website genuinely broken or non-functional (be conservative - temporary issues don't count)
|
||||
- Required links/pages truly inaccessible (404, 403, etc.)
|
||||
- Task requires authentication/login but no credentials were provided
|
||||
- Task asks for functionality that doesn't exist on the target site
|
||||
- Other insurmountable external obstacles beyond the agent's control
|
||||
|
||||
Do NOT mark as impossible if:
|
||||
- Agent made poor decisions but task was achievable
|
||||
- Temporary page loading issues that could be retried
|
||||
- Agent didn't try the right approach
|
||||
- Website works but agent struggled with it
|
||||
|
||||
**CAPTCHA DETECTION:**
|
||||
Set `reached_captcha` to true if:
|
||||
- Screenshots show captcha challenges (reCAPTCHA, hCaptcha, etc.)
|
||||
- Agent reports being blocked by bot detection
|
||||
- Error messages indicate captcha/verification requirements
|
||||
- Any evidence the agent encountered anti-bot measures during execution
|
||||
|
||||
**IMPORTANT EVALUATION NOTES:**
|
||||
- **evaluate for action** - For each key step of the trace, double check whether the action that the agent tried to performed actually happened. If the required action did not actually occur, the verdict should be false.
|
||||
- **screenshot is not entire content** - The agent has the entire DOM content, but the screenshot is only part of the content. If the agent extracts information from the page, but you do not see it in the screenshot, you can assume this information is there.
|
||||
- **Penalize poor tool usage** - Wrong tools, inefficient approaches, ignoring available information.
|
||||
- **current date/time is {current_date}** - content with recent dates is real, not fabricated.
|
||||
- **IMPORTANT**: be very picky about the user's request - Have very high standard for the agent completing the task exactly to the user's request.
|
||||
- **IMPORTANT**: be initially doubtful of the agent's self reported success, be sure to verify that its methods are valid and fulfill the user's desires to a tee.
|
||||
|
||||
</evaluation_framework>
|
||||
|
||||
<response_format>
|
||||
Respond with EXACTLY this JSON structure (no additional text before or after):
|
||||
|
||||
{{
|
||||
"reasoning": "Breakdown of user task into key points. Detailed analysis covering: what went well, what didn't work, trajectory quality assessment, tool usage evaluation, output quality review, and overall user satisfaction prediction.",
|
||||
"verdict": true or false,
|
||||
"failure_reason": "Max 5 sentences explanation of why the task was not completed successfully in case of failure. If verdict is true, use an empty string.",
|
||||
"impossible_task": true or false,
|
||||
"reached_captcha": true or false
|
||||
}}
|
||||
</response_format>
|
||||
"""
|
||||
|
||||
# Build user prompt with conditional ground truth section
|
||||
ground_truth_prompt = ''
|
||||
if ground_truth:
|
||||
ground_truth_prompt = f"""
|
||||
<ground_truth>
|
||||
{ground_truth}
|
||||
</ground_truth>
|
||||
"""
|
||||
|
||||
user_prompt = f"""
|
||||
<task>
|
||||
{task_truncated or 'No task provided'}
|
||||
</task>
|
||||
{ground_truth_prompt}
|
||||
<agent_trajectory>
|
||||
{steps_text_truncated or 'No agent trajectory provided'}
|
||||
</agent_trajectory>
|
||||
|
||||
<final_result>
|
||||
{final_result_truncated or 'No final result provided'}
|
||||
</final_result>
|
||||
|
||||
{len(encoded_images)} screenshots from execution are attached.
|
||||
|
||||
Evaluate this agent execution given the criteria and respond with the exact JSON structure requested."""
|
||||
|
||||
# Build messages with screenshots
|
||||
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=user_prompt)]
|
||||
content_parts.extend(encoded_images)
|
||||
|
||||
return [
|
||||
SystemMessage(content=system_prompt),
|
||||
UserMessage(content=content_parts),
|
||||
]
|
||||
@@ -0,0 +1,597 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
from browser_use.agent.message_manager.views import (
|
||||
HistoryItem,
|
||||
)
|
||||
from browser_use.agent.prompts import AgentMessagePrompt
|
||||
from browser_use.agent.views import (
|
||||
ActionResult,
|
||||
AgentOutput,
|
||||
AgentStepInfo,
|
||||
MessageCompactionSettings,
|
||||
MessageManagerState,
|
||||
)
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.filesystem.file_system import FileSystem
|
||||
from browser_use.llm.base import BaseChatModel
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
ContentPartImageParam,
|
||||
ContentPartTextParam,
|
||||
SystemMessage,
|
||||
UserMessage,
|
||||
)
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import (
|
||||
collect_sensitive_data_values,
|
||||
match_url_with_domain_pattern,
|
||||
redact_sensitive_string,
|
||||
time_execution_sync,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ========== Logging Helper Functions ==========
|
||||
# These functions are used ONLY for formatting debug log output.
|
||||
# They do NOT affect the actual message content sent to the LLM.
|
||||
# All logging functions start with _log_ for easy identification.
|
||||
|
||||
|
||||
def _log_get_message_emoji(message: BaseMessage) -> str:
|
||||
"""Get emoji for a message type - used only for logging display"""
|
||||
emoji_map = {
|
||||
'UserMessage': '💬',
|
||||
'SystemMessage': '🧠',
|
||||
'AssistantMessage': '🔨',
|
||||
}
|
||||
return emoji_map.get(message.__class__.__name__, '🎮')
|
||||
|
||||
|
||||
def _log_format_message_line(message: BaseMessage, content: str, is_last_message: bool, terminal_width: int) -> list[str]:
|
||||
"""Format a single message for logging display"""
|
||||
try:
|
||||
lines = []
|
||||
|
||||
# Get emoji and token info
|
||||
emoji = _log_get_message_emoji(message)
|
||||
# token_str = str(message.metadata.tokens).rjust(4)
|
||||
# TODO: fix the token count
|
||||
token_str = '??? (TODO)'
|
||||
prefix = f'{emoji}[{token_str}]: '
|
||||
|
||||
# Calculate available width (emoji=2 visual cols + [token]: =8 chars)
|
||||
content_width = terminal_width - 10
|
||||
|
||||
# Handle last message wrapping
|
||||
if is_last_message and len(content) > content_width:
|
||||
# Find a good break point
|
||||
break_point = content.rfind(' ', 0, content_width)
|
||||
if break_point > content_width * 0.7: # Keep at least 70% of line
|
||||
first_line = content[:break_point]
|
||||
rest = content[break_point + 1 :]
|
||||
else:
|
||||
# No good break point, just truncate
|
||||
first_line = content[:content_width]
|
||||
rest = content[content_width:]
|
||||
|
||||
lines.append(prefix + first_line)
|
||||
|
||||
# Second line with 10-space indent
|
||||
if rest:
|
||||
if len(rest) > terminal_width - 10:
|
||||
rest = rest[: terminal_width - 10]
|
||||
lines.append(' ' * 10 + rest)
|
||||
else:
|
||||
# Single line - truncate if needed
|
||||
if len(content) > content_width:
|
||||
content = content[:content_width]
|
||||
lines.append(prefix + content)
|
||||
|
||||
return lines
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to format message line for logging: {e}')
|
||||
# Return a simple fallback line
|
||||
return ['❓[ ?]: [Error formatting message]']
|
||||
|
||||
|
||||
# ========== End of Logging Helper Functions ==========
|
||||
|
||||
|
||||
class MessageManager:
|
||||
vision_detail_level: Literal['auto', 'low', 'high']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task: str,
|
||||
system_message: SystemMessage,
|
||||
file_system: FileSystem,
|
||||
state: MessageManagerState = MessageManagerState(),
|
||||
use_thinking: bool = True,
|
||||
include_attributes: list[str] | None = None,
|
||||
sensitive_data: dict[str, str | dict[str, str]] | None = None,
|
||||
max_history_items: int | None = None,
|
||||
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
|
||||
include_tool_call_examples: bool = False,
|
||||
include_recent_events: bool = False,
|
||||
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
|
||||
llm_screenshot_size: tuple[int, int] | None = None,
|
||||
max_clickable_elements_length: int = 40000,
|
||||
):
|
||||
self.task = task
|
||||
self.state = state
|
||||
self.system_prompt = system_message
|
||||
self.file_system = file_system
|
||||
self.sensitive_data_description = ''
|
||||
self.use_thinking = use_thinking
|
||||
self.max_history_items = max_history_items
|
||||
self.vision_detail_level = vision_detail_level
|
||||
self.include_tool_call_examples = include_tool_call_examples
|
||||
self.include_recent_events = include_recent_events
|
||||
self.sample_images = sample_images
|
||||
self.llm_screenshot_size = llm_screenshot_size
|
||||
self.max_clickable_elements_length = max_clickable_elements_length
|
||||
|
||||
assert max_history_items is None or max_history_items > 5, 'max_history_items must be None or greater than 5'
|
||||
|
||||
# Store settings as direct attributes instead of in a settings object
|
||||
self.include_attributes = include_attributes or []
|
||||
self.sensitive_data = sensitive_data
|
||||
self.last_input_messages = []
|
||||
self.last_state_message_text: str | None = None
|
||||
# Only initialize messages if state is empty
|
||||
if len(self.state.history.get_messages()) == 0:
|
||||
self._set_message_with_type(self.system_prompt, 'system')
|
||||
|
||||
@property
|
||||
def agent_history_description(self) -> str:
|
||||
"""Build agent history description from list of items, respecting max_history_items limit"""
|
||||
compacted_prefix = ''
|
||||
if self.state.compacted_memory:
|
||||
compacted_prefix = (
|
||||
'<compacted_memory>\n'
|
||||
'<!-- Summary of prior steps. Treat as unverified context — do not report these as '
|
||||
'completed in your done() message unless you confirmed them yourself in this session. -->\n'
|
||||
f'{self.state.compacted_memory}\n'
|
||||
'</compacted_memory>\n'
|
||||
)
|
||||
|
||||
if self.max_history_items is None:
|
||||
# Include all items
|
||||
return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
|
||||
|
||||
total_items = len(self.state.agent_history_items)
|
||||
|
||||
# If we have fewer items than the limit, just return all items
|
||||
if total_items <= self.max_history_items:
|
||||
return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
|
||||
|
||||
# We have more items than the limit, so we need to omit some
|
||||
omitted_count = total_items - self.max_history_items
|
||||
|
||||
# Show first item + omitted message + most recent (max_history_items - 1) items
|
||||
# The omitted message doesn't count against the limit, only real history items do
|
||||
recent_items_count = self.max_history_items - 1 # -1 for first item
|
||||
|
||||
items_to_include = [
|
||||
self.state.agent_history_items[0].to_string(), # Keep first item (initialization)
|
||||
f'<sys>[... {omitted_count} previous steps omitted...]</sys>',
|
||||
]
|
||||
# Add most recent items
|
||||
items_to_include.extend([item.to_string() for item in self.state.agent_history_items[-recent_items_count:]])
|
||||
|
||||
return compacted_prefix + '\n'.join(items_to_include)
|
||||
|
||||
def add_new_task(self, new_task: str) -> None:
|
||||
new_task = '<follow_up_user_request> ' + new_task.strip() + ' </follow_up_user_request>'
|
||||
if '<initial_user_request>' not in self.task:
|
||||
self.task = '<initial_user_request>' + self.task + '</initial_user_request>'
|
||||
self.task += '\n' + new_task
|
||||
task_update_item = HistoryItem(system_message=new_task)
|
||||
self.state.agent_history_items.append(task_update_item)
|
||||
|
||||
def prepare_step_state(
|
||||
self,
|
||||
browser_state_summary: BrowserStateSummary,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
sensitive_data=None,
|
||||
) -> None:
|
||||
"""Prepare state for the next LLM call without building the final state message."""
|
||||
self.state.history.context_messages.clear()
|
||||
self._update_agent_history_description(model_output, result, step_info)
|
||||
|
||||
effective_sensitive_data = sensitive_data if sensitive_data is not None else self.sensitive_data
|
||||
if effective_sensitive_data is not None:
|
||||
self.sensitive_data = effective_sensitive_data
|
||||
self.sensitive_data_description = self._get_sensitive_data_description(browser_state_summary.url)
|
||||
|
||||
async def maybe_compact_messages(
|
||||
self,
|
||||
llm: BaseChatModel | None,
|
||||
settings: MessageCompactionSettings | None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
) -> bool:
|
||||
"""Summarize older history into a compact memory block.
|
||||
|
||||
Step interval is the primary trigger; char count is a minimum floor.
|
||||
"""
|
||||
if not settings or not settings.enabled:
|
||||
return False
|
||||
if llm is None:
|
||||
return False
|
||||
if step_info is None:
|
||||
return False
|
||||
|
||||
# Step cadence gate
|
||||
steps_since = step_info.step_number - (self.state.last_compaction_step or 0)
|
||||
if steps_since < settings.compact_every_n_steps:
|
||||
return False
|
||||
|
||||
# Char floor gate
|
||||
history_items = self.state.agent_history_items
|
||||
full_history_text = '\n'.join(item.to_string() for item in history_items).strip()
|
||||
trigger_char_count = settings.trigger_char_count or 40000
|
||||
if len(full_history_text) < trigger_char_count:
|
||||
return False
|
||||
|
||||
logger.debug(f'Compacting message history (items={len(history_items)}, chars={len(full_history_text)})')
|
||||
|
||||
# Build compaction input
|
||||
compaction_sections = []
|
||||
if self.state.compacted_memory:
|
||||
compaction_sections.append(
|
||||
f'<previous_compacted_memory>\n{self.state.compacted_memory}\n</previous_compacted_memory>'
|
||||
)
|
||||
compaction_sections.append(f'<agent_history>\n{full_history_text}\n</agent_history>')
|
||||
if settings.include_read_state and self.state.read_state_description:
|
||||
compaction_sections.append(f'<read_state>\n{self.state.read_state_description}\n</read_state>')
|
||||
compaction_input = '\n\n'.join(compaction_sections)
|
||||
|
||||
if self.sensitive_data:
|
||||
filtered = self._filter_sensitive_data(UserMessage(content=compaction_input))
|
||||
compaction_input = filtered.text
|
||||
|
||||
system_prompt = (
|
||||
'You are summarizing an agent run for prompt compaction.\n'
|
||||
'Capture task requirements, key facts, decisions, partial progress, errors, and next steps.\n'
|
||||
'Preserve important entities, values, URLs, and file paths.\n'
|
||||
'CRITICAL: Only mark a step as completed if you see explicit success confirmation in the history. '
|
||||
'If a step was started but not explicitly confirmed complete, mark it as "IN-PROGRESS". '
|
||||
'Never infer completion from context — only report what was confirmed.\n'
|
||||
'Return plain text only. Do not include tool calls or JSON.'
|
||||
)
|
||||
if settings.summary_max_chars:
|
||||
system_prompt += f' Keep under {settings.summary_max_chars} characters if possible.'
|
||||
|
||||
messages = [SystemMessage(content=system_prompt), UserMessage(content=compaction_input)]
|
||||
try:
|
||||
response = await llm.ainvoke(messages)
|
||||
summary = (response.completion or '').strip()
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to compact messages: {e}')
|
||||
return False
|
||||
|
||||
if not summary:
|
||||
return False
|
||||
|
||||
if settings.summary_max_chars and len(summary) > settings.summary_max_chars:
|
||||
summary = summary[: settings.summary_max_chars].rstrip() + '…'
|
||||
|
||||
self.state.compacted_memory = summary
|
||||
self.state.compaction_count += 1
|
||||
self.state.last_compaction_step = step_info.step_number
|
||||
|
||||
# Keep first item + most recent items
|
||||
keep_last = max(0, settings.keep_last_items)
|
||||
if len(history_items) > keep_last + 1:
|
||||
if keep_last == 0:
|
||||
self.state.agent_history_items = [history_items[0]]
|
||||
else:
|
||||
self.state.agent_history_items = [history_items[0]] + history_items[-keep_last:]
|
||||
|
||||
logger.debug(f'Compaction complete (summary_chars={len(summary)}, history_items={len(self.state.agent_history_items)})')
|
||||
|
||||
return True
|
||||
|
||||
def _update_agent_history_description(
|
||||
self,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
) -> None:
|
||||
"""Update the agent history description"""
|
||||
|
||||
if result is None:
|
||||
result = []
|
||||
step_number = step_info.step_number if step_info else None
|
||||
|
||||
self.state.read_state_description = ''
|
||||
self.state.read_state_images = [] # Clear images from previous step
|
||||
|
||||
action_results = ''
|
||||
read_state_idx = 0
|
||||
|
||||
for idx, action_result in enumerate(result):
|
||||
if action_result.include_extracted_content_only_once and action_result.extracted_content:
|
||||
self.state.read_state_description += (
|
||||
f'<read_state_{read_state_idx}>\n{action_result.extracted_content}\n</read_state_{read_state_idx}>\n'
|
||||
)
|
||||
read_state_idx += 1
|
||||
logger.debug(f'Added extracted_content to read_state_description: {action_result.extracted_content}')
|
||||
|
||||
# Store images for one-time inclusion in the next message
|
||||
if action_result.images:
|
||||
self.state.read_state_images.extend(action_result.images)
|
||||
logger.debug(f'Added {len(action_result.images)} image(s) to read_state_images')
|
||||
|
||||
if action_result.long_term_memory:
|
||||
action_results += f'{action_result.long_term_memory}\n'
|
||||
logger.debug(f'Added long_term_memory to action_results: {action_result.long_term_memory}')
|
||||
elif action_result.extracted_content and not action_result.include_extracted_content_only_once:
|
||||
action_results += f'{action_result.extracted_content}\n'
|
||||
logger.debug(f'Added extracted_content to action_results: {action_result.extracted_content}')
|
||||
|
||||
if action_result.error:
|
||||
if len(action_result.error) > 200:
|
||||
error_text = action_result.error[:100] + '......' + action_result.error[-100:]
|
||||
else:
|
||||
error_text = action_result.error
|
||||
action_results += f'{error_text}\n'
|
||||
logger.debug(f'Added error to action_results: {error_text}')
|
||||
|
||||
# Simple 60k character limit for read_state_description
|
||||
MAX_CONTENT_SIZE = 60000
|
||||
if len(self.state.read_state_description) > MAX_CONTENT_SIZE:
|
||||
self.state.read_state_description = (
|
||||
self.state.read_state_description[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
|
||||
)
|
||||
logger.debug(f'Truncated read_state_description to {MAX_CONTENT_SIZE} characters')
|
||||
|
||||
self.state.read_state_description = self.state.read_state_description.strip('\n')
|
||||
|
||||
if action_results:
|
||||
action_results = f'Result\n{action_results}'
|
||||
action_results = action_results.strip('\n') if action_results else None
|
||||
|
||||
# Simple 60k character limit for action_results
|
||||
if action_results and len(action_results) > MAX_CONTENT_SIZE:
|
||||
action_results = action_results[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
|
||||
logger.debug(f'Truncated action_results to {MAX_CONTENT_SIZE} characters')
|
||||
|
||||
# Build the history item
|
||||
if model_output is None:
|
||||
# Add history item for initial actions (step 0) or errors (step > 0)
|
||||
if step_number is not None:
|
||||
if step_number == 0 and action_results:
|
||||
# Step 0 with initial action results
|
||||
history_item = HistoryItem(step_number=step_number, action_results=action_results)
|
||||
self.state.agent_history_items.append(history_item)
|
||||
elif step_number > 0:
|
||||
# Error case for steps > 0
|
||||
history_item = HistoryItem(step_number=step_number, error='Agent failed to output in the right format.')
|
||||
self.state.agent_history_items.append(history_item)
|
||||
else:
|
||||
history_item = HistoryItem(
|
||||
step_number=step_number,
|
||||
evaluation_previous_goal=model_output.current_state.evaluation_previous_goal,
|
||||
memory=model_output.current_state.memory,
|
||||
next_goal=model_output.current_state.next_goal,
|
||||
action_results=action_results,
|
||||
)
|
||||
self.state.agent_history_items.append(history_item)
|
||||
|
||||
def _get_sensitive_data_description(self, current_page_url) -> str:
|
||||
sensitive_data = self.sensitive_data
|
||||
if not sensitive_data:
|
||||
return ''
|
||||
|
||||
# Collect placeholders for sensitive data
|
||||
placeholders: set[str] = set()
|
||||
|
||||
for key, value in sensitive_data.items():
|
||||
if isinstance(value, dict):
|
||||
# New format: {domain: {key: value}}
|
||||
if current_page_url and match_url_with_domain_pattern(current_page_url, key, True):
|
||||
placeholders.update(value.keys())
|
||||
else:
|
||||
# Old format: {key: value}
|
||||
placeholders.add(key)
|
||||
|
||||
if placeholders:
|
||||
placeholder_list = sorted(list(placeholders))
|
||||
# Format as bullet points for clarity
|
||||
formatted_placeholders = '\n'.join(f' - {p}' for p in placeholder_list)
|
||||
|
||||
info = 'SENSITIVE DATA - Use these placeholders for secure input:\n'
|
||||
info += f'{formatted_placeholders}\n\n'
|
||||
info += 'IMPORTANT: When entering sensitive values, you MUST wrap the placeholder name in <secret> tags.\n'
|
||||
info += f'Example: To enter the value for "{placeholder_list[0]}", use: <secret>{placeholder_list[0]}</secret>\n'
|
||||
info += 'The system will automatically replace these tags with the actual secret values.'
|
||||
return info
|
||||
|
||||
return ''
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='create_state_messages')
|
||||
@time_execution_sync('--create_state_messages')
|
||||
def create_state_messages(
|
||||
self,
|
||||
browser_state_summary: BrowserStateSummary,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
use_vision: bool | Literal['auto'] = True,
|
||||
page_filtered_actions: str | None = None,
|
||||
sensitive_data=None,
|
||||
available_file_paths: list[str] | None = None, # Always pass current available_file_paths
|
||||
unavailable_skills_info: str | None = None, # Information about skills that cannot be used yet
|
||||
plan_description: str | None = None, # Rendered plan for injection into agent state
|
||||
skip_state_update: bool = False,
|
||||
) -> None:
|
||||
"""Create single state message with all content"""
|
||||
|
||||
if not skip_state_update:
|
||||
self.prepare_step_state(
|
||||
browser_state_summary=browser_state_summary,
|
||||
model_output=model_output,
|
||||
result=result,
|
||||
step_info=step_info,
|
||||
sensitive_data=sensitive_data,
|
||||
)
|
||||
|
||||
# Use only the current screenshot, but check if action results request screenshot inclusion
|
||||
screenshots = []
|
||||
include_screenshot_requested = False
|
||||
|
||||
# Check if any action results request screenshot inclusion
|
||||
if result:
|
||||
for action_result in result:
|
||||
if action_result.metadata and action_result.metadata.get('include_screenshot'):
|
||||
include_screenshot_requested = True
|
||||
logger.debug('Screenshot inclusion requested by action result')
|
||||
break
|
||||
|
||||
# Handle different use_vision modes:
|
||||
# - "auto": Only include screenshot if explicitly requested by action (e.g., screenshot)
|
||||
# - True: Always include screenshot
|
||||
# - False: Never include screenshot
|
||||
include_screenshot = False
|
||||
if use_vision is True:
|
||||
# Always include screenshot when use_vision=True
|
||||
include_screenshot = True
|
||||
elif use_vision == 'auto':
|
||||
# Only include screenshot if explicitly requested by action when use_vision="auto"
|
||||
include_screenshot = include_screenshot_requested
|
||||
# else: use_vision is False, never include screenshot (include_screenshot stays False)
|
||||
|
||||
if include_screenshot and browser_state_summary.screenshot:
|
||||
screenshots.append(browser_state_summary.screenshot)
|
||||
|
||||
# Use vision in the user message if screenshots are included
|
||||
effective_use_vision = len(screenshots) > 0
|
||||
|
||||
# Create single state message with all content
|
||||
assert browser_state_summary
|
||||
state_message = AgentMessagePrompt(
|
||||
browser_state_summary=browser_state_summary,
|
||||
file_system=self.file_system,
|
||||
agent_history_description=self.agent_history_description,
|
||||
read_state_description=self.state.read_state_description,
|
||||
task=self.task,
|
||||
include_attributes=self.include_attributes,
|
||||
step_info=step_info,
|
||||
page_filtered_actions=page_filtered_actions,
|
||||
max_clickable_elements_length=self.max_clickable_elements_length,
|
||||
sensitive_data=self.sensitive_data_description,
|
||||
available_file_paths=available_file_paths,
|
||||
screenshots=screenshots,
|
||||
vision_detail_level=self.vision_detail_level,
|
||||
include_recent_events=self.include_recent_events,
|
||||
sample_images=self.sample_images,
|
||||
read_state_images=self.state.read_state_images,
|
||||
llm_screenshot_size=self.llm_screenshot_size,
|
||||
unavailable_skills_info=unavailable_skills_info,
|
||||
plan_description=plan_description,
|
||||
).get_user_message(effective_use_vision)
|
||||
|
||||
# Store state message text for history
|
||||
self.last_state_message_text = state_message.text
|
||||
|
||||
# Set the state message with caching enabled
|
||||
self._set_message_with_type(state_message, 'state')
|
||||
|
||||
def _log_history_lines(self) -> str:
|
||||
"""Generate a formatted log string of message history for debugging / printing to terminal"""
|
||||
# TODO: fix logging
|
||||
|
||||
# try:
|
||||
# total_input_tokens = 0
|
||||
# message_lines = []
|
||||
# terminal_width = shutil.get_terminal_size((80, 20)).columns
|
||||
|
||||
# for i, m in enumerate(self.state.history.messages):
|
||||
# try:
|
||||
# total_input_tokens += m.metadata.tokens
|
||||
# is_last_message = i == len(self.state.history.messages) - 1
|
||||
|
||||
# # Extract content for logging
|
||||
# content = _log_extract_message_content(m.message, is_last_message, m.metadata)
|
||||
|
||||
# # Format the message line(s)
|
||||
# lines = _log_format_message_line(m, content, is_last_message, terminal_width)
|
||||
# message_lines.extend(lines)
|
||||
# except Exception as e:
|
||||
# logger.warning(f'Failed to format message {i} for logging: {e}')
|
||||
# # Add a fallback line for this message
|
||||
# message_lines.append('❓[ ?]: [Error formatting this message]')
|
||||
|
||||
# # Build final log message
|
||||
# return (
|
||||
# f'📜 LLM Message history ({len(self.state.history.messages)} messages, {total_input_tokens} tokens):\n'
|
||||
# + '\n'.join(message_lines)
|
||||
# )
|
||||
# except Exception as e:
|
||||
# logger.warning(f'Failed to generate history log: {e}')
|
||||
# # Return a minimal fallback message
|
||||
# return f'📜 LLM Message history (error generating log: {e})'
|
||||
|
||||
return ''
|
||||
|
||||
@time_execution_sync('--get_messages')
|
||||
def get_messages(self) -> list[BaseMessage]:
|
||||
"""Get current message list, potentially trimmed to max tokens"""
|
||||
|
||||
# Log message history for debugging
|
||||
logger.debug(self._log_history_lines())
|
||||
self.last_input_messages = self.state.history.get_messages()
|
||||
return self.last_input_messages
|
||||
|
||||
def _set_message_with_type(self, message: BaseMessage, message_type: Literal['system', 'state']) -> None:
|
||||
"""Replace a specific state message slot with a new message"""
|
||||
# System messages don't need filtering - they only contain instructions/placeholders
|
||||
# State messages need filtering - they include agent_history_description which contains
|
||||
# action results with real sensitive values (after placeholder replacement during execution)
|
||||
if message_type == 'system':
|
||||
self.state.history.system_message = message
|
||||
elif message_type == 'state':
|
||||
if self.sensitive_data:
|
||||
message = self._filter_sensitive_data(message)
|
||||
self.state.history.state_message = message
|
||||
else:
|
||||
raise ValueError(f'Invalid state message type: {message_type}')
|
||||
|
||||
def _add_context_message(self, message: BaseMessage) -> None:
|
||||
"""Add a contextual message specific to this step (e.g., validation errors, retry instructions, timeout warnings)"""
|
||||
# Context messages typically contain error messages and validation info, not action results
|
||||
# with sensitive data, so filtering is not needed here
|
||||
self.state.history.context_messages.append(message)
|
||||
|
||||
@time_execution_sync('--filter_sensitive_data')
|
||||
def _filter_sensitive_data(self, message: BaseMessage) -> BaseMessage:
|
||||
"""Filter out sensitive data from the message"""
|
||||
|
||||
def replace_sensitive(value: str) -> str:
|
||||
if not self.sensitive_data:
|
||||
return value
|
||||
|
||||
sensitive_values = collect_sensitive_data_values(self.sensitive_data)
|
||||
|
||||
# If there are no valid sensitive data entries, just return the original value
|
||||
if not sensitive_values:
|
||||
logger.warning('No valid entries found in sensitive_data dictionary')
|
||||
return value
|
||||
|
||||
return redact_sensitive_string(value, sensitive_values)
|
||||
|
||||
if isinstance(message.content, str):
|
||||
message.content = replace_sensitive(message.content)
|
||||
elif isinstance(message.content, list):
|
||||
for i, item in enumerate(message.content):
|
||||
if isinstance(item, ContentPartTextParam):
|
||||
item.text = replace_sensitive(item.text)
|
||||
message.content[i] = item
|
||||
return message
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import anyio
|
||||
|
||||
from browser_use.llm.messages import BaseMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def save_conversation(
|
||||
input_messages: list[BaseMessage],
|
||||
response: Any,
|
||||
target: str | Path,
|
||||
encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Save conversation history to file asynchronously."""
|
||||
target_path = Path(target)
|
||||
# create folders if not exists
|
||||
if target_path.parent:
|
||||
await anyio.Path(target_path.parent).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
await anyio.Path(target_path).write_text(
|
||||
await _format_conversation(input_messages, response),
|
||||
encoding=encoding or 'utf-8',
|
||||
)
|
||||
|
||||
|
||||
async def _format_conversation(messages: list[BaseMessage], response: Any) -> str:
|
||||
"""Format the conversation including messages and response."""
|
||||
lines = []
|
||||
|
||||
# Format messages
|
||||
for message in messages:
|
||||
lines.append(f' {message.role} ')
|
||||
|
||||
lines.append(message.text)
|
||||
lines.append('') # Empty line after each message
|
||||
|
||||
# Format response
|
||||
lines.append(json.dumps(json.loads(response.model_dump_json(exclude_unset=True)), indent=2, ensure_ascii=False))
|
||||
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
# Note: _write_messages_to_file and _write_response_to_file have been merged into _format_conversation
|
||||
# This is more efficient for async operations and reduces file I/O
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class HistoryItem(BaseModel):
|
||||
"""Represents a single agent history item with its data and string representation"""
|
||||
|
||||
step_number: int | None = None
|
||||
evaluation_previous_goal: str | None = None
|
||||
memory: str | None = None
|
||||
next_goal: str | None = None
|
||||
action_results: str | None = None
|
||||
error: str | None = None
|
||||
system_message: str | None = None
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
|
||||
|
||||
def model_post_init(self, __context) -> None:
|
||||
"""Validate that error and system_message are not both provided"""
|
||||
if self.error is not None and self.system_message is not None:
|
||||
raise ValueError('Cannot have both error and system_message at the same time')
|
||||
|
||||
def to_string(self) -> str:
|
||||
"""Get string representation of the history item"""
|
||||
step_str = 'step' if self.step_number is not None else 'step_unknown'
|
||||
|
||||
if self.error:
|
||||
return f"""<{step_str}>
|
||||
{self.error}"""
|
||||
elif self.system_message:
|
||||
return self.system_message
|
||||
else:
|
||||
content_parts = []
|
||||
|
||||
# Only include evaluation_previous_goal if it's not None/empty
|
||||
if self.evaluation_previous_goal:
|
||||
content_parts.append(f'{self.evaluation_previous_goal}')
|
||||
|
||||
# Always include memory
|
||||
if self.memory:
|
||||
content_parts.append(f'{self.memory}')
|
||||
|
||||
# Only include next_goal if it's not None/empty
|
||||
if self.next_goal:
|
||||
content_parts.append(f'{self.next_goal}')
|
||||
|
||||
if self.action_results:
|
||||
content_parts.append(self.action_results)
|
||||
|
||||
content = '\n'.join(content_parts)
|
||||
|
||||
return f"""<{step_str}>
|
||||
{content}"""
|
||||
|
||||
|
||||
class MessageHistory(BaseModel):
|
||||
"""History of messages"""
|
||||
|
||||
system_message: BaseMessage | None = None
|
||||
state_message: BaseMessage | None = None
|
||||
context_messages: list[BaseMessage] = Field(default_factory=list)
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
def get_messages(self) -> list[BaseMessage]:
|
||||
"""Get all messages in the correct order: system -> state -> contextual"""
|
||||
messages = []
|
||||
if self.system_message:
|
||||
messages.append(self.system_message)
|
||||
if self.state_message:
|
||||
messages.append(self.state_message)
|
||||
messages.extend(self.context_messages)
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
class MessageManagerState(BaseModel):
|
||||
"""Holds the state for MessageManager"""
|
||||
|
||||
history: MessageHistory = Field(default_factory=MessageHistory)
|
||||
tool_id: int = 1
|
||||
agent_history_items: list[HistoryItem] = Field(
|
||||
default_factory=lambda: [HistoryItem(step_number=0, system_message='Agent initialized')]
|
||||
)
|
||||
read_state_description: str = ''
|
||||
# Images to include in the next state message (cleared after each step)
|
||||
read_state_images: list[dict[str, Any]] = Field(default_factory=list)
|
||||
compacted_memory: str | None = None
|
||||
compaction_count: int = 0
|
||||
last_compaction_step: int | None = None
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
@@ -0,0 +1,588 @@
|
||||
import importlib.resources
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Literal, Optional
|
||||
|
||||
from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
|
||||
from browser_use.dom.views import NodeType, SimplifiedNode
|
||||
from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import is_new_tab_page, sanitize_surrogates
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.agent.views import AgentStepInfo
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.filesystem.file_system import FileSystem
|
||||
|
||||
|
||||
def _is_anthropic_4_5_model(model_name: str | None) -> bool:
|
||||
"""Check if the model is Claude Opus 4.5 or Haiku 4.5 (requires 4096+ token prompts for caching)."""
|
||||
if not model_name:
|
||||
return False
|
||||
model_lower = model_name.lower()
|
||||
# Check for Opus 4.5 or Haiku 4.5 variants
|
||||
is_opus_4_5 = 'opus' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
|
||||
is_haiku_4_5 = 'haiku' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
|
||||
return is_opus_4_5 or is_haiku_4_5
|
||||
|
||||
|
||||
class SystemPrompt:
|
||||
def __init__(
|
||||
self,
|
||||
max_actions_per_step: int = 3,
|
||||
override_system_message: str | None = None,
|
||||
extend_system_message: str | None = None,
|
||||
use_thinking: bool = True,
|
||||
flash_mode: bool = False,
|
||||
is_anthropic: bool = False,
|
||||
is_browser_use_model: bool = False,
|
||||
model_name: str | None = None,
|
||||
):
|
||||
self.max_actions_per_step = max_actions_per_step
|
||||
self.use_thinking = use_thinking
|
||||
self.flash_mode = flash_mode
|
||||
self.is_anthropic = is_anthropic
|
||||
self.is_browser_use_model = is_browser_use_model
|
||||
self.model_name = model_name
|
||||
# Check if this is an Anthropic 4.5 model that needs longer prompts for caching
|
||||
self.is_anthropic_4_5 = _is_anthropic_4_5_model(model_name)
|
||||
prompt = ''
|
||||
if override_system_message is not None:
|
||||
prompt = override_system_message
|
||||
else:
|
||||
self._load_prompt_template()
|
||||
prompt = self.prompt_template.format(max_actions=self.max_actions_per_step)
|
||||
|
||||
if extend_system_message:
|
||||
prompt += f'\n{extend_system_message}'
|
||||
|
||||
self.system_message = SystemMessage(content=prompt, cache=True)
|
||||
|
||||
def _load_prompt_template(self) -> None:
|
||||
"""Load the prompt template from the markdown file."""
|
||||
try:
|
||||
# Choose the appropriate template based on model type and mode
|
||||
# Browser-use models use simplified prompts optimized for fine-tuned models
|
||||
if self.is_browser_use_model:
|
||||
if self.flash_mode:
|
||||
template_filename = 'system_prompt_browser_use_flash.md'
|
||||
elif self.use_thinking:
|
||||
template_filename = 'system_prompt_browser_use.md'
|
||||
else:
|
||||
template_filename = 'system_prompt_browser_use_no_thinking.md'
|
||||
# Anthropic 4.5 models (Opus 4.5, Haiku 4.5) need 4096+ token prompts for caching
|
||||
elif self.is_anthropic_4_5 and self.flash_mode:
|
||||
template_filename = 'system_prompt_anthropic_flash.md'
|
||||
elif self.flash_mode and self.is_anthropic:
|
||||
template_filename = 'system_prompt_flash_anthropic.md'
|
||||
elif self.flash_mode:
|
||||
template_filename = 'system_prompt_flash.md'
|
||||
elif self.use_thinking:
|
||||
template_filename = 'system_prompt.md'
|
||||
else:
|
||||
template_filename = 'system_prompt_no_thinking.md'
|
||||
|
||||
# This works both in development and when installed as a package
|
||||
with (
|
||||
importlib.resources.files('browser_use.agent.system_prompts')
|
||||
.joinpath(template_filename)
|
||||
.open('r', encoding='utf-8') as f
|
||||
):
|
||||
self.prompt_template = f.read()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Failed to load system prompt template: {e}')
|
||||
|
||||
def get_system_message(self) -> SystemMessage:
|
||||
"""
|
||||
Get the system prompt for the agent.
|
||||
|
||||
Returns:
|
||||
SystemMessage: Formatted system prompt
|
||||
"""
|
||||
return self.system_message
|
||||
|
||||
|
||||
class AgentMessagePrompt:
|
||||
vision_detail_level: Literal['auto', 'low', 'high']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
browser_state_summary: 'BrowserStateSummary',
|
||||
file_system: 'FileSystem',
|
||||
agent_history_description: str | None = None,
|
||||
read_state_description: str | None = None,
|
||||
task: str | None = None,
|
||||
include_attributes: list[str] | None = None,
|
||||
step_info: Optional['AgentStepInfo'] = None,
|
||||
page_filtered_actions: str | None = None,
|
||||
max_clickable_elements_length: int = 40000,
|
||||
sensitive_data: str | None = None,
|
||||
available_file_paths: list[str] | None = None,
|
||||
screenshots: list[str] | None = None,
|
||||
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
|
||||
include_recent_events: bool = False,
|
||||
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
|
||||
read_state_images: list[dict] | None = None,
|
||||
llm_screenshot_size: tuple[int, int] | None = None,
|
||||
unavailable_skills_info: str | None = None,
|
||||
plan_description: str | None = None,
|
||||
):
|
||||
self.browser_state: 'BrowserStateSummary' = browser_state_summary
|
||||
self.file_system: 'FileSystem | None' = file_system
|
||||
self.agent_history_description: str | None = agent_history_description
|
||||
self.read_state_description: str | None = read_state_description
|
||||
self.task: str | None = task
|
||||
self.include_attributes = include_attributes
|
||||
self.step_info = step_info
|
||||
self.page_filtered_actions: str | None = page_filtered_actions
|
||||
self.max_clickable_elements_length: int = max_clickable_elements_length
|
||||
self.sensitive_data: str | None = sensitive_data
|
||||
self.available_file_paths: list[str] | None = available_file_paths
|
||||
self.screenshots = screenshots or []
|
||||
self.vision_detail_level = vision_detail_level
|
||||
self.include_recent_events = include_recent_events
|
||||
self.sample_images = sample_images or []
|
||||
self.read_state_images = read_state_images or []
|
||||
self.unavailable_skills_info: str | None = unavailable_skills_info
|
||||
self.plan_description: str | None = plan_description
|
||||
self.llm_screenshot_size = llm_screenshot_size
|
||||
assert self.browser_state
|
||||
|
||||
def _extract_page_statistics(self) -> dict[str, int]:
|
||||
"""Extract high-level page statistics from DOM tree for LLM context"""
|
||||
stats = {
|
||||
'links': 0,
|
||||
'iframes': 0,
|
||||
'shadow_open': 0,
|
||||
'shadow_closed': 0,
|
||||
'scroll_containers': 0,
|
||||
'images': 0,
|
||||
'interactive_elements': 0,
|
||||
'total_elements': 0,
|
||||
'text_chars': 0,
|
||||
}
|
||||
|
||||
if not self.browser_state.dom_state or not self.browser_state.dom_state._root:
|
||||
return stats
|
||||
|
||||
def traverse_node(node: SimplifiedNode) -> None:
|
||||
"""Recursively traverse simplified DOM tree to count elements"""
|
||||
if not node or not node.original_node:
|
||||
return
|
||||
|
||||
original = node.original_node
|
||||
stats['total_elements'] += 1
|
||||
|
||||
# Count by node type and tag
|
||||
if original.node_type == NodeType.ELEMENT_NODE:
|
||||
tag = original.tag_name.lower() if original.tag_name else ''
|
||||
|
||||
if tag == 'a':
|
||||
stats['links'] += 1
|
||||
elif tag in ('iframe', 'frame'):
|
||||
stats['iframes'] += 1
|
||||
elif tag == 'img':
|
||||
stats['images'] += 1
|
||||
|
||||
# Check if scrollable
|
||||
if original.is_actually_scrollable:
|
||||
stats['scroll_containers'] += 1
|
||||
|
||||
# Check if interactive
|
||||
if node.is_interactive:
|
||||
stats['interactive_elements'] += 1
|
||||
|
||||
# Check if this element hosts shadow DOM
|
||||
if node.is_shadow_host:
|
||||
# Check if any shadow children are closed
|
||||
has_closed_shadow = any(
|
||||
child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE
|
||||
and child.original_node.shadow_root_type
|
||||
and child.original_node.shadow_root_type.lower() == 'closed'
|
||||
for child in node.children
|
||||
)
|
||||
if has_closed_shadow:
|
||||
stats['shadow_closed'] += 1
|
||||
else:
|
||||
stats['shadow_open'] += 1
|
||||
|
||||
elif original.node_type == NodeType.TEXT_NODE:
|
||||
stats['text_chars'] += len(original.node_value.strip())
|
||||
|
||||
elif original.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
|
||||
# Shadow DOM fragment - these are the actual shadow roots
|
||||
# But don't double-count since we count them at the host level above
|
||||
pass
|
||||
|
||||
# Traverse children
|
||||
for child in node.children:
|
||||
traverse_node(child)
|
||||
|
||||
traverse_node(self.browser_state.dom_state._root)
|
||||
return stats
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')
|
||||
def _get_browser_state_description(self) -> str:
|
||||
# Extract page statistics first
|
||||
page_stats = self._extract_page_statistics()
|
||||
|
||||
# Format statistics
|
||||
stats_text = '<page_stats>'
|
||||
if page_stats['total_elements'] < 10:
|
||||
stats_text += 'Page appears empty (SPA not loaded?) - '
|
||||
# Skeleton screen: many elements but almost no text = loading placeholders
|
||||
elif page_stats['total_elements'] > 20 and page_stats['text_chars'] < page_stats['total_elements'] * 5:
|
||||
stats_text += 'Page appears to show skeleton/placeholder content (still loading?) - '
|
||||
stats_text += f'{page_stats["links"]} links, {page_stats["interactive_elements"]} interactive, '
|
||||
stats_text += f'{page_stats["iframes"]} iframes'
|
||||
if page_stats['shadow_open'] > 0 or page_stats['shadow_closed'] > 0:
|
||||
stats_text += f', {page_stats["shadow_open"]} shadow(open), {page_stats["shadow_closed"]} shadow(closed)'
|
||||
if page_stats['images'] > 0:
|
||||
stats_text += f', {page_stats["images"]} images'
|
||||
stats_text += f', {page_stats["total_elements"]} total elements'
|
||||
stats_text += '</page_stats>\n'
|
||||
|
||||
elements_text = self.browser_state.dom_state.llm_representation(include_attributes=self.include_attributes)
|
||||
|
||||
if len(elements_text) > self.max_clickable_elements_length:
|
||||
elements_text = elements_text[: self.max_clickable_elements_length]
|
||||
truncated_text = f' (truncated to {self.max_clickable_elements_length} characters)'
|
||||
else:
|
||||
truncated_text = ''
|
||||
|
||||
has_content_above = False
|
||||
has_content_below = False
|
||||
# Enhanced page information for the model
|
||||
page_info_text = ''
|
||||
if self.browser_state.page_info:
|
||||
pi = self.browser_state.page_info
|
||||
# Compute page statistics dynamically
|
||||
pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
has_content_above = pages_above > 0
|
||||
has_content_below = pages_below > 0
|
||||
page_info_text = '<page_info>'
|
||||
page_info_text += f'{pages_above:.1f} pages above, {pages_below:.1f} pages below'
|
||||
if pages_below > 0.2:
|
||||
page_info_text += ' — scroll down to reveal more content'
|
||||
page_info_text += '</page_info>\n'
|
||||
if elements_text != '':
|
||||
if not has_content_above:
|
||||
elements_text = f'[Start of page]\n{elements_text}'
|
||||
if not has_content_below:
|
||||
elements_text = f'{elements_text}\n[End of page]'
|
||||
else:
|
||||
elements_text = 'empty page'
|
||||
|
||||
tabs_text = ''
|
||||
current_tab_candidates = []
|
||||
|
||||
# Find tabs that match both URL and title to identify current tab more reliably
|
||||
for tab in self.browser_state.tabs:
|
||||
if tab.url == self.browser_state.url and tab.title == self.browser_state.title:
|
||||
current_tab_candidates.append(tab.target_id)
|
||||
|
||||
# If we have exactly one match, mark it as current
|
||||
# Otherwise, don't mark any tab as current to avoid confusion
|
||||
current_target_id = current_tab_candidates[0] if len(current_tab_candidates) == 1 else None
|
||||
|
||||
for tab in self.browser_state.tabs:
|
||||
tabs_text += f'Tab {tab.target_id[-4:]}: {tab.url} - {tab.title[:30]}\n'
|
||||
|
||||
current_tab_text = f'Current tab: {current_target_id[-4:]}' if current_target_id is not None else ''
|
||||
|
||||
# Check if current page is a PDF viewer and add appropriate message
|
||||
pdf_message = ''
|
||||
if self.browser_state.is_pdf_viewer:
|
||||
pdf_message = (
|
||||
'PDF viewer cannot be rendered. In this page, DO NOT use the extract action as PDF content cannot be rendered. '
|
||||
)
|
||||
pdf_message += (
|
||||
'Use the read_file action on the downloaded PDF in available_file_paths to read the full text content.\n\n'
|
||||
)
|
||||
|
||||
# Add recent events if available and requested
|
||||
recent_events_text = ''
|
||||
if self.include_recent_events and self.browser_state.recent_events:
|
||||
recent_events_text = f'Recent browser events: {self.browser_state.recent_events}\n'
|
||||
|
||||
# Add closed popup messages if any
|
||||
closed_popups_text = ''
|
||||
if self.browser_state.closed_popup_messages:
|
||||
closed_popups_text = 'Auto-closed JavaScript dialogs:\n'
|
||||
for popup_msg in self.browser_state.closed_popup_messages:
|
||||
closed_popups_text += f' - {popup_msg}\n'
|
||||
closed_popups_text += '\n'
|
||||
|
||||
browser_state = f"""{stats_text}{current_tab_text}
|
||||
Available tabs:
|
||||
{tabs_text}
|
||||
{page_info_text}
|
||||
{recent_events_text}{closed_popups_text}{pdf_message}Interactive elements{truncated_text}:
|
||||
{elements_text}
|
||||
"""
|
||||
return browser_state
|
||||
|
||||
def _get_agent_state_description(self) -> str:
|
||||
_todo_contents = self.file_system.get_todo_contents() if self.file_system else ''
|
||||
if not len(_todo_contents):
|
||||
_todo_contents = '[empty todo.md, fill it when applicable]'
|
||||
|
||||
agent_state = f"""
|
||||
<file_system>
|
||||
{self.file_system.describe() if self.file_system else 'No file system available'}
|
||||
</file_system>
|
||||
<todo_contents>
|
||||
{_todo_contents}
|
||||
</todo_contents>
|
||||
"""
|
||||
if self.plan_description:
|
||||
agent_state += f'<plan>\n{self.plan_description}\n</plan>\n'
|
||||
|
||||
if self.sensitive_data:
|
||||
agent_state += f'<sensitive_data>{self.sensitive_data}</sensitive_data>\n'
|
||||
|
||||
if self.available_file_paths:
|
||||
available_file_paths_text = '\n'.join(self.available_file_paths)
|
||||
agent_state += f'<available_file_paths>{available_file_paths_text}\nUse with absolute paths</available_file_paths>\n'
|
||||
return agent_state
|
||||
|
||||
def _get_user_request_description(self) -> str:
|
||||
return f'<user_request>\n{self.task}\n</user_request>\n\n'
|
||||
|
||||
def _get_step_meta_description(self) -> str:
|
||||
# Per-step varying metadata (step counter, wall-clock date). Kept out of <agent_state> so it
|
||||
# lives at the tail of the user message — anything before this block can in principle be
|
||||
# treated as the cacheable prefix.
|
||||
if self.step_info:
|
||||
step_info_description = f'Step{self.step_info.step_number + 1} maximum:{self.step_info.max_steps}\n'
|
||||
else:
|
||||
step_info_description = ''
|
||||
step_info_description += f'Today:{datetime.now().strftime("%Y-%m-%d")}'
|
||||
return f'<step_info>{step_info_description}</step_info>\n'
|
||||
|
||||
def _resize_screenshot(self, screenshot_b64: str) -> str:
|
||||
"""Resize screenshot to llm_screenshot_size if configured."""
|
||||
if not self.llm_screenshot_size:
|
||||
return screenshot_b64
|
||||
|
||||
try:
|
||||
import base64
|
||||
import logging
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
img = Image.open(BytesIO(base64.b64decode(screenshot_b64)))
|
||||
if img.size == self.llm_screenshot_size:
|
||||
return screenshot_b64
|
||||
|
||||
logging.getLogger(__name__).info(
|
||||
f'🔄 Resizing screenshot from {img.size[0]}x{img.size[1]} to {self.llm_screenshot_size[0]}x{self.llm_screenshot_size[1]} for LLM'
|
||||
)
|
||||
|
||||
img_resized = img.resize(self.llm_screenshot_size, Image.Resampling.LANCZOS)
|
||||
buffer = BytesIO()
|
||||
img_resized.save(buffer, format='PNG')
|
||||
return base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(f'Failed to resize screenshot: {e}, using original')
|
||||
return screenshot_b64
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')
|
||||
def get_user_message(self, use_vision: bool = True) -> UserMessage:
|
||||
"""Get complete state as a single cached message"""
|
||||
# New-tab pages only carry placeholder screenshots, even later in a multi-tab session.
|
||||
if is_new_tab_page(self.browser_state.url):
|
||||
use_vision = False
|
||||
|
||||
# Build complete state description
|
||||
state_description = (
|
||||
self._get_user_request_description()
|
||||
+ '<agent_history>\n'
|
||||
+ (self.agent_history_description.strip('\n') if self.agent_history_description else '')
|
||||
+ '\n</agent_history>\n\n'
|
||||
)
|
||||
state_description += '<agent_state>\n' + self._get_agent_state_description().strip('\n') + '\n</agent_state>\n'
|
||||
state_description += '<browser_state>\n' + self._get_browser_state_description().strip('\n') + '\n</browser_state>\n'
|
||||
# Only add read_state if it has content
|
||||
read_state_description = self.read_state_description.strip('\n').strip() if self.read_state_description else ''
|
||||
if read_state_description:
|
||||
state_description += '<read_state>\n' + read_state_description + '\n</read_state>\n'
|
||||
|
||||
if self.page_filtered_actions:
|
||||
state_description += '<page_specific_actions>\n'
|
||||
state_description += self.page_filtered_actions + '\n'
|
||||
state_description += '</page_specific_actions>\n'
|
||||
|
||||
# Add unavailable skills information if any
|
||||
if self.unavailable_skills_info:
|
||||
state_description += '\n' + self.unavailable_skills_info + '\n'
|
||||
|
||||
# Per-step varying metadata (step counter, date) lives at the tail of the message so that
|
||||
# everything above can in principle be treated as a cacheable prefix.
|
||||
state_description += self._get_step_meta_description()
|
||||
|
||||
# Sanitize surrogates from all text content
|
||||
state_description = sanitize_surrogates(state_description)
|
||||
|
||||
# Check if we have images to include (from read_file action)
|
||||
has_images = bool(self.read_state_images)
|
||||
screenshots = [screenshot for screenshot in self.screenshots if screenshot != PLACEHOLDER_4PX_SCREENSHOT]
|
||||
|
||||
if (use_vision is True and screenshots) or has_images:
|
||||
# Start with text description
|
||||
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=state_description)]
|
||||
|
||||
# Add sample images
|
||||
content_parts.extend(self.sample_images)
|
||||
|
||||
# Add screenshots with labels
|
||||
for i, screenshot in enumerate(screenshots):
|
||||
if i == len(screenshots) - 1:
|
||||
label = 'Current screenshot:'
|
||||
else:
|
||||
# Use simple, accurate labeling since we don't have actual step timing info
|
||||
label = 'Previous screenshot:'
|
||||
|
||||
# Add label as text content
|
||||
content_parts.append(ContentPartTextParam(text=label))
|
||||
|
||||
# Resize screenshot if llm_screenshot_size is configured
|
||||
processed_screenshot = self._resize_screenshot(screenshot)
|
||||
|
||||
# Add the screenshot
|
||||
content_parts.append(
|
||||
ContentPartImageParam(
|
||||
image_url=ImageURL(
|
||||
url=f'data:image/png;base64,{processed_screenshot}',
|
||||
media_type='image/png',
|
||||
detail=self.vision_detail_level,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Add read_state images (from read_file action) before screenshots
|
||||
for img_data in self.read_state_images:
|
||||
img_name = img_data.get('name', 'unknown')
|
||||
img_base64 = img_data.get('data', '')
|
||||
|
||||
if not img_base64:
|
||||
continue
|
||||
|
||||
# Detect image format from name
|
||||
if img_name.lower().endswith('.png'):
|
||||
media_type = 'image/png'
|
||||
else:
|
||||
media_type = 'image/jpeg'
|
||||
|
||||
# Add label
|
||||
content_parts.append(ContentPartTextParam(text=f'Image from file: {img_name}'))
|
||||
|
||||
# Add the image
|
||||
content_parts.append(
|
||||
ContentPartImageParam(
|
||||
image_url=ImageURL(
|
||||
url=f'data:{media_type};base64,{img_base64}',
|
||||
media_type=media_type,
|
||||
detail=self.vision_detail_level,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return UserMessage(content=content_parts, cache=True)
|
||||
|
||||
return UserMessage(content=state_description, cache=True)
|
||||
|
||||
|
||||
def get_rerun_summary_prompt(original_task: str, total_steps: int, success_count: int, error_count: int) -> str:
|
||||
return f'''You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary.
|
||||
|
||||
Original task: {original_task}
|
||||
|
||||
Execution statistics:
|
||||
- Total steps: {total_steps}
|
||||
- Successful steps: {success_count}
|
||||
- Failed steps: {error_count}
|
||||
|
||||
Analyze the screenshot to determine:
|
||||
1. Whether the task completed successfully
|
||||
2. What the final state shows
|
||||
3. Overall completion status (complete/partial/failed)
|
||||
|
||||
Respond with:
|
||||
- summary: A clear, concise summary of what happened during the rerun
|
||||
- success: Whether the task completed successfully (true/false)
|
||||
- completion_status: One of "complete", "partial", or "failed"'''
|
||||
|
||||
|
||||
def get_rerun_summary_message(prompt: str, screenshot_b64: str | None = None) -> UserMessage:
|
||||
"""
|
||||
Build a UserMessage for rerun summary generation.
|
||||
|
||||
Args:
|
||||
prompt: The prompt text
|
||||
screenshot_b64: Optional base64-encoded screenshot
|
||||
|
||||
Returns:
|
||||
UserMessage with prompt and optional screenshot
|
||||
"""
|
||||
if screenshot_b64:
|
||||
# With screenshot: use multi-part content
|
||||
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [
|
||||
ContentPartTextParam(type='text', text=prompt),
|
||||
ContentPartImageParam(
|
||||
type='image_url',
|
||||
image_url=ImageURL(url=f'data:image/png;base64,{screenshot_b64}'),
|
||||
),
|
||||
]
|
||||
return UserMessage(content=content_parts)
|
||||
else:
|
||||
# Without screenshot: use simple string content
|
||||
return UserMessage(content=prompt)
|
||||
|
||||
|
||||
def get_ai_step_system_prompt() -> str:
|
||||
"""
|
||||
Get system prompt for AI step action used during rerun.
|
||||
|
||||
Returns:
|
||||
System prompt string for AI step
|
||||
"""
|
||||
return """
|
||||
You are an expert at extracting data from webpages.
|
||||
|
||||
<input>
|
||||
You will be given:
|
||||
1. A query describing what to extract
|
||||
2. The markdown of the webpage (filtered to remove noise)
|
||||
3. Optionally, a screenshot of the current page state
|
||||
</input>
|
||||
|
||||
<instructions>
|
||||
- Extract information from the webpage that is relevant to the query
|
||||
- ONLY use the information available in the webpage - do not make up information
|
||||
- If the information is not available, mention that clearly
|
||||
- If the query asks for all items, list all of them
|
||||
</instructions>
|
||||
|
||||
<output>
|
||||
- Present ALL relevant information in a concise way
|
||||
- Do not use conversational format - directly output the relevant information
|
||||
- If information is unavailable, state that clearly
|
||||
</output>
|
||||
""".strip()
|
||||
|
||||
|
||||
def get_ai_step_user_prompt(query: str, stats_summary: str, content: str) -> str:
|
||||
"""
|
||||
Build user prompt for AI step action.
|
||||
|
||||
Args:
|
||||
query: What to extract or analyze
|
||||
stats_summary: Content statistics summary
|
||||
content: Page markdown content
|
||||
|
||||
Returns:
|
||||
Formatted prompt string
|
||||
"""
|
||||
return f'<query>\n{query}\n</query>\n\n<content_stats>\n{stats_summary}\n</content_stats>\n\n<webpage_content>\n{content}\n</webpage_content>'
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
# System prompt templates for browser-use agent
|
||||
@@ -0,0 +1,270 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operate effectively in an agent loop
|
||||
6. Efficiently performing diverse web tasks
|
||||
</intro>
|
||||
<language_settings>
|
||||
- Default working language: **English**
|
||||
- Always respond in the same language as the user request
|
||||
</language_settings>
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <user_request>: Your ultimate objective.
|
||||
2. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
|
||||
4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
|
||||
6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
and system messages wrapped in <sys> tag.
|
||||
</agent_history>
|
||||
<browser_state>
|
||||
1. Browser State will be given as:
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their ids.
|
||||
Interactive Elements: All interactive elements will be provided in a tree-style XML format:
|
||||
- Format: `[index]<tagname attribute=value />` for interactive elements
|
||||
- Text content appears as child nodes on separate lines (not inside tags)
|
||||
- Indentation with tabs shows parent/child relationships
|
||||
Examples:
|
||||
[33]<div />
|
||||
User form
|
||||
[35]<input type=text placeholder=Enter name />
|
||||
*[38]<button aria-label=Submit form />
|
||||
Submit
|
||||
[40]<a />
|
||||
About us
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
|
||||
- Pure text elements without [] are not interactive
|
||||
- `|SCROLL|` prefix indicates scrollable containers with scroll position info
|
||||
- `|SHADOW(open)|` or `|SHADOW(closed)|` prefix indicates shadow DOM elements
|
||||
</browser_state>
|
||||
<browser_vision>
|
||||
If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
Use screenshot if you are unsure or simply want more information.
|
||||
</browser_vision>
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed.
|
||||
- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
|
||||
- If the page is not fully loaded, use the wait action.
|
||||
- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- Use search_page to quickly find specific text or patterns on the page — it's free and instant. Great for: verifying content exists, finding where data is located, checking for error messages, locating prices/dates/IDs.
|
||||
- Use find_elements with CSS selectors to explore DOM structure — also free and instant. Great for: counting items (e.g. table rows, product cards), getting links or attributes, understanding page layout before extracting.
|
||||
- Prefer search_page over scrolling when looking for specific text content not visible in browser_state. Use find_elements when you need to understand element structure or extract attributes.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks always first think which type of request you are dealing with:
|
||||
1. Very specific step by step instructions:
|
||||
- Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
|
||||
- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
|
||||
- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
|
||||
</browser_rules>
|
||||
<file_system>
|
||||
- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
<planning>
|
||||
Decide whether to plan based on task complexity:
|
||||
- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
|
||||
- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
|
||||
- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
|
||||
When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
|
||||
Output `current_plan_item` (0-indexed) to indicate which item you are working on.
|
||||
Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
|
||||
Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
|
||||
</planning>
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
- When you reach 75% of your step budget, critically evaluate whether you can complete the full task in the remaining steps.
|
||||
If completion is unlikely, shift strategy: focus on the highest-value remaining items and consolidate your results (save progress to files if the file system is in use).
|
||||
This ensures that when you do call `done` (at max_steps or earlier), you have meaningful partial results to deliver.
|
||||
- For large multi-item tasks (e.g. "search 50 items"), estimate the per-item cost from the first few items.
|
||||
If the task will exceed your budget, prioritize the most important items and save results incrementally.
|
||||
<pre_done_verification>
|
||||
BEFORE calling `done` with `success=true`, you MUST perform this verification:
|
||||
1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
|
||||
2. **Check each requirement against your results:**
|
||||
- Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
|
||||
- Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
|
||||
- Does your output match the requested format exactly?
|
||||
3. **Verify actions actually completed:**
|
||||
- If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
|
||||
- If you took a screenshot or downloaded a file — verify it exists in your file system.
|
||||
4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
|
||||
5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
|
||||
6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
|
||||
Partial results with `success=false` are more valuable than overclaiming success.
|
||||
</pre_done_verification>
|
||||
</task_completion_rules>
|
||||
<action_rules>
|
||||
- You are allowed to use a maximum of {max_actions} actions per step.
|
||||
If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
|
||||
- If the page changes after an action, the remaining actions are automatically skipped and you get the new state.
|
||||
Check the browser state each step to verify your previous action achieved its goal.
|
||||
</action_rules>
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
|
||||
**Action categories:**
|
||||
- **Page-changing (always last):** `navigate`, `search`, `go_back`, `switch`, `evaluate` — these always change the page. Remaining actions after them are skipped automatically. Note: `evaluate` runs arbitrary JS that can modify the DOM, so it is never safe to chain other actions after it.
|
||||
- **Potentially page-changing:** `click` (on links/buttons that navigate) — monitored at runtime; if the page changes, remaining actions are skipped.
|
||||
- **Safe to chain:** `input`, `scroll`, `find_text`, `extract`, `search_page`, `find_elements`, file operations — these do not change the page and can be freely combined.
|
||||
|
||||
**Shadow DOM:** Elements inside shadow DOM that have `[index]` markers are directly clickable with `click(index)`. Do NOT use `evaluate` to click them.
|
||||
|
||||
**Recommended combinations:**
|
||||
- `input` + `input` + `input` + `click` → Fill multiple form fields then submit
|
||||
- `input` + `input` → Fill multiple form fields
|
||||
- `scroll` + `scroll` → Scroll further down the page
|
||||
- `click` + `click` → Navigate multi-step flows (only when clicks do not navigate)
|
||||
- File operations + browser actions
|
||||
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Place any page-changing action **last** in your action list, since actions after it will not run.
|
||||
</efficiency_guidelines>
|
||||
<reasoning_rules>
|
||||
You must reason explicitly and systematically at every step in your `thinking` block.
|
||||
Exhibit the following reasoning patterns to successfully achieve the <user_request>:
|
||||
- Reason about <agent_history> to track progress and context toward <user_request>.
|
||||
- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
|
||||
- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
|
||||
- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
|
||||
- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
|
||||
- Analyze `todo.md` to guide and track your progress.
|
||||
- If any todo.md items are finished, mark them as complete in the file.
|
||||
- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
|
||||
- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
|
||||
- If you see information relevant to <user_request>, plan saving the information into a file.
|
||||
- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
|
||||
- Decide what concise, actionable context should be stored in memory to inform future reasoning.
|
||||
- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
|
||||
- Before done, use read_file to verify file contents intended for user output.
|
||||
- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
|
||||
</reasoning_rules>
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
<evaluation_examples>
|
||||
- Positive Examples:
|
||||
"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
|
||||
"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
|
||||
- Negative Examples:
|
||||
"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
|
||||
"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
|
||||
</evaluation_examples>
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
|
||||
"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
|
||||
"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
|
||||
"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
|
||||
"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
|
||||
</memory_examples>
|
||||
<next_goal_examples>
|
||||
"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
|
||||
"next_goal": "Extract details from the first item on the page."
|
||||
"next_goal": "Close the popup that appeared blocking the main content."
|
||||
"next_goal": "Apply price filter to narrow results to items under $50."
|
||||
</next_goal_examples>
|
||||
</examples>
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"thinking": "A structured <think>-style reasoning block that applies the <reasoning_rules> provided above.",
|
||||
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"current_plan_item": 0,
|
||||
"plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
|
||||
"action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
`current_plan_item` and `plan_update` are optional. See <planning> for details.
|
||||
</output>
|
||||
<critical_reminders>
|
||||
1. ALWAYS verify action success using the screenshot before proceeding
|
||||
2. ALWAYS handle popups/modals/cookie banners before other actions
|
||||
3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
|
||||
4. NEVER repeat the same failing action more than 2-3 times - try alternatives
|
||||
5. NEVER assume success - always verify from screenshot or browser state
|
||||
6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
|
||||
7. Put ALL relevant findings in done action's text field
|
||||
8. Match user's requested output format exactly
|
||||
9. Track progress in memory to avoid loops
|
||||
10. When at max_steps, call done with whatever results you have
|
||||
11. Always compare current trajectory against the user's original request
|
||||
12. Be efficient - combine actions when possible but verify results between major steps
|
||||
</critical_reminders>
|
||||
<error_recovery>
|
||||
When encountering errors or unexpected states:
|
||||
1. First, verify the current state using screenshot as ground truth
|
||||
2. Check if a popup, modal, or overlay is blocking interaction
|
||||
3. If an element is not found, scroll to reveal more content
|
||||
4. If an action fails repeatedly (2-3 times), try an alternative approach
|
||||
5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
|
||||
6. If the page structure is different than expected, re-analyze and adapt
|
||||
7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
|
||||
8. If max_steps is approaching, prioritize completing the most important parts of the task
|
||||
</error_recovery>
|
||||
@@ -0,0 +1,247 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information from web pages
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operating effectively in an agent loop with persistent state
|
||||
6. Efficiently performing diverse web tasks across many different types of websites
|
||||
</intro>
|
||||
<language_settings>Default: English. Match user's language.</language_settings>
|
||||
<user_request>Ultimate objective. Specific tasks: follow each step precisely. Open-ended: plan your own approach.</user_request>
|
||||
<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new element since last step.</browser_state>
|
||||
<file_system>
|
||||
PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
<action_rules>
|
||||
You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
|
||||
</action_rules>
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided in the current browser state.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed. Scroll to see more elements if needed.
|
||||
- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
|
||||
- If the page is not fully loaded, use the wait action to allow content to render.
|
||||
- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results. This is critical for efficiency.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks:
|
||||
1. Very specific step by step instructions: Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first. Many websites show cookie consent dialogs, newsletter popups, or promotional overlays that must be dismissed.
|
||||
- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation. Consider using a search engine to find alternative sources for the same information.
|
||||
- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
|
||||
- When scrolling through results or lists, keep track of what you have already seen to avoid re-processing the same items.
|
||||
- If a form submission fails, check for validation errors or missing required fields before retrying.
|
||||
- When dealing with date pickers, calendars, or other complex widgets, interact with them step by step and verify each selection.
|
||||
</browser_rules>
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
**Recommended Action Combinations:**
|
||||
- `input` + `click` → Fill form field and submit/search in one step
|
||||
- `input` + `input` → Fill multiple form fields sequentially
|
||||
- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
|
||||
- File operations + browser actions → Save data while continuing to browse
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
|
||||
- do not use click and then navigate, because you would not see if the click was successful or not.
|
||||
- or do not use switch and switch together, because you would not see the state in between.
|
||||
- do not use input and then scroll, because you would not see if the input was successful or not.
|
||||
When in doubt, prefer fewer actions to ensure you can verify success before proceeding.
|
||||
</efficiency_guidelines>
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
<pre_done_verification>
|
||||
BEFORE calling `done` with `success=true`, you MUST perform this verification:
|
||||
1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
|
||||
2. **Check each requirement against your results:**
|
||||
- Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
|
||||
- Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
|
||||
- Does your output match the requested format exactly?
|
||||
3. **Verify actions actually completed:**
|
||||
- If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
|
||||
- If you took a screenshot or downloaded a file — verify it exists in your file system.
|
||||
4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
|
||||
5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
|
||||
6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
|
||||
Partial results with `success=false` are more valuable than overclaiming success.
|
||||
</pre_done_verification>
|
||||
</task_completion_rules>
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <user_request>: Your ultimate objective.
|
||||
2. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
|
||||
4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. This is your GROUND TRUTH.
|
||||
6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
and system messages wrapped in <sys> tag.
|
||||
Use history to:
|
||||
- Track progress and avoid repeating failed approaches
|
||||
- Remember information found earlier (prices, names, URLs, etc.)
|
||||
- Verify that your trajectory matches the user's request
|
||||
- Learn from previous failures and successes
|
||||
</agent_history>
|
||||
<browser_state_details>
|
||||
Browser State format:
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their ids.
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, link, div, etc.)
|
||||
- text: Element description or content
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
\t*[35]<button aria-label='Submit form'>Submit</button>
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared since the last step
|
||||
- Pure text elements without [] are not interactive
|
||||
- The index numbers may change between steps as the page updates
|
||||
</browser_state_details>
|
||||
<browser_vision_details>
|
||||
If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: use it to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
Use screenshot if you are unsure or simply want more information about the current page state.
|
||||
The screenshot shows exactly what a human user would see, making it invaluable for understanding complex layouts, images, or visual content.
|
||||
</browser_vision_details>
|
||||
<output>You must call the AgentOutput tool with the following schema for the arguments:
|
||||
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
|
||||
"action": [
|
||||
{{
|
||||
"action_name": {{
|
||||
"parameter1": "value1",
|
||||
"parameter2": "value2"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
Always put `memory` field before the `action` field.
|
||||
</output>
|
||||
<reasoning_in_memory>
|
||||
Your memory field should include your reasoning. Apply these patterns:
|
||||
- Did the previous action succeed? Verify using screenshot as ground truth.
|
||||
- What is the current state relative to the user request?
|
||||
- Are there any obstacles (popups, login walls)? CAPTCHAs are solved automatically.
|
||||
- What specific next step will make progress toward the goal?
|
||||
- If stuck, what alternative approach should you try?
|
||||
- What information should be remembered for later steps?
|
||||
Never assume an action succeeded just because you attempted it. Always verify from the screenshot or browser state.
|
||||
Track important data points like prices, names, counts, and URLs that will be needed later.
|
||||
</reasoning_in_memory>
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
|
||||
"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
|
||||
"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
|
||||
"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
|
||||
"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
|
||||
"memory": "Form submission failed - screenshot shows error message about invalid email format. Need to correct the email field."
|
||||
"memory": "Successfully added item to cart. Screenshot confirms cart count is now 1. Next step is to proceed to checkout."
|
||||
"memory": "Dropdown menu appeared after clicking. Need to select the 'Electronics' category from the options shown."
|
||||
"memory": "Page loaded but content is different from expected. URL shows login redirect. Will look for alternative access or report limitation."
|
||||
"memory": "Scrolled through first 10 results, found 3 matching items. Need to continue scrolling to find more options."
|
||||
</memory_examples>
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
</examples>
|
||||
<action_reference>
|
||||
Common actions you can use:
|
||||
- navigate: Go to a specific URL
|
||||
- click: Click on an element by index
|
||||
- input: Type text into an input field
|
||||
- scroll: Scroll the page up or down
|
||||
- wait: Wait for the page to load
|
||||
- extract: Extract structured information from the page
|
||||
- screenshot: Take a screenshot for visual verification
|
||||
- switch_tab: Switch between browser tabs
|
||||
- go_back: Navigate back in browser history
|
||||
- done: Complete the task and report results
|
||||
- write_file: Write content to a file
|
||||
- read_file: Read content from a file
|
||||
- replace_file_str: Replace text in a file
|
||||
Each action has specific parameters - refer to the action schema for details.
|
||||
</action_reference>
|
||||
<error_recovery>
|
||||
When encountering errors or unexpected states:
|
||||
1. First, verify the current state using screenshot as ground truth
|
||||
2. Check if a popup, modal, or overlay is blocking interaction
|
||||
3. If an element is not found, scroll to reveal more content
|
||||
4. If an action fails repeatedly (2-3 times), try an alternative approach
|
||||
5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
|
||||
6. If the page structure is different than expected, re-analyze and adapt
|
||||
7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
|
||||
8. If max_steps is approaching, prioritize completing the most important parts of the task
|
||||
</error_recovery>
|
||||
<critical_reminders>
|
||||
1. ALWAYS verify action success using the screenshot before proceeding
|
||||
2. ALWAYS handle popups/modals/cookie banners before other actions
|
||||
3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
|
||||
4. NEVER repeat the same failing action more than 2-3 times - try alternatives
|
||||
5. NEVER assume success - always verify from screenshot or browser state
|
||||
6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
|
||||
7. Put ALL relevant findings in done action's text field
|
||||
8. Match user's requested output format exactly
|
||||
9. Track progress in memory to avoid loops
|
||||
10. When at max_steps, call done with whatever results you have
|
||||
11. Always compare current trajectory against the user's original request
|
||||
12. Be efficient - combine actions when possible but verify results between major steps
|
||||
</critical_reminders>
|
||||
@@ -0,0 +1,18 @@
|
||||
You are a browser-use agent operating in thinking mode. You automate browser tasks by outputting structured JSON actions.
|
||||
|
||||
<constraint_enforcement>
|
||||
Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
|
||||
</constraint_enforcement>
|
||||
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"thinking": "A structured reasoning block analyzing: current page state, what was attempted, what worked/failed, and strategic planning for next steps.",
|
||||
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"action": [{{"action_name": {{...params...}}}}]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
@@ -0,0 +1,15 @@
|
||||
You are a browser-use agent operating in flash mode. You automate browser tasks by outputting structured JSON actions.
|
||||
|
||||
<constraint_enforcement>
|
||||
Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
|
||||
</constraint_enforcement>
|
||||
|
||||
<output>
|
||||
You must respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer.",
|
||||
"action": [{{"action_name": {{...params...}}}}]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
@@ -0,0 +1,17 @@
|
||||
You are a browser-use agent. You automate browser tasks by outputting structured JSON actions.
|
||||
|
||||
<constraint_enforcement>
|
||||
Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
|
||||
</constraint_enforcement>
|
||||
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"action": [{{"action_name": {{...params...}}}}]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
@@ -0,0 +1,16 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<language_settings>Default: English. Match user's language.</language_settings>
|
||||
<user_request>Ultimate objective. Specific tasks: follow each step. Open-ended: plan approach.</user_request>
|
||||
<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.</browser_state>
|
||||
<file_system>- PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. When writing CSV, use double quotes for commas. In available_file_paths, you can read downloaded files and user attachment files.</file_system>
|
||||
<action_rules>
|
||||
You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
</action_rules>
|
||||
<output>You must respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
|
||||
"action":[{{"navigate": {{ "url": "url_value"}}}}]
|
||||
}}
|
||||
Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found in the browser state or tool outputs, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
@@ -0,0 +1,31 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<user_request>
|
||||
User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.
|
||||
</user_request>
|
||||
<browser_state>
|
||||
Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.
|
||||
</browser_state>
|
||||
<file_system>
|
||||
PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
|
||||
</file_system>
|
||||
<action_rules>
|
||||
You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
</action_rules>
|
||||
<output>You must call the AgentOutput tool with the following schema for the arguments:
|
||||
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
|
||||
"action": [
|
||||
{{
|
||||
"action_name": {{
|
||||
"parameter1": "value1",
|
||||
"parameter2": "value2"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
Always put `memory` field before the `action` field.
|
||||
Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
@@ -0,0 +1,246 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operate effectively in an agent loop
|
||||
6. Efficiently performing diverse web tasks
|
||||
</intro>
|
||||
<language_settings>
|
||||
- Default working language: **English**
|
||||
- Always respond in the same language as the user request
|
||||
</language_settings>
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <user_request>: Your ultimate objective.
|
||||
2. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
3. <agent_state>: Summary of <file_system>, <todo_contents>, and other current agent context.
|
||||
4. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
5. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
|
||||
6. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
and system messages wrapped in <sys> tag.
|
||||
</agent_history>
|
||||
<browser_state>
|
||||
1. Browser State will be given as:
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their ids.
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, etc.)
|
||||
- text: Element description
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
\t*[35]<button aria-label='Submit form'>Submit</button>
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
|
||||
- Pure text elements without [] are not interactive.
|
||||
</browser_state>
|
||||
<browser_vision>
|
||||
If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
Use screenshot if you are unsure or simply want more information.
|
||||
</browser_vision>
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed.
|
||||
- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
|
||||
- If the page is not fully loaded, use the wait action.
|
||||
- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks always first think which type of request you are dealing with:
|
||||
1. Very specific step by step instructions:
|
||||
- Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
|
||||
- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
|
||||
- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
|
||||
</browser_rules>
|
||||
<file_system>
|
||||
- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
<planning>
|
||||
Decide whether to plan based on task complexity:
|
||||
- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
|
||||
- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
|
||||
- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
|
||||
When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
|
||||
Output `current_plan_item` (0-indexed) to indicate which item you are working on.
|
||||
Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
|
||||
Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
|
||||
</planning>
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
<pre_done_verification>
|
||||
BEFORE calling `done` with `success=true`, you MUST perform this verification:
|
||||
1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
|
||||
2. **Check each requirement against your results:**
|
||||
- Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
|
||||
- Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
|
||||
- Does your output match the requested format exactly?
|
||||
3. **Verify actions actually completed:**
|
||||
- If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
|
||||
- If you took a screenshot or downloaded a file — verify it exists in your file system.
|
||||
4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
|
||||
5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
|
||||
6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
|
||||
Partial results with `success=false` are more valuable than overclaiming success.
|
||||
</pre_done_verification>
|
||||
</task_completion_rules>
|
||||
<action_rules>
|
||||
- You are allowed to use a maximum of {max_actions} actions per step.
|
||||
If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
|
||||
- If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
|
||||
Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
</action_rules>
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
**Recommended Action Combinations:**
|
||||
- `input` + `click` → Fill form field and submit/search in one step
|
||||
- `input` + `input` → Fill multiple form fields
|
||||
- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
|
||||
- File operations + browser actions
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
|
||||
- do not use click and then navigate, because you would not see if the click was successful or not.
|
||||
- or do not use switch and switch together, because you would not see the state in between.
|
||||
- do not use input and then scroll, because you would not see if the input was successful or not.
|
||||
</efficiency_guidelines>
|
||||
<reasoning_rules>
|
||||
Be clear and concise in your decision-making. Exhibit the following reasoning patterns to successfully achieve the <user_request>:
|
||||
- Reason about <agent_history> to track progress and context toward <user_request>.
|
||||
- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
|
||||
- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
|
||||
- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
|
||||
- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
|
||||
- Analyze `todo.md` to guide and track your progress.
|
||||
- If any todo.md items are finished, mark them as complete in the file.
|
||||
- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
|
||||
- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
|
||||
- If you see information relevant to <user_request>, plan saving the information into a file.
|
||||
- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
|
||||
- Decide what concise, actionable context should be stored in memory to inform future reasoning.
|
||||
- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
|
||||
- Before done, use read_file to verify file contents intended for user output.
|
||||
- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
|
||||
</reasoning_rules>
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
<evaluation_examples>
|
||||
- Positive Examples:
|
||||
"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
|
||||
"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
|
||||
- Negative Examples:
|
||||
"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
|
||||
"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
|
||||
</evaluation_examples>
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
|
||||
"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
|
||||
"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
|
||||
"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
|
||||
"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
|
||||
</memory_examples>
|
||||
<next_goal_examples>
|
||||
"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
|
||||
"next_goal": "Extract details from the first item on the page."
|
||||
"next_goal": "Close the popup that appeared blocking the main content."
|
||||
"next_goal": "Apply price filter to narrow results to items under $50."
|
||||
</next_goal_examples>
|
||||
</examples>
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"evaluation_previous_goal": "One-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"current_plan_item": 0,
|
||||
"plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
|
||||
"action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
`current_plan_item` and `plan_update` are optional. See <planning> for details.
|
||||
</output>
|
||||
<critical_reminders>
|
||||
1. ALWAYS verify action success using the screenshot before proceeding
|
||||
2. ALWAYS handle popups/modals/cookie banners before other actions
|
||||
3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
|
||||
4. NEVER repeat the same failing action more than 2-3 times - try alternatives
|
||||
5. NEVER assume success - always verify from screenshot or browser state
|
||||
6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
|
||||
7. Put ALL relevant findings in done action's text field
|
||||
8. Match user's requested output format exactly
|
||||
9. Track progress in memory to avoid loops
|
||||
10. When at max_steps, call done with whatever results you have
|
||||
11. Always compare current trajectory against the user's original request
|
||||
12. Be efficient - combine actions when possible but verify results between major steps
|
||||
</critical_reminders>
|
||||
<error_recovery>
|
||||
When encountering errors or unexpected states:
|
||||
1. First, verify the current state using screenshot as ground truth
|
||||
2. Check if a popup, modal, or overlay is blocking interaction
|
||||
3. If an element is not found, scroll to reveal more content
|
||||
4. If an action fails repeatedly (2-3 times), try an alternative approach
|
||||
5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
|
||||
6. If the page structure is different than expected, re-analyze and adapt
|
||||
7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
|
||||
8. If max_steps is approaching, prioritize completing the most important parts of the task
|
||||
</error_recovery>
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Detect variables in agent history for reuse"""
|
||||
|
||||
import re
|
||||
|
||||
from browser_use.agent.views import AgentHistoryList, DetectedVariable
|
||||
from browser_use.dom.views import DOMInteractedElement
|
||||
|
||||
|
||||
def detect_variables_in_history(history: AgentHistoryList) -> dict[str, DetectedVariable]:
|
||||
"""
|
||||
Analyze agent history and detect reusable variables.
|
||||
|
||||
Uses two strategies:
|
||||
1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
|
||||
2. Value pattern matching (email, phone, date formats) - fallback
|
||||
|
||||
Returns:
|
||||
Dictionary mapping variable names to DetectedVariable objects
|
||||
"""
|
||||
detected: dict[str, DetectedVariable] = {}
|
||||
detected_values: set[str] = set() # Track which values we've already detected
|
||||
|
||||
for step_idx, history_item in enumerate(history.history):
|
||||
if not history_item.model_output:
|
||||
continue
|
||||
|
||||
for action_idx, action in enumerate(history_item.model_output.action):
|
||||
# Convert action to dict - handle both Pydantic models and dict-like objects
|
||||
if hasattr(action, 'model_dump'):
|
||||
action_dict = action.model_dump()
|
||||
elif isinstance(action, dict):
|
||||
action_dict = action
|
||||
else:
|
||||
# For SimpleNamespace or similar objects
|
||||
action_dict = vars(action)
|
||||
|
||||
# Get the interacted element for this action (if available)
|
||||
element = None
|
||||
if history_item.state and history_item.state.interacted_element:
|
||||
if len(history_item.state.interacted_element) > action_idx:
|
||||
element = history_item.state.interacted_element[action_idx]
|
||||
|
||||
# Detect variables in this action
|
||||
_detect_in_action(action_dict, element, detected, detected_values)
|
||||
|
||||
return detected
|
||||
|
||||
|
||||
def _detect_in_action(
|
||||
action_dict: dict,
|
||||
element: DOMInteractedElement | None,
|
||||
detected: dict[str, DetectedVariable],
|
||||
detected_values: set[str],
|
||||
) -> None:
|
||||
"""Detect variables in a single action using element context"""
|
||||
|
||||
# Extract action type and parameters
|
||||
for action_type, params in action_dict.items():
|
||||
if not isinstance(params, dict):
|
||||
continue
|
||||
|
||||
# Check fields that commonly contain variables
|
||||
fields_to_check = ['text', 'query']
|
||||
|
||||
for field in fields_to_check:
|
||||
if field not in params:
|
||||
continue
|
||||
|
||||
value = params[field]
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
continue
|
||||
|
||||
# Skip if we already detected this exact value
|
||||
if value in detected_values:
|
||||
continue
|
||||
|
||||
# Try to detect variable type (with element context)
|
||||
var_info = _detect_variable_type(value, element)
|
||||
if not var_info:
|
||||
continue
|
||||
|
||||
var_name, var_format = var_info
|
||||
|
||||
# Ensure unique variable name
|
||||
var_name = _ensure_unique_name(var_name, detected)
|
||||
|
||||
# Add detected variable
|
||||
detected[var_name] = DetectedVariable(
|
||||
name=var_name,
|
||||
original_value=value,
|
||||
type='string',
|
||||
format=var_format,
|
||||
)
|
||||
|
||||
detected_values.add(value)
|
||||
|
||||
|
||||
def _detect_variable_type(
|
||||
value: str,
|
||||
element: DOMInteractedElement | None = None,
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""
|
||||
Detect if a value looks like a variable, using element context when available.
|
||||
|
||||
Priority:
|
||||
1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
|
||||
2. Value pattern matching (email, phone, date formats) - fallback
|
||||
|
||||
Returns:
|
||||
(variable_name, format) or None if not detected
|
||||
"""
|
||||
|
||||
# STRATEGY 1: Use element attributes (most reliable)
|
||||
if element and element.attributes:
|
||||
attr_detection = _detect_from_attributes(element.attributes)
|
||||
if attr_detection:
|
||||
return attr_detection
|
||||
|
||||
# STRATEGY 2: Pattern matching on value (fallback)
|
||||
return _detect_from_value_pattern(value)
|
||||
|
||||
|
||||
def _detect_from_attributes(attributes: dict[str, str]) -> tuple[str, str | None] | None:
|
||||
"""
|
||||
Detect variable from element attributes.
|
||||
|
||||
Check attributes in priority order:
|
||||
1. type attribute (HTML5 input types - most specific)
|
||||
2. id, name, placeholder, aria-label (semantic hints)
|
||||
"""
|
||||
|
||||
# Check 'type' attribute first (HTML5 input types)
|
||||
input_type = attributes.get('type', '').lower()
|
||||
if input_type == 'email':
|
||||
return ('email', 'email')
|
||||
elif input_type == 'tel':
|
||||
return ('phone', 'phone')
|
||||
elif input_type == 'date':
|
||||
return ('date', 'date')
|
||||
elif input_type == 'number':
|
||||
return ('number', 'number')
|
||||
elif input_type == 'url':
|
||||
return ('url', 'url')
|
||||
|
||||
# Combine semantic attributes for keyword matching
|
||||
semantic_attrs = [
|
||||
attributes.get('id', ''),
|
||||
attributes.get('name', ''),
|
||||
attributes.get('placeholder', ''),
|
||||
attributes.get('aria-label', ''),
|
||||
]
|
||||
|
||||
combined_text = ' '.join(semantic_attrs).lower()
|
||||
|
||||
# Address detection
|
||||
if any(keyword in combined_text for keyword in ['address', 'street', 'addr']):
|
||||
if 'billing' in combined_text:
|
||||
return ('billing_address', None)
|
||||
elif 'shipping' in combined_text:
|
||||
return ('shipping_address', None)
|
||||
else:
|
||||
return ('address', None)
|
||||
|
||||
# Comment/Note detection
|
||||
if any(keyword in combined_text for keyword in ['comment', 'note', 'message', 'description']):
|
||||
return ('comment', None)
|
||||
|
||||
# Email detection
|
||||
if 'email' in combined_text or 'e-mail' in combined_text:
|
||||
return ('email', 'email')
|
||||
|
||||
# Phone detection
|
||||
if any(keyword in combined_text for keyword in ['phone', 'tel', 'mobile', 'cell']):
|
||||
return ('phone', 'phone')
|
||||
|
||||
# Name detection (order matters - check specific before general)
|
||||
if 'first' in combined_text and 'name' in combined_text:
|
||||
return ('first_name', None)
|
||||
elif 'last' in combined_text and 'name' in combined_text:
|
||||
return ('last_name', None)
|
||||
elif 'full' in combined_text and 'name' in combined_text:
|
||||
return ('full_name', None)
|
||||
elif 'name' in combined_text:
|
||||
return ('name', None)
|
||||
|
||||
# Date detection
|
||||
if any(keyword in combined_text for keyword in ['date', 'dob', 'birth']):
|
||||
return ('date', 'date')
|
||||
|
||||
# City detection
|
||||
if 'city' in combined_text:
|
||||
return ('city', None)
|
||||
|
||||
# State/Province detection
|
||||
if 'state' in combined_text or 'province' in combined_text:
|
||||
return ('state', None)
|
||||
|
||||
# Country detection
|
||||
if 'country' in combined_text:
|
||||
return ('country', None)
|
||||
|
||||
# Zip code detection
|
||||
if any(keyword in combined_text for keyword in ['zip', 'postal', 'postcode']):
|
||||
return ('zip_code', 'postal_code')
|
||||
|
||||
# Company detection
|
||||
if 'company' in combined_text or 'organization' in combined_text:
|
||||
return ('company', None)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _detect_from_value_pattern(value: str) -> tuple[str, str | None] | None:
|
||||
"""
|
||||
Detect variable type from value pattern (fallback when no element context).
|
||||
|
||||
Patterns:
|
||||
- Email: contains @ and . with valid format
|
||||
- Phone: digits with separators, 10+ chars
|
||||
- Date: YYYY-MM-DD format
|
||||
- Name: Capitalized word(s), 2-30 chars, letters only
|
||||
- Number: Pure digits, 1-9 chars
|
||||
"""
|
||||
|
||||
# Email detection - most specific first
|
||||
if '@' in value and '.' in value:
|
||||
# Basic email validation
|
||||
if re.match(r'^[\w\.-]+@[\w\.-]+\.\w+$', value):
|
||||
return ('email', 'email')
|
||||
|
||||
# Phone detection (digits with separators, 10+ chars)
|
||||
if re.match(r'^[\d\s\-\(\)\+]+$', value):
|
||||
# Remove separators and check length
|
||||
digits_only = re.sub(r'[\s\-\(\)\+]', '', value)
|
||||
if len(digits_only) >= 10:
|
||||
return ('phone', 'phone')
|
||||
|
||||
# Date detection (YYYY-MM-DD or similar)
|
||||
if re.match(r'^\d{4}-\d{2}-\d{2}$', value):
|
||||
return ('date', 'date')
|
||||
|
||||
# Name detection (capitalized, only letters/spaces, 2-30 chars)
|
||||
if value and value[0].isupper() and value.replace(' ', '').replace('-', '').isalpha() and 2 <= len(value) <= 30:
|
||||
words = value.split()
|
||||
if len(words) == 1:
|
||||
return ('first_name', None)
|
||||
elif len(words) == 2:
|
||||
return ('full_name', None)
|
||||
else:
|
||||
return ('name', None)
|
||||
|
||||
# Number detection (pure digits, not phone length)
|
||||
if value.isdigit() and 1 <= len(value) <= 9:
|
||||
return ('number', 'number')
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_unique_name(base_name: str, existing: dict[str, DetectedVariable]) -> str:
|
||||
"""
|
||||
Ensure variable name is unique by adding suffix if needed.
|
||||
|
||||
Examples:
|
||||
first_name → first_name
|
||||
first_name (exists) → first_name_2
|
||||
first_name_2 (exists) → first_name_3
|
||||
"""
|
||||
if base_name not in existing:
|
||||
return base_name
|
||||
|
||||
# Add numeric suffix
|
||||
counter = 2
|
||||
while f'{base_name}_{counter}' in existing:
|
||||
counter += 1
|
||||
|
||||
return f'{base_name}_{counter}'
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
||||
"""Beta Browser Use integration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.beta.service import Agent, BetaAgentError, find_browser_use_terminal_binary
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser import BrowserProfile, BrowserSession
|
||||
from browser_use.browser import BrowserSession as Browser
|
||||
from browser_use.llm.anthropic.chat import ChatAnthropic
|
||||
from browser_use.llm.azure.chat import ChatAzureOpenAI
|
||||
from browser_use.llm.browser_use.chat import ChatBrowserUse
|
||||
from browser_use.llm.google.chat import ChatGoogle
|
||||
from browser_use.llm.groq.chat import ChatGroq
|
||||
from browser_use.llm.litellm.chat import ChatLiteLLM
|
||||
from browser_use.llm.mistral.chat import ChatMistral
|
||||
from browser_use.llm.oci_raw.chat import ChatOCIRaw
|
||||
from browser_use.llm.ollama.chat import ChatOllama
|
||||
from browser_use.llm.openai.chat import ChatOpenAI
|
||||
from browser_use.llm.vercel.chat import ChatVercel
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
'Browser': ('browser_use.browser', 'BrowserSession'),
|
||||
'BrowserProfile': ('browser_use.browser', 'BrowserProfile'),
|
||||
'BrowserSession': ('browser_use.browser', 'BrowserSession'),
|
||||
'ChatOpenAI': ('browser_use.llm.openai.chat', 'ChatOpenAI'),
|
||||
'ChatGoogle': ('browser_use.llm.google.chat', 'ChatGoogle'),
|
||||
'ChatAnthropic': ('browser_use.llm.anthropic.chat', 'ChatAnthropic'),
|
||||
'ChatBrowserUse': ('browser_use.llm.browser_use.chat', 'ChatBrowserUse'),
|
||||
'ChatGroq': ('browser_use.llm.groq.chat', 'ChatGroq'),
|
||||
'ChatLiteLLM': ('browser_use.llm.litellm.chat', 'ChatLiteLLM'),
|
||||
'ChatMistral': ('browser_use.llm.mistral.chat', 'ChatMistral'),
|
||||
'ChatAzureOpenAI': ('browser_use.llm.azure.chat', 'ChatAzureOpenAI'),
|
||||
'ChatOCIRaw': ('browser_use.llm.oci_raw.chat', 'ChatOCIRaw'),
|
||||
'ChatOllama': ('browser_use.llm.ollama.chat', 'ChatOllama'),
|
||||
'ChatVercel': ('browser_use.llm.vercel.chat', 'ChatVercel'),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_path, attr_name = _LAZY_IMPORTS[name]
|
||||
from importlib import import_module
|
||||
|
||||
module = import_module(module_path)
|
||||
attr = getattr(module, attr_name)
|
||||
globals()[name] = attr
|
||||
return attr
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Agent',
|
||||
'BetaAgentError',
|
||||
'Browser',
|
||||
'BrowserProfile',
|
||||
'BrowserSession',
|
||||
'ChatAnthropic',
|
||||
'ChatAzureOpenAI',
|
||||
'ChatBrowserUse',
|
||||
'ChatGoogle',
|
||||
'ChatGroq',
|
||||
'ChatLiteLLM',
|
||||
'ChatMistral',
|
||||
'ChatOCIRaw',
|
||||
'ChatOllama',
|
||||
'ChatOpenAI',
|
||||
'ChatVercel',
|
||||
'find_browser_use_terminal_binary',
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,41 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
# Type stubs for lazy imports
|
||||
if TYPE_CHECKING:
|
||||
from .profile import BrowserProfile, ProxySettings
|
||||
from .session import BrowserSession
|
||||
|
||||
|
||||
# Lazy imports mapping for heavy browser components
|
||||
_LAZY_IMPORTS = {
|
||||
'ProxySettings': ('.profile', 'ProxySettings'),
|
||||
'BrowserProfile': ('.profile', 'BrowserProfile'),
|
||||
'BrowserSession': ('.session', 'BrowserSession'),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism for heavy browser components."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_path, attr_name = _LAZY_IMPORTS[name]
|
||||
try:
|
||||
from importlib import import_module
|
||||
|
||||
# Use relative import for current package
|
||||
full_module_path = f'browser_use.browser{module_path}'
|
||||
module = import_module(full_module_path)
|
||||
attr = getattr(module, attr_name)
|
||||
# Cache the imported attribute in the module's globals
|
||||
globals()[name] = attr
|
||||
return attr
|
||||
except ImportError as e:
|
||||
raise ImportError(f'Failed to import {name} from {full_module_path}: {e}') from e
|
||||
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
__all__ = [
|
||||
'BrowserSession',
|
||||
'BrowserProfile',
|
||||
'ProxySettings',
|
||||
]
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Per-CDP-request timeout wrapper around cdp_use.CDPClient.
|
||||
|
||||
cdp_use's `send_raw()` awaits a future that only resolves when the browser
|
||||
sends a matching response. If the server goes silent mid-session (observed
|
||||
failure mode against remote cloud browsers: WebSocket stays "alive" at the
|
||||
TCP/keepalive layer while the browser container is dead or the proxy has
|
||||
lost its upstream) the future never resolves and the whole agent hangs.
|
||||
|
||||
This module provides a thin subclass that wraps each `send_raw()` in
|
||||
`asyncio.wait_for`. Any CDP method that doesn't get a response within the
|
||||
cap raises `TimeoutError`, which propagates through existing
|
||||
error-handling paths in browser-use instead of hanging indefinitely.
|
||||
|
||||
Configure the cap via:
|
||||
- `BROWSER_USE_CDP_TIMEOUT_S` env var (process-wide default)
|
||||
- `TimeoutWrappedCDPClient(..., cdp_request_timeout_s=...)` constructor arg
|
||||
|
||||
Default (60s) is generous for slow operations like `Page.captureScreenshot`
|
||||
or `Page.printToPDF` on heavy pages, but well below the 180s agent step
|
||||
timeout and the typical outer agent watchdog.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from cdp_use import CDPClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CDP_TIMEOUT_FALLBACK_S = 60.0
|
||||
|
||||
|
||||
def _parse_env_cdp_timeout(raw: str | None) -> float:
|
||||
"""Parse BROWSER_USE_CDP_TIMEOUT_S defensively.
|
||||
|
||||
Accepts only finite positive values; everything else falls back to the
|
||||
hardcoded default with a warning. Mirrors the guard on
|
||||
BROWSER_USE_ACTION_TIMEOUT_S in tools/service.py — a bad env value here
|
||||
would otherwise make every CDP call time out immediately (nan) or never
|
||||
(inf / negative / zero).
|
||||
"""
|
||||
if raw is None or raw == '':
|
||||
return _CDP_TIMEOUT_FALLBACK_S
|
||||
try:
|
||||
parsed = float(raw)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
'Invalid BROWSER_USE_CDP_TIMEOUT_S=%r; falling back to %.0fs',
|
||||
raw,
|
||||
_CDP_TIMEOUT_FALLBACK_S,
|
||||
)
|
||||
return _CDP_TIMEOUT_FALLBACK_S
|
||||
if not math.isfinite(parsed) or parsed <= 0:
|
||||
logger.warning(
|
||||
'BROWSER_USE_CDP_TIMEOUT_S=%r is not a finite positive number; falling back to %.0fs',
|
||||
raw,
|
||||
_CDP_TIMEOUT_FALLBACK_S,
|
||||
)
|
||||
return _CDP_TIMEOUT_FALLBACK_S
|
||||
return parsed
|
||||
|
||||
|
||||
DEFAULT_CDP_REQUEST_TIMEOUT_S: float = _parse_env_cdp_timeout(os.getenv('BROWSER_USE_CDP_TIMEOUT_S'))
|
||||
|
||||
|
||||
def _coerce_valid_timeout(value: float | None) -> float:
|
||||
"""Normalize a user-supplied timeout to a finite positive value.
|
||||
|
||||
None / nan / inf / non-positive values all fall back to the env-derived
|
||||
default with a warning. This mirrors _parse_env_cdp_timeout so callers that
|
||||
pass cdp_request_timeout_s directly get the same defensive behaviour as
|
||||
callers that set the env var.
|
||||
"""
|
||||
if value is None:
|
||||
return DEFAULT_CDP_REQUEST_TIMEOUT_S
|
||||
if not math.isfinite(value) or value <= 0:
|
||||
logger.warning(
|
||||
'cdp_request_timeout_s=%r is not a finite positive number; falling back to %.0fs',
|
||||
value,
|
||||
DEFAULT_CDP_REQUEST_TIMEOUT_S,
|
||||
)
|
||||
return DEFAULT_CDP_REQUEST_TIMEOUT_S
|
||||
return float(value)
|
||||
|
||||
|
||||
class TimeoutWrappedCDPClient(CDPClient):
|
||||
"""CDPClient subclass that enforces a per-request timeout on send_raw.
|
||||
|
||||
Any CDP method that doesn't receive a response within `cdp_request_timeout_s`
|
||||
raises `TimeoutError` instead of hanging forever. This turns silent-hang
|
||||
failure modes (cloud proxy alive, browser dead) into fast observable errors.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args: Any,
|
||||
cdp_request_timeout_s: float | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._cdp_request_timeout_s: float = _coerce_valid_timeout(cdp_request_timeout_s)
|
||||
|
||||
async def send_raw(
|
||||
self,
|
||||
method: str,
|
||||
params: Any | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
super().send_raw(method=method, params=params, session_id=session_id),
|
||||
timeout=self._cdp_request_timeout_s,
|
||||
)
|
||||
except TimeoutError as e:
|
||||
# Raise a plain TimeoutError so existing `except TimeoutError`
|
||||
# handlers in browser-use / tools treat this uniformly.
|
||||
raise TimeoutError(
|
||||
f'CDP method {method!r} did not respond within {self._cdp_request_timeout_s:.0f}s. '
|
||||
f'The browser may be unresponsive (silent WebSocket — container crashed or proxy lost upstream).'
|
||||
) from e
|
||||
@@ -0,0 +1,126 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _chrome_user_data_dir_for_executable(executable_path: str | None) -> Path | None:
|
||||
if executable_path is None:
|
||||
return None
|
||||
|
||||
system = platform.system()
|
||||
path = Path(executable_path)
|
||||
if system == 'Darwin':
|
||||
app_path = str(path)
|
||||
base = Path.home() / 'Library' / 'Application Support'
|
||||
if 'Chromium.app' in app_path:
|
||||
return base / 'Chromium'
|
||||
if 'Google Chrome Canary.app' in app_path:
|
||||
return base / 'Google' / 'Chrome Canary'
|
||||
if 'Google Chrome.app' in app_path:
|
||||
return base / 'Google' / 'Chrome'
|
||||
if system == 'Linux':
|
||||
name = path.name
|
||||
base = Path.home() / '.config'
|
||||
if name in {'chromium', 'chromium-browser'}:
|
||||
return base / 'chromium'
|
||||
if name in {'google-chrome', 'google-chrome-stable'}:
|
||||
return base / 'google-chrome'
|
||||
if system == 'Windows' and path.name.lower() == 'chrome.exe':
|
||||
return Path(os.path.expandvars(r'%LocalAppData%\Google\Chrome\User Data'))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_chrome_executable() -> str | None:
|
||||
"""Find Chrome/Chromium executable on the system."""
|
||||
system = platform.system()
|
||||
|
||||
if system == 'Darwin':
|
||||
for path in (
|
||||
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
||||
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
||||
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
|
||||
):
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
elif system == 'Linux':
|
||||
for cmd in ('google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser'):
|
||||
try:
|
||||
result = subprocess.run(['which', cmd], capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
elif system == 'Windows':
|
||||
for path in (
|
||||
os.path.expandvars(r'%ProgramFiles%\Google\Chrome\Application\chrome.exe'),
|
||||
os.path.expandvars(r'%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe'),
|
||||
os.path.expandvars(r'%LocalAppData%\Google\Chrome\Application\chrome.exe'),
|
||||
):
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_chrome_profile_path(profile: str | None, executable_path: str | None = None) -> str | None:
|
||||
"""Get Chrome user data directory, or return a specific profile directory name."""
|
||||
if profile is not None:
|
||||
return profile
|
||||
|
||||
if browser_user_data_dir := _chrome_user_data_dir_for_executable(executable_path):
|
||||
return str(browser_user_data_dir)
|
||||
|
||||
system = platform.system()
|
||||
if system == 'Darwin':
|
||||
return str(Path.home() / 'Library' / 'Application Support' / 'Google' / 'Chrome')
|
||||
if system == 'Linux':
|
||||
base = Path.home() / '.config'
|
||||
for name in ('google-chrome', 'chromium'):
|
||||
if (base / name).is_dir():
|
||||
return str(base / name)
|
||||
return str(base / 'google-chrome')
|
||||
if system == 'Windows':
|
||||
return os.path.expandvars(r'%LocalAppData%\Google\Chrome\User Data')
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def list_chrome_profiles() -> list[dict[str, str]]:
|
||||
"""List available Chrome profiles with their display names."""
|
||||
user_data_dir = get_chrome_profile_path(None, executable_path=find_chrome_executable())
|
||||
if user_data_dir is None:
|
||||
return []
|
||||
|
||||
local_state_path = Path(user_data_dir) / 'Local State'
|
||||
if not local_state_path.exists():
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(local_state_path, encoding='utf-8') as f:
|
||||
local_state = json.load(f)
|
||||
|
||||
if not isinstance(local_state, dict):
|
||||
return []
|
||||
info_cache = local_state.get('profile', {}).get('info_cache', {})
|
||||
if not isinstance(info_cache, dict):
|
||||
return []
|
||||
return sorted(
|
||||
[
|
||||
{
|
||||
'directory': directory,
|
||||
'name': info.get('name', directory),
|
||||
}
|
||||
for directory, info in info_cache.items()
|
||||
if isinstance(info, dict)
|
||||
],
|
||||
key=lambda profile: profile['directory'],
|
||||
)
|
||||
except (json.JSONDecodeError, KeyError, OSError, TypeError):
|
||||
return []
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Cloud browser service integration for browser-use.
|
||||
|
||||
This module provides integration with the browser-use cloud browser service.
|
||||
When cloud_browser=True, it automatically creates a cloud browser instance
|
||||
and returns the CDP URL for connection.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
from browser_use.browser.cloud.views import CloudBrowserAuthError, CloudBrowserError, CloudBrowserResponse, CreateBrowserRequest
|
||||
from browser_use.sync.auth import CloudAuthConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CloudBrowserClient:
|
||||
"""Client for browser-use cloud browser service."""
|
||||
|
||||
def __init__(self, api_base_url: str = 'https://api.browser-use.com'):
|
||||
self.api_base_url = api_base_url
|
||||
self.client = httpx.AsyncClient(timeout=30.0)
|
||||
self.current_session_id: str | None = None
|
||||
|
||||
async def create_browser(
|
||||
self, request: CreateBrowserRequest, extra_headers: dict[str, str] | None = None
|
||||
) -> CloudBrowserResponse:
|
||||
"""Create a new cloud browser instance. For full docs refer to https://docs.cloud.browser-use.com/api-reference/v-2-api-current/browsers/create-browser-session-browsers-post
|
||||
|
||||
Args:
|
||||
request: CreateBrowserRequest object containing browser creation parameters
|
||||
|
||||
Returns:
|
||||
CloudBrowserResponse: Contains CDP URL and other browser info
|
||||
"""
|
||||
url = f'{self.api_base_url}/api/v2/browsers'
|
||||
|
||||
# Try to get API key from environment variable first, then auth config
|
||||
api_token = os.getenv('BROWSER_USE_API_KEY')
|
||||
|
||||
if not api_token:
|
||||
# Fallback to auth config file
|
||||
try:
|
||||
auth_config = CloudAuthConfig.load_from_file()
|
||||
api_token = auth_config.api_token
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not api_token:
|
||||
raise CloudBrowserAuthError(
|
||||
'BROWSER_USE_API_KEY is not set. To use cloud browsers, get a key at:\n'
|
||||
'https://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=use_cloud'
|
||||
)
|
||||
|
||||
headers = {'X-Browser-Use-API-Key': api_token, 'Content-Type': 'application/json', **(extra_headers or {})}
|
||||
|
||||
# Convert request to dictionary and exclude unset fields
|
||||
request_body = request.model_dump(exclude_unset=True)
|
||||
|
||||
try:
|
||||
logger.info('🌤️ Creating cloud browser instance...')
|
||||
|
||||
response = await self.client.post(url, headers=headers, json=request_body)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise CloudBrowserAuthError(
|
||||
'BROWSER_USE_API_KEY is invalid. Get a new key at:\n'
|
||||
'https://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=use_cloud'
|
||||
)
|
||||
elif response.status_code == 403:
|
||||
raise CloudBrowserAuthError('Access forbidden. Please check your browser-use cloud subscription status.')
|
||||
elif not response.is_success:
|
||||
error_msg = f'Failed to create cloud browser: HTTP {response.status_code}'
|
||||
try:
|
||||
error_data = response.json()
|
||||
if 'detail' in error_data:
|
||||
error_msg += f' - {error_data["detail"]}'
|
||||
except Exception:
|
||||
pass
|
||||
raise CloudBrowserError(error_msg)
|
||||
|
||||
browser_data = response.json()
|
||||
browser_response = CloudBrowserResponse(**browser_data)
|
||||
|
||||
# Store session ID for cleanup
|
||||
self.current_session_id = browser_response.id
|
||||
|
||||
logger.info(f'🌤️ Cloud browser created successfully: {browser_response.id}')
|
||||
logger.debug(f'🌤️ CDP URL: {browser_response.cdpUrl}')
|
||||
# Cyan color for live URL
|
||||
logger.info(f'\033[36m🔗 Live URL: {browser_response.liveUrl}\033[0m')
|
||||
|
||||
return browser_response
|
||||
|
||||
except httpx.TimeoutException:
|
||||
raise CloudBrowserError('Timeout while creating cloud browser. Please try again.')
|
||||
except httpx.ConnectError:
|
||||
raise CloudBrowserError('Failed to connect to cloud browser service. Please check your internet connection.')
|
||||
except Exception as e:
|
||||
if isinstance(e, (CloudBrowserError, CloudBrowserAuthError)):
|
||||
raise
|
||||
raise CloudBrowserError(f'Unexpected error creating cloud browser: {e}')
|
||||
|
||||
async def stop_browser(
|
||||
self, session_id: str | None = None, extra_headers: dict[str, str] | None = None
|
||||
) -> CloudBrowserResponse:
|
||||
"""Stop a cloud browser session.
|
||||
|
||||
Args:
|
||||
session_id: Session ID to stop. If None, uses current session.
|
||||
|
||||
Returns:
|
||||
CloudBrowserResponse: Updated browser info with stopped status
|
||||
|
||||
Raises:
|
||||
CloudBrowserAuthError: If authentication fails
|
||||
CloudBrowserError: If stopping fails
|
||||
"""
|
||||
if session_id is None:
|
||||
session_id = self.current_session_id
|
||||
|
||||
if not session_id:
|
||||
raise CloudBrowserError('No session ID provided and no current session available')
|
||||
|
||||
url = f'{self.api_base_url}/api/v2/browsers/{session_id}'
|
||||
|
||||
# Try to get API key from environment variable first, then auth config
|
||||
api_token = os.getenv('BROWSER_USE_API_KEY')
|
||||
|
||||
if not api_token:
|
||||
# Fallback to auth config file
|
||||
try:
|
||||
auth_config = CloudAuthConfig.load_from_file()
|
||||
api_token = auth_config.api_token
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not api_token:
|
||||
raise CloudBrowserAuthError(
|
||||
'BROWSER_USE_API_KEY is not set. To use cloud browsers, get a key at:\n'
|
||||
'https://cloud.browser-use.com/new-api-key?utm_source=oss&utm_medium=use_cloud'
|
||||
)
|
||||
|
||||
headers = {'X-Browser-Use-API-Key': api_token, 'Content-Type': 'application/json', **(extra_headers or {})}
|
||||
|
||||
request_body = {'action': 'stop'}
|
||||
|
||||
try:
|
||||
logger.info(f'🌤️ Stopping cloud browser session: {session_id}')
|
||||
|
||||
response = await self.client.patch(url, headers=headers, json=request_body)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise CloudBrowserAuthError(
|
||||
'Authentication failed. Please make sure you have set the BROWSER_USE_API_KEY environment variable to authenticate with the cloud service.'
|
||||
)
|
||||
elif response.status_code == 404:
|
||||
# Session already stopped or doesn't exist - treating as error and clearing session
|
||||
logger.debug(f'🌤️ Cloud browser session {session_id} not found (already stopped)')
|
||||
# Clear current session if it was this one
|
||||
if session_id == self.current_session_id:
|
||||
self.current_session_id = None
|
||||
raise CloudBrowserError(f'Cloud browser session {session_id} not found')
|
||||
elif not response.is_success:
|
||||
error_msg = f'Failed to stop cloud browser: HTTP {response.status_code}'
|
||||
try:
|
||||
error_data = response.json()
|
||||
if 'detail' in error_data:
|
||||
error_msg += f' - {error_data["detail"]}'
|
||||
except Exception:
|
||||
pass
|
||||
raise CloudBrowserError(error_msg)
|
||||
|
||||
browser_data = response.json()
|
||||
browser_response = CloudBrowserResponse(**browser_data)
|
||||
|
||||
# Clear current session if it was this one
|
||||
if session_id == self.current_session_id:
|
||||
self.current_session_id = None
|
||||
|
||||
logger.info(f'🌤️ Cloud browser session stopped: {browser_response.id}')
|
||||
logger.debug(f'🌤️ Status: {browser_response.status}')
|
||||
|
||||
return browser_response
|
||||
|
||||
except httpx.TimeoutException:
|
||||
raise CloudBrowserError('Timeout while stopping cloud browser. Please try again.')
|
||||
except httpx.ConnectError:
|
||||
raise CloudBrowserError('Failed to connect to cloud browser service. Please check your internet connection.')
|
||||
except Exception as e:
|
||||
if isinstance(e, (CloudBrowserError, CloudBrowserAuthError)):
|
||||
raise
|
||||
raise CloudBrowserError(f'Unexpected error stopping cloud browser: {e}')
|
||||
|
||||
async def close(self):
|
||||
"""Close the HTTP client and cleanup any active sessions.
|
||||
|
||||
Safe to call multiple times — subsequent calls are no-ops.
|
||||
"""
|
||||
# Try to stop current session if active
|
||||
if self.current_session_id:
|
||||
try:
|
||||
await self.stop_browser()
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to stop cloud browser session during cleanup: {e}')
|
||||
|
||||
if not self.client.is_closed:
|
||||
await self.client.aclose()
|
||||
@@ -0,0 +1,96 @@
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
ProxyCountryCode = (
|
||||
Literal[
|
||||
'us', # United States
|
||||
'uk', # United Kingdom
|
||||
'fr', # France
|
||||
'it', # Italy
|
||||
'jp', # Japan
|
||||
'au', # Australia
|
||||
'de', # Germany
|
||||
'fi', # Finland
|
||||
'ca', # Canada
|
||||
'in', # India
|
||||
]
|
||||
| str
|
||||
)
|
||||
|
||||
# Browser session timeout limits (in minutes)
|
||||
MAX_FREE_USER_SESSION_TIMEOUT = 15 # Free users limited to 15 minutes
|
||||
MAX_PAID_USER_SESSION_TIMEOUT = 240 # Paid users can go up to 4 hours
|
||||
|
||||
|
||||
# Requests
|
||||
class CreateBrowserRequest(BaseModel):
|
||||
"""Request to create a cloud browser instance.
|
||||
|
||||
Args:
|
||||
cloud_profile_id: The ID of the profile to use for the session
|
||||
cloud_proxy_country_code: Country code for proxy location
|
||||
cloud_timeout: The timeout for the session in minutes
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra='forbid', populate_by_name=True)
|
||||
|
||||
profile_id: UUID | str | None = Field(
|
||||
default=None,
|
||||
alias='cloud_profile_id',
|
||||
description='The ID of the profile to use for the session. Can be a UUID or a string of UUID.',
|
||||
title='Cloud Profile ID',
|
||||
)
|
||||
|
||||
proxy_country_code: ProxyCountryCode | None = Field(
|
||||
default=None,
|
||||
alias='cloud_proxy_country_code',
|
||||
description='Country code for proxy location.',
|
||||
title='Cloud Proxy Country Code',
|
||||
)
|
||||
|
||||
timeout: int | None = Field(
|
||||
ge=1,
|
||||
le=MAX_PAID_USER_SESSION_TIMEOUT,
|
||||
default=None,
|
||||
alias='cloud_timeout',
|
||||
description=f'The timeout for the session in minutes. Free users are limited to {MAX_FREE_USER_SESSION_TIMEOUT} minutes, paid users can use up to {MAX_PAID_USER_SESSION_TIMEOUT} minutes ({MAX_PAID_USER_SESSION_TIMEOUT // 60} hours).',
|
||||
title='Cloud Timeout',
|
||||
)
|
||||
|
||||
enable_recording: bool = Field(
|
||||
default=False,
|
||||
alias='enableRecording',
|
||||
description='Enable session recording for playback in the cloud dashboard.',
|
||||
title='Enable Recording',
|
||||
)
|
||||
|
||||
|
||||
CloudBrowserParams = CreateBrowserRequest # alias for easier readability
|
||||
|
||||
|
||||
# Responses
|
||||
class CloudBrowserResponse(BaseModel):
|
||||
"""Response from cloud browser API."""
|
||||
|
||||
id: str
|
||||
status: str
|
||||
liveUrl: str = Field(alias='liveUrl')
|
||||
cdpUrl: str = Field(alias='cdpUrl')
|
||||
timeoutAt: str = Field(alias='timeoutAt')
|
||||
startedAt: str = Field(alias='startedAt')
|
||||
finishedAt: str | None = Field(alias='finishedAt', default=None)
|
||||
|
||||
|
||||
# Errors
|
||||
class CloudBrowserError(Exception):
|
||||
"""Exception raised when cloud browser operations fail."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class CloudBrowserAuthError(CloudBrowserError):
|
||||
"""Exception raised when cloud browser authentication fails."""
|
||||
|
||||
pass
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,667 @@
|
||||
"""Event definitions for browser communication."""
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from typing import Any, Literal
|
||||
|
||||
from bubus import BaseEvent
|
||||
from bubus.models import T_EventResultType
|
||||
from cdp_use.cdp.target import TargetID
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.dom.views import EnhancedDOMTreeNode
|
||||
|
||||
|
||||
def _get_timeout(env_var: str, default: float) -> float | None:
|
||||
"""
|
||||
Safely parse environment variable timeout values with robust error handling.
|
||||
|
||||
Args:
|
||||
env_var: Environment variable name (e.g. 'TIMEOUT_NavigateToUrlEvent')
|
||||
default: Default timeout value as float (e.g. 15.0)
|
||||
|
||||
Returns:
|
||||
Parsed float value or the default if parsing fails
|
||||
|
||||
Raises:
|
||||
ValueError: Only if both env_var and default are invalid (should not happen with valid defaults)
|
||||
"""
|
||||
# Try environment variable first
|
||||
env_value = os.getenv(env_var)
|
||||
if env_value:
|
||||
try:
|
||||
parsed = float(env_value)
|
||||
if parsed < 0:
|
||||
print(f'Warning: {env_var}={env_value} is negative, using default {default}')
|
||||
return default
|
||||
return parsed
|
||||
except (ValueError, TypeError):
|
||||
print(f'Warning: {env_var}={env_value} is not a valid number, using default {default}')
|
||||
|
||||
# Fall back to default
|
||||
return default
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Agent/Tools -> BrowserSession Events (High-level browser actions)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class ElementSelectedEvent(BaseEvent[T_EventResultType]):
|
||||
"""An element was selected."""
|
||||
|
||||
node: EnhancedDOMTreeNode
|
||||
|
||||
@field_validator('node', mode='before')
|
||||
@classmethod
|
||||
def serialize_node(cls, data: EnhancedDOMTreeNode | None) -> EnhancedDOMTreeNode | None:
|
||||
if data is None:
|
||||
return None
|
||||
return EnhancedDOMTreeNode(
|
||||
node_id=data.node_id,
|
||||
backend_node_id=data.backend_node_id,
|
||||
session_id=data.session_id,
|
||||
frame_id=data.frame_id,
|
||||
target_id=data.target_id,
|
||||
node_type=data.node_type,
|
||||
node_name=data.node_name,
|
||||
node_value=data.node_value,
|
||||
attributes=data.attributes,
|
||||
is_scrollable=data.is_scrollable,
|
||||
is_visible=data.is_visible,
|
||||
absolute_position=data.absolute_position,
|
||||
# override the circular reference fields in EnhancedDOMTreeNode as they cant be serialized and aren't needed by event handlers
|
||||
# only used internally by the DOM service during DOM tree building process, not intended public API use
|
||||
content_document=None,
|
||||
shadow_root_type=None,
|
||||
shadow_roots=[],
|
||||
parent_node=None,
|
||||
children_nodes=[],
|
||||
ax_node=None,
|
||||
snapshot_node=None,
|
||||
)
|
||||
|
||||
|
||||
# TODO: add page handle to events
|
||||
# class PageHandle(share a base with browser.session.CDPSession?):
|
||||
# url: str
|
||||
# target_id: TargetID
|
||||
# @classmethod
|
||||
# def from_target_id(cls, target_id: TargetID) -> Self:
|
||||
# return cls(target_id=target_id)
|
||||
# @classmethod
|
||||
# def from_target_id(cls, target_id: TargetID) -> Self:
|
||||
# return cls(target_id=target_id)
|
||||
# @classmethod
|
||||
# def from_url(cls, url: str) -> Self:
|
||||
# @property
|
||||
# def root_frame_id(self) -> str:
|
||||
# return self.target_id
|
||||
# @property
|
||||
# def session_id(self) -> str:
|
||||
# return browser_session.get_or_create_cdp_session(self.target_id).session_id
|
||||
|
||||
# class PageSelectedEvent(BaseEvent[T_EventResultType]):
|
||||
# """An event like SwitchToTabEvent(page=PageHandle) or CloseTabEvent(page=PageHandle)"""
|
||||
# page: PageHandle
|
||||
|
||||
|
||||
class NavigateToUrlEvent(BaseEvent[None]):
|
||||
"""Navigate to a specific URL."""
|
||||
|
||||
url: str
|
||||
wait_until: Literal['load', 'domcontentloaded', 'networkidle', 'commit'] = 'load'
|
||||
timeout_ms: int | None = None
|
||||
new_tab: bool = Field(
|
||||
default=False, description='Set True to leave the current tab alone and open a new tab in the foreground for the new URL'
|
||||
)
|
||||
# existing_tab: PageHandle | None = None # TODO
|
||||
|
||||
# time limits enforced by bubus, not exposed to LLM:
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigateToUrlEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class ClickElementEvent(ElementSelectedEvent[dict[str, Any] | None]):
|
||||
"""Click an element."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
button: Literal['left', 'right', 'middle'] = 'left'
|
||||
# click_count: int = 1 # TODO
|
||||
# expect_download: bool = False # moved to downloads_watchdog.py
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ClickElementEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class ClickCoordinateEvent(BaseEvent[dict]):
|
||||
"""Click at specific coordinates."""
|
||||
|
||||
coordinate_x: int
|
||||
coordinate_y: int
|
||||
button: Literal['left', 'right', 'middle'] = 'left'
|
||||
force: bool = False # If True, skip safety checks (file input, print, select)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ClickCoordinateEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class TypeTextEvent(ElementSelectedEvent[dict | None]):
|
||||
"""Type text into an element."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
text: str
|
||||
clear: bool = True
|
||||
is_sensitive: bool = False # Flag to indicate if text contains sensitive data
|
||||
sensitive_key_name: str | None = None # Name of the sensitive key being typed (e.g., 'username', 'password')
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TypeTextEvent', 60.0)) # seconds
|
||||
|
||||
|
||||
class ScrollEvent(ElementSelectedEvent[None]):
|
||||
"""Scroll the page or element."""
|
||||
|
||||
direction: Literal['up', 'down', 'left', 'right']
|
||||
amount: int # pixels
|
||||
node: 'EnhancedDOMTreeNode | None' = None # None means scroll page
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollEvent', 8.0)) # seconds
|
||||
|
||||
|
||||
class SwitchTabEvent(BaseEvent[TargetID]):
|
||||
"""Switch to a different tab."""
|
||||
|
||||
target_id: TargetID | None = Field(default=None, description='None means switch to the most recently opened tab')
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SwitchTabEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class CloseTabEvent(BaseEvent[None]):
|
||||
"""Close a tab."""
|
||||
|
||||
target_id: TargetID
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CloseTabEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class ScreenshotEvent(BaseEvent[str]):
|
||||
"""Request to take a screenshot."""
|
||||
|
||||
full_page: bool = False
|
||||
clip: dict[str, float] | None = None # {x, y, width, height}
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScreenshotEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class BrowserStateRequestEvent(BaseEvent[BrowserStateSummary]):
|
||||
"""Request current browser state."""
|
||||
|
||||
include_dom: bool = True
|
||||
include_screenshot: bool = True
|
||||
include_recent_events: bool = False
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStateRequestEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# class WaitForConditionEvent(BaseEvent):
|
||||
# """Wait for a condition."""
|
||||
|
||||
# condition: Literal['navigation', 'selector', 'timeout', 'load_state']
|
||||
# timeout: float = 30000
|
||||
# selector: str | None = None
|
||||
# state: Literal['attached', 'detached', 'visible', 'hidden'] | None = None
|
||||
|
||||
|
||||
class GoBackEvent(BaseEvent[None]):
|
||||
"""Navigate back in browser history."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_GoBackEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class GoForwardEvent(BaseEvent[None]):
|
||||
"""Navigate forward in browser history."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_GoForwardEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class RefreshEvent(BaseEvent[None]):
|
||||
"""Refresh/reload the current page."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_RefreshEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class WaitEvent(BaseEvent[None]):
|
||||
"""Wait for a specified number of seconds."""
|
||||
|
||||
seconds: float = 3.0
|
||||
max_seconds: float = 10.0 # Safety cap
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_WaitEvent', 60.0)) # seconds
|
||||
|
||||
|
||||
class SendKeysEvent(BaseEvent[None]):
|
||||
"""Send keyboard keys/shortcuts."""
|
||||
|
||||
keys: str # e.g., "ctrl+a", "cmd+c", "Enter"
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SendKeysEvent', 60.0)) # seconds
|
||||
|
||||
|
||||
class UploadFileEvent(ElementSelectedEvent[None]):
|
||||
"""Upload a file to an element."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
file_path: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_UploadFileEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class GetDropdownOptionsEvent(ElementSelectedEvent[dict[str, str]]):
|
||||
"""Get all options from any dropdown (native <select>, ARIA menus, or custom dropdowns).
|
||||
|
||||
Returns a dict containing dropdown type, options list, and element metadata."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
|
||||
event_timeout: float | None = Field(
|
||||
default_factory=lambda: _get_timeout('TIMEOUT_GetDropdownOptionsEvent', 15.0)
|
||||
) # some dropdowns lazy-load the list of options on first interaction, so we need to wait for them to load (e.g. table filter lists can have thousands of options)
|
||||
|
||||
|
||||
class SelectDropdownOptionEvent(ElementSelectedEvent[dict[str, str]]):
|
||||
"""Select a dropdown option by exact text from any dropdown type.
|
||||
|
||||
Returns a dict containing success status and selection details."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
text: str # The option text to select
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SelectDropdownOptionEvent', 8.0)) # seconds
|
||||
|
||||
|
||||
class ScrollToTextEvent(BaseEvent[None]):
|
||||
"""Scroll to specific text on the page. Raises exception if text not found."""
|
||||
|
||||
text: str
|
||||
direction: Literal['up', 'down'] = 'down'
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollToTextEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BrowserStartEvent(BaseEvent):
|
||||
"""Start/connect to browser."""
|
||||
|
||||
cdp_url: str | None = None
|
||||
launch_options: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStartEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserStopEvent(BaseEvent):
|
||||
"""Stop/disconnect from browser."""
|
||||
|
||||
force: bool = False
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStopEvent', 45.0)) # seconds
|
||||
|
||||
|
||||
class BrowserLaunchResult(BaseModel):
|
||||
"""Result of launching a browser."""
|
||||
|
||||
# TODO: add browser executable_path, pid, version, latency, user_data_dir, X11 $DISPLAY, host IP address, etc.
|
||||
cdp_url: str
|
||||
|
||||
|
||||
class BrowserLaunchEvent(BaseEvent[BrowserLaunchResult]):
|
||||
"""Launch a local browser process."""
|
||||
|
||||
# TODO: add executable_path, proxy settings, preferences, extra launch args, etc.
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserLaunchEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserKillEvent(BaseEvent):
|
||||
"""Kill local browser subprocess."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserKillEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# TODO: replace all Runtime.evaluate() calls with this event
|
||||
# class ExecuteJavaScriptEvent(BaseEvent):
|
||||
# """Execute JavaScript in page context."""
|
||||
|
||||
# target_id: TargetID
|
||||
# expression: str
|
||||
# await_promise: bool = True
|
||||
|
||||
# event_timeout: float | None = 60.0 # seconds
|
||||
|
||||
# TODO: add this and use the old BrowserProfile.viewport options to set it
|
||||
# class SetViewportEvent(BaseEvent):
|
||||
# """Set the viewport size."""
|
||||
|
||||
# width: int
|
||||
# height: int
|
||||
# device_scale_factor: float = 1.0
|
||||
|
||||
# event_timeout: float | None = 15.0 # seconds
|
||||
|
||||
|
||||
# Moved to storage state
|
||||
# class SetCookiesEvent(BaseEvent):
|
||||
# """Set browser cookies."""
|
||||
|
||||
# cookies: list[dict[str, Any]]
|
||||
|
||||
# event_timeout: float | None = (
|
||||
# 30.0 # only long to support the edge case of restoring a big localStorage / on many origins (has to O(n) visit each origin to restore)
|
||||
# )
|
||||
|
||||
|
||||
# class GetCookiesEvent(BaseEvent):
|
||||
# """Get browser cookies."""
|
||||
|
||||
# urls: list[str] | None = None
|
||||
|
||||
# event_timeout: float | None = 30.0 # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# DOM-related Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BrowserConnectedEvent(BaseEvent):
|
||||
"""Browser has started/connected."""
|
||||
|
||||
cdp_url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserConnectedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserStoppedEvent(BaseEvent):
|
||||
"""Browser has stopped/disconnected."""
|
||||
|
||||
reason: str | None = None
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStoppedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class TabCreatedEvent(BaseEvent):
|
||||
"""A new tab was created."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TabCreatedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class TabClosedEvent(BaseEvent):
|
||||
"""A tab was closed."""
|
||||
|
||||
target_id: TargetID
|
||||
|
||||
# TODO:
|
||||
# new_focus_target_id: int | None = None
|
||||
# new_focus_url: str | None = None
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TabClosedEvent', 3.0)) # seconds
|
||||
|
||||
|
||||
# TODO: emit this when DOM changes significantly, inner frame navigates, form submits, history.pushState(), etc.
|
||||
# class TabUpdatedEvent(BaseEvent):
|
||||
# """Tab information updated (URL changed, etc.)."""
|
||||
|
||||
# target_id: TargetID
|
||||
# url: str
|
||||
|
||||
|
||||
class AgentFocusChangedEvent(BaseEvent):
|
||||
"""Agent focus changed to a different tab."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_AgentFocusChangedEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class TargetCrashedEvent(BaseEvent):
|
||||
"""A target has crashed."""
|
||||
|
||||
target_id: TargetID
|
||||
error: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TargetCrashedEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class NavigationStartedEvent(BaseEvent):
|
||||
"""Navigation started."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigationStartedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class NavigationCompleteEvent(BaseEvent):
|
||||
"""Navigation completed."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
status: int | None = None
|
||||
error_message: str | None = None # Error/timeout message if navigation had issues
|
||||
loading_status: str | None = None # Detailed loading status (e.g., network timeout info)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigationCompleteEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Error Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BrowserErrorEvent(BaseEvent):
|
||||
"""An error occurred in the browser layer."""
|
||||
|
||||
error_type: str
|
||||
message: str
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserErrorEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserReconnectingEvent(BaseEvent):
|
||||
"""WebSocket reconnection attempt is starting."""
|
||||
|
||||
cdp_url: str
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserReconnectingEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserReconnectedEvent(BaseEvent):
|
||||
"""WebSocket reconnection succeeded."""
|
||||
|
||||
cdp_url: str
|
||||
attempt: int
|
||||
downtime_seconds: float
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserReconnectedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Storage State Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class SaveStorageStateEvent(BaseEvent):
|
||||
"""Request to save browser storage state."""
|
||||
|
||||
path: str | None = None # Optional path, uses profile default if not provided
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SaveStorageStateEvent', 45.0)) # seconds
|
||||
|
||||
|
||||
class StorageStateSavedEvent(BaseEvent):
|
||||
"""Notification that storage state was saved."""
|
||||
|
||||
path: str
|
||||
cookies_count: int
|
||||
origins_count: int
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_StorageStateSavedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class LoadStorageStateEvent(BaseEvent):
|
||||
"""Request to load browser storage state."""
|
||||
|
||||
path: str | None = None # Optional path, uses profile default if not provided
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_LoadStorageStateEvent', 45.0)) # seconds
|
||||
|
||||
|
||||
# TODO: refactor this to:
|
||||
# - on_BrowserConnectedEvent() -> dispatch(LoadStorageStateEvent()) -> _copy_storage_state_from_json_to_browser(json_file, new_cdp_session) + return storage_state from handler
|
||||
# - on_BrowserStopEvent() -> dispatch(SaveStorageStateEvent()) -> _copy_storage_state_from_browser_to_json(new_cdp_session, json_file)
|
||||
# and get rid of StorageStateSavedEvent and StorageStateLoadedEvent, have the original events + provide handler return values for any results
|
||||
class StorageStateLoadedEvent(BaseEvent):
|
||||
"""Notification that storage state was loaded."""
|
||||
|
||||
path: str
|
||||
cookies_count: int
|
||||
origins_count: int
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_StorageStateLoadedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# File Download Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class DownloadStartedEvent(BaseEvent):
|
||||
"""A file download has started (CDP downloadWillBegin received)."""
|
||||
|
||||
guid: str # CDP download GUID to correlate with FileDownloadedEvent
|
||||
url: str
|
||||
suggested_filename: str
|
||||
auto_download: bool = False # Whether this was triggered automatically
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_DownloadStartedEvent', 5.0)) # seconds
|
||||
|
||||
|
||||
class DownloadProgressEvent(BaseEvent):
|
||||
"""A file download progress update (CDP downloadProgress received)."""
|
||||
|
||||
guid: str # CDP download GUID to correlate with other download events
|
||||
received_bytes: int
|
||||
total_bytes: int # 0 if unknown
|
||||
state: str # 'inProgress', 'completed', or 'canceled'
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_DownloadProgressEvent', 5.0)) # seconds
|
||||
|
||||
|
||||
class FileDownloadedEvent(BaseEvent):
|
||||
"""A file has been downloaded."""
|
||||
|
||||
guid: str | None = None # CDP download GUID to correlate with DownloadStartedEvent
|
||||
url: str
|
||||
path: str
|
||||
file_name: str
|
||||
file_size: int
|
||||
file_type: str | None = None # e.g., 'pdf', 'zip', 'docx', etc.
|
||||
mime_type: str | None = None # e.g., 'application/pdf'
|
||||
from_cache: bool = False
|
||||
auto_download: bool = False # Whether this was an automatic download (e.g., PDF auto-download)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_FileDownloadedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class AboutBlankDVDScreensaverShownEvent(BaseEvent):
|
||||
"""AboutBlankWatchdog has shown DVD screensaver animation on an about:blank tab."""
|
||||
|
||||
target_id: TargetID
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class DialogOpenedEvent(BaseEvent):
|
||||
"""Event dispatched when a JavaScript dialog is opened and handled."""
|
||||
|
||||
dialog_type: str # 'alert', 'confirm', 'prompt', or 'beforeunload'
|
||||
message: str
|
||||
url: str
|
||||
frame_id: str | None = None # Can be None when frameId is not provided by CDP
|
||||
# target_id: TargetID # TODO: add this to avoid needing target_id_from_frame() later
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Captcha Solver Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class CaptchaSolverStartedEvent(BaseEvent):
|
||||
"""Captcha solving started by the browser proxy.
|
||||
|
||||
Emitted when the browser proxy detects a CAPTCHA and begins solving it.
|
||||
The agent should wait for a corresponding CaptchaSolverFinishedEvent before proceeding.
|
||||
"""
|
||||
|
||||
target_id: TargetID
|
||||
vendor: str # e.g. 'cloudflare', 'recaptcha', 'hcaptcha', 'datadome', 'perimeterx', 'geetest'
|
||||
url: str
|
||||
started_at: int # Unix millis
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CaptchaSolverStartedEvent', 5.0))
|
||||
|
||||
|
||||
class CaptchaSolverFinishedEvent(BaseEvent):
|
||||
"""Captcha solving finished by the browser proxy.
|
||||
|
||||
Emitted when the browser proxy finishes solving a CAPTCHA (successfully or not).
|
||||
"""
|
||||
|
||||
target_id: TargetID
|
||||
vendor: str
|
||||
url: str
|
||||
duration_ms: int
|
||||
finished_at: int # Unix millis
|
||||
success: bool # Whether the captcha was solved successfully
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CaptchaSolverFinishedEvent', 5.0))
|
||||
|
||||
|
||||
# Note: Model rebuilding for forward references is handled in the importing modules
|
||||
# Events with 'EnhancedDOMTreeNode' forward references (ClickElementEvent, TypeTextEvent,
|
||||
# ScrollEvent, UploadFileEvent) need model_rebuild() called after imports are complete
|
||||
|
||||
|
||||
def _check_event_names_dont_overlap():
|
||||
"""
|
||||
check that event names defined in this file are valid and non-overlapping
|
||||
(naiively n^2 so it's pretty slow but ok for now, optimize when >20 events)
|
||||
"""
|
||||
event_names = {
|
||||
name.split('[')[0]
|
||||
for name in globals().keys()
|
||||
if not name.startswith('_')
|
||||
and inspect.isclass(globals()[name])
|
||||
and issubclass(globals()[name], BaseEvent)
|
||||
and name != 'BaseEvent'
|
||||
}
|
||||
for name_a in event_names:
|
||||
assert name_a.endswith('Event'), f'Event with name {name_a} does not end with "Event"'
|
||||
for name_b in event_names:
|
||||
if name_a != name_b: # Skip self-comparison
|
||||
assert name_a not in name_b, (
|
||||
f'Event with name {name_a} is a substring of {name_b}, all events must be completely unique to avoid find-and-replace accidents'
|
||||
)
|
||||
|
||||
|
||||
# overlapping event names are a nightmare to trace and rename later, dont do it!
|
||||
# e.g. prevent ClickEvent and FailedClickEvent are terrible names because one is a substring of the other,
|
||||
# must be ClickEvent and ClickFailedEvent to preserve the usefulnes of codebase grep/sed/awk as refactoring tools.
|
||||
# at import time, we do a quick check that all event names defined above are valid and non-overlapping.
|
||||
# this is hand written in blood by a human! not LLM slop. feel free to optimize but do not remove it without a good reason.
|
||||
_check_event_names_dont_overlap()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,548 @@
|
||||
"""Python-based highlighting system for drawing bounding boxes on screenshots.
|
||||
|
||||
This module replaces JavaScript-based highlighting with fast Python image processing
|
||||
to draw bounding boxes around interactive elements directly on screenshots.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from browser_use.dom.views import DOMSelectorMap, EnhancedDOMTreeNode
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import time_execution_async
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Font cache to prevent repeated font loading and reduce memory usage
|
||||
_FONT_CACHE: dict[tuple[str, int], ImageFont.FreeTypeFont | None] = {}
|
||||
|
||||
# Cross-platform font paths
|
||||
_FONT_PATHS = [
|
||||
'/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', # Linux (Debian/Ubuntu)
|
||||
'/usr/share/fonts/TTF/DejaVuSans-Bold.ttf', # Linux (Arch/Fedora)
|
||||
'/System/Library/Fonts/Arial.ttf', # macOS
|
||||
'C:\\Windows\\Fonts\\arial.ttf', # Windows
|
||||
'arial.ttf', # Windows (system path)
|
||||
'Arial Bold.ttf', # macOS alternative
|
||||
'/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf', # Linux alternative
|
||||
]
|
||||
|
||||
|
||||
def get_cross_platform_font(font_size: int) -> ImageFont.FreeTypeFont | None:
|
||||
"""Get a cross-platform compatible font with caching to prevent memory leaks.
|
||||
|
||||
Args:
|
||||
font_size: Size of the font to load
|
||||
|
||||
Returns:
|
||||
ImageFont object or None if no system fonts are available
|
||||
"""
|
||||
# Use cache key based on font size
|
||||
cache_key = ('system_font', font_size)
|
||||
|
||||
# Return cached font if available
|
||||
if cache_key in _FONT_CACHE:
|
||||
return _FONT_CACHE[cache_key]
|
||||
|
||||
# Try to load a system font
|
||||
font = None
|
||||
for font_path in _FONT_PATHS:
|
||||
try:
|
||||
font = ImageFont.truetype(font_path, font_size)
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
# Cache the result (even if None) to avoid repeated attempts
|
||||
_FONT_CACHE[cache_key] = font
|
||||
return font
|
||||
|
||||
|
||||
def cleanup_font_cache() -> None:
|
||||
"""Clean up the font cache to prevent memory leaks in long-running applications."""
|
||||
global _FONT_CACHE
|
||||
_FONT_CACHE.clear()
|
||||
|
||||
|
||||
# Color scheme for different element types
|
||||
ELEMENT_COLORS = {
|
||||
'button': '#FF6B6B', # Red for buttons
|
||||
'input': '#4ECDC4', # Teal for inputs
|
||||
'select': '#45B7D1', # Blue for dropdowns
|
||||
'a': '#96CEB4', # Green for links
|
||||
'textarea': '#FF8C42', # Orange for text areas (was yellow, now more visible)
|
||||
'default': '#DDA0DD', # Light purple for other interactive elements
|
||||
}
|
||||
|
||||
# Element type mappings
|
||||
ELEMENT_TYPE_MAP = {
|
||||
'button': 'button',
|
||||
'input': 'input',
|
||||
'select': 'select',
|
||||
'a': 'a',
|
||||
'textarea': 'textarea',
|
||||
}
|
||||
|
||||
|
||||
def get_element_color(tag_name: str, element_type: str | None = None) -> str:
|
||||
"""Get color for element based on tag name and type."""
|
||||
# Check input type first
|
||||
if tag_name == 'input' and element_type:
|
||||
if element_type in ['button', 'submit']:
|
||||
return ELEMENT_COLORS['button']
|
||||
|
||||
# Use tag-based color
|
||||
return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])
|
||||
|
||||
|
||||
def should_show_index_overlay(backend_node_id: int | None) -> bool:
|
||||
"""Determine if index overlay should be shown."""
|
||||
return backend_node_id is not None
|
||||
|
||||
|
||||
def draw_enhanced_bounding_box_with_text(
|
||||
draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
|
||||
bbox: tuple[int, int, int, int],
|
||||
color: str,
|
||||
text: str | None = None,
|
||||
font: ImageFont.FreeTypeFont | None = None,
|
||||
element_type: str = 'div',
|
||||
image_size: tuple[int, int] = (2000, 1500),
|
||||
device_pixel_ratio: float = 1.0,
|
||||
) -> None:
|
||||
"""Draw an enhanced bounding box with much bigger index containers and dashed borders."""
|
||||
x1, y1, x2, y2 = bbox
|
||||
|
||||
# Draw dashed bounding box with pattern: 1 line, 2 spaces, 1 line, 2 spaces...
|
||||
dash_length = 4
|
||||
gap_length = 8
|
||||
line_width = 2
|
||||
|
||||
# Helper function to draw dashed line
|
||||
def draw_dashed_line(start_x, start_y, end_x, end_y):
|
||||
if start_x == end_x: # Vertical line
|
||||
y = start_y
|
||||
while y < end_y:
|
||||
dash_end = min(y + dash_length, end_y)
|
||||
draw.line([(start_x, y), (start_x, dash_end)], fill=color, width=line_width)
|
||||
y += dash_length + gap_length
|
||||
else: # Horizontal line
|
||||
x = start_x
|
||||
while x < end_x:
|
||||
dash_end = min(x + dash_length, end_x)
|
||||
draw.line([(x, start_y), (dash_end, start_y)], fill=color, width=line_width)
|
||||
x += dash_length + gap_length
|
||||
|
||||
# Draw dashed rectangle
|
||||
draw_dashed_line(x1, y1, x2, y1) # Top
|
||||
draw_dashed_line(x2, y1, x2, y2) # Right
|
||||
draw_dashed_line(x2, y2, x1, y2) # Bottom
|
||||
draw_dashed_line(x1, y2, x1, y1) # Left
|
||||
|
||||
# Draw much bigger index overlay if we have index text
|
||||
if text:
|
||||
try:
|
||||
# Scale font size for appropriate sizing across different resolutions
|
||||
img_width, img_height = image_size
|
||||
|
||||
css_width = img_width # / device_pixel_ratio
|
||||
# Much smaller scaling - 1% of CSS viewport width, max 16px to prevent huge highlights
|
||||
base_font_size = max(10, min(20, int(css_width * 0.01)))
|
||||
# Use shared font loading function with caching
|
||||
big_font = get_cross_platform_font(base_font_size)
|
||||
if big_font is None:
|
||||
big_font = font # Fallback to original font if no system fonts found
|
||||
|
||||
# Get text size with bigger font
|
||||
if big_font:
|
||||
bbox_text = draw.textbbox((0, 0), text, font=big_font)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
else:
|
||||
# Fallback for default font
|
||||
bbox_text = draw.textbbox((0, 0), text)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
|
||||
# Scale padding appropriately for different resolutions
|
||||
padding = max(4, min(10, int(css_width * 0.005))) # 0.3% of CSS width, max 4px
|
||||
element_width = x2 - x1
|
||||
element_height = y2 - y1
|
||||
|
||||
# Container dimensions
|
||||
container_width = text_width + padding * 2
|
||||
container_height = text_height + padding * 2
|
||||
|
||||
# Position in top center - for small elements, place further up to avoid blocking content
|
||||
# Center horizontally within the element
|
||||
bg_x1 = x1 + (element_width - container_width) // 2
|
||||
|
||||
# Simple rule: if element is small, place index further up to avoid blocking icons
|
||||
if element_width < 60 or element_height < 30:
|
||||
# Small element: place well above to avoid blocking content
|
||||
bg_y1 = max(0, y1 - container_height - 5)
|
||||
else:
|
||||
# Regular element: place inside with small offset
|
||||
bg_y1 = y1 + 2
|
||||
|
||||
bg_x2 = bg_x1 + container_width
|
||||
bg_y2 = bg_y1 + container_height
|
||||
|
||||
# Center the number within the index box with proper baseline handling
|
||||
text_x = bg_x1 + (container_width - text_width) // 2
|
||||
# Add extra vertical space to prevent clipping
|
||||
text_y = bg_y1 + (container_height - text_height) // 2 - bbox_text[1] # Subtract top offset
|
||||
|
||||
# Ensure container stays within image bounds
|
||||
img_width, img_height = image_size
|
||||
if bg_x1 < 0:
|
||||
offset = -bg_x1
|
||||
bg_x1 += offset
|
||||
bg_x2 += offset
|
||||
text_x += offset
|
||||
if bg_y1 < 0:
|
||||
offset = -bg_y1
|
||||
bg_y1 += offset
|
||||
bg_y2 += offset
|
||||
text_y += offset
|
||||
if bg_x2 > img_width:
|
||||
offset = bg_x2 - img_width
|
||||
bg_x1 -= offset
|
||||
bg_x2 -= offset
|
||||
text_x -= offset
|
||||
if bg_y2 > img_height:
|
||||
offset = bg_y2 - img_height
|
||||
bg_y1 -= offset
|
||||
bg_y2 -= offset
|
||||
text_y -= offset
|
||||
|
||||
# Draw bigger background rectangle with thicker border
|
||||
draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill=color, outline='white', width=2)
|
||||
|
||||
# Draw white text centered in the index box
|
||||
draw.text((text_x, text_y), text, fill='white', font=big_font or font)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to draw enhanced text overlay: {e}')
|
||||
|
||||
|
||||
def draw_bounding_box_with_text(
|
||||
draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
|
||||
bbox: tuple[int, int, int, int],
|
||||
color: str,
|
||||
text: str | None = None,
|
||||
font: ImageFont.FreeTypeFont | None = None,
|
||||
) -> None:
|
||||
"""Draw a bounding box with optional text overlay."""
|
||||
x1, y1, x2, y2 = bbox
|
||||
|
||||
# Draw dashed bounding box
|
||||
dash_length = 2
|
||||
gap_length = 6
|
||||
|
||||
# Top edge
|
||||
x = x1
|
||||
while x < x2:
|
||||
end_x = min(x + dash_length, x2)
|
||||
draw.line([(x, y1), (end_x, y1)], fill=color, width=2)
|
||||
draw.line([(x, y1 + 1), (end_x, y1 + 1)], fill=color, width=2)
|
||||
x += dash_length + gap_length
|
||||
|
||||
# Bottom edge
|
||||
x = x1
|
||||
while x < x2:
|
||||
end_x = min(x + dash_length, x2)
|
||||
draw.line([(x, y2), (end_x, y2)], fill=color, width=2)
|
||||
draw.line([(x, y2 - 1), (end_x, y2 - 1)], fill=color, width=2)
|
||||
x += dash_length + gap_length
|
||||
|
||||
# Left edge
|
||||
y = y1
|
||||
while y < y2:
|
||||
end_y = min(y + dash_length, y2)
|
||||
draw.line([(x1, y), (x1, end_y)], fill=color, width=2)
|
||||
draw.line([(x1 + 1, y), (x1 + 1, end_y)], fill=color, width=2)
|
||||
y += dash_length + gap_length
|
||||
|
||||
# Right edge
|
||||
y = y1
|
||||
while y < y2:
|
||||
end_y = min(y + dash_length, y2)
|
||||
draw.line([(x2, y), (x2, end_y)], fill=color, width=2)
|
||||
draw.line([(x2 - 1, y), (x2 - 1, end_y)], fill=color, width=2)
|
||||
y += dash_length + gap_length
|
||||
|
||||
# Draw index overlay if we have index text
|
||||
if text:
|
||||
try:
|
||||
# Get text size
|
||||
if font:
|
||||
bbox_text = draw.textbbox((0, 0), text, font=font)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
else:
|
||||
# Fallback for default font
|
||||
bbox_text = draw.textbbox((0, 0), text)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
|
||||
# Smart positioning based on element size
|
||||
padding = 5
|
||||
element_width = x2 - x1
|
||||
element_height = y2 - y1
|
||||
element_area = element_width * element_height
|
||||
index_box_area = (text_width + padding * 2) * (text_height + padding * 2)
|
||||
|
||||
# Calculate size ratio to determine positioning strategy
|
||||
size_ratio = element_area / max(index_box_area, 1)
|
||||
|
||||
if size_ratio < 4:
|
||||
# Very small elements: place outside in bottom-right corner
|
||||
text_x = x2 + padding
|
||||
text_y = y2 - text_height
|
||||
# Ensure it doesn't go off screen
|
||||
text_x = min(text_x, 1200 - text_width - padding)
|
||||
text_y = max(text_y, 0)
|
||||
elif size_ratio < 16:
|
||||
# Medium elements: place in bottom-right corner inside
|
||||
text_x = x2 - text_width - padding
|
||||
text_y = y2 - text_height - padding
|
||||
else:
|
||||
# Large elements: place in center
|
||||
text_x = x1 + (element_width - text_width) // 2
|
||||
text_y = y1 + (element_height - text_height) // 2
|
||||
|
||||
# Ensure text stays within bounds
|
||||
text_x = max(0, min(text_x, 1200 - text_width))
|
||||
text_y = max(0, min(text_y, 800 - text_height))
|
||||
|
||||
# Draw background rectangle for maximum contrast
|
||||
bg_x1 = text_x - padding
|
||||
bg_y1 = text_y - padding
|
||||
bg_x2 = text_x + text_width + padding
|
||||
bg_y2 = text_y + text_height + padding
|
||||
|
||||
# Use white background with thick black border for maximum visibility
|
||||
draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill='white', outline='black', width=2)
|
||||
|
||||
# Draw bold dark text on light background for best contrast
|
||||
draw.text((text_x, text_y), text, fill='black', font=font)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to draw text overlay: {e}')
|
||||
|
||||
|
||||
def process_element_highlight(
|
||||
element_id: int,
|
||||
element: EnhancedDOMTreeNode,
|
||||
draw,
|
||||
device_pixel_ratio: float,
|
||||
font,
|
||||
filter_highlight_ids: bool,
|
||||
image_size: tuple[int, int],
|
||||
) -> None:
|
||||
"""Process a single element for highlighting."""
|
||||
try:
|
||||
# Use absolute_position coordinates directly
|
||||
if not element.absolute_position:
|
||||
return
|
||||
|
||||
bounds = element.absolute_position
|
||||
|
||||
# Scale coordinates from CSS pixels to device pixels for screenshot
|
||||
# The screenshot is captured at device pixel resolution, but coordinates are in CSS pixels
|
||||
x1 = int(bounds.x * device_pixel_ratio)
|
||||
y1 = int(bounds.y * device_pixel_ratio)
|
||||
x2 = int((bounds.x + bounds.width) * device_pixel_ratio)
|
||||
y2 = int((bounds.y + bounds.height) * device_pixel_ratio)
|
||||
|
||||
# Ensure coordinates are within image bounds
|
||||
img_width, img_height = image_size
|
||||
x1 = max(0, min(x1, img_width))
|
||||
y1 = max(0, min(y1, img_height))
|
||||
x2 = max(x1, min(x2, img_width))
|
||||
y2 = max(y1, min(y2, img_height))
|
||||
|
||||
# Skip if bounding box is too small or invalid
|
||||
if x2 - x1 < 2 or y2 - y1 < 2:
|
||||
return
|
||||
|
||||
# Get element color based on type
|
||||
tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
|
||||
element_type = None
|
||||
if hasattr(element, 'attributes') and element.attributes:
|
||||
element_type = element.attributes.get('type')
|
||||
|
||||
color = get_element_color(tag_name, element_type)
|
||||
|
||||
# Get element index for overlay and apply filtering
|
||||
backend_node_id = getattr(element, 'backend_node_id', None)
|
||||
index_text = None
|
||||
|
||||
if backend_node_id is not None:
|
||||
if filter_highlight_ids:
|
||||
# Use the meaningful text that matches what the LLM sees
|
||||
meaningful_text = element.get_meaningful_text_for_llm()
|
||||
# Show ID only if meaningful text is less than 5 characters
|
||||
if len(meaningful_text) < 3:
|
||||
index_text = str(backend_node_id)
|
||||
else:
|
||||
# Always show ID when filter is disabled
|
||||
index_text = str(backend_node_id)
|
||||
|
||||
# Draw enhanced bounding box with bigger index
|
||||
draw_enhanced_bounding_box_with_text(
|
||||
draw, (x1, y1, x2, y2), color, index_text, font, tag_name, image_size, device_pixel_ratio
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to draw highlight for element {element_id}: {e}')
|
||||
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='create_highlighted_screenshot')
|
||||
@time_execution_async('create_highlighted_screenshot')
|
||||
async def create_highlighted_screenshot(
|
||||
screenshot_b64: str,
|
||||
selector_map: DOMSelectorMap,
|
||||
device_pixel_ratio: float = 1.0,
|
||||
viewport_offset_x: int = 0,
|
||||
viewport_offset_y: int = 0,
|
||||
filter_highlight_ids: bool = True,
|
||||
) -> str:
|
||||
"""Create a highlighted screenshot with bounding boxes around interactive elements.
|
||||
|
||||
Args:
|
||||
screenshot_b64: Base64 encoded screenshot
|
||||
selector_map: Map of interactive elements with their positions
|
||||
device_pixel_ratio: Device pixel ratio for scaling coordinates
|
||||
viewport_offset_x: X offset for viewport positioning
|
||||
viewport_offset_y: Y offset for viewport positioning
|
||||
|
||||
Returns:
|
||||
Base64 encoded highlighted screenshot
|
||||
"""
|
||||
try:
|
||||
# Decode screenshot
|
||||
screenshot_data = base64.b64decode(screenshot_b64)
|
||||
image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')
|
||||
|
||||
# Create drawing context
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
# Load font using shared function with caching
|
||||
font = get_cross_platform_font(12)
|
||||
# If no system fonts found, font remains None and will use default font
|
||||
|
||||
# Process elements sequentially to avoid ImageDraw thread safety issues
|
||||
# PIL ImageDraw is not thread-safe, so we process elements one by one
|
||||
for element_id, element in selector_map.items():
|
||||
process_element_highlight(element_id, element, draw, device_pixel_ratio, font, filter_highlight_ids, image.size)
|
||||
|
||||
# Convert back to base64
|
||||
output_buffer = io.BytesIO()
|
||||
try:
|
||||
image.save(output_buffer, format='PNG')
|
||||
output_buffer.seek(0)
|
||||
highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')
|
||||
|
||||
logger.debug(f'Successfully created highlighted screenshot with {len(selector_map)} elements')
|
||||
return highlighted_b64
|
||||
finally:
|
||||
# Explicit cleanup to prevent memory leaks
|
||||
output_buffer.close()
|
||||
if 'image' in locals():
|
||||
image.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to create highlighted screenshot: {e}')
|
||||
# Clean up on error as well
|
||||
if 'image' in locals():
|
||||
image.close()
|
||||
# Return original screenshot on error
|
||||
return screenshot_b64
|
||||
|
||||
|
||||
async def get_viewport_info_from_cdp(cdp_session) -> tuple[float, int, int]:
|
||||
"""Get viewport information from CDP session.
|
||||
|
||||
Returns:
|
||||
Tuple of (device_pixel_ratio, scroll_x, scroll_y)
|
||||
"""
|
||||
try:
|
||||
# Get layout metrics which includes viewport info and device pixel ratio
|
||||
metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
|
||||
|
||||
# Extract viewport information
|
||||
visual_viewport = metrics.get('visualViewport', {})
|
||||
css_visual_viewport = metrics.get('cssVisualViewport', {})
|
||||
css_layout_viewport = metrics.get('cssLayoutViewport', {})
|
||||
|
||||
# Calculate device pixel ratio
|
||||
css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))
|
||||
device_width = visual_viewport.get('clientWidth', css_width)
|
||||
device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
|
||||
|
||||
# Get scroll position in CSS pixels
|
||||
scroll_x = int(css_visual_viewport.get('pageX', 0))
|
||||
scroll_y = int(css_visual_viewport.get('pageY', 0))
|
||||
|
||||
return float(device_pixel_ratio), scroll_x, scroll_y
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to get viewport info from CDP: {e}')
|
||||
return 1.0, 0, 0
|
||||
|
||||
|
||||
@time_execution_async('create_highlighted_screenshot_async')
|
||||
async def create_highlighted_screenshot_async(
|
||||
screenshot_b64: str, selector_map: DOMSelectorMap, cdp_session=None, filter_highlight_ids: bool = True
|
||||
) -> str:
|
||||
"""Async wrapper for creating highlighted screenshots.
|
||||
|
||||
Args:
|
||||
screenshot_b64: Base64 encoded screenshot
|
||||
selector_map: Map of interactive elements
|
||||
cdp_session: CDP session for getting viewport info
|
||||
filter_highlight_ids: Whether to filter element IDs based on meaningful text
|
||||
|
||||
Returns:
|
||||
Base64 encoded highlighted screenshot
|
||||
"""
|
||||
# Get viewport information if CDP session is available
|
||||
device_pixel_ratio = 1.0
|
||||
viewport_offset_x = 0
|
||||
viewport_offset_y = 0
|
||||
|
||||
if cdp_session:
|
||||
try:
|
||||
device_pixel_ratio, viewport_offset_x, viewport_offset_y = await get_viewport_info_from_cdp(cdp_session)
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to get viewport info from CDP: {e}')
|
||||
|
||||
# Create highlighted screenshot with async processing
|
||||
final_screenshot = await create_highlighted_screenshot(
|
||||
screenshot_b64, selector_map, device_pixel_ratio, viewport_offset_x, viewport_offset_y, filter_highlight_ids
|
||||
)
|
||||
|
||||
filename = os.getenv('BROWSER_USE_SCREENSHOT_FILE')
|
||||
if filename:
|
||||
|
||||
def _write_screenshot():
|
||||
try:
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(base64.b64decode(final_screenshot))
|
||||
logger.debug('Saved screenshot to ' + str(filename))
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to save screenshot to {filename}: {e}')
|
||||
|
||||
await asyncio.to_thread(_write_screenshot)
|
||||
return final_screenshot
|
||||
|
||||
|
||||
# Export the cleanup function for external use in long-running applications
|
||||
__all__ = ['create_highlighted_screenshot', 'create_highlighted_screenshot_async', 'cleanup_font_cache']
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,918 @@
|
||||
"""Event-driven CDP session management.
|
||||
|
||||
Manages CDP sessions by listening to Target.attachedToTarget and Target.detachedFromTarget
|
||||
events, ensuring the session pool always reflects the current browser state.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from cdp_use.cdp.target import AttachedToTargetEvent, DetachedFromTargetEvent, SessionID, TargetID
|
||||
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser.session import BrowserSession, CDPSession, Target
|
||||
|
||||
|
||||
class SessionManager:
|
||||
"""Event-driven CDP session manager.
|
||||
|
||||
Automatically synchronizes the CDP session pool with browser state via CDP events.
|
||||
|
||||
Key features:
|
||||
- Sessions added/removed automatically via Target attach/detach events
|
||||
- Multiple sessions can attach to the same target
|
||||
- Targets only removed when ALL sessions detach
|
||||
- No stale sessions - pool always reflects browser reality
|
||||
|
||||
SessionManager is the SINGLE SOURCE OF TRUTH for all targets and sessions.
|
||||
"""
|
||||
|
||||
def __init__(self, browser_session: 'BrowserSession'):
|
||||
self.browser_session = browser_session
|
||||
self.logger = browser_session.logger
|
||||
|
||||
# All targets (entities: pages, iframes, workers)
|
||||
self._targets: dict[TargetID, 'Target'] = {}
|
||||
|
||||
# All sessions (communication channels)
|
||||
self._sessions: dict[SessionID, 'CDPSession'] = {}
|
||||
|
||||
# Mapping: target -> sessions attached to it
|
||||
self._target_sessions: dict[TargetID, set[SessionID]] = {}
|
||||
|
||||
# Reverse mapping: session -> target it belongs to
|
||||
self._session_to_target: dict[SessionID, TargetID] = {}
|
||||
|
||||
# Page lifecycle events per target, fed by ONE global Page.lifecycleEvent handler
|
||||
# registered in start_monitoring(). cdp-use's event registry is single-slot per
|
||||
# CDP method, so per-session handler registrations would replace each other and
|
||||
# leave every tab but the most recently attached one without lifecycle events.
|
||||
self._lifecycle_events: dict[TargetID, deque[dict[str, Any]]] = {}
|
||||
|
||||
self._lock = asyncio.Lock()
|
||||
self._recovery_lock = asyncio.Lock()
|
||||
|
||||
# Focus recovery coordination - event-driven instead of polling
|
||||
self._recovery_in_progress: bool = False
|
||||
self._recovery_complete_event: asyncio.Event | None = None
|
||||
self._recovery_task: asyncio.Task | None = None
|
||||
|
||||
async def start_monitoring(self) -> None:
|
||||
"""Start monitoring Target attach/detach events.
|
||||
|
||||
Registers CDP event handlers to keep the session pool synchronized with browser state.
|
||||
Also discovers and initializes all existing targets on startup.
|
||||
"""
|
||||
if not self.browser_session._cdp_client_root:
|
||||
raise RuntimeError('CDP client not initialized')
|
||||
|
||||
# Capture cdp_client_root in closure to avoid type errors
|
||||
cdp_client = self.browser_session._cdp_client_root
|
||||
|
||||
# Enable target discovery to receive targetInfoChanged events automatically
|
||||
# This eliminates the need for getTargetInfo() polling calls
|
||||
await cdp_client.send.Target.setDiscoverTargets(
|
||||
params={'discover': True, 'filter': [{'type': 'page'}, {'type': 'iframe'}]}
|
||||
)
|
||||
|
||||
# Register synchronous event handlers (CDP requirement)
|
||||
def on_attached(event: AttachedToTargetEvent, session_id: SessionID | None = None):
|
||||
# _handle_target_attached() handles:
|
||||
# - setAutoAttach for children
|
||||
# - Create CDPSession
|
||||
# - Enable monitoring (for pages/tabs)
|
||||
# - Add to pool
|
||||
create_task_with_error_handling(
|
||||
self._handle_target_attached(event),
|
||||
name='handle_target_attached',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
def on_detached(event: DetachedFromTargetEvent, session_id: SessionID | None = None):
|
||||
create_task_with_error_handling(
|
||||
self._handle_target_detached(event),
|
||||
name='handle_target_detached',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
def on_target_info_changed(event, session_id: SessionID | None = None):
|
||||
# Update session info from targetInfoChanged events (no polling needed!)
|
||||
create_task_with_error_handling(
|
||||
self._handle_target_info_changed(event),
|
||||
name='handle_target_info_changed',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
def on_lifecycle_event(event, session_id: SessionID | None = None):
|
||||
# ONE global handler for all targets: route by session_id -> target_id.
|
||||
# Registering per-session closures instead would clobber each other in
|
||||
# cdp-use's single-slot registry (one handler per CDP method).
|
||||
if not session_id:
|
||||
return
|
||||
target_id = self.get_target_id_from_session_id(session_id)
|
||||
if not target_id:
|
||||
return
|
||||
self.get_lifecycle_events(target_id).append(
|
||||
{
|
||||
'name': event.get('name', 'unknown'),
|
||||
'loaderId': event.get('loaderId'),
|
||||
'timestamp': asyncio.get_event_loop().time(),
|
||||
}
|
||||
)
|
||||
|
||||
cdp_client.register.Target.attachedToTarget(on_attached)
|
||||
cdp_client.register.Target.detachedFromTarget(on_detached)
|
||||
cdp_client.register.Target.targetInfoChanged(on_target_info_changed)
|
||||
cdp_client.register.Page.lifecycleEvent(on_lifecycle_event)
|
||||
|
||||
self.logger.debug('[SessionManager] Event monitoring started')
|
||||
|
||||
# Discover and initialize ALL existing targets
|
||||
await self._initialize_existing_targets()
|
||||
|
||||
def get_lifecycle_events(self, target_id: TargetID) -> 'deque[dict[str, Any]]':
|
||||
"""Get (creating if needed) the lifecycle event buffer for a target."""
|
||||
events = self._lifecycle_events.get(target_id)
|
||||
if events is None:
|
||||
events = deque(maxlen=50)
|
||||
self._lifecycle_events[target_id] = events
|
||||
return events
|
||||
|
||||
def _get_session_for_target(self, target_id: TargetID) -> 'CDPSession | None':
|
||||
"""Internal: Get ANY valid session for a target (picks first available).
|
||||
|
||||
⚠️ INTERNAL API - Use browser_session.get_or_create_cdp_session() instead!
|
||||
This method has no validation, no focus management, no recovery.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to get session for
|
||||
|
||||
Returns:
|
||||
CDPSession if exists, None if target has detached
|
||||
"""
|
||||
session_ids = self._target_sessions.get(target_id, set())
|
||||
if not session_ids:
|
||||
# Check if this is the focused target - indicates stale focus that needs cleanup
|
||||
if self.browser_session.agent_focus_target_id == target_id:
|
||||
self.logger.warning(
|
||||
f'[SessionManager] ⚠️ Attempted to get session for stale focused target {target_id[:8]}... '
|
||||
f'Clearing stale focus and triggering recovery.'
|
||||
)
|
||||
|
||||
# Clear stale focus immediately (defense in depth)
|
||||
self.browser_session.agent_focus_target_id = None
|
||||
|
||||
# Trigger recovery if not already in progress
|
||||
if not self._recovery_in_progress:
|
||||
self.logger.warning('[SessionManager] Recovery was not in progress! Triggering now.')
|
||||
self._recovery_task = create_task_with_error_handling(
|
||||
self._recover_agent_focus(target_id),
|
||||
name='recover_agent_focus_from_stale_get',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=False,
|
||||
)
|
||||
return None
|
||||
return self._sessions.get(next(iter(session_ids)))
|
||||
|
||||
def get_all_page_targets(self) -> list:
|
||||
"""Get all page/tab targets using owned data.
|
||||
|
||||
Returns:
|
||||
List of Target objects for all page/tab targets
|
||||
"""
|
||||
page_targets = []
|
||||
for target in self._targets.values():
|
||||
if target.target_type in ('page', 'tab'):
|
||||
page_targets.append(target)
|
||||
return page_targets
|
||||
|
||||
async def validate_session(self, target_id: TargetID) -> bool:
|
||||
"""Check if a target still has active sessions.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to validate
|
||||
|
||||
Returns:
|
||||
True if target has active sessions, False if it should be removed
|
||||
"""
|
||||
if target_id not in self._target_sessions:
|
||||
return False
|
||||
return len(self._target_sessions[target_id]) > 0
|
||||
|
||||
async def clear(self) -> None:
|
||||
"""Clear all owned data structures for cleanup."""
|
||||
async with self._lock:
|
||||
# Clear owned data (single source of truth)
|
||||
self._targets.clear()
|
||||
self._sessions.clear()
|
||||
self._target_sessions.clear()
|
||||
self._session_to_target.clear()
|
||||
|
||||
self.logger.info('[SessionManager] Cleared all owned data (targets, sessions, mappings)')
|
||||
|
||||
async def is_target_valid(self, target_id: TargetID) -> bool:
|
||||
"""Check if a target is still valid and has active sessions.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to validate
|
||||
|
||||
Returns:
|
||||
True if target is valid and has active sessions, False otherwise
|
||||
"""
|
||||
if target_id not in self._target_sessions:
|
||||
return False
|
||||
return len(self._target_sessions[target_id]) > 0
|
||||
|
||||
def get_target_id_from_session_id(self, session_id: SessionID) -> TargetID | None:
|
||||
"""Look up which target a session belongs to.
|
||||
|
||||
Args:
|
||||
session_id: The session ID to look up
|
||||
|
||||
Returns:
|
||||
Target ID if found, None otherwise
|
||||
"""
|
||||
return self._session_to_target.get(session_id)
|
||||
|
||||
def get_target(self, target_id: TargetID) -> 'Target | None':
|
||||
"""Get target from owned data.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to get
|
||||
|
||||
Returns:
|
||||
Target object if found, None otherwise
|
||||
"""
|
||||
return self._targets.get(target_id)
|
||||
|
||||
def get_all_targets(self) -> dict[TargetID, 'Target']:
|
||||
"""Get all targets (read-only access to owned data).
|
||||
|
||||
Returns:
|
||||
Dict mapping target_id to Target objects
|
||||
"""
|
||||
return self._targets
|
||||
|
||||
def get_all_target_ids(self) -> list[TargetID]:
|
||||
"""Get all target IDs from owned data.
|
||||
|
||||
Returns:
|
||||
List of all target IDs
|
||||
"""
|
||||
return list(self._targets.keys())
|
||||
|
||||
def get_all_sessions(self) -> dict[SessionID, 'CDPSession']:
|
||||
"""Get all sessions (read-only access to owned data).
|
||||
|
||||
Returns:
|
||||
Dict mapping session_id to CDPSession objects
|
||||
"""
|
||||
return self._sessions
|
||||
|
||||
def get_session(self, session_id: SessionID) -> 'CDPSession | None':
|
||||
"""Get session from owned data.
|
||||
|
||||
Args:
|
||||
session_id: Session ID to get
|
||||
|
||||
Returns:
|
||||
CDPSession object if found, None otherwise
|
||||
"""
|
||||
return self._sessions.get(session_id)
|
||||
|
||||
def get_all_sessions_for_target(self, target_id: TargetID) -> list['CDPSession']:
|
||||
"""Get ALL sessions attached to a target from owned data.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to get sessions for
|
||||
|
||||
Returns:
|
||||
List of all CDPSession objects for this target
|
||||
"""
|
||||
session_ids = self._target_sessions.get(target_id, set())
|
||||
return [self._sessions[sid] for sid in session_ids if sid in self._sessions]
|
||||
|
||||
def get_target_sessions_mapping(self) -> dict[TargetID, set[SessionID]]:
|
||||
"""Get target->sessions mapping (read-only access).
|
||||
|
||||
Returns:
|
||||
Dict mapping target_id to set of session_ids
|
||||
"""
|
||||
return self._target_sessions
|
||||
|
||||
def get_focused_target(self) -> 'Target | None':
|
||||
"""Get the target that currently has agent focus.
|
||||
|
||||
Convenience method that uses browser_session.agent_focus_target_id.
|
||||
|
||||
Returns:
|
||||
Target object if agent has focus, None otherwise
|
||||
"""
|
||||
if not self.browser_session.agent_focus_target_id:
|
||||
return None
|
||||
return self.get_target(self.browser_session.agent_focus_target_id)
|
||||
|
||||
async def ensure_valid_focus(self, timeout: float = 3.0) -> bool:
|
||||
"""Ensure agent_focus_target_id points to a valid, attached CDP session.
|
||||
|
||||
If the focus target is stale (detached), this method waits for automatic recovery.
|
||||
Uses event-driven coordination instead of polling for efficiency.
|
||||
|
||||
Args:
|
||||
timeout: Maximum time to wait for recovery in seconds (default: 3.0)
|
||||
|
||||
Returns:
|
||||
True if focus is valid or successfully recovered, False if no focus or recovery failed
|
||||
"""
|
||||
if not self.browser_session.agent_focus_target_id:
|
||||
# No focus at all - might be initial state or complete failure
|
||||
if self._recovery_in_progress and self._recovery_complete_event:
|
||||
# Recovery is happening, wait for it
|
||||
try:
|
||||
await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=timeout)
|
||||
# Check again after recovery - simple existence check
|
||||
focus_id = self.browser_session.agent_focus_target_id
|
||||
return bool(focus_id and self._get_session_for_target(focus_id))
|
||||
except TimeoutError:
|
||||
self.logger.error(f'[SessionManager] ❌ Timed out waiting for recovery after {timeout}s')
|
||||
return False
|
||||
return False
|
||||
|
||||
# Simple existence check - does the focused target have a session?
|
||||
cdp_session = self._get_session_for_target(self.browser_session.agent_focus_target_id)
|
||||
if cdp_session:
|
||||
# Session exists - validate it's still active
|
||||
is_valid = await self.validate_session(self.browser_session.agent_focus_target_id)
|
||||
if is_valid:
|
||||
return True
|
||||
|
||||
# Focus is stale - wait for recovery using event instead of polling
|
||||
stale_target_id = self.browser_session.agent_focus_target_id
|
||||
self.logger.warning(
|
||||
f'[SessionManager] ⚠️ Stale agent_focus detected (target {stale_target_id[:8] if stale_target_id else "None"}... detached), '
|
||||
f'waiting for recovery...'
|
||||
)
|
||||
|
||||
# Check if recovery is already in progress
|
||||
if not self._recovery_in_progress:
|
||||
self.logger.warning(
|
||||
'[SessionManager] ⚠️ Recovery not in progress for stale focus! '
|
||||
'This indicates a bug - recovery should have been triggered.'
|
||||
)
|
||||
return False
|
||||
|
||||
# Wait for recovery complete event (event-driven, not polling!)
|
||||
if self._recovery_complete_event:
|
||||
try:
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=timeout)
|
||||
elapsed = asyncio.get_event_loop().time() - start_time
|
||||
|
||||
# Verify recovery succeeded - simple existence check
|
||||
focus_id = self.browser_session.agent_focus_target_id
|
||||
if focus_id and self._get_session_for_target(focus_id):
|
||||
self.logger.info(
|
||||
f'[SessionManager] ✅ Agent focus recovered to {self.browser_session.agent_focus_target_id[:8]}... '
|
||||
f'after {elapsed * 1000:.0f}ms'
|
||||
)
|
||||
return True
|
||||
else:
|
||||
self.logger.error(
|
||||
f'[SessionManager] ❌ Recovery completed but focus still invalid after {elapsed * 1000:.0f}ms'
|
||||
)
|
||||
return False
|
||||
|
||||
except TimeoutError:
|
||||
self.logger.error(
|
||||
f'[SessionManager] ❌ Recovery timed out after {timeout}s '
|
||||
f'(was: {stale_target_id[:8] if stale_target_id else "None"}..., '
|
||||
f'now: {self.browser_session.agent_focus_target_id[:8] if self.browser_session.agent_focus_target_id else "None"})'
|
||||
)
|
||||
return False
|
||||
else:
|
||||
self.logger.error('[SessionManager] ❌ Recovery event not initialized')
|
||||
return False
|
||||
|
||||
async def _handle_target_attached(self, event: AttachedToTargetEvent) -> None:
|
||||
"""Handle Target.attachedToTarget event.
|
||||
|
||||
Called automatically by Chrome when a new target/session is created.
|
||||
This is the ONLY place where sessions are added to the pool.
|
||||
"""
|
||||
target_id = event['targetInfo']['targetId']
|
||||
session_id = event['sessionId']
|
||||
target_type = event['targetInfo']['type']
|
||||
target_info = event['targetInfo']
|
||||
waiting_for_debugger = event.get('waitingForDebugger', False)
|
||||
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Target attached: {target_id[:8]}... (session={session_id[:8]}..., '
|
||||
f'type={target_type}, waitingForDebugger={waiting_for_debugger})'
|
||||
)
|
||||
|
||||
# Defensive check: browser may be shutting down and _cdp_client_root could be None
|
||||
if self.browser_session._cdp_client_root is None:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Skipping target attach for {target_id[:8]}... - browser shutting down (no CDP client)'
|
||||
)
|
||||
return
|
||||
|
||||
# Enable auto-attach for this session's children (do this FIRST, outside lock)
|
||||
try:
|
||||
await self.browser_session._cdp_client_root.send.Target.setAutoAttach(
|
||||
params={'autoAttach': True, 'waitForDebuggerOnStart': False, 'flatten': True}, session_id=session_id
|
||||
)
|
||||
except Exception as e:
|
||||
error_str = str(e)
|
||||
# Expected for short-lived targets (workers, temp iframes) that detach before this executes
|
||||
if '-32001' not in error_str and 'Session with given id not found' not in error_str:
|
||||
self.logger.debug(f'[SessionManager] Auto-attach failed for {target_type}: {e}')
|
||||
|
||||
from browser_use.browser.session import Target
|
||||
|
||||
async with self._lock:
|
||||
# Track this session for the target
|
||||
if target_id not in self._target_sessions:
|
||||
self._target_sessions[target_id] = set()
|
||||
|
||||
self._target_sessions[target_id].add(session_id)
|
||||
self._session_to_target[session_id] = target_id
|
||||
|
||||
# Create or update Target inside the same lock so that get_target() is never
|
||||
# called in the window between _target_sessions being set and _targets being set.
|
||||
if target_id not in self._targets:
|
||||
target = Target(
|
||||
target_id=target_id,
|
||||
target_type=target_type,
|
||||
url=target_info.get('url', 'about:blank'),
|
||||
title=target_info.get('title', 'Unknown title'),
|
||||
)
|
||||
self._targets[target_id] = target
|
||||
self.logger.debug(f'[SessionManager] Created target {target_id[:8]}... (type={target_type})')
|
||||
else:
|
||||
# Update existing target info
|
||||
existing_target = self._targets[target_id]
|
||||
existing_target.url = target_info.get('url', existing_target.url)
|
||||
existing_target.title = target_info.get('title', existing_target.title)
|
||||
|
||||
# Create CDPSession (communication channel)
|
||||
from browser_use.browser.session import CDPSession
|
||||
|
||||
assert self.browser_session._cdp_client_root is not None, 'Root CDP client required'
|
||||
|
||||
cdp_session = CDPSession(
|
||||
cdp_client=self.browser_session._cdp_client_root,
|
||||
target_id=target_id,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
# Add to sessions dict
|
||||
self._sessions[session_id] = cdp_session
|
||||
|
||||
# If proxy auth is configured, enable Fetch auth handling on this session
|
||||
# Avoids overwriting Target.attachedToTarget handlers elsewhere
|
||||
try:
|
||||
proxy_cfg = self.browser_session.browser_profile.proxy
|
||||
username = proxy_cfg.username if proxy_cfg else None
|
||||
password = proxy_cfg.password if proxy_cfg else None
|
||||
if username and password:
|
||||
await cdp_session.cdp_client.send.Fetch.enable(
|
||||
params={'handleAuthRequests': True},
|
||||
session_id=cdp_session.session_id,
|
||||
)
|
||||
self.logger.debug(f'[SessionManager] Fetch.enable(handleAuthRequests=True) on session {session_id[:8]}...')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'[SessionManager] Fetch.enable on attached session failed: {type(e).__name__}: {e}')
|
||||
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Created session {session_id[:8]}... for target {target_id[:8]}... '
|
||||
f'(total sessions: {len(self._sessions)})'
|
||||
)
|
||||
|
||||
# Enable lifecycle events and network monitoring for page targets
|
||||
if target_type in ('page', 'tab'):
|
||||
await self._enable_page_monitoring(cdp_session)
|
||||
|
||||
# Resume execution if waiting for debugger
|
||||
if waiting_for_debugger:
|
||||
try:
|
||||
assert self.browser_session._cdp_client_root is not None
|
||||
await self.browser_session._cdp_client_root.send.Runtime.runIfWaitingForDebugger(session_id=session_id)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'[SessionManager] Failed to resume execution: {e}')
|
||||
|
||||
async def _handle_target_info_changed(self, event: dict) -> None:
|
||||
"""Handle Target.targetInfoChanged event.
|
||||
|
||||
Updates target title/URL without polling getTargetInfo().
|
||||
Chrome fires this automatically when title or URL changes.
|
||||
"""
|
||||
target_info = event.get('targetInfo', {})
|
||||
target_id = target_info.get('targetId')
|
||||
|
||||
if not target_id:
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
# Update target if it exists (source of truth for url/title)
|
||||
if target_id in self._targets:
|
||||
target = self._targets[target_id]
|
||||
|
||||
target.title = target_info.get('title', target.title)
|
||||
target.url = target_info.get('url', target.url)
|
||||
|
||||
async def _handle_target_detached(self, event: DetachedFromTargetEvent) -> None:
|
||||
"""Handle Target.detachedFromTarget event.
|
||||
|
||||
Called automatically by Chrome when a target/session is destroyed.
|
||||
This is the ONLY place where sessions are removed from the pool.
|
||||
"""
|
||||
session_id = event['sessionId']
|
||||
target_id = event.get('targetId') # May be empty
|
||||
|
||||
# If targetId not in event, look it up via session mapping
|
||||
if not target_id:
|
||||
async with self._lock:
|
||||
target_id = self._session_to_target.get(session_id)
|
||||
|
||||
if not target_id:
|
||||
self.logger.warning(f'[SessionManager] Session detached but target unknown (session={session_id[:8]}...)')
|
||||
return
|
||||
|
||||
agent_focus_lost = False
|
||||
target_fully_removed = False
|
||||
target_type = None
|
||||
|
||||
async with self._lock:
|
||||
# Remove this session from target's session set
|
||||
if target_id in self._target_sessions:
|
||||
self._target_sessions[target_id].discard(session_id)
|
||||
|
||||
remaining_sessions = len(self._target_sessions[target_id])
|
||||
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Session detached: target={target_id[:8]}... '
|
||||
f'session={session_id[:8]}... (remaining={remaining_sessions})'
|
||||
)
|
||||
|
||||
# Only remove target when NO sessions remain
|
||||
if remaining_sessions == 0:
|
||||
self.logger.debug(f'[SessionManager] No sessions remain for target {target_id[:8]}..., removing target')
|
||||
|
||||
target_fully_removed = True
|
||||
|
||||
# Check if agent_focus points to this target
|
||||
agent_focus_lost = self.browser_session.agent_focus_target_id == target_id
|
||||
|
||||
# Immediately clear stale focus to prevent operations on detached target
|
||||
if agent_focus_lost:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Clearing stale agent_focus_target_id {target_id[:8]}... '
|
||||
f'to prevent operations on detached target'
|
||||
)
|
||||
self.browser_session.agent_focus_target_id = None
|
||||
|
||||
# Get target type before removing (needed for TabClosedEvent dispatch)
|
||||
target = self._targets.get(target_id)
|
||||
target_type = target.target_type if target else None
|
||||
|
||||
# Remove target (entity) from owned data
|
||||
if target_id in self._targets:
|
||||
self._targets.pop(target_id)
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Removed target {target_id[:8]}... (remaining targets: {len(self._targets)})'
|
||||
)
|
||||
|
||||
# Clean up tracking
|
||||
del self._target_sessions[target_id]
|
||||
self._lifecycle_events.pop(target_id, None)
|
||||
else:
|
||||
# Target not tracked - already removed or never attached
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Session detached from untracked target: target={target_id[:8]}... '
|
||||
f'session={session_id[:8]}... (target was already removed or attach event was missed)'
|
||||
)
|
||||
|
||||
# Remove session from owned sessions dict
|
||||
if session_id in self._sessions:
|
||||
self._sessions.pop(session_id)
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Removed session {session_id[:8]}... (remaining sessions: {len(self._sessions)})'
|
||||
)
|
||||
|
||||
# Remove from reverse mapping
|
||||
if session_id in self._session_to_target:
|
||||
del self._session_to_target[session_id]
|
||||
|
||||
# Dispatch TabClosedEvent only for page/tab targets that are fully removed (not iframes/workers or partial detaches)
|
||||
if target_fully_removed:
|
||||
if target_type in ('page', 'tab'):
|
||||
from browser_use.browser.events import TabClosedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(TabClosedEvent(target_id=target_id))
|
||||
self.logger.debug(f'[SessionManager] Dispatched TabClosedEvent for page target {target_id[:8]}...')
|
||||
elif target_type:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Target {target_id[:8]}... fully removed (type={target_type}) - not dispatching TabClosedEvent'
|
||||
)
|
||||
|
||||
# Auto-recover agent_focus outside the lock to avoid blocking other operations
|
||||
if agent_focus_lost:
|
||||
# Create recovery task instead of awaiting directly - allows concurrent operations to wait on same recovery
|
||||
if not self._recovery_in_progress:
|
||||
self._recovery_task = create_task_with_error_handling(
|
||||
self._recover_agent_focus(target_id),
|
||||
name='recover_agent_focus',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=False,
|
||||
)
|
||||
|
||||
async def _recover_agent_focus(self, crashed_target_id: TargetID) -> None:
|
||||
"""Auto-recover agent_focus when the focused target crashes/detaches.
|
||||
|
||||
Uses recovery lock to prevent concurrent recovery attempts from creating multiple emergency tabs.
|
||||
Coordinates with ensure_valid_focus() via events for efficient waiting.
|
||||
|
||||
Args:
|
||||
crashed_target_id: The target ID that was lost
|
||||
"""
|
||||
try:
|
||||
# Prevent concurrent recovery attempts
|
||||
async with self._recovery_lock:
|
||||
# Set recovery state INSIDE lock to prevent race conditions
|
||||
if self._recovery_in_progress:
|
||||
self.logger.debug('[SessionManager] Recovery already in progress, waiting for it to complete')
|
||||
# Wait for ongoing recovery instead of starting a new one
|
||||
if self._recovery_complete_event:
|
||||
try:
|
||||
await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=5.0)
|
||||
except TimeoutError:
|
||||
self.logger.error('[SessionManager] Timed out waiting for ongoing recovery')
|
||||
return
|
||||
|
||||
# Set recovery state
|
||||
self._recovery_in_progress = True
|
||||
self._recovery_complete_event = asyncio.Event()
|
||||
|
||||
if self.browser_session._cdp_client_root is None:
|
||||
self.logger.debug('[SessionManager] Skipping focus recovery - browser shutting down (no CDP client)')
|
||||
return
|
||||
|
||||
# Check if another recovery already fixed agent_focus
|
||||
if self.browser_session.agent_focus_target_id and self.browser_session.agent_focus_target_id != crashed_target_id:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Agent focus already recovered by concurrent operation '
|
||||
f'(now: {self.browser_session.agent_focus_target_id[:8]}...), skipping recovery'
|
||||
)
|
||||
return
|
||||
|
||||
# Note: agent_focus_target_id may already be None (cleared in _handle_target_detached)
|
||||
current_focus_desc = (
|
||||
f'{self.browser_session.agent_focus_target_id[:8]}...'
|
||||
if self.browser_session.agent_focus_target_id
|
||||
else 'None (already cleared)'
|
||||
)
|
||||
|
||||
self.logger.warning(
|
||||
f'[SessionManager] Agent focus target {crashed_target_id[:8]}... detached! '
|
||||
f'Current focus: {current_focus_desc}. Auto-recovering by switching to another target...'
|
||||
)
|
||||
|
||||
# Perform recovery (outside lock to allow concurrent operations)
|
||||
# Try to find another valid page target
|
||||
page_targets = self.get_all_page_targets()
|
||||
|
||||
new_target_id = None
|
||||
is_existing_tab = False
|
||||
|
||||
if page_targets:
|
||||
# Switch to most recent page that's not the crashed one
|
||||
new_target_id = page_targets[-1].target_id
|
||||
is_existing_tab = True
|
||||
self.logger.info(f'[SessionManager] Switching agent_focus to existing tab {new_target_id[:8]}...')
|
||||
else:
|
||||
# No pages exist - create a new one
|
||||
self.logger.warning('[SessionManager] No tabs remain! Creating new tab for agent...')
|
||||
new_target_id = await self.browser_session._cdp_create_new_page('about:blank')
|
||||
self.logger.info(f'[SessionManager] Created new tab {new_target_id[:8]}... for agent')
|
||||
|
||||
# Dispatch TabCreatedEvent so watchdogs can initialize
|
||||
from browser_use.browser.events import TabCreatedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(TabCreatedEvent(url='about:blank', target_id=new_target_id))
|
||||
|
||||
# Wait for CDP attach event to create session
|
||||
# Note: This polling is necessary - waiting for external Chrome CDP event
|
||||
# _handle_target_attached will add session to pool when Chrome fires attachedToTarget
|
||||
new_session = None
|
||||
for attempt in range(20): # Wait up to 2 seconds
|
||||
await asyncio.sleep(0.1)
|
||||
new_session = self._get_session_for_target(new_target_id)
|
||||
if new_session:
|
||||
break
|
||||
|
||||
if new_session:
|
||||
self.browser_session.agent_focus_target_id = new_target_id
|
||||
self.logger.info(f'[SessionManager] ✅ Agent focus recovered: {new_target_id[:8]}...')
|
||||
|
||||
# Visually activate the tab in browser (only for existing tabs)
|
||||
if is_existing_tab:
|
||||
try:
|
||||
assert self.browser_session._cdp_client_root is not None
|
||||
await self.browser_session._cdp_client_root.send.Target.activateTarget(params={'targetId': new_target_id})
|
||||
self.logger.debug(f'[SessionManager] Activated tab {new_target_id[:8]}... in browser UI')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'[SessionManager] Failed to activate tab visually: {e}')
|
||||
|
||||
# Get target to access url (from owned data)
|
||||
target = self.get_target(new_target_id)
|
||||
target_url = target.url if target else 'about:blank'
|
||||
|
||||
# Dispatch focus changed event
|
||||
from browser_use.browser.events import AgentFocusChangedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(AgentFocusChangedEvent(target_id=new_target_id, url=target_url))
|
||||
return
|
||||
|
||||
# Recovery failed - create emergency fallback tab
|
||||
self.logger.error(
|
||||
f'[SessionManager] ❌ Failed to get session for {new_target_id[:8]}... after 2s, creating emergency fallback tab'
|
||||
)
|
||||
|
||||
fallback_target_id = await self.browser_session._cdp_create_new_page('about:blank')
|
||||
self.logger.warning(f'[SessionManager] Created emergency fallback tab {fallback_target_id[:8]}...')
|
||||
|
||||
# Try one more time with fallback
|
||||
# Note: This polling is necessary - waiting for external Chrome CDP event
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.1)
|
||||
fallback_session = self._get_session_for_target(fallback_target_id)
|
||||
if fallback_session:
|
||||
self.browser_session.agent_focus_target_id = fallback_target_id
|
||||
self.logger.warning(f'[SessionManager] ⚠️ Agent focus set to emergency fallback: {fallback_target_id[:8]}...')
|
||||
|
||||
from browser_use.browser.events import AgentFocusChangedEvent, TabCreatedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(TabCreatedEvent(url='about:blank', target_id=fallback_target_id))
|
||||
self.browser_session.event_bus.dispatch(
|
||||
AgentFocusChangedEvent(target_id=fallback_target_id, url='about:blank')
|
||||
)
|
||||
return
|
||||
|
||||
# Complete failure - this should never happen
|
||||
self.logger.critical(
|
||||
'[SessionManager] 🚨 CRITICAL: Failed to recover agent_focus even with fallback! Agent may be in broken state.'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[SessionManager] ❌ Error during agent_focus recovery: {type(e).__name__}: {e}')
|
||||
finally:
|
||||
# Always signal completion and reset recovery state
|
||||
# This allows all waiting operations to proceed (success or failure)
|
||||
if self._recovery_complete_event:
|
||||
self._recovery_complete_event.set()
|
||||
self._recovery_in_progress = False
|
||||
self._recovery_task = None
|
||||
self.logger.debug('[SessionManager] Recovery state reset')
|
||||
|
||||
async def _initialize_existing_targets(self) -> None:
|
||||
"""Discover and initialize all existing targets at startup.
|
||||
|
||||
Attaches to each target and initializes it SYNCHRONOUSLY.
|
||||
Chrome will also fire attachedToTarget events, but _handle_target_attached() is
|
||||
idempotent (checks if target already in pool), so duplicate handling is safe.
|
||||
|
||||
This eliminates race conditions - monitoring is guaranteed ready before navigation.
|
||||
"""
|
||||
cdp_client = self.browser_session._cdp_client_root
|
||||
assert cdp_client is not None
|
||||
|
||||
# Get all existing targets
|
||||
targets_result = await cdp_client.send.Target.getTargets()
|
||||
existing_targets = targets_result.get('targetInfos', [])
|
||||
|
||||
self.logger.debug(f'[SessionManager] Discovered {len(existing_targets)} existing targets')
|
||||
|
||||
# Track target IDs for verification
|
||||
target_ids_to_wait_for = []
|
||||
|
||||
# Just attach to ALL existing targets - Chrome fires attachedToTarget events
|
||||
# The on_attached handler (via create_task) does ALL the work
|
||||
for target in existing_targets:
|
||||
target_id = target['targetId']
|
||||
target_type = target.get('type', 'unknown')
|
||||
|
||||
try:
|
||||
# Just attach - event handler does everything
|
||||
await cdp_client.send.Target.attachToTarget(params={'targetId': target_id, 'flatten': True})
|
||||
target_ids_to_wait_for.append(target_id)
|
||||
except Exception as e:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Failed to attach to existing target {target_id[:8]}... (type={target_type}): {e}'
|
||||
)
|
||||
|
||||
# Wait for event handlers to complete their work (they run via create_task)
|
||||
# Use event-driven approach instead of polling for better performance
|
||||
ready_event = asyncio.Event()
|
||||
|
||||
async def check_all_ready():
|
||||
"""Check if all sessions are ready and signal completion."""
|
||||
while True:
|
||||
ready_count = 0
|
||||
for tid in target_ids_to_wait_for:
|
||||
session = self._get_session_for_target(tid)
|
||||
if session:
|
||||
target = self._targets.get(tid)
|
||||
target_type = target.target_type if target else 'unknown'
|
||||
# For pages, verify monitoring is enabled
|
||||
if target_type in ('page', 'tab'):
|
||||
if hasattr(session, '_lifecycle_events') and session._lifecycle_events is not None:
|
||||
ready_count += 1
|
||||
else:
|
||||
# Non-page targets don't need monitoring
|
||||
ready_count += 1
|
||||
|
||||
if ready_count == len(target_ids_to_wait_for):
|
||||
ready_event.set()
|
||||
return
|
||||
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
# Start checking in background
|
||||
check_task = create_task_with_error_handling(
|
||||
check_all_ready(), name='check_all_targets_ready', logger_instance=self.logger
|
||||
)
|
||||
|
||||
try:
|
||||
# Wait for completion with timeout
|
||||
await asyncio.wait_for(ready_event.wait(), timeout=2.0)
|
||||
except TimeoutError:
|
||||
# Timeout - count what's ready
|
||||
ready_count = 0
|
||||
for tid in target_ids_to_wait_for:
|
||||
session = self._get_session_for_target(tid)
|
||||
if session:
|
||||
target = self._targets.get(tid)
|
||||
target_type = target.target_type if target else 'unknown'
|
||||
# For pages, verify monitoring is enabled
|
||||
if target_type in ('page', 'tab'):
|
||||
if hasattr(session, '_lifecycle_events') and session._lifecycle_events is not None:
|
||||
ready_count += 1
|
||||
else:
|
||||
# Non-page targets don't need monitoring
|
||||
ready_count += 1
|
||||
self.logger.warning(
|
||||
f'[SessionManager] Initialization timeout after 2.0s: {ready_count}/{len(target_ids_to_wait_for)} sessions ready'
|
||||
)
|
||||
finally:
|
||||
check_task.cancel()
|
||||
try:
|
||||
await check_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _enable_page_monitoring(self, cdp_session: 'CDPSession') -> None:
|
||||
"""Enable lifecycle events and network monitoring for a page target.
|
||||
|
||||
This is called once per page when it's created, avoiding handler accumulation.
|
||||
Registers a SINGLE lifecycle handler per session that stores events for navigations to consume.
|
||||
|
||||
Args:
|
||||
cdp_session: The CDP session to enable monitoring on
|
||||
"""
|
||||
try:
|
||||
# Enable Page domain first (required for lifecycle events)
|
||||
await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
|
||||
|
||||
# Enable lifecycle events (load, DOMContentLoaded, networkIdle, etc.)
|
||||
await cdp_session.cdp_client.send.Page.setLifecycleEventsEnabled(
|
||||
params={'enabled': True}, session_id=cdp_session.session_id
|
||||
)
|
||||
|
||||
# Enable network monitoring for networkIdle detection
|
||||
await cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)
|
||||
|
||||
# Event storage and the Page.lifecycleEvent handler live in SessionManager
|
||||
# (one global handler registered in start_monitoring, routed by session_id):
|
||||
# cdp-use's registry is single-slot per method, so a per-session registration
|
||||
# here would replace the previous tab's handler and freeze its event buffer.
|
||||
# Expose the shared per-target buffer on the session for readiness checks.
|
||||
cdp_session._lifecycle_events = self.get_lifecycle_events(cdp_session.target_id)
|
||||
|
||||
except Exception as e:
|
||||
# Don't fail - target might be short-lived or already detached
|
||||
error_str = str(e)
|
||||
if '-32001' in error_str or 'Session with given id not found' in error_str:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Target {cdp_session.target_id[:8]}... detached before monitoring could be enabled (normal for short-lived targets)'
|
||||
)
|
||||
else:
|
||||
self.logger.warning(
|
||||
f'[SessionManager] Failed to enable monitoring for target {cdp_session.target_id[:8]}...: {e}'
|
||||
)
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Video Recording Service for Browser Use Sessions."""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from browser_use.browser.profile import ViewportSize
|
||||
|
||||
try:
|
||||
import imageio.v2 as iio # type: ignore[import-not-found]
|
||||
import numpy as np # type: ignore[import-not-found]
|
||||
from imageio.core.format import Format # type: ignore[import-not-found]
|
||||
from PIL import Image
|
||||
|
||||
IMAGEIO_AVAILABLE = True
|
||||
except ImportError:
|
||||
IMAGEIO_AVAILABLE = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_padded_size(size: ViewportSize, macro_block_size: int = 16) -> ViewportSize:
|
||||
"""Calculates the dimensions padded to the nearest multiple of macro_block_size."""
|
||||
width = int(math.ceil(size['width'] / macro_block_size)) * macro_block_size
|
||||
height = int(math.ceil(size['height'] / macro_block_size)) * macro_block_size
|
||||
return ViewportSize(width=width, height=height)
|
||||
|
||||
|
||||
class VideoRecorderService:
|
||||
"""
|
||||
Handles the video encoding process for a browser session using imageio.
|
||||
|
||||
This service captures individual frames from the CDP screencast, decodes them,
|
||||
and appends them to a video file using a pip-installable ffmpeg backend.
|
||||
It automatically resizes frames to match the target video dimensions.
|
||||
"""
|
||||
|
||||
def __init__(self, output_path: Path, size: ViewportSize, framerate: int):
|
||||
"""
|
||||
Initializes the video recorder.
|
||||
|
||||
Args:
|
||||
output_path: The full path where the video will be saved.
|
||||
size: A ViewportSize object specifying the width and height of the video.
|
||||
framerate: The desired framerate for the output video.
|
||||
"""
|
||||
self.output_path = output_path
|
||||
self.size = size
|
||||
self.framerate = framerate
|
||||
self._writer: Optional['Format.Writer'] = None
|
||||
self._is_active = False
|
||||
self.padded_size = _get_padded_size(self.size)
|
||||
|
||||
def start(self) -> None:
|
||||
"""
|
||||
Prepares and starts the video writer.
|
||||
|
||||
If the required optional dependencies are not installed, this method will
|
||||
log an error and do nothing.
|
||||
"""
|
||||
if not IMAGEIO_AVAILABLE:
|
||||
logger.error(
|
||||
'MP4 recording requires optional dependencies. Please install them with: pip install "browser-use[video]"'
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# The macro_block_size is set to None because we handle padding ourselves
|
||||
self._writer = iio.get_writer(
|
||||
str(self.output_path),
|
||||
fps=self.framerate,
|
||||
codec='libx264',
|
||||
quality=8, # A good balance of quality and file size (1-10 scale)
|
||||
pixelformat='yuv420p', # Ensures compatibility with most players
|
||||
macro_block_size=None,
|
||||
)
|
||||
self._is_active = True
|
||||
logger.debug(f'Video recorder started. Output will be saved to {self.output_path}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to initialize video writer: {e}')
|
||||
self._is_active = False
|
||||
|
||||
def add_frame(self, frame_data_b64: str) -> None:
|
||||
"""
|
||||
Decodes a base64-encoded PNG frame, resizes it, pads it to be codec-compatible,
|
||||
and appends it to the video.
|
||||
|
||||
Args:
|
||||
frame_data_b64: A base64-encoded string of the PNG frame data.
|
||||
"""
|
||||
if not self._is_active or not self._writer:
|
||||
return
|
||||
|
||||
try:
|
||||
frame_bytes = base64.b64decode(frame_data_b64)
|
||||
|
||||
# Use PIL to handle image processing in memory - much faster than spawning ffmpeg subprocess per frame
|
||||
with Image.open(io.BytesIO(frame_bytes)) as img:
|
||||
# 1. Resize if needed to target viewport size
|
||||
if img.size != (self.size['width'], self.size['height']):
|
||||
# Use BICUBIC as it's faster than LANCZOS and good enough for screen recordings
|
||||
img = img.resize((self.size['width'], self.size['height']), Image.Resampling.BICUBIC)
|
||||
|
||||
# 2. Handle Padding (Macro block alignment for codecs)
|
||||
# Check if padding is actually needed
|
||||
if self.padded_size['width'] != self.size['width'] or self.padded_size['height'] != self.size['height']:
|
||||
new_img = Image.new('RGB', (self.padded_size['width'], self.padded_size['height']), (0, 0, 0))
|
||||
# Center the image
|
||||
x_offset = (self.padded_size['width'] - self.size['width']) // 2
|
||||
y_offset = (self.padded_size['height'] - self.size['height']) // 2
|
||||
new_img.paste(img, (x_offset, y_offset))
|
||||
img = new_img
|
||||
|
||||
# 3. Convert to numpy array for imageio
|
||||
img_array = np.array(img)
|
||||
|
||||
self._writer.append_data(img_array)
|
||||
except Exception as e:
|
||||
logger.warning(f'Could not process and add video frame: {e}')
|
||||
|
||||
def stop_and_save(self) -> None:
|
||||
"""
|
||||
Finalizes the video file by closing the writer.
|
||||
|
||||
This method should be called when the recording session is complete.
|
||||
"""
|
||||
if not self._is_active or not self._writer:
|
||||
return
|
||||
|
||||
try:
|
||||
self._writer.close()
|
||||
logger.info(f'📹 Video recording saved successfully to: {self.output_path}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to finalize and save video: {e}')
|
||||
finally:
|
||||
self._is_active = False
|
||||
self._writer = None
|
||||
@@ -0,0 +1,200 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.target import TargetID
|
||||
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, field_serializer
|
||||
|
||||
from browser_use.dom.views import DOMInteractedElement, SerializedDOMState
|
||||
|
||||
# Known placeholder image data for about:blank pages - a 4x4 white PNG
|
||||
PLACEHOLDER_4PX_SCREENSHOT = (
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII='
|
||||
)
|
||||
|
||||
|
||||
# Pydantic
|
||||
class TabInfo(BaseModel):
|
||||
"""Represents information about a browser tab"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
extra='forbid',
|
||||
validate_by_name=True,
|
||||
validate_by_alias=True,
|
||||
populate_by_name=True,
|
||||
)
|
||||
|
||||
# Original fields
|
||||
url: str
|
||||
title: str
|
||||
target_id: TargetID = Field(serialization_alias='tab_id', validation_alias=AliasChoices('tab_id', 'target_id'))
|
||||
parent_target_id: TargetID | None = Field(
|
||||
default=None, serialization_alias='parent_tab_id', validation_alias=AliasChoices('parent_tab_id', 'parent_target_id')
|
||||
) # parent page that contains this popup or cross-origin iframe
|
||||
|
||||
@field_serializer('target_id')
|
||||
def serialize_target_id(self, target_id: TargetID, _info: Any) -> str:
|
||||
return target_id[-4:]
|
||||
|
||||
@field_serializer('parent_target_id')
|
||||
def serialize_parent_target_id(self, parent_target_id: TargetID | None, _info: Any) -> str | None:
|
||||
return parent_target_id[-4:] if parent_target_id else None
|
||||
|
||||
|
||||
class PageInfo(BaseModel):
|
||||
"""Comprehensive page size and scroll information"""
|
||||
|
||||
# Current viewport dimensions
|
||||
viewport_width: int
|
||||
viewport_height: int
|
||||
|
||||
# Total page dimensions
|
||||
page_width: int
|
||||
page_height: int
|
||||
|
||||
# Current scroll position
|
||||
scroll_x: int
|
||||
scroll_y: int
|
||||
|
||||
# Calculated scroll information
|
||||
pixels_above: int
|
||||
pixels_below: int
|
||||
pixels_left: int
|
||||
pixels_right: int
|
||||
|
||||
# Page statistics are now computed dynamically instead of stored
|
||||
|
||||
|
||||
@dataclass
|
||||
class NetworkRequest:
|
||||
"""Information about a pending network request"""
|
||||
|
||||
url: str
|
||||
method: str = 'GET'
|
||||
loading_duration_ms: float = 0.0 # How long this request has been loading (ms since request started, max 10s)
|
||||
resource_type: str | None = None # e.g., 'Document', 'Stylesheet', 'Image', 'Script', 'XHR', 'Fetch'
|
||||
|
||||
|
||||
@dataclass
|
||||
class PaginationButton:
|
||||
"""Information about a pagination button detected on the page"""
|
||||
|
||||
button_type: str # 'next', 'prev', 'first', 'last', 'page_number'
|
||||
backend_node_id: int # Backend node ID for clicking
|
||||
text: str # Button text/label
|
||||
selector: str # XPath or other selector to locate the element
|
||||
is_disabled: bool = False # Whether the button appears disabled
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserStateSummary:
|
||||
"""The summary of the browser's current state designed for an LLM to process"""
|
||||
|
||||
# provided by SerializedDOMState:
|
||||
dom_state: SerializedDOMState
|
||||
|
||||
url: str
|
||||
title: str
|
||||
tabs: list[TabInfo]
|
||||
screenshot: str | None = field(default=None, repr=False)
|
||||
page_info: PageInfo | None = None # Enhanced page information
|
||||
|
||||
# Keep legacy fields for backward compatibility
|
||||
pixels_above: int = 0
|
||||
pixels_below: int = 0
|
||||
browser_errors: list[str] = field(default_factory=list)
|
||||
is_pdf_viewer: bool = False # Whether the current page is a PDF viewer
|
||||
recent_events: str | None = None # Text summary of recent browser events
|
||||
pending_network_requests: list[NetworkRequest] = field(default_factory=list) # Currently loading network requests
|
||||
pagination_buttons: list[PaginationButton] = field(default_factory=list) # Detected pagination buttons
|
||||
closed_popup_messages: list[str] = field(default_factory=list) # Messages from auto-closed JavaScript dialogs
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserStateHistory:
|
||||
"""The summary of the browser's state at a past point in time to usse in LLM message history"""
|
||||
|
||||
url: str
|
||||
title: str
|
||||
tabs: list[TabInfo]
|
||||
interacted_element: list[DOMInteractedElement | None] | list[None]
|
||||
screenshot_path: str | None = None
|
||||
|
||||
def get_screenshot(self) -> str | None:
|
||||
"""Load screenshot from disk and return as base64 string"""
|
||||
if not self.screenshot_path:
|
||||
return None
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
path_obj = Path(self.screenshot_path)
|
||||
if not path_obj.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(path_obj, 'rb') as f:
|
||||
screenshot_data = f.read()
|
||||
return base64.b64encode(screenshot_data).decode('utf-8')
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
data = {}
|
||||
data['tabs'] = [tab.model_dump() for tab in self.tabs]
|
||||
data['screenshot_path'] = self.screenshot_path
|
||||
data['interacted_element'] = [el.to_dict() if el else None for el in self.interacted_element]
|
||||
data['url'] = self.url
|
||||
data['title'] = self.title
|
||||
return data
|
||||
|
||||
|
||||
class BrowserError(Exception):
|
||||
"""Browser error with structured memory for LLM context management.
|
||||
|
||||
This exception class provides separate memory contexts for browser actions:
|
||||
- short_term_memory: Immediate context shown once to the LLM for the next action
|
||||
- long_term_memory: Persistent error information stored across steps
|
||||
"""
|
||||
|
||||
message: str
|
||||
short_term_memory: str | None = None
|
||||
long_term_memory: str | None = None
|
||||
details: dict[str, Any] | None = None
|
||||
while_handling_event: BaseEvent[Any] | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
short_term_memory: str | None = None,
|
||||
long_term_memory: str | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
event: BaseEvent[Any] | None = None,
|
||||
):
|
||||
"""Initialize a BrowserError with structured memory contexts.
|
||||
|
||||
Args:
|
||||
message: Technical error message for logging and debugging
|
||||
short_term_memory: Context shown once to LLM (e.g., available actions, options)
|
||||
long_term_memory: Persistent error info stored in agent memory
|
||||
details: Additional metadata for debugging
|
||||
event: The browser event that triggered this error
|
||||
"""
|
||||
self.message = message
|
||||
self.short_term_memory = short_term_memory
|
||||
self.long_term_memory = long_term_memory
|
||||
self.details = details
|
||||
self.while_handling_event = event
|
||||
super().__init__(message)
|
||||
|
||||
def __str__(self) -> str:
|
||||
parts = [self.message]
|
||||
if self.details:
|
||||
parts.append(f'({self.details})')
|
||||
if self.while_handling_event:
|
||||
parts.append(f'during: {self.while_handling_event}')
|
||||
return ' '.join(parts)
|
||||
|
||||
|
||||
class URLNotAllowedError(BrowserError):
|
||||
"""Error raised when a URL is not allowed"""
|
||||
@@ -0,0 +1,321 @@
|
||||
"""Base watchdog class for browser monitoring components."""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent, EventBus
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from browser_use.browser.session import BrowserSession
|
||||
|
||||
|
||||
class BaseWatchdog(BaseModel):
|
||||
"""Base class for all browser watchdogs.
|
||||
|
||||
Watchdogs monitor browser state and emit events based on changes.
|
||||
They automatically register event handlers based on method names.
|
||||
|
||||
Handler methods should be named: on_EventTypeName(self, event: EventTypeName)
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
arbitrary_types_allowed=True, # allow non-serializable objects like EventBus/BrowserSession in fields
|
||||
extra='forbid', # dont allow implicit class/instance state, everything must be a properly typed Field or PrivateAttr
|
||||
validate_assignment=False, # avoid re-triggering __init__ / validators on values on every assignment
|
||||
revalidate_instances='never', # avoid re-triggering __init__ / validators and erasing private attrs
|
||||
)
|
||||
|
||||
# Class variables to statically define the list of events relevant to each watchdog
|
||||
# (not enforced, just to make it easier to understand the code and debug watchdogs at runtime)
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [] # Events this watchdog listens to
|
||||
EMITS: ClassVar[list[type[BaseEvent[Any]]]] = [] # Events this watchdog emits
|
||||
|
||||
# Core dependencies
|
||||
event_bus: EventBus = Field()
|
||||
browser_session: BrowserSession = Field()
|
||||
|
||||
# Shared state that other watchdogs might need to access should not be defined on BrowserSession, not here!
|
||||
# Shared helper methods needed by other watchdogs should be defined on BrowserSession, not here!
|
||||
# Alternatively, expose some events on the watchdog to allow access to state/helpers via event_bus system.
|
||||
|
||||
# Private state internal to the watchdog can be defined like this on BaseWatchdog subclasses:
|
||||
# _screenshot_cache: dict[str, bytes] = PrivateAttr(default_factory=dict)
|
||||
# _browser_crash_watcher_task: asyncio.Task | None = PrivateAttr(default=None)
|
||||
# _cdp_download_tasks: WeakSet[asyncio.Task] = PrivateAttr(default_factory=WeakSet)
|
||||
# ...
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
"""Get the logger from the browser session."""
|
||||
return self.browser_session.logger
|
||||
|
||||
@staticmethod
|
||||
def attach_handler_to_session(browser_session: 'BrowserSession', event_class: type[BaseEvent[Any]], handler) -> None:
|
||||
"""Attach a single event handler to a browser session.
|
||||
|
||||
Args:
|
||||
browser_session: The browser session to attach to
|
||||
event_class: The event class to listen for
|
||||
handler: The handler method (must start with 'on_' and end with event type)
|
||||
"""
|
||||
event_bus = browser_session.event_bus
|
||||
|
||||
# Validate handler naming convention
|
||||
assert hasattr(handler, '__name__'), 'Handler must have a __name__ attribute'
|
||||
assert handler.__name__.startswith('on_'), f'Handler {handler.__name__} must start with "on_"'
|
||||
assert handler.__name__.endswith(event_class.__name__), (
|
||||
f'Handler {handler.__name__} must end with event type {event_class.__name__}'
|
||||
)
|
||||
|
||||
# Get the watchdog instance if this is a bound method
|
||||
watchdog_instance = getattr(handler, '__self__', None)
|
||||
watchdog_class_name = watchdog_instance.__class__.__name__ if watchdog_instance else 'Unknown'
|
||||
|
||||
# Events that should always run even when CDP is disconnected (lifecycle management)
|
||||
LIFECYCLE_EVENT_NAMES = frozenset(
|
||||
{
|
||||
'BrowserStartEvent',
|
||||
'BrowserStopEvent',
|
||||
'BrowserStoppedEvent',
|
||||
'BrowserLaunchEvent',
|
||||
'BrowserErrorEvent',
|
||||
'BrowserKillEvent',
|
||||
'BrowserReconnectingEvent',
|
||||
'BrowserReconnectedEvent',
|
||||
}
|
||||
)
|
||||
|
||||
# Create a wrapper function with unique name to avoid duplicate handler warnings
|
||||
# Capture handler by value to avoid closure issues
|
||||
def make_unique_handler(actual_handler):
|
||||
async def unique_handler(event):
|
||||
# Circuit breaker: skip handler if CDP WebSocket is dead
|
||||
# (prevents handlers from hanging on broken connections until timeout)
|
||||
# Lifecycle events are exempt — they manage browser start/stop
|
||||
if event.event_type not in LIFECYCLE_EVENT_NAMES and not browser_session.is_cdp_connected:
|
||||
# If reconnection is in progress, wait for it instead of silently skipping
|
||||
if browser_session.is_reconnecting:
|
||||
wait_timeout = browser_session.RECONNECT_WAIT_TIMEOUT
|
||||
browser_session.logger.debug(
|
||||
f'🚌 [{watchdog_class_name}.{actual_handler.__name__}] ⏳ Waiting for reconnection ({wait_timeout}s)...'
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(browser_session._reconnect_event.wait(), timeout=wait_timeout)
|
||||
except TimeoutError:
|
||||
raise ConnectionError(
|
||||
f'[{watchdog_class_name}.{actual_handler.__name__}] '
|
||||
f'Reconnection wait timed out after {wait_timeout}s'
|
||||
)
|
||||
# After wait: check if reconnection actually succeeded
|
||||
if not browser_session.is_cdp_connected:
|
||||
raise ConnectionError(
|
||||
f'[{watchdog_class_name}.{actual_handler.__name__}] Reconnection failed — CDP still not connected'
|
||||
)
|
||||
# Reconnection succeeded — fall through to execute handler normally
|
||||
else:
|
||||
# Not reconnecting — intentional stop, backward compat silent skip
|
||||
browser_session.logger.debug(
|
||||
f'🚌 [{watchdog_class_name}.{actual_handler.__name__}] ⚡ Skipped — CDP not connected'
|
||||
)
|
||||
return None
|
||||
|
||||
# just for debug logging, not used for anything else
|
||||
parent_event = event_bus.event_history.get(event.event_parent_id) if event.event_parent_id else None
|
||||
grandparent_event = (
|
||||
event_bus.event_history.get(parent_event.event_parent_id)
|
||||
if parent_event and parent_event.event_parent_id
|
||||
else None
|
||||
)
|
||||
parent = (
|
||||
f'↲ triggered by on_{parent_event.event_type}#{parent_event.event_id[-4:]}'
|
||||
if parent_event
|
||||
else '👈 by Agent'
|
||||
)
|
||||
grandparent = (
|
||||
(
|
||||
f'↲ under {grandparent_event.event_type}#{grandparent_event.event_id[-4:]}'
|
||||
if grandparent_event
|
||||
else '👈 by Agent'
|
||||
)
|
||||
if parent_event
|
||||
else ''
|
||||
)
|
||||
event_str = f'#{event.event_id[-4:]}'
|
||||
time_start = time.time()
|
||||
watchdog_and_handler_str = f'[{watchdog_class_name}.{actual_handler.__name__}({event_str})]'.ljust(54)
|
||||
browser_session.logger.debug(f'🚌 {watchdog_and_handler_str} ⏳ Starting... {parent} {grandparent}')
|
||||
|
||||
try:
|
||||
# **EXECUTE THE EVENT HANDLER FUNCTION**
|
||||
result = await actual_handler(event)
|
||||
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
|
||||
# just for debug logging, not used for anything else
|
||||
time_end = time.time()
|
||||
time_elapsed = time_end - time_start
|
||||
result_summary = '' if result is None else f' ➡️ <{type(result).__name__}>'
|
||||
parents_summary = f' {parent}'.replace('↲ triggered by ', '⤴ returned to ').replace(
|
||||
'👈 by Agent', '👉 returned to Agent'
|
||||
)
|
||||
browser_session.logger.debug(
|
||||
f'🚌 {watchdog_and_handler_str} Succeeded ({time_elapsed:.2f}s){result_summary}{parents_summary}'
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
time_end = time.time()
|
||||
time_elapsed = time_end - time_start
|
||||
original_error = e
|
||||
browser_session.logger.error(
|
||||
f'🚌 {watchdog_and_handler_str} ❌ Failed ({time_elapsed:.2f}s): {type(e).__name__}: {e}'
|
||||
)
|
||||
|
||||
# attempt to repair potentially crashed CDP session
|
||||
try:
|
||||
if browser_session.agent_focus_target_id:
|
||||
# With event-driven sessions, Chrome will send detach/attach events
|
||||
# SessionManager handles pool cleanup automatically
|
||||
target_id_to_restore = browser_session.agent_focus_target_id
|
||||
browser_session.logger.debug(
|
||||
f'🚌 {watchdog_and_handler_str} ⚠️ Session error detected, waiting for CDP events to sync (target: {target_id_to_restore})'
|
||||
)
|
||||
|
||||
# Wait for new attach event to restore the session
|
||||
# This will raise ValueError if target doesn't re-attach
|
||||
await browser_session.get_or_create_cdp_session(target_id=target_id_to_restore, focus=True)
|
||||
else:
|
||||
# Try to get any available session
|
||||
await browser_session.get_or_create_cdp_session(target_id=None, focus=True)
|
||||
except Exception as sub_error:
|
||||
if 'ConnectionClosedError' in str(type(sub_error)) or 'ConnectionError' in str(type(sub_error)):
|
||||
browser_session.logger.error(
|
||||
f'🚌 {watchdog_and_handler_str} ❌ Browser closed or CDP Connection disconnected by remote. {type(sub_error).__name__}: {sub_error}\n'
|
||||
)
|
||||
raise
|
||||
else:
|
||||
browser_session.logger.error(
|
||||
f'🚌 {watchdog_and_handler_str} ❌ CDP connected but failed to re-create CDP session after error "{type(original_error).__name__}: {original_error}" in {actual_handler.__name__}({event.event_type}#{event.event_id[-4:]}): due to {type(sub_error).__name__}: {sub_error}\n'
|
||||
)
|
||||
|
||||
# Always re-raise the original error with its traceback preserved
|
||||
raise
|
||||
|
||||
return unique_handler
|
||||
|
||||
unique_handler = make_unique_handler(handler)
|
||||
unique_handler.__name__ = f'{watchdog_class_name}.{handler.__name__}'
|
||||
|
||||
# Check if this handler is already registered - throw error if duplicate
|
||||
existing_handlers = event_bus.handlers.get(event_class.__name__, [])
|
||||
handler_names = [getattr(h, '__name__', str(h)) for h in existing_handlers]
|
||||
|
||||
if unique_handler.__name__ in handler_names:
|
||||
raise RuntimeError(
|
||||
f'[{watchdog_class_name}] Duplicate handler registration attempted! '
|
||||
f'Handler {unique_handler.__name__} is already registered for {event_class.__name__}. '
|
||||
f'This likely means attach_to_session() was called multiple times.'
|
||||
)
|
||||
|
||||
event_bus.on(event_class, unique_handler)
|
||||
|
||||
@staticmethod
|
||||
def detach_handler_from_session(browser_session: 'BrowserSession', event_class: type[BaseEvent[Any]], handler) -> None:
|
||||
"""Detach a single event handler from a browser session."""
|
||||
event_bus = browser_session.event_bus
|
||||
|
||||
# Get the watchdog instance if this is a bound method
|
||||
watchdog_instance = getattr(handler, '__self__', None)
|
||||
watchdog_class_name = watchdog_instance.__class__.__name__ if watchdog_instance else 'Unknown'
|
||||
|
||||
# Find and remove the handler by its unique name pattern
|
||||
unique_handler_name = f'{watchdog_class_name}.{handler.__name__}'
|
||||
|
||||
existing_handlers = event_bus.handlers.get(event_class.__name__, [])
|
||||
for existing_handler in existing_handlers[:]: # copy list to allow modification during iteration
|
||||
if getattr(existing_handler, '__name__', '') == unique_handler_name:
|
||||
existing_handlers.remove(existing_handler)
|
||||
break
|
||||
|
||||
def attach_to_session(self) -> None:
|
||||
"""Attach watchdog to its browser session and start monitoring.
|
||||
|
||||
This method handles event listener registration. The watchdog is already
|
||||
bound to a browser session via self.browser_session from initialization.
|
||||
"""
|
||||
# Register event handlers automatically based on method names
|
||||
assert self.browser_session is not None, 'Root CDP client not initialized - browser may not be connected yet'
|
||||
|
||||
from browser_use.browser import events
|
||||
|
||||
event_classes = {}
|
||||
for name in dir(events):
|
||||
obj = getattr(events, name)
|
||||
if inspect.isclass(obj) and issubclass(obj, BaseEvent) and obj is not BaseEvent:
|
||||
event_classes[name] = obj
|
||||
|
||||
# Find all handler methods (on_EventName)
|
||||
registered_events = set()
|
||||
for method_name in dir(self):
|
||||
if method_name.startswith('on_') and callable(getattr(self, method_name)):
|
||||
# Extract event name from method name (on_EventName -> EventName)
|
||||
event_name = method_name[3:] # Remove 'on_' prefix
|
||||
|
||||
if event_name in event_classes:
|
||||
event_class = event_classes[event_name]
|
||||
|
||||
# ASSERTION: If LISTENS_TO is defined, enforce it
|
||||
if self.LISTENS_TO:
|
||||
assert event_class in self.LISTENS_TO, (
|
||||
f'[{self.__class__.__name__}] Handler {method_name} listens to {event_name} '
|
||||
f'but {event_name} is not declared in LISTENS_TO: {[e.__name__ for e in self.LISTENS_TO]}'
|
||||
)
|
||||
|
||||
handler = getattr(self, method_name)
|
||||
|
||||
# Use the static helper to attach the handler
|
||||
self.attach_handler_to_session(self.browser_session, event_class, handler)
|
||||
registered_events.add(event_class)
|
||||
|
||||
# ASSERTION: If LISTENS_TO is defined, ensure all declared events have handlers
|
||||
if self.LISTENS_TO:
|
||||
missing_handlers = set(self.LISTENS_TO) - registered_events
|
||||
if missing_handlers:
|
||||
missing_names = [e.__name__ for e in missing_handlers]
|
||||
self.logger.warning(
|
||||
f'[{self.__class__.__name__}] LISTENS_TO declares {missing_names} '
|
||||
f'but no handlers found (missing on_{"_, on_".join(missing_names)} methods)'
|
||||
)
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""Clean up any running tasks during garbage collection."""
|
||||
|
||||
# A BIT OF MAGIC: Cancel any private attributes that look like asyncio tasks
|
||||
try:
|
||||
for attr_name in dir(self):
|
||||
# e.g. _browser_crash_watcher_task = asyncio.Task
|
||||
if attr_name.startswith('_') and attr_name.endswith('_task'):
|
||||
try:
|
||||
task = getattr(self, attr_name)
|
||||
if hasattr(task, 'cancel') and callable(task.cancel) and not task.done():
|
||||
task.cancel()
|
||||
# self.logger.debug(f'[{self.__class__.__name__}] Cancelled {attr_name} during cleanup')
|
||||
except Exception:
|
||||
pass # Ignore errors during cleanup
|
||||
|
||||
# e.g. _cdp_download_tasks = WeakSet[asyncio.Task] or list[asyncio.Task]
|
||||
if attr_name.startswith('_') and attr_name.endswith('_tasks') and isinstance(getattr(self, attr_name), Iterable):
|
||||
for task in getattr(self, attr_name):
|
||||
try:
|
||||
if hasattr(task, 'cancel') and callable(task.cancel) and not task.done():
|
||||
task.cancel()
|
||||
# self.logger.debug(f'[{self.__class__.__name__}] Cancelled {attr_name} during cleanup')
|
||||
except Exception:
|
||||
pass # Ignore errors during cleanup
|
||||
except Exception as e:
|
||||
from browser_use.utils import logger
|
||||
|
||||
logger.error(f'⚠️ Error during BrowserSession {self.__class__.__name__} garbage collection __del__(): {type(e)}: {e}')
|
||||
@@ -0,0 +1,259 @@
|
||||
"""About:blank watchdog for managing about:blank tabs with DVD screensaver."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.target import TargetID
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
AboutBlankDVDScreensaverShownEvent,
|
||||
BrowserStopEvent,
|
||||
BrowserStoppedEvent,
|
||||
CloseTabEvent,
|
||||
NavigateToUrlEvent,
|
||||
TabClosedEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class AboutBlankWatchdog(BaseWatchdog):
|
||||
"""Ensures there's always exactly one about:blank tab with DVD screensaver."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserStopEvent,
|
||||
BrowserStoppedEvent,
|
||||
TabCreatedEvent,
|
||||
TabClosedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
NavigateToUrlEvent,
|
||||
CloseTabEvent,
|
||||
AboutBlankDVDScreensaverShownEvent,
|
||||
]
|
||||
|
||||
_stopping: bool = PrivateAttr(default=False)
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""Handle browser stop request - stop creating new tabs."""
|
||||
# logger.info('[AboutBlankWatchdog] Browser stop requested, stopping tab creation')
|
||||
self._stopping = True
|
||||
|
||||
async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
|
||||
"""Handle browser stopped event."""
|
||||
# logger.info('[AboutBlankWatchdog] Browser stopped')
|
||||
self._stopping = True
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Check tabs when a new tab is created."""
|
||||
# logger.debug(f'[AboutBlankWatchdog] ➕ New tab created: {event.url}')
|
||||
|
||||
# If an about:blank tab was created, show DVD screensaver on all about:blank tabs
|
||||
if event.url == 'about:blank':
|
||||
await self._show_dvd_screensaver_on_about_blank_tabs()
|
||||
|
||||
async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
|
||||
"""Check tabs when a tab is closed and proactively create about:blank if needed."""
|
||||
# Don't create new tabs if browser is shutting down
|
||||
if self._stopping:
|
||||
return
|
||||
|
||||
# Don't attempt CDP operations if the WebSocket is dead — dispatching
|
||||
# NavigateToUrlEvent on a broken connection will hang until timeout
|
||||
if not self.browser_session.is_cdp_connected:
|
||||
self.logger.debug('[AboutBlankWatchdog] CDP not connected, skipping tab recovery')
|
||||
return
|
||||
|
||||
# Check if we're about to close the last tab (event happens BEFORE tab closes)
|
||||
# Use _cdp_get_all_pages for quick check without fetching titles
|
||||
page_targets = await self.browser_session._cdp_get_all_pages()
|
||||
if len(page_targets) < 1:
|
||||
self.logger.debug(
|
||||
'[AboutBlankWatchdog] Last tab closing, creating new about:blank tab to avoid closing entire browser'
|
||||
)
|
||||
# Create the animation tab since no tabs should remain
|
||||
navigate_event = self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
|
||||
await navigate_event
|
||||
# Show DVD screensaver on the new tab
|
||||
await self._show_dvd_screensaver_on_about_blank_tabs()
|
||||
else:
|
||||
# Multiple tabs exist, check after close
|
||||
await self._check_and_ensure_about_blank_tab()
|
||||
|
||||
async def attach_to_target(self, target_id: TargetID) -> None:
|
||||
"""AboutBlankWatchdog doesn't monitor individual targets."""
|
||||
pass
|
||||
|
||||
async def _check_and_ensure_about_blank_tab(self) -> None:
|
||||
"""Check current tabs and ensure exactly one about:blank tab with animation exists."""
|
||||
try:
|
||||
if not self.browser_session.is_cdp_connected:
|
||||
return
|
||||
|
||||
# For quick checks, just get page targets without titles to reduce noise
|
||||
page_targets = await self.browser_session._cdp_get_all_pages()
|
||||
|
||||
# If no tabs exist at all, create one to keep browser alive
|
||||
if len(page_targets) == 0:
|
||||
# Only create a new tab if there are no tabs at all
|
||||
self.logger.debug('[AboutBlankWatchdog] No tabs exist, creating new about:blank DVD screensaver tab')
|
||||
navigate_event = self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
|
||||
await navigate_event
|
||||
# Show DVD screensaver on the new tab
|
||||
await self._show_dvd_screensaver_on_about_blank_tabs()
|
||||
# Otherwise there are tabs, don't create new ones to avoid interfering
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[AboutBlankWatchdog] Error ensuring about:blank tab: {e}')
|
||||
|
||||
async def _show_dvd_screensaver_on_about_blank_tabs(self) -> None:
|
||||
"""Show DVD screensaver on all about:blank pages only."""
|
||||
try:
|
||||
# Get just the page targets without expensive title fetching
|
||||
page_targets = await self.browser_session._cdp_get_all_pages()
|
||||
browser_session_label = str(self.browser_session.id)[-4:]
|
||||
|
||||
for page_target in page_targets:
|
||||
target_id = page_target['targetId']
|
||||
url = page_target['url']
|
||||
|
||||
# Only target about:blank pages specifically
|
||||
if url == 'about:blank':
|
||||
await self._show_dvd_screensaver_loading_animation_cdp(target_id, browser_session_label)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[AboutBlankWatchdog] Error showing DVD screensaver: {e}')
|
||||
|
||||
async def _show_dvd_screensaver_loading_animation_cdp(self, target_id: TargetID, browser_session_label: str) -> None:
|
||||
"""
|
||||
Injects a DVD screensaver-style bouncing logo loading animation overlay into the target using CDP.
|
||||
This is used to visually indicate that the browser is setting up or waiting.
|
||||
"""
|
||||
try:
|
||||
# Create temporary session for this target without switching focus
|
||||
temp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
|
||||
|
||||
# Inject the DVD screensaver script (from main branch with idempotency added)
|
||||
script = f"""
|
||||
(function(browser_session_label) {{
|
||||
// Idempotency check
|
||||
if (window.__dvdAnimationRunning) {{
|
||||
return; // Already running, don't add another
|
||||
}}
|
||||
window.__dvdAnimationRunning = true;
|
||||
|
||||
// Ensure document.body exists before proceeding
|
||||
if (!document.body) {{
|
||||
// Try again after DOM is ready
|
||||
window.__dvdAnimationRunning = false; // Reset flag to retry
|
||||
if (document.readyState === 'loading') {{
|
||||
document.addEventListener('DOMContentLoaded', () => arguments.callee(browser_session_label));
|
||||
}}
|
||||
return;
|
||||
}}
|
||||
|
||||
const animated_title = `Starting agent ${{browser_session_label}}...`;
|
||||
if (document.title === animated_title) {{
|
||||
return; // already run on this tab, dont run again
|
||||
}}
|
||||
document.title = animated_title;
|
||||
|
||||
// Create the main overlay
|
||||
const loadingOverlay = document.createElement('div');
|
||||
loadingOverlay.id = 'pretty-loading-animation';
|
||||
loadingOverlay.style.position = 'fixed';
|
||||
loadingOverlay.style.top = '0';
|
||||
loadingOverlay.style.left = '0';
|
||||
loadingOverlay.style.width = '100vw';
|
||||
loadingOverlay.style.height = '100vh';
|
||||
loadingOverlay.style.background = '#000';
|
||||
loadingOverlay.style.zIndex = '99999';
|
||||
loadingOverlay.style.overflow = 'hidden';
|
||||
|
||||
// Create the image element
|
||||
const img = document.createElement('img');
|
||||
img.src = 'https://cf.browser-use.com/logo.svg';
|
||||
img.alt = 'Browser-Use';
|
||||
img.style.width = '200px';
|
||||
img.style.height = 'auto';
|
||||
img.style.position = 'absolute';
|
||||
img.style.left = '0px';
|
||||
img.style.top = '0px';
|
||||
img.style.zIndex = '2';
|
||||
img.style.opacity = '0.8';
|
||||
|
||||
loadingOverlay.appendChild(img);
|
||||
document.body.appendChild(loadingOverlay);
|
||||
|
||||
// DVD screensaver bounce logic
|
||||
let x = Math.random() * (window.innerWidth - 300);
|
||||
let y = Math.random() * (window.innerHeight - 300);
|
||||
let dx = 1.2 + Math.random() * 0.4; // px per frame
|
||||
let dy = 1.2 + Math.random() * 0.4;
|
||||
// Randomize direction
|
||||
if (Math.random() > 0.5) dx = -dx;
|
||||
if (Math.random() > 0.5) dy = -dy;
|
||||
|
||||
function animate() {{
|
||||
const imgWidth = img.offsetWidth || 300;
|
||||
const imgHeight = img.offsetHeight || 300;
|
||||
x += dx;
|
||||
y += dy;
|
||||
|
||||
if (x <= 0) {{
|
||||
x = 0;
|
||||
dx = Math.abs(dx);
|
||||
}} else if (x + imgWidth >= window.innerWidth) {{
|
||||
x = window.innerWidth - imgWidth;
|
||||
dx = -Math.abs(dx);
|
||||
}}
|
||||
if (y <= 0) {{
|
||||
y = 0;
|
||||
dy = Math.abs(dy);
|
||||
}} else if (y + imgHeight >= window.innerHeight) {{
|
||||
y = window.innerHeight - imgHeight;
|
||||
dy = -Math.abs(dy);
|
||||
}}
|
||||
|
||||
img.style.left = `${{x}}px`;
|
||||
img.style.top = `${{y}}px`;
|
||||
|
||||
requestAnimationFrame(animate);
|
||||
}}
|
||||
animate();
|
||||
|
||||
// Responsive: update bounds on resize
|
||||
window.addEventListener('resize', () => {{
|
||||
x = Math.min(x, window.innerWidth - img.offsetWidth);
|
||||
y = Math.min(y, window.innerHeight - img.offsetHeight);
|
||||
}});
|
||||
|
||||
// Add a little CSS for smoothness
|
||||
const style = document.createElement('style');
|
||||
style.textContent = `
|
||||
#pretty-loading-animation {{
|
||||
/*backdrop-filter: blur(2px) brightness(0.9);*/
|
||||
}}
|
||||
#pretty-loading-animation img {{
|
||||
user-select: none;
|
||||
pointer-events: none;
|
||||
}}
|
||||
`;
|
||||
document.head.appendChild(style);
|
||||
}})('{browser_session_label}');
|
||||
"""
|
||||
|
||||
await temp_session.cdp_client.send.Runtime.evaluate(params={'expression': script}, session_id=temp_session.session_id)
|
||||
|
||||
# No need to detach - session is cached
|
||||
|
||||
# Dispatch event
|
||||
self.event_bus.dispatch(AboutBlankDVDScreensaverShownEvent(target_id=target_id))
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[AboutBlankWatchdog] Error injecting DVD screensaver: {e}')
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Captcha solver watchdog — monitors captcha events from the browser proxy.
|
||||
|
||||
Listens for BrowserUse.captchaSolverStarted/Finished CDP events and exposes a
|
||||
wait_if_captcha_solving() method that the agent step loop uses to block until
|
||||
a captcha is resolved (with a configurable timeout).
|
||||
|
||||
NOTE: Only a single captcha solve is tracked at a time. If multiple captchas
|
||||
overlap (e.g. rapid successive navigations), only the latest one is tracked and
|
||||
earlier in-flight waits may return prematurely.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar, Literal
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.browseruse.events import CaptchaSolverFinishedEvent as CDPCaptchaSolverFinishedEvent
|
||||
from cdp_use.cdp.browseruse.events import CaptchaSolverStartedEvent as CDPCaptchaSolverStartedEvent
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserConnectedEvent,
|
||||
BrowserStoppedEvent,
|
||||
CaptchaSolverFinishedEvent,
|
||||
CaptchaSolverStartedEvent,
|
||||
_get_timeout,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
CaptchaResultType = Literal['success', 'failed', 'timeout', 'unknown']
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptchaWaitResult:
|
||||
"""Result returned by wait_if_captcha_solving() when the agent had to wait."""
|
||||
|
||||
waited: bool
|
||||
vendor: str
|
||||
url: str
|
||||
duration_ms: int
|
||||
result: CaptchaResultType
|
||||
|
||||
|
||||
class CaptchaWatchdog(BaseWatchdog):
|
||||
"""Monitors captcha solver events from the browser proxy.
|
||||
|
||||
When the proxy detects a CAPTCHA and starts solving it, a CDP event
|
||||
``BrowserUse.captchaSolverStarted`` is sent over the WebSocket. This
|
||||
watchdog catches that event and blocks the agent's step loop (via
|
||||
``wait_if_captcha_solving``) until ``BrowserUse.captchaSolverFinished``
|
||||
arrives or the configurable timeout expires.
|
||||
"""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
BrowserStoppedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
CaptchaSolverStartedEvent,
|
||||
CaptchaSolverFinishedEvent,
|
||||
]
|
||||
|
||||
# --- private state ---
|
||||
_captcha_solving: bool = PrivateAttr(default=False)
|
||||
_captcha_solved_event: asyncio.Event = PrivateAttr(default_factory=asyncio.Event)
|
||||
_captcha_info: dict[str, Any] = PrivateAttr(default_factory=dict)
|
||||
_captcha_result: CaptchaResultType = PrivateAttr(default='unknown')
|
||||
_captcha_duration_ms: int = PrivateAttr(default=0)
|
||||
_cdp_handlers_registered: bool = PrivateAttr(default=False)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
# Start in "not blocked" state so callers never wait when there is no captcha.
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Event handlers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Register CDP event handlers for BrowserUse captcha solver events."""
|
||||
if self._cdp_handlers_registered:
|
||||
self.logger.debug('CaptchaWatchdog: CDP handlers already registered, skipping')
|
||||
return
|
||||
|
||||
cdp_client = self.browser_session.cdp_client
|
||||
|
||||
def _on_captcha_started(event_data: CDPCaptchaSolverStartedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
self._captcha_solving = True
|
||||
self._captcha_result = 'unknown'
|
||||
self._captcha_duration_ms = 0
|
||||
self._captcha_info = {
|
||||
'vendor': event_data.get('vendor', 'unknown'),
|
||||
'url': event_data.get('url', ''),
|
||||
'targetId': event_data.get('targetId', ''),
|
||||
'startedAt': event_data.get('startedAt', 0),
|
||||
}
|
||||
# Block any waiter
|
||||
self._captcha_solved_event.clear()
|
||||
|
||||
vendor = self._captcha_info['vendor']
|
||||
url = self._captcha_info['url']
|
||||
self.logger.info(f'🔒 Captcha solving started: {vendor} on {url}')
|
||||
|
||||
self.event_bus.dispatch(
|
||||
CaptchaSolverStartedEvent(
|
||||
target_id=event_data.get('targetId', ''),
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
started_at=event_data.get('startedAt', 0),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('Error handling captchaSolverStarted CDP event')
|
||||
# Ensure consistent state: unblock any waiter
|
||||
self._captcha_solving = False
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
def _on_captcha_finished(event_data: CDPCaptchaSolverFinishedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
success = event_data.get('success', False)
|
||||
self._captcha_solving = False
|
||||
self._captcha_duration_ms = event_data.get('durationMs', 0)
|
||||
self._captcha_result = 'success' if success else 'failed'
|
||||
|
||||
vendor = event_data.get('vendor', self._captcha_info.get('vendor', 'unknown'))
|
||||
url = event_data.get('url', self._captcha_info.get('url', ''))
|
||||
duration_s = self._captcha_duration_ms / 1000
|
||||
|
||||
self.logger.info(f'🔓 Captcha solving finished: {self._captcha_result} — {vendor} on {url} ({duration_s:.1f}s)')
|
||||
|
||||
# Unblock any waiter
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
self.event_bus.dispatch(
|
||||
CaptchaSolverFinishedEvent(
|
||||
target_id=event_data.get('targetId', ''),
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
duration_ms=self._captcha_duration_ms,
|
||||
finished_at=event_data.get('finishedAt', 0),
|
||||
success=success,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('Error handling captchaSolverFinished CDP event')
|
||||
# Ensure consistent state: unblock any waiter
|
||||
self._captcha_solving = False
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
cdp_client.register.BrowserUse.captchaSolverStarted(_on_captcha_started)
|
||||
cdp_client.register.BrowserUse.captchaSolverFinished(_on_captcha_finished)
|
||||
self._cdp_handlers_registered = True
|
||||
self.logger.debug('🔒 CaptchaWatchdog: registered CDP event handlers for BrowserUse captcha events')
|
||||
|
||||
async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
|
||||
"""Clear captcha state when the browser disconnects so nothing hangs."""
|
||||
self._captcha_solving = False
|
||||
self._captcha_result = 'unknown'
|
||||
self._captcha_duration_ms = 0
|
||||
self._captcha_info = {}
|
||||
self._captcha_solved_event.set()
|
||||
self._cdp_handlers_registered = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def wait_if_captcha_solving(self, timeout: float | None = None) -> CaptchaWaitResult | None:
|
||||
"""Wait if a captcha is currently being solved.
|
||||
|
||||
Returns:
|
||||
``None`` if no captcha was in progress.
|
||||
A ``CaptchaWaitResult`` with the outcome otherwise.
|
||||
"""
|
||||
if not self._captcha_solving:
|
||||
return None
|
||||
|
||||
if timeout is None:
|
||||
timeout = _get_timeout('TIMEOUT_CaptchaSolverWait', 120.0)
|
||||
assert timeout is not None
|
||||
vendor = self._captcha_info.get('vendor', 'unknown')
|
||||
url = self._captcha_info.get('url', '')
|
||||
self.logger.info(f'⏳ Waiting for {vendor} captcha to be solved on {url} (timeout={timeout}s)...')
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(self._captcha_solved_event.wait(), timeout=timeout)
|
||||
return CaptchaWaitResult(
|
||||
waited=True,
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
duration_ms=self._captcha_duration_ms,
|
||||
result=self._captcha_result,
|
||||
)
|
||||
except TimeoutError:
|
||||
# Timed out — unblock and report
|
||||
self._captcha_solving = False
|
||||
self._captcha_solved_event.set()
|
||||
self.logger.warning(f'⏰ Captcha wait timed out after {timeout}s for {vendor} on {url}')
|
||||
return CaptchaWaitResult(
|
||||
waited=True,
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
duration_ms=int(timeout * 1000),
|
||||
result='timeout',
|
||||
)
|
||||
@@ -0,0 +1,336 @@
|
||||
"""Browser watchdog for monitoring crashes and network timeouts using CDP."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
import psutil
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.target import SessionID, TargetID
|
||||
from cdp_use.cdp.target.events import TargetCrashedEvent
|
||||
from pydantic import Field, PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserConnectedEvent,
|
||||
BrowserErrorEvent,
|
||||
BrowserStoppedEvent,
|
||||
TabClosedEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class NetworkRequestTracker:
|
||||
"""Tracks ongoing network requests."""
|
||||
|
||||
def __init__(self, request_id: str, start_time: float, url: str, method: str, resource_type: str | None = None):
|
||||
self.request_id = request_id
|
||||
self.start_time = start_time
|
||||
self.url = url
|
||||
self.method = method
|
||||
self.resource_type = resource_type
|
||||
|
||||
|
||||
class CrashWatchdog(BaseWatchdog):
|
||||
"""Monitors browser health for crashes and network timeouts using CDP."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
BrowserStoppedEvent,
|
||||
TabCreatedEvent,
|
||||
TabClosedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [BrowserErrorEvent]
|
||||
|
||||
# Configuration
|
||||
network_timeout_seconds: float = Field(default=10.0)
|
||||
check_interval_seconds: float = Field(default=5.0) # Reduced frequency to reduce noise
|
||||
|
||||
# Private state
|
||||
_active_requests: dict[str, NetworkRequestTracker] = PrivateAttr(default_factory=dict)
|
||||
_monitoring_task: asyncio.Task | None = PrivateAttr(default=None)
|
||||
_last_responsive_checks: dict[str, float] = PrivateAttr(default_factory=dict) # target_url -> timestamp
|
||||
_cdp_event_tasks: set[asyncio.Task] = PrivateAttr(default_factory=set) # Track CDP event handler tasks
|
||||
_targets_with_listeners: set[str] = PrivateAttr(default_factory=set) # Track targets that already have event listeners
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Start monitoring when browser is connected."""
|
||||
# logger.debug('[CrashWatchdog] Browser connected event received, beginning monitoring')
|
||||
|
||||
create_task_with_error_handling(
|
||||
self._start_monitoring(), name='start_crash_monitoring', logger_instance=self.logger, suppress_exceptions=True
|
||||
)
|
||||
# logger.debug(f'[CrashWatchdog] Monitoring task started: {self._monitoring_task and not self._monitoring_task.done()}')
|
||||
|
||||
async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
|
||||
"""Stop monitoring when browser stops."""
|
||||
# logger.debug('[CrashWatchdog] Browser stopped, ending monitoring')
|
||||
await self._stop_monitoring()
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Attach to new tab."""
|
||||
assert self.browser_session.agent_focus_target_id is not None, 'No current target ID'
|
||||
await self.attach_to_target(self.browser_session.agent_focus_target_id)
|
||||
|
||||
async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
|
||||
"""Clean up tracking when tab closes."""
|
||||
# Remove target from listener tracking to prevent memory leak
|
||||
if event.target_id in self._targets_with_listeners:
|
||||
self._targets_with_listeners.discard(event.target_id)
|
||||
self.logger.debug(f'[CrashWatchdog] Removed target {event.target_id[:8]}... from monitoring')
|
||||
|
||||
async def attach_to_target(self, target_id: TargetID) -> None:
|
||||
"""Set up crash monitoring for a specific target using CDP."""
|
||||
try:
|
||||
# Check if we already have listeners for this target
|
||||
if target_id in self._targets_with_listeners:
|
||||
self.logger.debug(f'[CrashWatchdog] Event listeners already exist for target: {target_id[:8]}...')
|
||||
return
|
||||
|
||||
# Create temporary session for monitoring without switching focus
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
|
||||
|
||||
# Register crash event handler
|
||||
def on_target_crashed(event: TargetCrashedEvent, session_id: SessionID | None = None):
|
||||
# Create and track the task
|
||||
task = create_task_with_error_handling(
|
||||
self._on_target_crash_cdp(target_id),
|
||||
name='handle_target_crash',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
self._cdp_event_tasks.add(task)
|
||||
# Remove from set when done
|
||||
task.add_done_callback(lambda t: self._cdp_event_tasks.discard(t))
|
||||
|
||||
cdp_session.cdp_client.register.Target.targetCrashed(on_target_crashed)
|
||||
|
||||
# Track that we've added listeners to this target
|
||||
self._targets_with_listeners.add(target_id)
|
||||
|
||||
target = self.browser_session.session_manager.get_target(target_id)
|
||||
if target:
|
||||
self.logger.debug(f'[CrashWatchdog] Added target to monitoring: {target.url}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f'[CrashWatchdog] Failed to attach to target {target_id}: {e}')
|
||||
|
||||
async def _on_request_cdp(self, event: dict) -> None:
|
||||
"""Track new network request from CDP event."""
|
||||
request_id = event.get('requestId', '')
|
||||
request = event.get('request', {})
|
||||
|
||||
self._active_requests[request_id] = NetworkRequestTracker(
|
||||
request_id=request_id,
|
||||
start_time=time.time(),
|
||||
url=request.get('url', ''),
|
||||
method=request.get('method', ''),
|
||||
resource_type=event.get('type'),
|
||||
)
|
||||
# logger.debug(f'[CrashWatchdog] Tracking request: {request.get("method", "")} {request.get("url", "")[:50]}...')
|
||||
|
||||
def _on_response_cdp(self, event: dict) -> None:
|
||||
"""Remove request from tracking on response."""
|
||||
request_id = event.get('requestId', '')
|
||||
if request_id in self._active_requests:
|
||||
elapsed = time.time() - self._active_requests[request_id].start_time
|
||||
response = event.get('response', {})
|
||||
self.logger.debug(f'[CrashWatchdog] Request completed in {elapsed:.2f}s: {response.get("url", "")[:50]}...')
|
||||
# Don't remove yet - wait for loadingFinished
|
||||
|
||||
def _on_request_failed_cdp(self, event: dict) -> None:
|
||||
"""Remove request from tracking on failure."""
|
||||
request_id = event.get('requestId', '')
|
||||
if request_id in self._active_requests:
|
||||
elapsed = time.time() - self._active_requests[request_id].start_time
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Request failed after {elapsed:.2f}s: {self._active_requests[request_id].url[:50]}...'
|
||||
)
|
||||
del self._active_requests[request_id]
|
||||
|
||||
def _on_request_finished_cdp(self, event: dict) -> None:
|
||||
"""Remove request from tracking when loading is finished."""
|
||||
request_id = event.get('requestId', '')
|
||||
self._active_requests.pop(request_id, None)
|
||||
|
||||
async def _on_target_crash_cdp(self, target_id: TargetID) -> None:
|
||||
"""Handle target crash detected via CDP."""
|
||||
self.logger.debug(f'[CrashWatchdog] Target crashed: {target_id[:8]}..., waiting for detach event')
|
||||
|
||||
target = self.browser_session.session_manager.get_target(target_id)
|
||||
|
||||
is_agent_focus = (
|
||||
target
|
||||
and self.browser_session.agent_focus_target_id
|
||||
and target.target_id == self.browser_session.agent_focus_target_id
|
||||
)
|
||||
|
||||
if is_agent_focus:
|
||||
self.logger.error(f'[CrashWatchdog] 💥 Agent focus tab crashed: {target.url} (SessionManager will auto-recover)')
|
||||
|
||||
# Emit browser error event
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='TargetCrash',
|
||||
message=f'Target crashed: {target_id}',
|
||||
details={
|
||||
'url': target.url if target else None,
|
||||
'target_id': target_id,
|
||||
'was_agent_focus': is_agent_focus,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
async def _start_monitoring(self) -> None:
|
||||
"""Start the monitoring loop."""
|
||||
assert self.browser_session.cdp_client is not None, 'Root CDP client not initialized - browser may not be connected yet'
|
||||
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
# logger.info('[CrashWatchdog] Monitoring already running')
|
||||
return
|
||||
|
||||
self._monitoring_task = create_task_with_error_handling(
|
||||
self._monitoring_loop(), name='crash_monitoring_loop', logger_instance=self.logger, suppress_exceptions=True
|
||||
)
|
||||
# logger.debug('[CrashWatchdog] Monitoring loop created and started')
|
||||
|
||||
async def _stop_monitoring(self) -> None:
|
||||
"""Stop the monitoring loop and clean up all tracking."""
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
self._monitoring_task.cancel()
|
||||
try:
|
||||
await self._monitoring_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.logger.debug('[CrashWatchdog] Monitoring loop stopped')
|
||||
|
||||
# Cancel all CDP event handler tasks
|
||||
for task in list(self._cdp_event_tasks):
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
# Wait for all tasks to complete cancellation
|
||||
if self._cdp_event_tasks:
|
||||
await asyncio.gather(*self._cdp_event_tasks, return_exceptions=True)
|
||||
self._cdp_event_tasks.clear()
|
||||
|
||||
# Clear all tracking
|
||||
self._active_requests.clear()
|
||||
self._targets_with_listeners.clear()
|
||||
self._last_responsive_checks.clear()
|
||||
|
||||
async def _monitoring_loop(self) -> None:
|
||||
"""Main monitoring loop."""
|
||||
await asyncio.sleep(10) # give browser time to start up and load the first page after first LLM call
|
||||
while True:
|
||||
try:
|
||||
await self._check_network_timeouts()
|
||||
await self._check_browser_health()
|
||||
await asyncio.sleep(self.check_interval_seconds)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.error(f'[CrashWatchdog] Error in monitoring loop: {e}')
|
||||
|
||||
async def _check_network_timeouts(self) -> None:
|
||||
"""Check for network requests exceeding timeout."""
|
||||
current_time = time.time()
|
||||
timed_out_requests = []
|
||||
|
||||
# Debug logging
|
||||
if self._active_requests:
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Checking {len(self._active_requests)} active requests for timeouts (threshold: {self.network_timeout_seconds}s)'
|
||||
)
|
||||
|
||||
for request_id, tracker in self._active_requests.items():
|
||||
elapsed = current_time - tracker.start_time
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Request {tracker.url[:30]}... elapsed: {elapsed:.1f}s, timeout: {self.network_timeout_seconds}s'
|
||||
)
|
||||
if elapsed >= self.network_timeout_seconds:
|
||||
timed_out_requests.append((request_id, tracker))
|
||||
|
||||
# Emit events for timed out requests
|
||||
for request_id, tracker in timed_out_requests:
|
||||
self.logger.warning(
|
||||
f'[CrashWatchdog] Network request timeout after {self.network_timeout_seconds}s: '
|
||||
f'{tracker.method} {tracker.url[:100]}...'
|
||||
)
|
||||
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='NetworkTimeout',
|
||||
message=f'Network request timed out after {self.network_timeout_seconds}s',
|
||||
details={
|
||||
'url': tracker.url,
|
||||
'method': tracker.method,
|
||||
'resource_type': tracker.resource_type,
|
||||
'elapsed_seconds': current_time - tracker.start_time,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# Remove from tracking
|
||||
del self._active_requests[request_id]
|
||||
|
||||
async def _check_browser_health(self) -> None:
|
||||
"""Check if browser and targets are still responsive."""
|
||||
|
||||
try:
|
||||
self.logger.debug(f'[CrashWatchdog] Checking browser health for target {self.browser_session.agent_focus_target_id}')
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
|
||||
for target in self.browser_session.session_manager.get_all_page_targets():
|
||||
if self._is_new_tab_page(target.url) and target.url != 'about:blank':
|
||||
self.logger.debug(f'[CrashWatchdog] Redirecting chrome://new-tab-page/ to about:blank {target.url}')
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(target_id=target.target_id)
|
||||
await cdp_session.cdp_client.send.Page.navigate(
|
||||
params={'url': 'about:blank'}, session_id=cdp_session.session_id
|
||||
)
|
||||
|
||||
# Quick ping to check if session is alive
|
||||
self.logger.debug(f'[CrashWatchdog] Attempting to run simple JS test expression in session {cdp_session} 1+1')
|
||||
await asyncio.wait_for(
|
||||
cdp_session.cdp_client.send.Runtime.evaluate(params={'expression': '1+1'}, session_id=cdp_session.session_id),
|
||||
timeout=1.0,
|
||||
)
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Browser health check passed for target {self.browser_session.agent_focus_target_id}'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(
|
||||
f'[CrashWatchdog] ❌ Crashed/unresponsive session detected for target {self.browser_session.agent_focus_target_id} '
|
||||
f'error: {type(e).__name__}: {e} (Chrome will send detach event, SessionManager will auto-recover)'
|
||||
)
|
||||
|
||||
# Check browser process if we have PID
|
||||
if self.browser_session._local_browser_watchdog and (proc := self.browser_session._local_browser_watchdog._subprocess):
|
||||
try:
|
||||
if proc.status() in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD):
|
||||
self.logger.error(f'[CrashWatchdog] Browser process {proc.pid} has crashed')
|
||||
|
||||
# Browser process crashed - SessionManager will clean up via detach events
|
||||
# Just dispatch error event and stop monitoring
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='BrowserProcessCrashed',
|
||||
message=f'Browser process {proc.pid} has crashed',
|
||||
details={'pid': proc.pid, 'status': proc.status()},
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.warning('[CrashWatchdog] Browser process dead - stopping health monitoring')
|
||||
await self._stop_monitoring()
|
||||
return
|
||||
except Exception:
|
||||
pass # psutil not available or process doesn't exist
|
||||
|
||||
@staticmethod
|
||||
def _is_new_tab_page(url: str) -> bool:
|
||||
"""Check if URL is a new tab page."""
|
||||
return url in ['about:blank', 'chrome://new-tab-page/', 'chrome://newtab/']
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,865 @@
|
||||
"""DOM watchdog for browser DOM tree management using CDP."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserErrorEvent,
|
||||
BrowserStateRequestEvent,
|
||||
ScreenshotEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.dom.views import (
|
||||
EnhancedDOMTreeNode,
|
||||
SerializedDOMState,
|
||||
)
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import create_task_with_error_handling, time_execution_async
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser.views import BrowserStateSummary, NetworkRequest, PageInfo, PaginationButton
|
||||
|
||||
|
||||
class DOMWatchdog(BaseWatchdog):
|
||||
"""Handles DOM tree building, serialization, and element access via CDP.
|
||||
|
||||
This watchdog acts as a bridge between the event-driven browser session
|
||||
and the DomService implementation, maintaining cached state and providing
|
||||
helper methods for other watchdogs.
|
||||
"""
|
||||
|
||||
LISTENS_TO = [TabCreatedEvent, BrowserStateRequestEvent]
|
||||
EMITS = [BrowserErrorEvent]
|
||||
|
||||
# Public properties for other watchdogs
|
||||
selector_map: dict[int, EnhancedDOMTreeNode] | None = None
|
||||
current_dom_state: SerializedDOMState | None = None
|
||||
enhanced_dom_tree: EnhancedDOMTreeNode | None = None
|
||||
|
||||
# Internal DOM service
|
||||
_dom_service: DomService | None = None
|
||||
|
||||
# Network tracking - maps request_id to (url, start_time, method, resource_type)
|
||||
_pending_requests: dict[str, tuple[str, float, str, str | None]] = {}
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
# self.logger.debug('Setting up init scripts in browser')
|
||||
return None
|
||||
|
||||
def _get_recent_events_str(self, limit: int = 10) -> str | None:
|
||||
"""Get the most recent events from the event bus as JSON.
|
||||
|
||||
Args:
|
||||
limit: Maximum number of recent events to include
|
||||
|
||||
Returns:
|
||||
JSON string of recent events or None if not available
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
# Get all events from history, sorted by creation time (most recent first)
|
||||
all_events = sorted(
|
||||
self.browser_session.event_bus.event_history.values(), key=lambda e: e.event_created_at.timestamp(), reverse=True
|
||||
)
|
||||
|
||||
# Take the most recent events and create JSON-serializable data
|
||||
recent_events_data = []
|
||||
for event in all_events[:limit]:
|
||||
event_data = {
|
||||
'event_type': event.event_type,
|
||||
'timestamp': event.event_created_at.isoformat(),
|
||||
}
|
||||
# Add specific fields for certain event types
|
||||
if hasattr(event, 'url'):
|
||||
event_data['url'] = getattr(event, 'url')
|
||||
if hasattr(event, 'error_message'):
|
||||
event_data['error_message'] = getattr(event, 'error_message')
|
||||
if hasattr(event, 'target_id'):
|
||||
event_data['target_id'] = getattr(event, 'target_id')
|
||||
recent_events_data.append(event_data)
|
||||
|
||||
return json.dumps(recent_events_data) # Return empty array if no events
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get recent events: {e}')
|
||||
|
||||
return json.dumps([]) # Return empty JSON array on error
|
||||
|
||||
async def _get_pending_network_requests(self) -> list['NetworkRequest']:
|
||||
"""Get list of currently pending network requests.
|
||||
|
||||
Uses document.readyState and performance API to detect pending requests.
|
||||
Filters out ads, tracking, and other noise.
|
||||
|
||||
Returns:
|
||||
List of NetworkRequest objects representing currently loading resources
|
||||
"""
|
||||
from browser_use.browser.views import NetworkRequest
|
||||
|
||||
try:
|
||||
# get_or_create_cdp_session() now handles focus validation automatically
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
|
||||
|
||||
# Use performance API to get pending requests
|
||||
js_code = """
|
||||
(function() {
|
||||
const now = performance.now();
|
||||
const resources = performance.getEntriesByType('resource');
|
||||
const pending = [];
|
||||
|
||||
// Check document readyState
|
||||
const docLoading = document.readyState !== 'complete';
|
||||
|
||||
// Common ad/tracking domains and patterns to filter out
|
||||
const adDomains = [
|
||||
// Standard ad/tracking networks
|
||||
'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',
|
||||
'facebook.net', 'analytics', 'ads', 'tracking', 'pixel',
|
||||
'hotjar.com', 'clarity.ms', 'mixpanel.com', 'segment.com',
|
||||
// Analytics platforms
|
||||
'demdex.net', 'omtrdc.net', 'adobedtm.com', 'ensighten.com',
|
||||
'newrelic.com', 'nr-data.net', 'google-analytics.com',
|
||||
// Social media trackers
|
||||
'connect.facebook.net', 'platform.twitter.com', 'platform.linkedin.com',
|
||||
// CDN/image hosts (usually not critical for functionality)
|
||||
'.cloudfront.net/image/', '.akamaized.net/image/',
|
||||
// Common tracking paths
|
||||
'/tracker/', '/collector/', '/beacon/', '/telemetry/', '/log/',
|
||||
'/events/', '/eventBatch', '/track.', '/metrics/'
|
||||
];
|
||||
|
||||
// Get resources that are still loading (responseEnd is 0)
|
||||
let totalResourcesChecked = 0;
|
||||
let filteredByResponseEnd = 0;
|
||||
const allDomains = new Set();
|
||||
|
||||
for (const entry of resources) {
|
||||
totalResourcesChecked++;
|
||||
|
||||
// Track all domains from recent resources (for logging)
|
||||
try {
|
||||
const hostname = new URL(entry.name).hostname;
|
||||
if (hostname) allDomains.add(hostname);
|
||||
} catch (e) {}
|
||||
|
||||
if (entry.responseEnd === 0) {
|
||||
filteredByResponseEnd++;
|
||||
const url = entry.name;
|
||||
|
||||
// Filter out ads and tracking
|
||||
const isAd = adDomains.some(domain => url.includes(domain));
|
||||
if (isAd) continue;
|
||||
|
||||
// Filter out data: URLs and very long URLs (often inline resources)
|
||||
if (url.startsWith('data:') || url.length > 500) continue;
|
||||
|
||||
const loadingDuration = now - entry.startTime;
|
||||
|
||||
// Skip requests that have been loading for >10 seconds (likely stuck/polling)
|
||||
if (loadingDuration > 10000) continue;
|
||||
|
||||
const resourceType = entry.initiatorType || 'unknown';
|
||||
|
||||
// Filter out non-critical resources (images, fonts, icons) if loading >3 seconds
|
||||
const nonCriticalTypes = ['img', 'image', 'icon', 'font'];
|
||||
if (nonCriticalTypes.includes(resourceType) && loadingDuration > 3000) continue;
|
||||
|
||||
// Filter out image URLs even if type is unknown
|
||||
const isImageUrl = /\\.(jpg|jpeg|png|gif|webp|svg|ico)(\\?|$)/i.test(url);
|
||||
if (isImageUrl && loadingDuration > 3000) continue;
|
||||
|
||||
pending.push({
|
||||
url: url,
|
||||
method: 'GET',
|
||||
loading_duration_ms: Math.round(loadingDuration),
|
||||
resource_type: resourceType
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
pending_requests: pending,
|
||||
document_loading: docLoading,
|
||||
document_ready_state: document.readyState,
|
||||
debug: {
|
||||
total_resources: totalResourcesChecked,
|
||||
with_response_end_zero: filteredByResponseEnd,
|
||||
after_all_filters: pending.length,
|
||||
all_domains: Array.from(allDomains)
|
||||
}
|
||||
};
|
||||
})()
|
||||
"""
|
||||
|
||||
result = await cdp_session.cdp_client.send.Runtime.evaluate(
|
||||
params={'expression': js_code, 'returnByValue': True}, session_id=cdp_session.session_id
|
||||
)
|
||||
|
||||
if result.get('result', {}).get('type') == 'object':
|
||||
data = result['result'].get('value', {})
|
||||
pending = data.get('pending_requests', [])
|
||||
doc_state = data.get('document_ready_state', 'unknown')
|
||||
doc_loading = data.get('document_loading', False)
|
||||
debug_info = data.get('debug', {})
|
||||
|
||||
# Get all domains that had recent activity (from JS)
|
||||
all_domains = debug_info.get('all_domains', [])
|
||||
all_domains_str = ', '.join(sorted(all_domains)[:5]) if all_domains else 'none'
|
||||
if len(all_domains) > 5:
|
||||
all_domains_str += f' +{len(all_domains) - 5} more'
|
||||
|
||||
# Debug logging
|
||||
self.logger.debug(
|
||||
f'🔍 Network check: document.readyState={doc_state}, loading={doc_loading}, '
|
||||
f'total_resources={debug_info.get("total_resources", 0)}, '
|
||||
f'responseEnd=0: {debug_info.get("with_response_end_zero", 0)}, '
|
||||
f'after_filters={len(pending)}, domains=[{all_domains_str}]'
|
||||
)
|
||||
|
||||
# Convert to NetworkRequest objects
|
||||
network_requests = []
|
||||
for req in pending[:20]: # Limit to 20 to avoid overwhelming the context
|
||||
network_requests.append(
|
||||
NetworkRequest(
|
||||
url=req['url'],
|
||||
method=req.get('method', 'GET'),
|
||||
loading_duration_ms=req.get('loading_duration_ms', 0.0),
|
||||
resource_type=req.get('resource_type'),
|
||||
)
|
||||
)
|
||||
|
||||
return network_requests
|
||||
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get pending network requests: {e}')
|
||||
|
||||
return []
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='browser_state_request_event')
|
||||
async def on_BrowserStateRequestEvent(self, event: BrowserStateRequestEvent) -> 'BrowserStateSummary':
|
||||
"""Handle browser state request by coordinating DOM building and screenshot capture.
|
||||
|
||||
This is the main entry point for getting the complete browser state.
|
||||
|
||||
Args:
|
||||
event: The browser state request event with options
|
||||
|
||||
Returns:
|
||||
Complete BrowserStateSummary with DOM, screenshot, and target info
|
||||
"""
|
||||
from browser_use.browser.views import BrowserStateSummary, PageInfo
|
||||
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: STARTING browser state request')
|
||||
page_url = await self.browser_session.get_current_page_url()
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page URL: {page_url}')
|
||||
|
||||
# Get focused session for logging (validation already done by get_current_page_url)
|
||||
if self.browser_session.agent_focus_target_id:
|
||||
self.logger.debug(f'Current page URL: {page_url}, target_id: {self.browser_session.agent_focus_target_id}')
|
||||
|
||||
# check if we should skip DOM tree build for pointless pages
|
||||
not_a_meaningful_website = page_url.lower().split(':', 1)[0] not in ('http', 'https')
|
||||
|
||||
# Check for pending network requests BEFORE waiting (so we can see what's loading)
|
||||
# Timeout after 2s — on slow CI machines or heavy pages, this call can hang
|
||||
# for 15s+ eating into the 30s BrowserStateRequestEvent budget.
|
||||
pending_requests_before_wait = []
|
||||
if not not_a_meaningful_website:
|
||||
try:
|
||||
pending_requests_before_wait = await asyncio.wait_for(self._get_pending_network_requests(), timeout=2.0)
|
||||
if pending_requests_before_wait:
|
||||
self.logger.debug(f'🔍 Found {len(pending_requests_before_wait)} pending requests before stability wait')
|
||||
except TimeoutError:
|
||||
self.logger.debug('Pending network request check timed out (2s), skipping')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get pending requests before wait: {e}')
|
||||
pending_requests = pending_requests_before_wait
|
||||
# Wait for page stability using browser profile settings (main branch pattern)
|
||||
if not not_a_meaningful_website:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ⏳ Waiting for page stability...')
|
||||
try:
|
||||
if pending_requests_before_wait:
|
||||
# Reduced from 1s to 0.3s for faster DOM builds while still allowing critical resources to load
|
||||
await asyncio.sleep(0.3)
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Page stability complete')
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Network waiting failed: {e}, continuing anyway...'
|
||||
)
|
||||
|
||||
# Get tabs info once at the beginning for all paths
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting tabs info...')
|
||||
tabs_info = await self.browser_session.get_tabs()
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got {len(tabs_info)} tabs')
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Tabs info: {tabs_info}')
|
||||
|
||||
# Get viewport / scroll position info, remember changing scroll position should invalidate selector_map cache because it only includes visible elements
|
||||
# cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
|
||||
# scroll_info = await cdp_session.cdp_client.send.Runtime.evaluate(
|
||||
# params={'expression': 'JSON.stringify({y: document.body.scrollTop, x: document.body.scrollLeft, width: document.documentElement.clientWidth, height: document.documentElement.clientHeight})'},
|
||||
# session_id=cdp_session.session_id,
|
||||
# )
|
||||
# self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got scroll info: {scroll_info["result"]}')
|
||||
|
||||
try:
|
||||
# Fast path for empty pages
|
||||
if not_a_meaningful_website:
|
||||
self.logger.debug(f'⚡ Skipping BuildDOMTree for empty target: {page_url}')
|
||||
self.logger.debug(f'📸 Not taking screenshot for empty page: {page_url} (non-http/https URL)')
|
||||
|
||||
# Create minimal DOM state
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
|
||||
# Skip screenshot for empty pages
|
||||
screenshot_b64 = None
|
||||
|
||||
# Try to get page info from CDP, fall back to defaults if unavailable
|
||||
try:
|
||||
page_info = await self._get_page_info()
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get page info from CDP for empty page: {e}, using fallback')
|
||||
# Use default viewport dimensions
|
||||
viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720}
|
||||
page_info = PageInfo(
|
||||
viewport_width=viewport['width'],
|
||||
viewport_height=viewport['height'],
|
||||
page_width=viewport['width'],
|
||||
page_height=viewport['height'],
|
||||
scroll_x=0,
|
||||
scroll_y=0,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
pixels_left=0,
|
||||
pixels_right=0,
|
||||
)
|
||||
|
||||
return BrowserStateSummary(
|
||||
dom_state=content,
|
||||
url=page_url,
|
||||
title='Empty Tab',
|
||||
tabs=tabs_info,
|
||||
screenshot=screenshot_b64,
|
||||
page_info=page_info,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
browser_errors=[],
|
||||
is_pdf_viewer=False,
|
||||
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
|
||||
pending_network_requests=[], # Empty page has no pending requests
|
||||
pagination_buttons=[], # Empty page has no pagination
|
||||
closed_popup_messages=self.browser_session._closed_popup_messages.copy(),
|
||||
)
|
||||
|
||||
# Execute DOM building and screenshot capture in parallel
|
||||
dom_task = None
|
||||
screenshot_task = None
|
||||
|
||||
# Start DOM building task if requested
|
||||
if event.include_dom:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')
|
||||
|
||||
previous_state = (
|
||||
self.browser_session._cached_browser_state_summary.dom_state
|
||||
if self.browser_session._cached_browser_state_summary
|
||||
else None
|
||||
)
|
||||
|
||||
dom_task = create_task_with_error_handling(
|
||||
self._build_dom_tree_without_highlights(previous_state),
|
||||
name='build_dom_tree',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
# Start clean screenshot task if requested (without JS highlights)
|
||||
if event.include_screenshot:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...')
|
||||
screenshot_task = create_task_with_error_handling(
|
||||
self._capture_clean_screenshot(),
|
||||
name='capture_screenshot',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
# Wait for both tasks to complete
|
||||
content = None
|
||||
screenshot_b64 = None
|
||||
|
||||
if dom_task:
|
||||
try:
|
||||
content = await dom_task
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
else:
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
|
||||
if screenshot_task:
|
||||
try:
|
||||
screenshot_b64 = await screenshot_task
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')
|
||||
screenshot_b64 = None
|
||||
|
||||
# Add browser-side highlights for user visibility
|
||||
if content and content.selector_map and self.browser_session.browser_profile.dom_highlight_elements:
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Adding browser-side highlights...')
|
||||
await self.browser_session.add_highlights(content.selector_map)
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Added browser highlights for {len(content.selector_map)} elements'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Browser highlighting failed: {e}')
|
||||
|
||||
# Ensure we have valid content
|
||||
if not content:
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
|
||||
# Tabs info already fetched at the beginning
|
||||
|
||||
# Get target title safely
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page title...')
|
||||
title = await asyncio.wait_for(self.browser_session.get_current_page_title(), timeout=1.0)
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got title: {title}')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get title: {e}')
|
||||
title = 'Page'
|
||||
|
||||
# Get comprehensive page info from CDP with timeout
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page info from CDP...')
|
||||
page_info = await asyncio.wait_for(self._get_page_info(), timeout=1.0)
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page info from CDP: {page_info}')
|
||||
except Exception as e:
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get page info from CDP: {e}, using fallback'
|
||||
)
|
||||
# Fallback to default viewport dimensions
|
||||
viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720}
|
||||
page_info = PageInfo(
|
||||
viewport_width=viewport['width'],
|
||||
viewport_height=viewport['height'],
|
||||
page_width=viewport['width'],
|
||||
page_height=viewport['height'],
|
||||
scroll_x=0,
|
||||
scroll_y=0,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
pixels_left=0,
|
||||
pixels_right=0,
|
||||
)
|
||||
|
||||
# Check for PDF viewer
|
||||
is_pdf_viewer = page_url.endswith('.pdf') or '/pdf/' in page_url
|
||||
|
||||
# Detect pagination buttons from the DOM
|
||||
pagination_buttons_data = []
|
||||
if content and content.selector_map:
|
||||
pagination_buttons_data = self._detect_pagination_buttons(content.selector_map)
|
||||
|
||||
# Build and cache the browser state summary
|
||||
if screenshot_b64:
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary with screenshot, length: {len(screenshot_b64)}'
|
||||
)
|
||||
else:
|
||||
self.logger.debug(
|
||||
'🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary WITHOUT screenshot'
|
||||
)
|
||||
|
||||
browser_state = BrowserStateSummary(
|
||||
dom_state=content,
|
||||
url=page_url,
|
||||
title=title,
|
||||
tabs=tabs_info,
|
||||
screenshot=screenshot_b64,
|
||||
page_info=page_info,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
browser_errors=[],
|
||||
is_pdf_viewer=is_pdf_viewer,
|
||||
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
|
||||
pending_network_requests=pending_requests,
|
||||
pagination_buttons=pagination_buttons_data,
|
||||
closed_popup_messages=self.browser_session._closed_popup_messages.copy(),
|
||||
)
|
||||
|
||||
# Cache the state
|
||||
self.browser_session._cached_browser_state_summary = browser_state
|
||||
|
||||
# Cache viewport size for coordinate conversion (if llm_screenshot_size is enabled)
|
||||
if page_info:
|
||||
self.browser_session._original_viewport_size = (page_info.viewport_width, page_info.viewport_height)
|
||||
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ COMPLETED - Returning browser state')
|
||||
return browser_state
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'Failed to get browser state: {e}')
|
||||
|
||||
# Return minimal recovery state
|
||||
return BrowserStateSummary(
|
||||
dom_state=SerializedDOMState(_root=None, selector_map={}),
|
||||
url=page_url if 'page_url' in locals() else '',
|
||||
title='Error',
|
||||
tabs=[],
|
||||
screenshot=None,
|
||||
page_info=PageInfo(
|
||||
viewport_width=1280,
|
||||
viewport_height=720,
|
||||
page_width=1280,
|
||||
page_height=720,
|
||||
scroll_x=0,
|
||||
scroll_y=0,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
pixels_left=0,
|
||||
pixels_right=0,
|
||||
),
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
browser_errors=[str(e)],
|
||||
is_pdf_viewer=False,
|
||||
recent_events=None,
|
||||
pending_network_requests=[], # Error state has no pending requests
|
||||
pagination_buttons=[], # Error state has no pagination
|
||||
closed_popup_messages=self.browser_session._closed_popup_messages.copy()
|
||||
if hasattr(self, 'browser_session') and self.browser_session is not None
|
||||
else [],
|
||||
)
|
||||
|
||||
@time_execution_async('build_dom_tree_without_highlights')
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='build_dom_tree_without_highlights')
|
||||
async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:
|
||||
"""Build DOM tree without injecting JavaScript highlights (for parallel execution)."""
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: STARTING DOM tree build')
|
||||
|
||||
# Create or reuse DOM service
|
||||
if self._dom_service is None:
|
||||
self._dom_service = DomService(
|
||||
browser_session=self.browser_session,
|
||||
logger=self.logger,
|
||||
cross_origin_iframes=self.browser_session.browser_profile.cross_origin_iframes,
|
||||
paint_order_filtering=self.browser_session.browser_profile.paint_order_filtering,
|
||||
max_iframes=self.browser_session.browser_profile.max_iframes,
|
||||
max_iframe_depth=self.browser_session.browser_profile.max_iframe_depth,
|
||||
)
|
||||
|
||||
# Get serialized DOM tree using the service
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Calling DomService.get_serialized_dom_tree...')
|
||||
start = time.time()
|
||||
self.current_dom_state, self.enhanced_dom_tree, timing_info = await self._dom_service.get_serialized_dom_tree(
|
||||
previous_cached_state=previous_state,
|
||||
)
|
||||
end = time.time()
|
||||
total_time_ms = (end - start) * 1000
|
||||
self.logger.debug(
|
||||
'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ DomService.get_serialized_dom_tree completed'
|
||||
)
|
||||
|
||||
# Build hierarchical timing breakdown as single multi-line string
|
||||
timing_lines = [f'⏱️ Total DOM tree time: {total_time_ms:.2f}ms', '📊 Timing breakdown:']
|
||||
|
||||
# get_all_trees breakdown
|
||||
get_all_trees_ms = timing_info.get('get_all_trees_total_ms', 0)
|
||||
if get_all_trees_ms > 0:
|
||||
timing_lines.append(f' ├─ get_all_trees: {get_all_trees_ms:.2f}ms')
|
||||
iframe_scroll_ms = timing_info.get('iframe_scroll_detection_ms', 0)
|
||||
cdp_parallel_ms = timing_info.get('cdp_parallel_calls_ms', 0)
|
||||
snapshot_proc_ms = timing_info.get('snapshot_processing_ms', 0)
|
||||
if iframe_scroll_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ iframe_scroll_detection: {iframe_scroll_ms:.2f}ms')
|
||||
if cdp_parallel_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ cdp_parallel_calls: {cdp_parallel_ms:.2f}ms')
|
||||
if snapshot_proc_ms > 0.01:
|
||||
timing_lines.append(f' │ └─ snapshot_processing: {snapshot_proc_ms:.2f}ms')
|
||||
|
||||
# build_ax_lookup
|
||||
build_ax_ms = timing_info.get('build_ax_lookup_ms', 0)
|
||||
if build_ax_ms > 0.01:
|
||||
timing_lines.append(f' ├─ build_ax_lookup: {build_ax_ms:.2f}ms')
|
||||
|
||||
# build_snapshot_lookup
|
||||
build_snapshot_ms = timing_info.get('build_snapshot_lookup_ms', 0)
|
||||
if build_snapshot_ms > 0.01:
|
||||
timing_lines.append(f' ├─ build_snapshot_lookup: {build_snapshot_ms:.2f}ms')
|
||||
|
||||
# construct_enhanced_tree
|
||||
construct_tree_ms = timing_info.get('construct_enhanced_tree_ms', 0)
|
||||
if construct_tree_ms > 0.01:
|
||||
timing_lines.append(f' ├─ construct_enhanced_tree: {construct_tree_ms:.2f}ms')
|
||||
|
||||
# serialize_accessible_elements breakdown
|
||||
serialize_total_ms = timing_info.get('serialize_accessible_elements_total_ms', 0)
|
||||
if serialize_total_ms > 0.01:
|
||||
timing_lines.append(f' ├─ serialize_accessible_elements: {serialize_total_ms:.2f}ms')
|
||||
create_simp_ms = timing_info.get('create_simplified_tree_ms', 0)
|
||||
paint_order_ms = timing_info.get('calculate_paint_order_ms', 0)
|
||||
optimize_ms = timing_info.get('optimize_tree_ms', 0)
|
||||
bbox_ms = timing_info.get('bbox_filtering_ms', 0)
|
||||
assign_idx_ms = timing_info.get('assign_interactive_indices_ms', 0)
|
||||
clickable_ms = timing_info.get('clickable_detection_time_ms', 0)
|
||||
|
||||
if create_simp_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ create_simplified_tree: {create_simp_ms:.2f}ms')
|
||||
if clickable_ms > 0.01:
|
||||
timing_lines.append(f' │ │ └─ clickable_detection: {clickable_ms:.2f}ms')
|
||||
if paint_order_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ calculate_paint_order: {paint_order_ms:.2f}ms')
|
||||
if optimize_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ optimize_tree: {optimize_ms:.2f}ms')
|
||||
if bbox_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ bbox_filtering: {bbox_ms:.2f}ms')
|
||||
if assign_idx_ms > 0.01:
|
||||
timing_lines.append(f' │ └─ assign_interactive_indices: {assign_idx_ms:.2f}ms')
|
||||
|
||||
# Overheads
|
||||
get_dom_overhead_ms = timing_info.get('get_dom_tree_overhead_ms', 0)
|
||||
serialize_overhead_ms = timing_info.get('serialization_overhead_ms', 0)
|
||||
get_serialized_overhead_ms = timing_info.get('get_serialized_dom_tree_overhead_ms', 0)
|
||||
|
||||
if get_dom_overhead_ms > 0.1:
|
||||
timing_lines.append(f' ├─ get_dom_tree_overhead: {get_dom_overhead_ms:.2f}ms')
|
||||
if serialize_overhead_ms > 0.1:
|
||||
timing_lines.append(f' ├─ serialization_overhead: {serialize_overhead_ms:.2f}ms')
|
||||
if get_serialized_overhead_ms > 0.1:
|
||||
timing_lines.append(f' └─ get_serialized_dom_tree_overhead: {get_serialized_overhead_ms:.2f}ms')
|
||||
|
||||
# Calculate total tracked time for validation
|
||||
main_operations_ms = (
|
||||
get_all_trees_ms
|
||||
+ build_ax_ms
|
||||
+ build_snapshot_ms
|
||||
+ construct_tree_ms
|
||||
+ serialize_total_ms
|
||||
+ get_dom_overhead_ms
|
||||
+ serialize_overhead_ms
|
||||
+ get_serialized_overhead_ms
|
||||
)
|
||||
untracked_time_ms = total_time_ms - main_operations_ms
|
||||
|
||||
if untracked_time_ms > 1.0: # Only log if significant
|
||||
timing_lines.append(f' ⚠️ untracked_time: {untracked_time_ms:.2f}ms')
|
||||
|
||||
# Single log call with all timing info
|
||||
self.logger.debug('\n'.join(timing_lines))
|
||||
|
||||
# Update selector map for other watchdogs
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Updating selector maps...')
|
||||
self.selector_map = self.current_dom_state.selector_map
|
||||
# Update BrowserSession's cached selector map
|
||||
if self.browser_session:
|
||||
self.browser_session.update_cached_selector_map(self.selector_map)
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ Selector maps updated, {len(self.selector_map)} elements'
|
||||
)
|
||||
|
||||
# Skip JavaScript highlighting injection - Python highlighting will be applied later
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ COMPLETED DOM tree build (no JS highlights)')
|
||||
return self.current_dom_state
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'Failed to build DOM tree without highlights: {e}')
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='DOMBuildFailed',
|
||||
message=str(e),
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
@time_execution_async('capture_clean_screenshot')
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='capture_clean_screenshot')
|
||||
async def _capture_clean_screenshot(self) -> str:
|
||||
"""Capture a clean screenshot without JavaScript highlights."""
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: Capturing clean screenshot...')
|
||||
|
||||
await self.browser_session.get_or_create_cdp_session(target_id=self.browser_session.agent_focus_target_id, focus=True)
|
||||
|
||||
# Check if handler is registered
|
||||
handlers = self.event_bus.handlers.get('ScreenshotEvent', [])
|
||||
handler_names = [getattr(h, '__name__', str(h)) for h in handlers]
|
||||
self.logger.debug(f'📸 ScreenshotEvent handlers registered: {len(handlers)} - {handler_names}')
|
||||
|
||||
screenshot_event = self.event_bus.dispatch(ScreenshotEvent(full_page=False))
|
||||
self.logger.debug('📸 Dispatched ScreenshotEvent, waiting for event to complete...')
|
||||
|
||||
# Wait for the event itself to complete (this waits for all handlers)
|
||||
await screenshot_event
|
||||
|
||||
# Get the single handler result
|
||||
screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=True)
|
||||
if screenshot_b64 is None:
|
||||
raise RuntimeError('Screenshot handler returned None')
|
||||
self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: ✅ Clean screenshot captured successfully')
|
||||
return str(screenshot_b64)
|
||||
|
||||
except TimeoutError:
|
||||
self.logger.warning('📸 Clean screenshot timed out after 6 seconds - no handler registered or slow page?')
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.warning(f'📸 Clean screenshot failed: {type(e).__name__}: {e}')
|
||||
raise
|
||||
|
||||
def _detect_pagination_buttons(self, selector_map: dict[int, EnhancedDOMTreeNode]) -> list['PaginationButton']:
|
||||
"""Detect pagination buttons from the DOM selector map.
|
||||
|
||||
Args:
|
||||
selector_map: Dictionary mapping element indices to DOM tree nodes
|
||||
|
||||
Returns:
|
||||
List of PaginationButton instances found in the DOM
|
||||
"""
|
||||
from browser_use.browser.views import PaginationButton
|
||||
|
||||
pagination_buttons_data = []
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog._detect_pagination_buttons: Detecting pagination buttons...')
|
||||
pagination_buttons_raw = DomService.detect_pagination_buttons(selector_map)
|
||||
# Convert to PaginationButton instances
|
||||
pagination_buttons_data = [
|
||||
PaginationButton(
|
||||
button_type=btn['button_type'], # type: ignore
|
||||
backend_node_id=btn['backend_node_id'], # type: ignore
|
||||
text=btn['text'], # type: ignore
|
||||
selector=btn['selector'], # type: ignore
|
||||
is_disabled=btn['is_disabled'], # type: ignore
|
||||
)
|
||||
for btn in pagination_buttons_raw
|
||||
]
|
||||
if pagination_buttons_data:
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog._detect_pagination_buttons: Found {len(pagination_buttons_data)} pagination buttons'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog._detect_pagination_buttons: Pagination detection failed: {e}')
|
||||
|
||||
return pagination_buttons_data
|
||||
|
||||
async def _get_page_info(self) -> 'PageInfo':
|
||||
"""Get comprehensive page information using a single CDP call.
|
||||
|
||||
TODO: should we make this an event as well?
|
||||
|
||||
Returns:
|
||||
PageInfo with all viewport, page dimensions, and scroll information
|
||||
"""
|
||||
|
||||
from browser_use.browser.views import PageInfo
|
||||
|
||||
# get_or_create_cdp_session() handles focus validation automatically
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(
|
||||
target_id=self.browser_session.agent_focus_target_id, focus=True
|
||||
)
|
||||
|
||||
# Get layout metrics which includes all the information we need
|
||||
metrics = await asyncio.wait_for(
|
||||
cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id), timeout=10.0
|
||||
)
|
||||
|
||||
# Extract different viewport types
|
||||
layout_viewport = metrics.get('layoutViewport', {})
|
||||
visual_viewport = metrics.get('visualViewport', {})
|
||||
css_visual_viewport = metrics.get('cssVisualViewport', {})
|
||||
css_layout_viewport = metrics.get('cssLayoutViewport', {})
|
||||
content_size = metrics.get('contentSize', {})
|
||||
|
||||
# Calculate device pixel ratio to convert between device pixels and CSS pixels
|
||||
# This matches the approach in dom/service.py _get_viewport_ratio method
|
||||
css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))
|
||||
device_width = visual_viewport.get('clientWidth', css_width)
|
||||
device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
|
||||
|
||||
# For viewport dimensions, use CSS pixels (what JavaScript sees)
|
||||
# Prioritize CSS layout viewport, then fall back to layout viewport
|
||||
viewport_width = int(css_layout_viewport.get('clientWidth') or layout_viewport.get('clientWidth', 1280))
|
||||
viewport_height = int(css_layout_viewport.get('clientHeight') or layout_viewport.get('clientHeight', 720))
|
||||
|
||||
# For total page dimensions, content size is typically in device pixels, so convert to CSS pixels
|
||||
# by dividing by device pixel ratio
|
||||
raw_page_width = content_size.get('width', viewport_width * device_pixel_ratio)
|
||||
raw_page_height = content_size.get('height', viewport_height * device_pixel_ratio)
|
||||
page_width = int(raw_page_width / device_pixel_ratio)
|
||||
page_height = int(raw_page_height / device_pixel_ratio)
|
||||
|
||||
# For scroll position, use CSS visual viewport if available, otherwise CSS layout viewport
|
||||
# These should already be in CSS pixels
|
||||
scroll_x = int(css_visual_viewport.get('pageX') or css_layout_viewport.get('pageX', 0))
|
||||
scroll_y = int(css_visual_viewport.get('pageY') or css_layout_viewport.get('pageY', 0))
|
||||
|
||||
# Calculate scroll information - pixels that are above/below/left/right of current viewport
|
||||
pixels_above = scroll_y
|
||||
pixels_below = max(0, page_height - viewport_height - scroll_y)
|
||||
pixels_left = scroll_x
|
||||
pixels_right = max(0, page_width - viewport_width - scroll_x)
|
||||
|
||||
page_info = PageInfo(
|
||||
viewport_width=viewport_width,
|
||||
viewport_height=viewport_height,
|
||||
page_width=page_width,
|
||||
page_height=page_height,
|
||||
scroll_x=scroll_x,
|
||||
scroll_y=scroll_y,
|
||||
pixels_above=pixels_above,
|
||||
pixels_below=pixels_below,
|
||||
pixels_left=pixels_left,
|
||||
pixels_right=pixels_right,
|
||||
)
|
||||
|
||||
return page_info
|
||||
|
||||
# ========== Public Helper Methods ==========
|
||||
|
||||
async def get_element_by_index(self, index: int) -> EnhancedDOMTreeNode | None:
|
||||
"""Get DOM element by index from cached selector map.
|
||||
|
||||
Builds DOM if not cached.
|
||||
|
||||
Returns:
|
||||
EnhancedDOMTreeNode or None if index not found
|
||||
"""
|
||||
if not self.selector_map:
|
||||
# Build DOM if not cached
|
||||
await self._build_dom_tree_without_highlights()
|
||||
|
||||
return self.selector_map.get(index) if self.selector_map else None
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear cached DOM state to force rebuild on next access."""
|
||||
self.selector_map = None
|
||||
self.current_dom_state = None
|
||||
self.enhanced_dom_tree = None
|
||||
# Keep the DOM service instance to reuse its CDP client connection
|
||||
|
||||
def is_file_input(self, element: EnhancedDOMTreeNode) -> bool:
|
||||
"""Check if element is a file input."""
|
||||
return element.node_name.upper() == 'INPUT' and element.attributes.get('type', '').lower() == 'file'
|
||||
|
||||
@staticmethod
|
||||
def is_element_visible_according_to_all_parents(node: EnhancedDOMTreeNode, html_frames: list[EnhancedDOMTreeNode]) -> bool:
|
||||
"""Check if the element is visible according to all its parent HTML frames.
|
||||
|
||||
Delegates to the DomService static method.
|
||||
"""
|
||||
return DomService.is_element_visible_according_to_all_parents(node, html_frames)
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
"""Clean up DOM service on exit."""
|
||||
if self._dom_service:
|
||||
await self._dom_service.__aexit__(exc_type, exc_value, traceback)
|
||||
self._dom_service = None
|
||||
|
||||
def __del__(self):
|
||||
"""Clean up DOM service on deletion."""
|
||||
super().__del__()
|
||||
# DOM service will clean up its own CDP client
|
||||
self._dom_service = None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,779 @@
|
||||
"""HAR Recording Watchdog for Browser-Use sessions.
|
||||
|
||||
Captures HTTPS network activity via CDP Network domain and writes a HAR 1.2
|
||||
file on browser shutdown. Respects `record_har_content` (omit/embed/attach)
|
||||
and `record_har_mode` (full/minimal).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from importlib import metadata as importlib_metadata
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.network.events import (
|
||||
DataReceivedEvent,
|
||||
LoadingFailedEvent,
|
||||
LoadingFinishedEvent,
|
||||
RequestWillBeSentEvent,
|
||||
ResponseReceivedEvent,
|
||||
)
|
||||
from cdp_use.cdp.page.events import FrameNavigatedEvent, LifecycleEventEvent
|
||||
|
||||
from browser_use.browser.events import BrowserConnectedEvent, BrowserStopEvent
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
|
||||
@dataclass
|
||||
class _HarContent:
|
||||
mime_type: str | None = None
|
||||
text_b64: str | None = None # for embed
|
||||
file_rel: str | None = None # for attach
|
||||
size: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _HarEntryBuilder:
|
||||
request_id: str = ''
|
||||
frame_id: str | None = None
|
||||
document_url: str | None = None
|
||||
url: str | None = None
|
||||
method: str | None = None
|
||||
request_headers: dict = field(default_factory=dict)
|
||||
request_body: bytes | None = None
|
||||
post_data: str | None = None # CDP postData field
|
||||
status: int | None = None
|
||||
status_text: str | None = None
|
||||
response_headers: dict = field(default_factory=dict)
|
||||
mime_type: str | None = None
|
||||
encoded_data: bytearray = field(default_factory=bytearray)
|
||||
failed: bool = False
|
||||
# timing info (CDP timestamps are monotonic seconds); wallTime is epoch seconds
|
||||
ts_request: float | None = None
|
||||
wall_time_request: float | None = None
|
||||
ts_response: float | None = None
|
||||
ts_finished: float | None = None
|
||||
encoded_data_length: int | None = None
|
||||
response_body: bytes | None = None
|
||||
content_length: int | None = None # From Content-Length header
|
||||
protocol: str | None = None
|
||||
server_ip_address: str | None = None
|
||||
server_port: int | None = None
|
||||
security_details: dict | None = None
|
||||
transfer_size: int | None = None
|
||||
|
||||
|
||||
def _is_https(url: str | None) -> bool:
|
||||
return bool(url and url.lower().startswith('https://'))
|
||||
|
||||
|
||||
def _origin(url: str) -> str:
|
||||
# Very small origin extractor, assumes https URLs
|
||||
# https://host[:port]/...
|
||||
if not url:
|
||||
return ''
|
||||
try:
|
||||
without_scheme = url.split('://', 1)[1]
|
||||
host_port = without_scheme.split('/', 1)[0]
|
||||
return f'https://{host_port}'
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
def _mime_to_extension(mime_type: str | None) -> str:
|
||||
"""Map MIME type to file extension, matching Playwright's behavior."""
|
||||
if not mime_type:
|
||||
return 'bin'
|
||||
|
||||
mime_lower = mime_type.lower().split(';')[0].strip()
|
||||
|
||||
# Common MIME type to extension mapping
|
||||
mime_map = {
|
||||
'text/html': 'html',
|
||||
'text/css': 'css',
|
||||
'text/javascript': 'js',
|
||||
'application/javascript': 'js',
|
||||
'application/x-javascript': 'js',
|
||||
'application/json': 'json',
|
||||
'application/xml': 'xml',
|
||||
'text/xml': 'xml',
|
||||
'text/plain': 'txt',
|
||||
'image/png': 'png',
|
||||
'image/jpeg': 'jpg',
|
||||
'image/jpg': 'jpg',
|
||||
'image/gif': 'gif',
|
||||
'image/webp': 'webp',
|
||||
'image/svg+xml': 'svg',
|
||||
'image/x-icon': 'ico',
|
||||
'font/woff': 'woff',
|
||||
'font/woff2': 'woff2',
|
||||
'application/font-woff': 'woff',
|
||||
'application/font-woff2': 'woff2',
|
||||
'application/x-font-woff': 'woff',
|
||||
'application/x-font-woff2': 'woff2',
|
||||
'font/ttf': 'ttf',
|
||||
'application/x-font-ttf': 'ttf',
|
||||
'font/otf': 'otf',
|
||||
'application/x-font-opentype': 'otf',
|
||||
'application/pdf': 'pdf',
|
||||
'application/zip': 'zip',
|
||||
'application/x-zip-compressed': 'zip',
|
||||
'video/mp4': 'mp4',
|
||||
'video/webm': 'webm',
|
||||
'audio/mpeg': 'mp3',
|
||||
'audio/mp3': 'mp3',
|
||||
'audio/wav': 'wav',
|
||||
'audio/ogg': 'ogg',
|
||||
}
|
||||
|
||||
return mime_map.get(mime_lower, 'bin')
|
||||
|
||||
|
||||
def _generate_har_filename(content: bytes, mime_type: str | None) -> str:
|
||||
"""Generate a hash-based filename for HAR attach mode, matching Playwright's format."""
|
||||
content_hash = hashlib.sha1(content).hexdigest()
|
||||
extension = _mime_to_extension(mime_type)
|
||||
return f'{content_hash}.{extension}'
|
||||
|
||||
|
||||
class HarRecordingWatchdog(BaseWatchdog):
|
||||
"""Collects HTTPS requests/responses and writes a HAR 1.2 file on stop."""
|
||||
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._enabled: bool = False
|
||||
self._entries: dict[str, _HarEntryBuilder] = {}
|
||||
self._top_level_pages: dict[
|
||||
str, dict
|
||||
] = {} # frameId -> {url, title, startedDateTime, monotonic_start, onContentLoad, onLoad}
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
profile = self.browser_session.browser_profile
|
||||
if not profile.record_har_path:
|
||||
return
|
||||
|
||||
# Normalize config
|
||||
self._content_mode = (profile.record_har_content or 'embed').lower()
|
||||
self._mode = (profile.record_har_mode or 'full').lower()
|
||||
self._har_path = Path(str(profile.record_har_path)).expanduser().resolve()
|
||||
self._har_dir = self._har_path.parent
|
||||
self._har_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Enable Network and Page domains for events
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
await cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)
|
||||
await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
|
||||
|
||||
# Query browser version for HAR log.browser
|
||||
try:
|
||||
version_info = await self.browser_session.cdp_client.send.Browser.getVersion()
|
||||
self._browser_name = version_info.get('product') or 'Chromium'
|
||||
self._browser_version = version_info.get('jsVersion') or ''
|
||||
except Exception:
|
||||
self._browser_name = 'Chromium'
|
||||
self._browser_version = ''
|
||||
|
||||
cdp = self.browser_session.cdp_client.register
|
||||
cdp.Network.requestWillBeSent(self._on_request_will_be_sent)
|
||||
cdp.Network.responseReceived(self._on_response_received)
|
||||
cdp.Network.dataReceived(self._on_data_received)
|
||||
cdp.Network.loadingFinished(self._on_loading_finished)
|
||||
cdp.Network.loadingFailed(self._on_loading_failed)
|
||||
cdp.Page.lifecycleEvent(self._on_lifecycle_event)
|
||||
cdp.Page.frameNavigated(self._on_frame_navigated)
|
||||
|
||||
self._enabled = True
|
||||
self.logger.info(f'📊 Starting HAR recording to {self._har_path}')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to enable HAR recording: {e}')
|
||||
self._enabled = False
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
if not self._enabled:
|
||||
return
|
||||
try:
|
||||
await self._write_har()
|
||||
self.logger.info(f'📊 HAR file saved: {self._har_path}')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to write HAR: {e}')
|
||||
|
||||
# =============== CDP Event Handlers (sync) ==================
|
||||
def _on_request_will_be_sent(self, params: RequestWillBeSentEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
req = params.get('request', {}) if hasattr(params, 'get') else getattr(params, 'request', {})
|
||||
url = req.get('url') if isinstance(req, dict) else getattr(req, 'url', None)
|
||||
if not _is_https(url):
|
||||
return # HTTPS-only requirement (only HTTPS requests are recorded for now)
|
||||
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id:
|
||||
return
|
||||
|
||||
entry = self._entries.setdefault(request_id, _HarEntryBuilder(request_id=request_id))
|
||||
entry.url = url
|
||||
entry.method = req.get('method') if isinstance(req, dict) else getattr(req, 'method', None)
|
||||
entry.post_data = req.get('postData') if isinstance(req, dict) else getattr(req, 'postData', None)
|
||||
|
||||
# Convert headers to plain dict, handling various formats
|
||||
headers_raw = req.get('headers') if isinstance(req, dict) else getattr(req, 'headers', None)
|
||||
if headers_raw is None:
|
||||
entry.request_headers = {}
|
||||
elif isinstance(headers_raw, dict):
|
||||
entry.request_headers = {k.lower(): str(v) for k, v in headers_raw.items()}
|
||||
elif isinstance(headers_raw, list):
|
||||
entry.request_headers = {
|
||||
h.get('name', '').lower(): str(h.get('value') or '') for h in headers_raw if isinstance(h, dict)
|
||||
}
|
||||
else:
|
||||
# Handle Headers type or other formats - convert to dict
|
||||
try:
|
||||
headers_dict = dict(headers_raw) if hasattr(headers_raw, '__iter__') else {}
|
||||
entry.request_headers = {k.lower(): str(v) for k, v in headers_dict.items()}
|
||||
except Exception:
|
||||
entry.request_headers = {}
|
||||
|
||||
entry.frame_id = params.get('frameId') if hasattr(params, 'get') else getattr(params, 'frameId', None)
|
||||
entry.document_url = (
|
||||
params.get('documentURL')
|
||||
if hasattr(params, 'get')
|
||||
else getattr(params, 'documentURL', None) or entry.document_url
|
||||
)
|
||||
|
||||
# Timing anchors
|
||||
entry.ts_request = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
|
||||
entry.wall_time_request = params.get('wallTime') if hasattr(params, 'get') else getattr(params, 'wallTime', None)
|
||||
|
||||
# Track top-level navigations for page context
|
||||
req_type = params.get('type') if hasattr(params, 'get') else getattr(params, 'type', None)
|
||||
is_same_doc = (
|
||||
params.get('isSameDocument', False) if hasattr(params, 'get') else getattr(params, 'isSameDocument', False)
|
||||
)
|
||||
if req_type == 'Document' and not is_same_doc:
|
||||
# best-effort: consider as navigation
|
||||
if entry.frame_id and url:
|
||||
if entry.frame_id not in self._top_level_pages:
|
||||
self._top_level_pages[entry.frame_id] = {
|
||||
'url': str(url),
|
||||
'title': str(url), # Default to URL, will be updated from DOM
|
||||
'startedDateTime': entry.wall_time_request,
|
||||
'monotonic_start': entry.ts_request, # Track monotonic start time for timing calculations
|
||||
'onContentLoad': -1,
|
||||
'onLoad': -1,
|
||||
}
|
||||
else:
|
||||
# Update startedDateTime and monotonic_start if this is earlier
|
||||
page_info = self._top_level_pages[entry.frame_id]
|
||||
if entry.wall_time_request and (
|
||||
page_info['startedDateTime'] is None or entry.wall_time_request < page_info['startedDateTime']
|
||||
):
|
||||
page_info['startedDateTime'] = entry.wall_time_request
|
||||
page_info['monotonic_start'] = entry.ts_request
|
||||
except Exception as e:
|
||||
self.logger.debug(f'requestWillBeSent handling error: {e}')
|
||||
|
||||
def _on_response_received(self, params: ResponseReceivedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id or request_id not in self._entries:
|
||||
return
|
||||
response = params.get('response', {}) if hasattr(params, 'get') else getattr(params, 'response', {})
|
||||
entry = self._entries[request_id]
|
||||
entry.status = response.get('status') if isinstance(response, dict) else getattr(response, 'status', None)
|
||||
entry.status_text = (
|
||||
response.get('statusText') if isinstance(response, dict) else getattr(response, 'statusText', None)
|
||||
)
|
||||
|
||||
# Extract Content-Length for compression calculation (before converting headers)
|
||||
headers_raw = response.get('headers') if isinstance(response, dict) else getattr(response, 'headers', None)
|
||||
if headers_raw:
|
||||
if isinstance(headers_raw, dict):
|
||||
cl_str = headers_raw.get('content-length') or headers_raw.get('Content-Length')
|
||||
elif isinstance(headers_raw, list):
|
||||
cl_header = next(
|
||||
(h for h in headers_raw if isinstance(h, dict) and h.get('name', '').lower() == 'content-length'), None
|
||||
)
|
||||
cl_str = cl_header.get('value') if cl_header else None
|
||||
else:
|
||||
cl_str = None
|
||||
if cl_str:
|
||||
try:
|
||||
entry.content_length = int(cl_str)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Convert headers to plain dict, handling various formats
|
||||
if headers_raw is None:
|
||||
entry.response_headers = {}
|
||||
elif isinstance(headers_raw, dict):
|
||||
entry.response_headers = {k.lower(): str(v) for k, v in headers_raw.items()}
|
||||
elif isinstance(headers_raw, list):
|
||||
entry.response_headers = {
|
||||
h.get('name', '').lower(): str(h.get('value') or '') for h in headers_raw if isinstance(h, dict)
|
||||
}
|
||||
else:
|
||||
# Handle Headers type or other formats - convert to dict
|
||||
try:
|
||||
headers_dict = dict(headers_raw) if hasattr(headers_raw, '__iter__') else {}
|
||||
entry.response_headers = {k.lower(): str(v) for k, v in headers_dict.items()}
|
||||
except Exception:
|
||||
entry.response_headers = {}
|
||||
|
||||
entry.mime_type = response.get('mimeType') if isinstance(response, dict) else getattr(response, 'mimeType', None)
|
||||
entry.ts_response = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
|
||||
|
||||
protocol_raw = response.get('protocol') if isinstance(response, dict) else getattr(response, 'protocol', None)
|
||||
if protocol_raw:
|
||||
protocol_lower = str(protocol_raw).lower()
|
||||
if protocol_lower == 'h2' or protocol_lower.startswith('http/2'):
|
||||
entry.protocol = 'HTTP/2.0'
|
||||
elif protocol_lower.startswith('http/1.1'):
|
||||
entry.protocol = 'HTTP/1.1'
|
||||
elif protocol_lower.startswith('http/1.0'):
|
||||
entry.protocol = 'HTTP/1.0'
|
||||
else:
|
||||
entry.protocol = str(protocol_raw).upper()
|
||||
|
||||
entry.server_ip_address = (
|
||||
response.get('remoteIPAddress') if isinstance(response, dict) else getattr(response, 'remoteIPAddress', None)
|
||||
)
|
||||
server_port_raw = response.get('remotePort') if isinstance(response, dict) else getattr(response, 'remotePort', None)
|
||||
if server_port_raw is not None:
|
||||
try:
|
||||
entry.server_port = int(server_port_raw)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Extract security details (TLS info)
|
||||
security_details_raw = (
|
||||
response.get('securityDetails') if isinstance(response, dict) else getattr(response, 'securityDetails', None)
|
||||
)
|
||||
if security_details_raw:
|
||||
try:
|
||||
entry.security_details = dict(security_details_raw)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.logger.debug(f'responseReceived handling error: {e}')
|
||||
|
||||
def _on_data_received(self, params: DataReceivedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id or request_id not in self._entries:
|
||||
return
|
||||
data = params.get('data') if hasattr(params, 'get') else getattr(params, 'data', None)
|
||||
if isinstance(data, str):
|
||||
try:
|
||||
self._entries[request_id].encoded_data.extend(data.encode('latin1'))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.logger.debug(f'dataReceived handling error: {e}')
|
||||
|
||||
def _on_loading_finished(self, params: LoadingFinishedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id or request_id not in self._entries:
|
||||
return
|
||||
entry = self._entries[request_id]
|
||||
entry.ts_finished = params.get('timestamp')
|
||||
# Fetch response body via CDP as dataReceived may be incomplete
|
||||
import asyncio as _asyncio
|
||||
|
||||
async def _fetch_body(self_ref, req_id, sess_id):
|
||||
try:
|
||||
resp = await self_ref.browser_session.cdp_client.send.Network.getResponseBody(
|
||||
params={'requestId': req_id}, session_id=sess_id
|
||||
)
|
||||
data = resp.get('body', b'')
|
||||
if resp.get('base64Encoded'):
|
||||
import base64 as _b64
|
||||
|
||||
data = _b64.b64decode(data)
|
||||
else:
|
||||
# Ensure data is bytes even if CDP returns a string
|
||||
if isinstance(data, str):
|
||||
data = data.encode('utf-8', errors='replace')
|
||||
# Ensure we always have bytes
|
||||
if not isinstance(data, bytes):
|
||||
data = bytes(data) if data else b''
|
||||
entry.response_body = data
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Always schedule the response body fetch task
|
||||
_asyncio.create_task(_fetch_body(self, request_id, session_id))
|
||||
|
||||
encoded_length = (
|
||||
params.get('encodedDataLength') if hasattr(params, 'get') else getattr(params, 'encodedDataLength', None)
|
||||
)
|
||||
if encoded_length is not None:
|
||||
try:
|
||||
entry.encoded_data_length = int(encoded_length)
|
||||
entry.transfer_size = entry.encoded_data_length
|
||||
except Exception:
|
||||
entry.encoded_data_length = None
|
||||
except Exception as e:
|
||||
self.logger.debug(f'loadingFinished handling error: {e}')
|
||||
|
||||
def _on_loading_failed(self, params: LoadingFailedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if request_id and request_id in self._entries:
|
||||
self._entries[request_id].failed = True
|
||||
except Exception as e:
|
||||
self.logger.debug(f'loadingFailed handling error: {e}')
|
||||
|
||||
# ===================== HAR Writing ==========================
|
||||
def _on_lifecycle_event(self, params: LifecycleEventEvent, session_id: str | None) -> None:
|
||||
"""Handle Page.lifecycleEvent for tracking page load timings."""
|
||||
try:
|
||||
frame_id = params.get('frameId') if hasattr(params, 'get') else getattr(params, 'frameId', None)
|
||||
name = params.get('name') if hasattr(params, 'get') else getattr(params, 'name', None)
|
||||
timestamp = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
|
||||
|
||||
if not frame_id or not name or frame_id not in self._top_level_pages:
|
||||
return
|
||||
|
||||
page_info = self._top_level_pages[frame_id]
|
||||
# Use monotonic_start instead of startedDateTime (wall-clock) for timing calculations
|
||||
monotonic_start = page_info.get('monotonic_start')
|
||||
|
||||
if name == 'DOMContentLoaded' and monotonic_start is not None:
|
||||
# Calculate milliseconds since page start using monotonic timestamps
|
||||
try:
|
||||
elapsed_ms = int(round((timestamp - monotonic_start) * 1000))
|
||||
page_info['onContentLoad'] = max(0, elapsed_ms)
|
||||
except Exception:
|
||||
pass
|
||||
elif name == 'load' and monotonic_start is not None:
|
||||
try:
|
||||
elapsed_ms = int(round((timestamp - monotonic_start) * 1000))
|
||||
page_info['onLoad'] = max(0, elapsed_ms)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.logger.debug(f'lifecycleEvent handling error: {e}')
|
||||
|
||||
def _on_frame_navigated(self, params: FrameNavigatedEvent, session_id: str | None) -> None:
|
||||
"""Handle Page.frameNavigated to update page title from DOM."""
|
||||
try:
|
||||
frame = params.get('frame') if hasattr(params, 'get') else getattr(params, 'frame', None)
|
||||
if not frame:
|
||||
return
|
||||
|
||||
frame_id = frame.get('id') if isinstance(frame, dict) else getattr(frame, 'id', None)
|
||||
title = (
|
||||
frame.get('name') or frame.get('url')
|
||||
if isinstance(frame, dict)
|
||||
else getattr(frame, 'name', None) or getattr(frame, 'url', None)
|
||||
)
|
||||
|
||||
if frame_id and frame_id in self._top_level_pages:
|
||||
# Try to get actual page title via Runtime.evaluate if possible
|
||||
# For now, use frame name or URL as fallback
|
||||
if title:
|
||||
self._top_level_pages[frame_id]['title'] = str(title)
|
||||
except Exception as e:
|
||||
self.logger.debug(f'frameNavigated handling error: {e}')
|
||||
|
||||
# ===================== HAR Writing ==========================
|
||||
async def _write_har(self) -> None:
|
||||
# Filter by mode and HTTPS already respected at collection time
|
||||
entries = [e for e in self._entries.values() if self._include_entry(e)]
|
||||
|
||||
har_entries = []
|
||||
sidecar_dir: Path | None = None
|
||||
if self._content_mode == 'attach':
|
||||
sidecar_dir = self._har_dir / f'{self._har_path.stem}_har_parts'
|
||||
sidecar_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for e in entries:
|
||||
content_obj: dict = {'mimeType': e.mime_type or ''}
|
||||
|
||||
# Get body data, preferring response_body over encoded_data
|
||||
if e.response_body is not None:
|
||||
body_data = e.response_body
|
||||
else:
|
||||
body_data = e.encoded_data
|
||||
|
||||
# Defensive conversion: ensure body_data is always bytes
|
||||
if isinstance(body_data, str):
|
||||
body_bytes = body_data.encode('utf-8', errors='replace')
|
||||
elif isinstance(body_data, bytearray):
|
||||
body_bytes = bytes(body_data)
|
||||
elif isinstance(body_data, bytes):
|
||||
body_bytes = body_data
|
||||
else:
|
||||
# Fallback: try to convert to bytes
|
||||
try:
|
||||
body_bytes = bytes(body_data) if body_data else b''
|
||||
except (TypeError, ValueError):
|
||||
body_bytes = b''
|
||||
|
||||
content_size = len(body_bytes)
|
||||
|
||||
# Calculate compression (bytes saved by compression)
|
||||
compression = 0
|
||||
if e.content_length is not None and e.encoded_data_length is not None:
|
||||
compression = max(0, e.content_length - e.encoded_data_length)
|
||||
|
||||
if self._content_mode == 'embed' and content_size > 0:
|
||||
# Prefer plain text; fallback to base64 only if decoding fails
|
||||
try:
|
||||
text_decoded = body_bytes.decode('utf-8')
|
||||
content_obj['text'] = text_decoded
|
||||
content_obj['size'] = content_size
|
||||
content_obj['compression'] = compression
|
||||
except UnicodeDecodeError:
|
||||
content_obj['text'] = base64.b64encode(body_bytes).decode('ascii')
|
||||
content_obj['encoding'] = 'base64'
|
||||
content_obj['size'] = content_size
|
||||
content_obj['compression'] = compression
|
||||
elif self._content_mode == 'attach' and content_size > 0 and sidecar_dir is not None:
|
||||
filename = _generate_har_filename(body_bytes, e.mime_type)
|
||||
(sidecar_dir / filename).write_bytes(body_bytes)
|
||||
content_obj['_file'] = filename
|
||||
content_obj['size'] = content_size
|
||||
content_obj['compression'] = compression
|
||||
else:
|
||||
# omit or empty
|
||||
content_obj['size'] = content_size
|
||||
if content_size > 0:
|
||||
content_obj['compression'] = compression
|
||||
|
||||
started_date_time, total_time_ms, timings = self._compute_timings(e)
|
||||
req_headers_list = [{'name': k, 'value': str(v)} for k, v in (e.request_headers or {}).items()]
|
||||
resp_headers_list = [{'name': k, 'value': str(v)} for k, v in (e.response_headers or {}).items()]
|
||||
request_headers_size = self._calc_headers_size(e.method or 'GET', e.url or '', req_headers_list)
|
||||
response_headers_size = self._calc_headers_size(None, None, resp_headers_list)
|
||||
request_body_size = self._calc_request_body_size(e)
|
||||
request_post_data = None
|
||||
if e.post_data and self._content_mode != 'omit':
|
||||
if self._content_mode == 'embed':
|
||||
request_post_data = {'mimeType': e.request_headers.get('content-type', ''), 'text': e.post_data}
|
||||
elif self._content_mode == 'attach' and sidecar_dir is not None:
|
||||
post_data_bytes = e.post_data.encode('utf-8')
|
||||
req_mime_type = e.request_headers.get('content-type', 'text/plain')
|
||||
req_filename = _generate_har_filename(post_data_bytes, req_mime_type)
|
||||
(sidecar_dir / req_filename).write_bytes(post_data_bytes)
|
||||
request_post_data = {
|
||||
'mimeType': req_mime_type,
|
||||
'_file': req_filename,
|
||||
}
|
||||
|
||||
http_version = e.protocol if e.protocol else 'HTTP/1.1'
|
||||
|
||||
response_body_size = e.transfer_size
|
||||
if response_body_size is None:
|
||||
response_body_size = e.encoded_data_length
|
||||
if response_body_size is None:
|
||||
response_body_size = content_size if content_size > 0 else -1
|
||||
|
||||
entry_dict = {
|
||||
'startedDateTime': started_date_time,
|
||||
'time': total_time_ms,
|
||||
'request': {
|
||||
'method': e.method or 'GET',
|
||||
'url': e.url or '',
|
||||
'httpVersion': http_version,
|
||||
'headers': req_headers_list,
|
||||
'queryString': [],
|
||||
'cookies': [],
|
||||
'headersSize': request_headers_size,
|
||||
'bodySize': request_body_size,
|
||||
'postData': request_post_data,
|
||||
},
|
||||
'response': {
|
||||
'status': e.status or 0,
|
||||
'statusText': e.status_text or '',
|
||||
'httpVersion': http_version,
|
||||
'headers': resp_headers_list,
|
||||
'cookies': [],
|
||||
'content': content_obj,
|
||||
'redirectURL': '',
|
||||
'headersSize': response_headers_size,
|
||||
'bodySize': response_body_size,
|
||||
},
|
||||
'cache': {},
|
||||
'timings': timings,
|
||||
'pageref': self._page_ref_for_entry(e),
|
||||
}
|
||||
|
||||
# Add security/TLS details if available
|
||||
if e.server_ip_address:
|
||||
entry_dict['serverIPAddress'] = e.server_ip_address
|
||||
if e.server_port is not None:
|
||||
entry_dict['_serverPort'] = e.server_port
|
||||
if e.security_details:
|
||||
# Filter to match Playwright's minimal security details set
|
||||
security_filtered = {}
|
||||
if 'protocol' in e.security_details:
|
||||
security_filtered['protocol'] = e.security_details['protocol']
|
||||
if 'subjectName' in e.security_details:
|
||||
security_filtered['subjectName'] = e.security_details['subjectName']
|
||||
if 'issuer' in e.security_details:
|
||||
security_filtered['issuer'] = e.security_details['issuer']
|
||||
if 'validFrom' in e.security_details:
|
||||
security_filtered['validFrom'] = e.security_details['validFrom']
|
||||
if 'validTo' in e.security_details:
|
||||
security_filtered['validTo'] = e.security_details['validTo']
|
||||
if security_filtered:
|
||||
entry_dict['_securityDetails'] = security_filtered
|
||||
if e.transfer_size is not None:
|
||||
entry_dict['response']['_transferSize'] = e.transfer_size
|
||||
|
||||
har_entries.append(entry_dict)
|
||||
|
||||
# Try to include our library version in creator
|
||||
try:
|
||||
bu_version = importlib_metadata.version('browser-use')
|
||||
except Exception:
|
||||
# Fallback when running from source without installed package metadata
|
||||
bu_version = 'dev'
|
||||
|
||||
har_obj = {
|
||||
'log': {
|
||||
'version': '1.2',
|
||||
'creator': {'name': 'browser-use', 'version': bu_version},
|
||||
'browser': {'name': self._browser_name, 'version': self._browser_version},
|
||||
'pages': [
|
||||
{
|
||||
'id': f'page@{pid}', # Use Playwright format: "page@{frame_id}"
|
||||
'title': page_info.get('title', page_info.get('url', '')),
|
||||
'startedDateTime': self._format_page_started_datetime(page_info.get('startedDateTime')),
|
||||
'pageTimings': (
|
||||
(lambda _ocl, _ol: ({k: v for k, v in (('onContentLoad', _ocl), ('onLoad', _ol)) if v is not None}))(
|
||||
(page_info.get('onContentLoad') if page_info.get('onContentLoad', -1) >= 0 else None),
|
||||
(page_info.get('onLoad') if page_info.get('onLoad', -1) >= 0 else None),
|
||||
)
|
||||
),
|
||||
}
|
||||
for pid, page_info in self._top_level_pages.items()
|
||||
],
|
||||
'entries': har_entries,
|
||||
}
|
||||
}
|
||||
|
||||
tmp_path = self._har_path.with_suffix(self._har_path.suffix + '.tmp')
|
||||
# Write as bytes explicitly to avoid any text/binary mode confusion in different environments
|
||||
tmp_path.write_bytes(json.dumps(har_obj, indent=2, ensure_ascii=False).encode('utf-8'))
|
||||
tmp_path.replace(self._har_path)
|
||||
|
||||
def _format_page_started_datetime(self, timestamp: float | None) -> str:
|
||||
"""Format page startedDateTime from timestamp."""
|
||||
if timestamp is None:
|
||||
return ''
|
||||
try:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat().replace('+00:00', 'Z')
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
def _page_ref_for_entry(self, e: _HarEntryBuilder) -> str | None:
|
||||
# Use Playwright format: "page@{frame_id}" if frame_id is known
|
||||
if e.frame_id and e.frame_id in self._top_level_pages:
|
||||
return f'page@{e.frame_id}'
|
||||
return None
|
||||
|
||||
def _include_entry(self, e: _HarEntryBuilder) -> bool:
|
||||
if not _is_https(e.url):
|
||||
return False
|
||||
# Filter out favicon requests (matching Playwright behavior)
|
||||
if e.url and '/favicon.ico' in e.url.lower():
|
||||
return False
|
||||
if getattr(self, '_mode', 'full') == 'full':
|
||||
return True
|
||||
# minimal: include main document and same-origin subresources
|
||||
if e.frame_id and e.frame_id in self._top_level_pages:
|
||||
page_info = self._top_level_pages[e.frame_id]
|
||||
page_url = page_info.get('url') if isinstance(page_info, dict) else page_info
|
||||
return _origin(e.url or '') == _origin(page_url or '')
|
||||
return False
|
||||
|
||||
# ===================== Helpers ==============================
|
||||
def _compute_timings(self, e: _HarEntryBuilder) -> tuple[str, int, dict]:
|
||||
# startedDateTime from wall_time_request in ISO8601 Z
|
||||
started = ''
|
||||
try:
|
||||
if e.wall_time_request is not None:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
started = datetime.fromtimestamp(e.wall_time_request, tz=timezone.utc).isoformat().replace('+00:00', 'Z')
|
||||
except Exception:
|
||||
started = ''
|
||||
|
||||
# Calculate timings - CDP doesn't always provide DNS/connect/SSL breakdown
|
||||
# Default to 0 for unavailable timings, calculate what we can from timestamps
|
||||
dns_ms = 0
|
||||
connect_ms = 0
|
||||
ssl_ms = 0
|
||||
send_ms = 0
|
||||
wait_ms = 0
|
||||
receive_ms = 0
|
||||
|
||||
if e.ts_request is not None and e.ts_response is not None:
|
||||
wait_ms = max(0, int(round((e.ts_response - e.ts_request) * 1000)))
|
||||
|
||||
if e.ts_response is not None and e.ts_finished is not None:
|
||||
receive_ms = max(0, int(round((e.ts_finished - e.ts_response) * 1000)))
|
||||
|
||||
# Note: DNS, connect, and SSL timings would require additional CDP events or ResourceTiming API
|
||||
# For now, we structure the timings dict to match Playwright format
|
||||
# but leave DNS/connect/SSL as 0 since CDP doesn't provide this breakdown directly
|
||||
|
||||
total = dns_ms + connect_ms + ssl_ms + send_ms + wait_ms + receive_ms
|
||||
return (
|
||||
started,
|
||||
total,
|
||||
{
|
||||
'dns': dns_ms,
|
||||
'connect': connect_ms,
|
||||
'ssl': ssl_ms,
|
||||
'send': send_ms,
|
||||
'wait': wait_ms,
|
||||
'receive': receive_ms,
|
||||
},
|
||||
)
|
||||
|
||||
def _calc_headers_size(self, method: str | None, url: str | None, headers_list: list[dict]) -> int:
|
||||
try:
|
||||
# Approximate per RFC: sum of header lines + CRLF; include request/status line only for request
|
||||
size = 0
|
||||
if method and url:
|
||||
# Use HTTP/1.1 request line approximation
|
||||
size += len(f'{method} {url} HTTP/1.1\r\n'.encode('latin1'))
|
||||
for h in headers_list:
|
||||
size += len(f'{h.get("name", "")}: {h.get("value", "")}\r\n'.encode('latin1'))
|
||||
size += len(b'\r\n')
|
||||
return size
|
||||
except Exception:
|
||||
return -1
|
||||
|
||||
def _calc_request_body_size(self, e: _HarEntryBuilder) -> int:
|
||||
# Try Content-Length header first; else post_data; else request_body; else 0 for GET/HEAD, -1 if unknown
|
||||
try:
|
||||
cl = None
|
||||
if e.request_headers:
|
||||
cl = e.request_headers.get('content-length') or e.request_headers.get('Content-Length')
|
||||
if cl is not None:
|
||||
return int(cl)
|
||||
if e.post_data:
|
||||
return len(e.post_data.encode('utf-8'))
|
||||
if e.request_body is not None:
|
||||
return len(e.request_body)
|
||||
# GET/HEAD requests typically have no body
|
||||
if e.method and e.method.upper() in ('GET', 'HEAD'):
|
||||
return 0
|
||||
except Exception:
|
||||
pass
|
||||
return -1
|
||||
@@ -0,0 +1,506 @@
|
||||
"""Local browser watchdog for managing browser subprocess lifecycle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import psutil
|
||||
from bubus import BaseEvent
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserKillEvent,
|
||||
BrowserLaunchEvent,
|
||||
BrowserLaunchResult,
|
||||
BrowserStopEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.observability import observe_debug
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser.profile import BrowserChannel
|
||||
|
||||
|
||||
class LocalBrowserWatchdog(BaseWatchdog):
|
||||
"""Manages local browser subprocess lifecycle."""
|
||||
|
||||
# Events this watchdog listens to
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [
|
||||
BrowserLaunchEvent,
|
||||
BrowserKillEvent,
|
||||
BrowserStopEvent,
|
||||
]
|
||||
|
||||
# Events this watchdog emits
|
||||
EMITS: ClassVar[list[type[BaseEvent[Any]]]] = []
|
||||
|
||||
# Private state for subprocess management
|
||||
_subprocess: psutil.Process | None = PrivateAttr(default=None)
|
||||
_owns_browser_resources: bool = PrivateAttr(default=True)
|
||||
_temp_dirs_to_cleanup: list[Path] = PrivateAttr(default_factory=list)
|
||||
_original_user_data_dir: str | None = PrivateAttr(default=None)
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='browser_launch_event')
|
||||
async def on_BrowserLaunchEvent(self, event: BrowserLaunchEvent) -> BrowserLaunchResult:
|
||||
"""Launch a local browser process."""
|
||||
|
||||
try:
|
||||
self.logger.debug('[LocalBrowserWatchdog] Received BrowserLaunchEvent, launching local browser...')
|
||||
|
||||
# self.logger.debug('[LocalBrowserWatchdog] Calling _launch_browser...')
|
||||
process, cdp_url = await self._launch_browser()
|
||||
self._subprocess = process
|
||||
# self.logger.debug(f'[LocalBrowserWatchdog] _launch_browser returned: process={process}, cdp_url={cdp_url}')
|
||||
|
||||
return BrowserLaunchResult(cdp_url=cdp_url)
|
||||
except Exception as e:
|
||||
self.logger.error(f'[LocalBrowserWatchdog] Exception in on_BrowserLaunchEvent: {e}', exc_info=True)
|
||||
raise
|
||||
|
||||
async def on_BrowserKillEvent(self, event: BrowserKillEvent) -> None:
|
||||
"""Kill the local browser subprocess."""
|
||||
self.logger.debug('[LocalBrowserWatchdog] Killing local browser process')
|
||||
|
||||
if self._subprocess:
|
||||
await self._cleanup_process(self._subprocess)
|
||||
self._subprocess = None
|
||||
|
||||
# Clean up temp directories if any were created
|
||||
for temp_dir in self._temp_dirs_to_cleanup:
|
||||
self._cleanup_temp_dir(temp_dir)
|
||||
self._temp_dirs_to_cleanup.clear()
|
||||
|
||||
# Restore original user_data_dir if it was modified
|
||||
if self._original_user_data_dir is not None:
|
||||
self.browser_session.browser_profile.user_data_dir = self._original_user_data_dir
|
||||
self._original_user_data_dir = None
|
||||
|
||||
self.logger.debug('[LocalBrowserWatchdog] Browser cleanup completed')
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""Listen for BrowserStopEvent and dispatch BrowserKillEvent without awaiting it."""
|
||||
if self.browser_session.is_local and self._subprocess:
|
||||
self.logger.debug('[LocalBrowserWatchdog] BrowserStopEvent received, dispatching BrowserKillEvent')
|
||||
# Dispatch BrowserKillEvent without awaiting so it gets processed after all BrowserStopEvent handlers
|
||||
self.event_bus.dispatch(BrowserKillEvent())
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='launch_browser_process')
|
||||
async def _launch_browser(self, max_retries: int = 3) -> tuple[psutil.Process, str]:
|
||||
"""Launch browser process and return (process, cdp_url).
|
||||
|
||||
Handles launch errors by falling back to temporary directories if needed.
|
||||
|
||||
Returns:
|
||||
Tuple of (psutil.Process, cdp_url)
|
||||
"""
|
||||
# Keep track of original user_data_dir to restore if needed
|
||||
profile = self.browser_session.browser_profile
|
||||
self._original_user_data_dir = str(profile.user_data_dir) if profile.user_data_dir else None
|
||||
self._temp_dirs_to_cleanup = []
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
# Get launch args from profile
|
||||
launch_args = profile.get_args()
|
||||
|
||||
# Add debugging port
|
||||
debug_port = self._find_free_port()
|
||||
launch_args.extend(
|
||||
[
|
||||
f'--remote-debugging-port={debug_port}',
|
||||
]
|
||||
)
|
||||
assert '--user-data-dir' in str(launch_args), (
|
||||
'User data dir must be set somewhere in launch args to a non-default path, otherwise Chrome will not let us attach via CDP'
|
||||
)
|
||||
|
||||
# Get browser executable
|
||||
# Priority: custom executable > fallback paths > playwright subprocess
|
||||
if profile.executable_path:
|
||||
browser_path = profile.executable_path
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 📦 Using custom local browser executable_path= {browser_path}')
|
||||
else:
|
||||
# self.logger.debug('[LocalBrowserWatchdog] 🔍 Looking for local browser binary path...')
|
||||
# Try fallback paths first (Playwright's Chromium preferred by default)
|
||||
browser_path = self._find_installed_browser_path(channel=profile.channel)
|
||||
if not browser_path:
|
||||
self.logger.error(
|
||||
'[LocalBrowserWatchdog] ⚠️ No local browser binary found, installing browser using playwright subprocess...'
|
||||
)
|
||||
browser_path = await self._install_browser_with_playwright()
|
||||
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 📦 Found local browser installed at executable_path= {browser_path}')
|
||||
if not browser_path:
|
||||
raise RuntimeError('No local Chrome/Chromium install found, and failed to install with playwright')
|
||||
|
||||
# Launch browser subprocess directly
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 🚀 Launching browser subprocess with {len(launch_args)} args...')
|
||||
self.logger.debug(
|
||||
f'[LocalBrowserWatchdog] 📂 user_data_dir={profile.user_data_dir}, profile_directory={profile.profile_directory}'
|
||||
)
|
||||
subprocess = await asyncio.create_subprocess_exec(
|
||||
browser_path,
|
||||
*launch_args,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
self.logger.debug(
|
||||
f'[LocalBrowserWatchdog] 🎭 Browser running with browser_pid= {subprocess.pid} 🔗 listening on CDP port :{debug_port}'
|
||||
)
|
||||
|
||||
# Convert to psutil.Process
|
||||
process = psutil.Process(subprocess.pid)
|
||||
|
||||
# Wait for CDP to be ready and get the URL
|
||||
cdp_url = await self._wait_for_cdp_url(debug_port)
|
||||
|
||||
# Success! Clean up only the temp dirs we created but didn't use
|
||||
currently_used_dir = str(profile.user_data_dir)
|
||||
unused_temp_dirs = [tmp_dir for tmp_dir in self._temp_dirs_to_cleanup if str(tmp_dir) != currently_used_dir]
|
||||
|
||||
for tmp_dir in unused_temp_dirs:
|
||||
try:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Keep only the in-use directory for cleanup during browser kill
|
||||
if currently_used_dir and 'browseruse-tmp-' in currently_used_dir:
|
||||
self._temp_dirs_to_cleanup = [Path(currently_used_dir)]
|
||||
else:
|
||||
self._temp_dirs_to_cleanup = []
|
||||
|
||||
return process, cdp_url
|
||||
|
||||
except Exception as e:
|
||||
error_str = str(e).lower()
|
||||
|
||||
# Check if this is a user_data_dir related error
|
||||
if any(err in error_str for err in ['singletonlock', 'user data directory', 'cannot create', 'already in use']):
|
||||
self.logger.warning(f'Browser launch failed (attempt {attempt + 1}/{max_retries}): {e}')
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
# Create a temporary directory for next attempt
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix='browseruse-tmp-'))
|
||||
self._temp_dirs_to_cleanup.append(tmp_dir)
|
||||
|
||||
# Update profile to use temp directory
|
||||
profile.user_data_dir = str(tmp_dir)
|
||||
self.logger.debug(f'Retrying with temporary user_data_dir: {tmp_dir}')
|
||||
|
||||
# Small delay before retry
|
||||
await asyncio.sleep(0.5)
|
||||
continue
|
||||
|
||||
# Not a recoverable error or last attempt failed
|
||||
# Restore original user_data_dir before raising
|
||||
if self._original_user_data_dir is not None:
|
||||
profile.user_data_dir = self._original_user_data_dir
|
||||
|
||||
# Clean up any temp dirs we created
|
||||
for tmp_dir in self._temp_dirs_to_cleanup:
|
||||
try:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
raise
|
||||
|
||||
# Should not reach here, but just in case
|
||||
if self._original_user_data_dir is not None:
|
||||
profile.user_data_dir = self._original_user_data_dir
|
||||
raise RuntimeError(f'Failed to launch browser after {max_retries} attempts')
|
||||
|
||||
@staticmethod
|
||||
def _find_installed_browser_path(channel: BrowserChannel | None = None) -> str | None:
|
||||
"""Try to find browser executable from common fallback locations.
|
||||
|
||||
If a channel is specified, paths for that browser are searched first.
|
||||
Falls back to all known browser paths if the channel-specific search fails.
|
||||
|
||||
Prioritizes:
|
||||
1. Channel-specific paths (if channel is set to a non-default value)
|
||||
2. Playwright bundled Chromium (when no channel or default channel specified)
|
||||
3. System Chrome stable
|
||||
4. Other system native browsers (Chromium -> Chrome Canary/Dev -> Brave -> Edge)
|
||||
5. Playwright headless-shell fallback
|
||||
|
||||
Returns:
|
||||
Path to browser executable or None if not found
|
||||
"""
|
||||
import glob
|
||||
import platform
|
||||
from pathlib import Path
|
||||
|
||||
from browser_use.browser.profile import BROWSERUSE_DEFAULT_CHANNEL, BrowserChannel
|
||||
|
||||
system = platform.system()
|
||||
|
||||
# Get playwright browsers path from environment variable if set
|
||||
playwright_path = os.environ.get('PLAYWRIGHT_BROWSERS_PATH')
|
||||
|
||||
# Build tagged pattern lists per OS: (browser_group, path)
|
||||
# browser_group is used to match against the requested channel
|
||||
if system == 'Darwin': # macOS
|
||||
if not playwright_path:
|
||||
playwright_path = '~/Library/Caches/ms-playwright'
|
||||
all_patterns = [
|
||||
('chrome', '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'),
|
||||
('chromium', f'{playwright_path}/chromium-*/chrome-mac*/Chromium.app/Contents/MacOS/Chromium'),
|
||||
('chromium', '/Applications/Chromium.app/Contents/MacOS/Chromium'),
|
||||
('chrome-canary', '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'),
|
||||
('brave', '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'),
|
||||
('msedge', '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'),
|
||||
('chromium', f'{playwright_path}/chromium_headless_shell-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium'),
|
||||
]
|
||||
elif system == 'Linux':
|
||||
if not playwright_path:
|
||||
playwright_path = '~/.cache/ms-playwright'
|
||||
all_patterns = [
|
||||
('chrome', '/usr/bin/google-chrome-stable'),
|
||||
('chrome', '/usr/bin/google-chrome'),
|
||||
('chrome', '/usr/local/bin/google-chrome'),
|
||||
('chromium', f'{playwright_path}/chromium-*/chrome-linux*/chrome'),
|
||||
('chromium', '/usr/bin/chromium'),
|
||||
('chromium', '/usr/bin/chromium-browser'),
|
||||
('chromium', '/usr/local/bin/chromium'),
|
||||
('chromium', '/snap/bin/chromium'),
|
||||
('chrome-beta', '/usr/bin/google-chrome-beta'),
|
||||
('chrome-dev', '/usr/bin/google-chrome-dev'),
|
||||
('brave', '/usr/bin/brave-browser'),
|
||||
('msedge', '/usr/bin/microsoft-edge-stable'),
|
||||
('msedge', '/usr/bin/microsoft-edge'),
|
||||
('chromium', f'{playwright_path}/chromium_headless_shell-*/chrome-linux*/chrome'),
|
||||
]
|
||||
elif system == 'Windows':
|
||||
if not playwright_path:
|
||||
playwright_path = r'%LOCALAPPDATA%\ms-playwright'
|
||||
all_patterns = [
|
||||
('chrome', r'C:\Program Files\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'%PROGRAMFILES%\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'%PROGRAMFILES(X86)%\Google\Chrome\Application\chrome.exe'),
|
||||
('chromium', f'{playwright_path}\\chromium-*\\chrome-win\\chrome.exe'),
|
||||
('chromium', r'C:\Program Files\Chromium\Application\chrome.exe'),
|
||||
('chromium', r'C:\Program Files (x86)\Chromium\Application\chrome.exe'),
|
||||
('chromium', r'%LOCALAPPDATA%\Chromium\Application\chrome.exe'),
|
||||
('brave', r'C:\Program Files\BraveSoftware\Brave-Browser\Application\brave.exe'),
|
||||
('brave', r'C:\Program Files (x86)\BraveSoftware\Brave-Browser\Application\brave.exe'),
|
||||
('msedge', r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe'),
|
||||
('msedge', r'C:\Program Files\Microsoft\Edge\Application\msedge.exe'),
|
||||
('msedge', r'%LOCALAPPDATA%\Microsoft\Edge\Application\msedge.exe'),
|
||||
('chromium', f'{playwright_path}\\chromium_headless_shell-*\\chrome-win\\chrome.exe'),
|
||||
]
|
||||
else:
|
||||
all_patterns = []
|
||||
|
||||
# Map channel enum values to browser group tags
|
||||
_channel_to_group: dict[BrowserChannel, str] = {
|
||||
BrowserChannel.CHROME: 'chrome',
|
||||
BrowserChannel.CHROME_BETA: 'chrome-beta',
|
||||
BrowserChannel.CHROME_DEV: 'chrome-dev',
|
||||
BrowserChannel.CHROME_CANARY: 'chrome-canary',
|
||||
BrowserChannel.CHROMIUM: 'chromium',
|
||||
BrowserChannel.MSEDGE: 'msedge',
|
||||
BrowserChannel.MSEDGE_BETA: 'msedge',
|
||||
BrowserChannel.MSEDGE_DEV: 'msedge',
|
||||
BrowserChannel.MSEDGE_CANARY: 'msedge',
|
||||
}
|
||||
|
||||
# Prioritize the target browser group, then fall back to the rest.
|
||||
if channel and channel != BROWSERUSE_DEFAULT_CHANNEL and channel in _channel_to_group:
|
||||
target_group = _channel_to_group[channel]
|
||||
else:
|
||||
target_group = _channel_to_group[BROWSERUSE_DEFAULT_CHANNEL]
|
||||
prioritized = [p for g, p in all_patterns if g == target_group]
|
||||
rest = [p for g, p in all_patterns if g != target_group]
|
||||
patterns = prioritized + rest
|
||||
|
||||
for pattern in patterns:
|
||||
# Expand user home directory
|
||||
expanded_pattern = Path(pattern).expanduser()
|
||||
|
||||
# Handle Windows environment variables
|
||||
if system == 'Windows':
|
||||
pattern_str = str(expanded_pattern)
|
||||
for env_var in ['%LOCALAPPDATA%', '%PROGRAMFILES%', '%PROGRAMFILES(X86)%']:
|
||||
if env_var in pattern_str:
|
||||
env_key = env_var.strip('%').replace('(X86)', ' (x86)')
|
||||
env_value = os.environ.get(env_key, '')
|
||||
if env_value:
|
||||
pattern_str = pattern_str.replace(env_var, env_value)
|
||||
expanded_pattern = Path(pattern_str)
|
||||
|
||||
# Convert to string for glob
|
||||
pattern_str = str(expanded_pattern)
|
||||
|
||||
# Check if pattern contains wildcards
|
||||
if '*' in pattern_str:
|
||||
# Use glob to expand the pattern
|
||||
matches = glob.glob(pattern_str)
|
||||
if matches:
|
||||
# Sort matches and take the last one (alphanumerically highest version)
|
||||
matches.sort()
|
||||
browser_path = matches[-1]
|
||||
if Path(browser_path).exists() and Path(browser_path).is_file():
|
||||
return browser_path
|
||||
else:
|
||||
# Direct path check
|
||||
if expanded_pattern.exists() and expanded_pattern.is_file():
|
||||
return str(expanded_pattern)
|
||||
|
||||
return None
|
||||
|
||||
async def _install_browser_with_playwright(self) -> str:
|
||||
"""Get browser executable path from playwright in a subprocess to avoid thread issues."""
|
||||
import platform
|
||||
|
||||
# Build command - only use --with-deps on Linux (it fails on Windows/macOS)
|
||||
cmd = ['uvx', 'playwright', 'install', 'chromium']
|
||||
if platform.system() == 'Linux':
|
||||
cmd.append('--with-deps')
|
||||
|
||||
# Run in subprocess with timeout
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=60.0)
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 📦 Playwright install output: {stdout}')
|
||||
browser_path = self._find_installed_browser_path()
|
||||
if browser_path:
|
||||
return browser_path
|
||||
self.logger.error(f'[LocalBrowserWatchdog] ❌ Playwright local browser installation error: \n{stdout}\n{stderr}')
|
||||
raise RuntimeError('No local browser path found after: uvx playwright install chromium')
|
||||
except TimeoutError:
|
||||
# Kill the subprocess if it times out
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise RuntimeError('Timeout getting browser path from playwright')
|
||||
except Exception as e:
|
||||
# Make sure subprocess is terminated
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise RuntimeError(f'Error getting browser path: {e}')
|
||||
|
||||
@staticmethod
|
||||
def _find_free_port() -> int:
|
||||
"""Find a free port for the debugging interface."""
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(('127.0.0.1', 0))
|
||||
s.listen(1)
|
||||
port = s.getsockname()[1]
|
||||
return port
|
||||
|
||||
@staticmethod
|
||||
async def _wait_for_cdp_url(port: int, timeout: float = 30) -> str:
|
||||
"""Wait for the browser to start and return the CDP URL."""
|
||||
import aiohttp
|
||||
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
|
||||
while asyncio.get_event_loop().time() - start_time < timeout:
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f'http://127.0.0.1:{port}/json/version') as resp:
|
||||
if resp.status == 200:
|
||||
# Chrome is ready
|
||||
return f'http://127.0.0.1:{port}/'
|
||||
else:
|
||||
# Chrome is starting up and returning 502/500 errors
|
||||
await asyncio.sleep(0.1)
|
||||
except Exception:
|
||||
# Connection error - Chrome might not be ready yet
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
raise TimeoutError(f'Browser did not start within {timeout} seconds')
|
||||
|
||||
@staticmethod
|
||||
async def _cleanup_process(process: psutil.Process) -> None:
|
||||
"""Clean up browser process.
|
||||
|
||||
Args:
|
||||
process: psutil.Process to terminate
|
||||
"""
|
||||
if not process:
|
||||
return
|
||||
|
||||
try:
|
||||
# Try graceful shutdown first
|
||||
process.terminate()
|
||||
|
||||
# Use async wait instead of blocking wait
|
||||
for _ in range(50): # Wait up to 5 seconds (50 * 0.1)
|
||||
if not process.is_running():
|
||||
return
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# If still running after 5 seconds, force kill
|
||||
if process.is_running():
|
||||
process.kill()
|
||||
# Give it a moment to die
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
except psutil.NoSuchProcess:
|
||||
# Process already gone
|
||||
pass
|
||||
except Exception:
|
||||
# Ignore any other errors during cleanup
|
||||
pass
|
||||
|
||||
def _cleanup_temp_dir(self, temp_dir: Path | str) -> None:
|
||||
"""Clean up temporary directory.
|
||||
|
||||
Args:
|
||||
temp_dir: Path to temporary directory to remove
|
||||
"""
|
||||
if not temp_dir:
|
||||
return
|
||||
|
||||
try:
|
||||
temp_path = Path(temp_dir)
|
||||
# Only remove if it's actually a temp directory we created
|
||||
if 'browseruse-tmp-' in str(temp_path):
|
||||
shutil.rmtree(temp_path, ignore_errors=True)
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to cleanup temp dir {temp_dir}: {e}')
|
||||
|
||||
@property
|
||||
def browser_pid(self) -> int | None:
|
||||
"""Get the browser process ID."""
|
||||
if self._subprocess:
|
||||
return self._subprocess.pid
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def get_browser_pid_via_cdp(browser) -> int | None:
|
||||
"""Get the browser process ID via CDP SystemInfo.getProcessInfo.
|
||||
|
||||
Args:
|
||||
browser: Playwright Browser instance
|
||||
|
||||
Returns:
|
||||
Process ID or None if failed
|
||||
"""
|
||||
try:
|
||||
cdp_session = await browser.new_browser_cdp_session()
|
||||
result = await cdp_session.send('SystemInfo.getProcessInfo')
|
||||
process_info = result.get('processInfo', {})
|
||||
pid = process_info.get('id')
|
||||
await cdp_session.detach()
|
||||
return pid
|
||||
except Exception:
|
||||
# If we can't get PID via CDP, it's not critical
|
||||
return None
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Permissions watchdog for granting browser permissions on connection."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
|
||||
from browser_use.browser.events import BrowserConnectedEvent
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class PermissionsWatchdog(BaseWatchdog):
|
||||
"""Grants browser permissions when browser connects."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Grant permissions when browser connects."""
|
||||
permissions = self.browser_session.browser_profile.permissions
|
||||
|
||||
if not permissions:
|
||||
self.logger.debug('No permissions to grant')
|
||||
return
|
||||
|
||||
self.logger.debug(f'🔓 Granting browser permissions: {permissions}')
|
||||
|
||||
try:
|
||||
# Grant permissions using CDP Browser.grantPermissions
|
||||
# origin=None means grant to all origins
|
||||
# Browser domain commands don't use session_id
|
||||
await self.browser_session.cdp_client.send.Browser.grantPermissions(
|
||||
params={'permissions': permissions} # type: ignore
|
||||
)
|
||||
self.logger.debug(f'✅ Successfully granted permissions: {permissions}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'❌ Failed to grant permissions: {str(e)}')
|
||||
# Don't raise - permissions are not critical to browser operation
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Watchdog for handling JavaScript dialogs (alert, confirm, prompt) automatically."""
|
||||
|
||||
import asyncio
|
||||
from typing import ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import TabCreatedEvent
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
|
||||
class PopupsWatchdog(BaseWatchdog):
|
||||
"""Handles JavaScript dialogs (alert, confirm, prompt) by automatically accepting them immediately."""
|
||||
|
||||
# Events this watchdog listens to and emits
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [TabCreatedEvent]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
# Track which targets have dialog handlers registered
|
||||
_dialog_listeners_registered: set[str] = PrivateAttr(default_factory=set)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.logger.debug(f'🚀 PopupsWatchdog initialized with browser_session={self.browser_session}, ID={id(self)}')
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Set up JavaScript dialog handling when a new tab is created."""
|
||||
target_id = event.target_id
|
||||
self.logger.debug(f'🎯 PopupsWatchdog received TabCreatedEvent for target {target_id}')
|
||||
|
||||
# Skip if we've already registered for this target
|
||||
if target_id in self._dialog_listeners_registered:
|
||||
self.logger.debug(f'Already registered dialog handlers for target {target_id}')
|
||||
return
|
||||
|
||||
self.logger.debug(f'📌 Starting dialog handler setup for target {target_id}')
|
||||
try:
|
||||
# Get all CDP sessions for this target and any child frames
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(
|
||||
target_id, focus=False
|
||||
) # don't auto-focus new tabs! sometimes we need to open tabs in background
|
||||
|
||||
# CRITICAL: Enable Page domain to receive dialog events
|
||||
try:
|
||||
await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
|
||||
self.logger.debug(f'✅ Enabled Page domain for session {cdp_session.session_id[-8:]}')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to enable Page domain: {e}')
|
||||
|
||||
# Also register for the root CDP client to catch dialogs from any frame
|
||||
if self.browser_session._cdp_client_root:
|
||||
self.logger.debug('📌 Also registering handler on root CDP client')
|
||||
try:
|
||||
# Enable Page domain on root client too
|
||||
await self.browser_session._cdp_client_root.send.Page.enable()
|
||||
self.logger.debug('✅ Enabled Page domain on root CDP client')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to enable Page domain on root: {e}')
|
||||
|
||||
# Set up async handler for JavaScript dialogs - accept immediately without event dispatch
|
||||
async def handle_dialog(event_data, session_id: str | None = None):
|
||||
"""Handle JavaScript dialog events - accept immediately."""
|
||||
try:
|
||||
dialog_type = event_data.get('type', 'alert')
|
||||
message = event_data.get('message', '')
|
||||
|
||||
# Store the popup message in browser session for inclusion in browser state
|
||||
if message:
|
||||
formatted_message = f'[{dialog_type}] {message}'
|
||||
self.browser_session._closed_popup_messages.append(formatted_message)
|
||||
self.logger.debug(f'📝 Stored popup message: {formatted_message[:100]}')
|
||||
|
||||
# Choose action based on dialog type:
|
||||
# - alert: accept=true (click OK to dismiss)
|
||||
# - confirm: accept=true (click OK to proceed - safer for automation)
|
||||
# - prompt: accept=false (click Cancel since we can't provide input)
|
||||
# - beforeunload: accept=true (allow navigation)
|
||||
should_accept = dialog_type in ('alert', 'confirm', 'beforeunload')
|
||||
|
||||
action_str = 'accepting (OK)' if should_accept else 'dismissing (Cancel)'
|
||||
self.logger.info(f"🔔 JavaScript {dialog_type} dialog: '{message[:100]}' - {action_str}...")
|
||||
|
||||
dismissed = False
|
||||
|
||||
# Approach 1: Use the session that detected the dialog (most reliable)
|
||||
if self.browser_session._cdp_client_root and session_id:
|
||||
try:
|
||||
self.logger.debug(f'🔄 Approach 1: Using detecting session {session_id[-8:]}')
|
||||
await asyncio.wait_for(
|
||||
self.browser_session._cdp_client_root.send.Page.handleJavaScriptDialog(
|
||||
params={'accept': should_accept},
|
||||
session_id=session_id,
|
||||
),
|
||||
timeout=0.5,
|
||||
)
|
||||
dismissed = True
|
||||
self.logger.info('✅ Dialog handled successfully via detecting session')
|
||||
except (TimeoutError, Exception) as e:
|
||||
self.logger.debug(f'Approach 1 failed: {type(e).__name__}')
|
||||
|
||||
# Approach 2: Try with current agent focus session
|
||||
if not dismissed and self.browser_session._cdp_client_root and self.browser_session.agent_focus_target_id:
|
||||
try:
|
||||
# Use public API with focus=False to avoid changing focus during popup dismissal
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(
|
||||
self.browser_session.agent_focus_target_id, focus=False
|
||||
)
|
||||
self.logger.debug(f'🔄 Approach 2: Using agent focus session {cdp_session.session_id[-8:]}')
|
||||
await asyncio.wait_for(
|
||||
self.browser_session._cdp_client_root.send.Page.handleJavaScriptDialog(
|
||||
params={'accept': should_accept},
|
||||
session_id=cdp_session.session_id,
|
||||
),
|
||||
timeout=0.5,
|
||||
)
|
||||
dismissed = True
|
||||
self.logger.info('✅ Dialog handled successfully via agent focus session')
|
||||
except (TimeoutError, Exception) as e:
|
||||
self.logger.debug(f'Approach 2 failed: {type(e).__name__}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'❌ Critical error in dialog handler: {type(e).__name__}: {e}')
|
||||
|
||||
# Register handler on the specific session
|
||||
cdp_session.cdp_client.register.Page.javascriptDialogOpening(handle_dialog) # type: ignore[arg-type]
|
||||
self.logger.debug(
|
||||
f'Successfully registered Page.javascriptDialogOpening handler for session {cdp_session.session_id}'
|
||||
)
|
||||
|
||||
# Also register on root CDP client to catch dialogs from any frame
|
||||
if hasattr(self.browser_session._cdp_client_root, 'register'):
|
||||
try:
|
||||
self.browser_session._cdp_client_root.register.Page.javascriptDialogOpening(handle_dialog) # type: ignore[arg-type]
|
||||
self.logger.debug('Successfully registered dialog handler on root CDP client for all frames')
|
||||
except Exception as root_error:
|
||||
self.logger.warning(f'Failed to register on root CDP client: {root_error}')
|
||||
|
||||
# Mark this target as having dialog handling set up
|
||||
self._dialog_listeners_registered.add(target_id)
|
||||
|
||||
self.logger.debug(f'Set up JavaScript dialog handling for tab {target_id}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to set up popup handling for tab {target_id}: {e}')
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Recording Watchdog for Browser Use Sessions."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.page.events import ScreencastFrameEvent
|
||||
from pydantic import PrivateAttr
|
||||
from uuid_extensions import uuid7str
|
||||
|
||||
from browser_use.browser.events import AgentFocusChangedEvent, BrowserConnectedEvent, BrowserStopEvent
|
||||
from browser_use.browser.profile import ViewportSize
|
||||
from browser_use.browser.video_recorder import VideoRecorderService
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
|
||||
class RecordingWatchdog(BaseWatchdog):
|
||||
"""
|
||||
Manages video recording of a browser session using CDP screencasting.
|
||||
"""
|
||||
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent, AgentFocusChangedEvent]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
_recorder: VideoRecorderService | None = PrivateAttr(default=None)
|
||||
_current_session_id: str | None = PrivateAttr(default=None)
|
||||
_screencast_params: dict[str, Any] | None = PrivateAttr(default=None)
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""
|
||||
Starts video recording if it is configured in the browser profile.
|
||||
"""
|
||||
profile = self.browser_session.browser_profile
|
||||
if not profile.record_video_dir:
|
||||
return
|
||||
|
||||
video_format = getattr(profile, 'record_video_format', 'mp4').strip('.')
|
||||
output_path = Path(profile.record_video_dir) / f'{uuid7str()}.{video_format}'
|
||||
try:
|
||||
await self.start_recording(output_path, size=profile.record_video_size, framerate=profile.record_video_framerate)
|
||||
except RuntimeError as e:
|
||||
# Preserve prior graceful degradation: a session configured with record_video_dir
|
||||
# should not fail startup when video deps are missing or viewport detection fails.
|
||||
self.logger.warning(f'Skipping video recording: {e}')
|
||||
|
||||
async def start_recording(
|
||||
self,
|
||||
output_path: Path,
|
||||
size: ViewportSize | None = None,
|
||||
framerate: int | None = None,
|
||||
) -> Path:
|
||||
"""
|
||||
Begin recording the current session to `output_path`. Safe to call at any time
|
||||
after the browser has connected.
|
||||
|
||||
Returns the resolved output path. Raises RuntimeError if recording is already active
|
||||
or if the viewport size could not be determined.
|
||||
"""
|
||||
if self._recorder is not None:
|
||||
raise RuntimeError(f'Recording already in progress (output: {self._recorder.output_path})')
|
||||
|
||||
if size is None:
|
||||
self.logger.debug('record size not specified, detecting viewport size...')
|
||||
size = await self._get_current_viewport_size()
|
||||
if not size:
|
||||
raise RuntimeError('Cannot start video recording: viewport size could not be determined.')
|
||||
|
||||
if framerate is None:
|
||||
framerate = self.browser_session.browser_profile.record_video_framerate
|
||||
|
||||
output_path = Path(output_path)
|
||||
self.logger.debug(f'Initializing video recorder → {output_path}')
|
||||
recorder = VideoRecorderService(output_path=output_path, size=size, framerate=framerate)
|
||||
recorder.start()
|
||||
if not recorder._is_active:
|
||||
raise RuntimeError(
|
||||
'Failed to initialize video recorder — ensure optional deps are installed (`pip install "browser-use[video]"`).'
|
||||
)
|
||||
|
||||
self._recorder = recorder
|
||||
self.browser_session.cdp_client.register.Page.screencastFrame(self.on_screencastFrame)
|
||||
self._screencast_params = {
|
||||
'format': 'png',
|
||||
'quality': 90,
|
||||
'maxWidth': size['width'],
|
||||
'maxHeight': size['height'],
|
||||
'everyNthFrame': 1,
|
||||
}
|
||||
await self._start_screencast()
|
||||
return output_path
|
||||
|
||||
async def stop_recording(self) -> Path | None:
|
||||
"""
|
||||
Stop any in-progress recording and finalize the output file.
|
||||
|
||||
Returns the path of the saved video, or None if no recording was active.
|
||||
"""
|
||||
if not self._recorder:
|
||||
return None
|
||||
|
||||
recorder = self._recorder
|
||||
session_id = self._current_session_id
|
||||
self._recorder = None
|
||||
self._current_session_id = None
|
||||
self._screencast_params = None
|
||||
|
||||
if session_id:
|
||||
try:
|
||||
await self.browser_session.cdp_client.send.Page.stopScreencast(session_id=session_id)
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to stop CDP screencast on {session_id}: {e}')
|
||||
|
||||
output_path = recorder.output_path
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, recorder.stop_and_save)
|
||||
return output_path
|
||||
|
||||
@property
|
||||
def is_recording(self) -> bool:
|
||||
"""Whether a recording is currently in progress."""
|
||||
return self._recorder is not None
|
||||
|
||||
async def on_AgentFocusChangedEvent(self, event: AgentFocusChangedEvent) -> None:
|
||||
"""
|
||||
Switches video recording to the new tab.
|
||||
"""
|
||||
if self._recorder:
|
||||
self.logger.debug(f'Agent focus changed to {event.target_id}, switching screencast...')
|
||||
await self._start_screencast()
|
||||
|
||||
async def _start_screencast(self) -> None:
|
||||
"""Starts screencast on the currently focused tab."""
|
||||
if not self._recorder or not self._screencast_params:
|
||||
return
|
||||
|
||||
try:
|
||||
# Get the current session (for the focused target)
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
|
||||
# If we are already recording this session, do nothing
|
||||
if self._current_session_id == cdp_session.session_id:
|
||||
return
|
||||
|
||||
# Stop recording on the previous session
|
||||
if self._current_session_id:
|
||||
try:
|
||||
# Use the root client to stop screencast on the specific session
|
||||
await self.browser_session.cdp_client.send.Page.stopScreencast(session_id=self._current_session_id)
|
||||
except Exception as e:
|
||||
# It's possible the session is already closed
|
||||
self.logger.debug(f'Failed to stop screencast on old session {self._current_session_id}: {e}')
|
||||
|
||||
self._current_session_id = cdp_session.session_id
|
||||
|
||||
# Start recording on the new session
|
||||
await cdp_session.cdp_client.send.Page.startScreencast(
|
||||
params=self._screencast_params, # type: ignore
|
||||
session_id=cdp_session.session_id,
|
||||
)
|
||||
self.logger.info(f'📹 Started/Switched video recording to target {cdp_session.target_id}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'Failed to switch screencast via CDP: {e}')
|
||||
# If we fail to start on the new tab, we reset current session id
|
||||
self._current_session_id = None
|
||||
|
||||
async def _get_current_viewport_size(self) -> ViewportSize | None:
|
||||
"""Gets the current viewport size directly from the browser via CDP."""
|
||||
try:
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
|
||||
|
||||
# Use cssVisualViewport for the most accurate representation of the visible area
|
||||
viewport = metrics.get('cssVisualViewport', {})
|
||||
width = viewport.get('clientWidth')
|
||||
height = viewport.get('clientHeight')
|
||||
|
||||
if width and height:
|
||||
self.logger.debug(f'Detected viewport size: {width}x{height}')
|
||||
return ViewportSize(width=int(width), height=int(height))
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to get viewport size from browser: {e}')
|
||||
|
||||
return None
|
||||
|
||||
def on_screencastFrame(self, event: ScreencastFrameEvent, session_id: str | None) -> None:
|
||||
"""
|
||||
Synchronous handler for incoming screencast frames.
|
||||
"""
|
||||
# Only process frames from the current session we intend to record
|
||||
# This handles race conditions where old session might still send frames before stop completes
|
||||
if self._current_session_id and session_id != self._current_session_id:
|
||||
return
|
||||
|
||||
if not self._recorder:
|
||||
return
|
||||
self._recorder.add_frame(event['data'])
|
||||
create_task_with_error_handling(
|
||||
self._ack_screencast_frame(event, session_id),
|
||||
name='ack_screencast_frame',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
async def _ack_screencast_frame(self, event: ScreencastFrameEvent, session_id: str | None) -> None:
|
||||
"""
|
||||
Asynchronously acknowledges a screencast frame.
|
||||
"""
|
||||
try:
|
||||
await self.browser_session.cdp_client.send.Page.screencastFrameAck(
|
||||
params={'sessionId': event['sessionId']}, session_id=session_id
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to acknowledge screencast frame: {e}')
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""
|
||||
Stops the video recording and finalizes the video file.
|
||||
"""
|
||||
if self._recorder:
|
||||
self.logger.debug('Stopping video recording and saving file...')
|
||||
await self.stop_recording()
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Screenshot watchdog for handling screenshot requests using CDP."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.page import CaptureScreenshotParameters
|
||||
|
||||
from browser_use.browser.events import ScreenshotEvent
|
||||
from browser_use.browser.views import BrowserError
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.observability import observe_debug
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class ScreenshotWatchdog(BaseWatchdog):
|
||||
"""Handles screenshot requests using CDP."""
|
||||
|
||||
# Events this watchdog listens to
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [ScreenshotEvent]
|
||||
|
||||
# Events this watchdog emits
|
||||
EMITS: ClassVar[list[type[BaseEvent[Any]]]] = []
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='screenshot_event_handler')
|
||||
async def on_ScreenshotEvent(self, event: ScreenshotEvent) -> str:
|
||||
"""Handle screenshot request using CDP.
|
||||
|
||||
Args:
|
||||
event: ScreenshotEvent with optional full_page and clip parameters
|
||||
|
||||
Returns:
|
||||
Dict with 'screenshot' key containing base64-encoded screenshot or None
|
||||
"""
|
||||
self.logger.debug('[ScreenshotWatchdog] Handler START - on_ScreenshotEvent called')
|
||||
try:
|
||||
# Validate focused target is a top-level page (not iframe/worker)
|
||||
# CDP Page.captureScreenshot only works on page/tab targets
|
||||
focused_target = self.browser_session.get_focused_target()
|
||||
|
||||
if focused_target and focused_target.target_type in ('page', 'tab'):
|
||||
target_id = focused_target.target_id
|
||||
else:
|
||||
# Focused target is iframe/worker/missing - fall back to any page target
|
||||
target_type_str = focused_target.target_type if focused_target else 'None'
|
||||
self.logger.warning(f'[ScreenshotWatchdog] Focused target is {target_type_str}, falling back to page target')
|
||||
page_targets = self.browser_session.get_page_targets()
|
||||
if not page_targets:
|
||||
raise BrowserError('[ScreenshotWatchdog] No page targets available for screenshot')
|
||||
target_id = page_targets[-1].target_id
|
||||
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=True)
|
||||
|
||||
# Remove highlights BEFORE taking the screenshot so they don't appear in the image.
|
||||
# Done here (not in finally) so CancelledError is never swallowed — any await in a
|
||||
# finally block can suppress external task cancellation.
|
||||
# remove_highlights() has its own asyncio.timeout(3.0) internally so it won't block.
|
||||
try:
|
||||
await self.browser_session.remove_highlights()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Prepare screenshot parameters
|
||||
params_dict: dict[str, Any] = {'format': 'png', 'captureBeyondViewport': event.full_page}
|
||||
if event.clip:
|
||||
params_dict['clip'] = {
|
||||
'x': event.clip['x'],
|
||||
'y': event.clip['y'],
|
||||
'width': event.clip['width'],
|
||||
'height': event.clip['height'],
|
||||
'scale': 1,
|
||||
}
|
||||
params = CaptureScreenshotParameters(**params_dict)
|
||||
|
||||
# Take screenshot using CDP
|
||||
self.logger.debug(f'[ScreenshotWatchdog] Taking screenshot with params: {params}')
|
||||
result = await cdp_session.cdp_client.send.Page.captureScreenshot(params=params, session_id=cdp_session.session_id)
|
||||
|
||||
# Return base64-encoded screenshot data
|
||||
if result and 'data' in result:
|
||||
self.logger.debug('[ScreenshotWatchdog] Screenshot captured successfully')
|
||||
return result['data']
|
||||
|
||||
raise BrowserError('[ScreenshotWatchdog] Screenshot result missing data')
|
||||
except Exception as e:
|
||||
self.logger.error(f'[ScreenshotWatchdog] Screenshot failed: {e}')
|
||||
raise
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Security watchdog for enforcing URL access policies."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserErrorEvent,
|
||||
NavigateToUrlEvent,
|
||||
NavigationCompleteEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
# Track if we've shown the glob warning
|
||||
_GLOB_WARNING_SHOWN = False
|
||||
|
||||
|
||||
class SecurityWatchdog(BaseWatchdog):
|
||||
"""Monitors and enforces security policies for URL access."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
NavigateToUrlEvent,
|
||||
NavigationCompleteEvent,
|
||||
TabCreatedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserErrorEvent,
|
||||
]
|
||||
|
||||
async def on_NavigateToUrlEvent(self, event: NavigateToUrlEvent) -> None:
|
||||
"""Check if navigation URL is allowed before navigation starts."""
|
||||
# Security check BEFORE navigation
|
||||
if not self._is_url_allowed(event.url):
|
||||
self.logger.warning(f'⛔️ Blocking navigation to disallowed URL: {event.url}')
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='NavigationBlocked',
|
||||
message=f'Navigation blocked to disallowed URL: {event.url}',
|
||||
details={'url': event.url, 'reason': 'not_in_allowed_domains'},
|
||||
)
|
||||
)
|
||||
# Stop event propagation by raising exception
|
||||
raise ValueError(f'Navigation to {event.url} blocked by security policy')
|
||||
|
||||
async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
|
||||
"""Check if navigated URL is allowed (catches redirects to blocked domains)."""
|
||||
# Check if the navigated URL is allowed (in case of redirects)
|
||||
if not self._is_url_allowed(event.url):
|
||||
self.logger.warning(f'⛔️ Navigation to non-allowed URL detected: {event.url}')
|
||||
|
||||
# Dispatch browser error
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='NavigationBlocked',
|
||||
message=f'Navigation blocked to non-allowed URL: {event.url} - redirecting to about:blank',
|
||||
details={'url': event.url, 'target_id': event.target_id},
|
||||
)
|
||||
)
|
||||
# Navigate to about:blank to keep session alive
|
||||
# Agent will see the error and can continue with other tasks
|
||||
try:
|
||||
session = await self.browser_session.get_or_create_cdp_session(target_id=event.target_id)
|
||||
await session.cdp_client.send.Page.navigate(params={'url': 'about:blank'}, session_id=session.session_id)
|
||||
self.logger.info(f'⛔️ Navigated to about:blank after blocked URL: {event.url}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'⛔️ Failed to navigate to about:blank: {type(e).__name__} {e}')
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Check if new tab URL is allowed."""
|
||||
if not self._is_url_allowed(event.url):
|
||||
self.logger.warning(f'⛔️ New tab created with disallowed URL: {event.url}')
|
||||
|
||||
# Dispatch error and try to close the tab
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='TabCreationBlocked',
|
||||
message=f'Tab created with non-allowed URL: {event.url}',
|
||||
details={'url': event.url, 'target_id': event.target_id},
|
||||
)
|
||||
)
|
||||
|
||||
# Try to close the offending tab
|
||||
try:
|
||||
await self.browser_session._cdp_close_page(event.target_id)
|
||||
self.logger.info(f'⛔️ Closed new tab with non-allowed URL: {event.url}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'⛔️ Failed to close new tab with non-allowed URL: {type(e).__name__} {e}')
|
||||
|
||||
def _is_root_domain(self, domain: str) -> bool:
|
||||
"""Check if a domain is a root domain (no subdomain present).
|
||||
|
||||
Simple heuristic: only add www for domains with exactly 1 dot (domain.tld).
|
||||
For complex cases like country TLDs or subdomains, users should configure explicitly.
|
||||
|
||||
Args:
|
||||
domain: The domain to check
|
||||
|
||||
Returns:
|
||||
True if it's a simple root domain, False otherwise
|
||||
"""
|
||||
# Skip if it contains wildcards or protocol
|
||||
if '*' in domain or '://' in domain:
|
||||
return False
|
||||
|
||||
return domain.count('.') == 1
|
||||
|
||||
def _log_glob_warning(self) -> None:
|
||||
"""Log a warning about glob patterns in allowed_domains."""
|
||||
global _GLOB_WARNING_SHOWN
|
||||
if not _GLOB_WARNING_SHOWN:
|
||||
_GLOB_WARNING_SHOWN = True
|
||||
self.logger.warning(
|
||||
'⚠️ Using glob patterns in allowed_domains. '
|
||||
'Note: Patterns like "*.example.com" will match both subdomains AND the main domain.'
|
||||
)
|
||||
|
||||
def _get_domain_variants(self, host: str) -> tuple[str, str]:
|
||||
"""Get both variants of a domain (with and without www prefix).
|
||||
|
||||
Args:
|
||||
host: The hostname to process
|
||||
|
||||
Returns:
|
||||
Tuple of (original_host, variant_host)
|
||||
- If host starts with www., variant is without www.
|
||||
- Otherwise, variant is with www. prefix
|
||||
"""
|
||||
if host.startswith('www.'):
|
||||
return (host, host[4:]) # ('www.example.com', 'example.com')
|
||||
else:
|
||||
return (host, f'www.{host}') # ('example.com', 'www.example.com')
|
||||
|
||||
def _is_ip_address(self, host: str) -> bool:
|
||||
"""True iff `host` matches an IPv4 or IPv6 the browser would resolve.
|
||||
|
||||
Mirrors WHATWG host canonicalization so non-standard IPv4 encodings
|
||||
(decimal, hex, octal, short-form, percent-encoded, Unicode digits)
|
||||
can't bypass `block_ip_addresses`. Never raises — unrecognizable
|
||||
hosts return False and fall through to domain-allowlist handling.
|
||||
"""
|
||||
import ipaddress
|
||||
import socket
|
||||
import unicodedata
|
||||
from urllib.parse import unquote
|
||||
|
||||
bare = host.strip('[]')
|
||||
try:
|
||||
bare = unquote(bare)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
bare = unicodedata.normalize('NFKC', bare)
|
||||
except Exception:
|
||||
pass
|
||||
# IDNA label separators NFKC misses (U+3002, U+FF61 → U+3002).
|
||||
bare = bare.replace('。', '.').replace('｡', '.')
|
||||
|
||||
try:
|
||||
ipaddress.ip_address(bare)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
# Non-standard IPv4 (decimal, hex, octal, short-form) — `inet_aton`
|
||||
# accepts the same liberal forms the kernel resolver does.
|
||||
try:
|
||||
socket.inet_aton(bare)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _is_url_allowed(self, url: str) -> bool:
|
||||
"""Check if a URL is allowed based on the allowed_domains configuration.
|
||||
|
||||
Args:
|
||||
url: The URL to check
|
||||
|
||||
Returns:
|
||||
True if the URL is allowed, False otherwise
|
||||
"""
|
||||
|
||||
# Always allow internal browser targets (before any other checks)
|
||||
if url in ['about:blank', 'chrome://new-tab-page/', 'chrome://new-tab-page', 'chrome://newtab/']:
|
||||
return True
|
||||
|
||||
# Parse the URL to extract components
|
||||
from urllib.parse import urlparse
|
||||
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except Exception:
|
||||
# Invalid URL
|
||||
return False
|
||||
|
||||
# Allow data: and blob: URLs (they don't have hostnames)
|
||||
if parsed.scheme in ['data', 'blob']:
|
||||
return True
|
||||
|
||||
# Get the actual host (domain)
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
return False
|
||||
|
||||
# Check if IP addresses should be blocked (before domain checks)
|
||||
if self.browser_session.browser_profile.block_ip_addresses:
|
||||
if self._is_ip_address(host):
|
||||
return False
|
||||
|
||||
# If no allowed_domains specified, allow all URLs
|
||||
if (
|
||||
not self.browser_session.browser_profile.allowed_domains
|
||||
and not self.browser_session.browser_profile.prohibited_domains
|
||||
):
|
||||
return True
|
||||
|
||||
# Check allowed domains (fast path for sets, slow path for lists with patterns)
|
||||
if self.browser_session.browser_profile.allowed_domains:
|
||||
allowed_domains = self.browser_session.browser_profile.allowed_domains
|
||||
|
||||
if isinstance(allowed_domains, set):
|
||||
# Fast path: O(1) exact hostname match - check both www and non-www variants
|
||||
host_variant, host_alt = self._get_domain_variants(host)
|
||||
return host_variant in allowed_domains or host_alt in allowed_domains
|
||||
else:
|
||||
# Slow path: O(n) pattern matching for lists
|
||||
for pattern in allowed_domains:
|
||||
if self._is_url_match(url, host, parsed.scheme, pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
# Check prohibited domains (fast path for sets, slow path for lists with patterns)
|
||||
if self.browser_session.browser_profile.prohibited_domains:
|
||||
prohibited_domains = self.browser_session.browser_profile.prohibited_domains
|
||||
|
||||
if isinstance(prohibited_domains, set):
|
||||
# Fast path: O(1) exact hostname match - check both www and non-www variants
|
||||
host_variant, host_alt = self._get_domain_variants(host)
|
||||
return host_variant not in prohibited_domains and host_alt not in prohibited_domains
|
||||
else:
|
||||
# Slow path: O(n) pattern matching for lists
|
||||
for pattern in prohibited_domains:
|
||||
if self._is_url_match(url, host, parsed.scheme, pattern):
|
||||
return False
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
def _is_url_match(self, url: str, host: str, scheme: str, pattern: str) -> bool:
|
||||
"""Check if a URL matches a pattern."""
|
||||
|
||||
# Full URL for matching (scheme + host)
|
||||
full_url_pattern = f'{scheme}://{host}'
|
||||
|
||||
# Handle glob patterns
|
||||
if '*' in pattern:
|
||||
self._log_glob_warning()
|
||||
import fnmatch
|
||||
|
||||
# Check if pattern matches the host
|
||||
if pattern.startswith('*.'):
|
||||
# Pattern like *.example.com should match subdomains and main domain
|
||||
domain_part = pattern[2:] # Remove *.
|
||||
if host == domain_part or host.endswith('.' + domain_part):
|
||||
# Only match http/https URLs for domain-only patterns
|
||||
if scheme in ['http', 'https']:
|
||||
return True
|
||||
elif pattern.endswith('/*'):
|
||||
# Pattern like brave://* or http*://example.com/*
|
||||
if fnmatch.fnmatch(url, pattern):
|
||||
return True
|
||||
else:
|
||||
# Use fnmatch for other glob patterns
|
||||
if fnmatch.fnmatch(
|
||||
full_url_pattern if '://' in pattern else host,
|
||||
pattern,
|
||||
):
|
||||
return True
|
||||
else:
|
||||
# Exact match
|
||||
if '://' in pattern:
|
||||
# Full URL pattern
|
||||
if url.startswith(pattern):
|
||||
return True
|
||||
else:
|
||||
# Domain-only pattern (case-insensitive comparison)
|
||||
if host.lower() == pattern.lower():
|
||||
return True
|
||||
# If pattern is a root domain, also check www subdomain
|
||||
if self._is_root_domain(pattern) and host.lower() == f'www.{pattern.lower()}':
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -0,0 +1,373 @@
|
||||
"""Storage state watchdog for managing browser cookies and storage persistence."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.network import Cookie
|
||||
from pydantic import Field, PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserConnectedEvent,
|
||||
BrowserStopEvent,
|
||||
LoadStorageStateEvent,
|
||||
SaveStorageStateEvent,
|
||||
StorageStateLoadedEvent,
|
||||
StorageStateSavedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
|
||||
class StorageStateWatchdog(BaseWatchdog):
|
||||
"""Monitors and persists browser storage state including cookies and localStorage."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
BrowserStopEvent,
|
||||
SaveStorageStateEvent,
|
||||
LoadStorageStateEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
StorageStateSavedEvent,
|
||||
StorageStateLoadedEvent,
|
||||
]
|
||||
|
||||
# Configuration
|
||||
auto_save_interval: float = Field(default=30.0) # Auto-save every 30 seconds
|
||||
save_on_change: bool = Field(default=True) # Save immediately when cookies change
|
||||
|
||||
# Private state
|
||||
_monitoring_task: asyncio.Task | None = PrivateAttr(default=None)
|
||||
_last_cookie_state: list[dict] = PrivateAttr(default_factory=list)
|
||||
_save_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock)
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Start monitoring when browser starts."""
|
||||
self.logger.debug('[StorageStateWatchdog] 🍪 Initializing auth/cookies sync <-> with storage_state.json file')
|
||||
|
||||
# Start monitoring
|
||||
await self._start_monitoring()
|
||||
|
||||
# Automatically load storage state after browser start
|
||||
await self.event_bus.dispatch(LoadStorageStateEvent())
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""Stop monitoring when browser stops."""
|
||||
self.logger.debug('[StorageStateWatchdog] Stopping storage_state monitoring')
|
||||
await self._stop_monitoring()
|
||||
|
||||
async def on_SaveStorageStateEvent(self, event: SaveStorageStateEvent) -> None:
|
||||
"""Handle storage state save request."""
|
||||
# Use provided path or fall back to profile default
|
||||
path = event.path
|
||||
if path is None:
|
||||
# Use profile default path if available
|
||||
if self.browser_session.browser_profile.storage_state:
|
||||
path = str(self.browser_session.browser_profile.storage_state)
|
||||
else:
|
||||
path = None # Skip saving if no path available
|
||||
await self._save_storage_state(path)
|
||||
|
||||
async def on_LoadStorageStateEvent(self, event: LoadStorageStateEvent) -> None:
|
||||
"""Handle storage state load request."""
|
||||
# Use provided path or fall back to profile default
|
||||
path = event.path
|
||||
if path is None:
|
||||
# Use profile default path if available
|
||||
if self.browser_session.browser_profile.storage_state:
|
||||
path = str(self.browser_session.browser_profile.storage_state)
|
||||
else:
|
||||
path = None # Skip loading if no path available
|
||||
await self._load_storage_state(path)
|
||||
|
||||
async def _start_monitoring(self) -> None:
|
||||
"""Start the monitoring task."""
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
return
|
||||
|
||||
assert self.browser_session.cdp_client is not None
|
||||
|
||||
self._monitoring_task = create_task_with_error_handling(
|
||||
self._monitor_storage_changes(), name='monitor_storage_changes', logger_instance=self.logger, suppress_exceptions=True
|
||||
)
|
||||
# self.logger'[StorageStateWatchdog] Started storage monitoring task')
|
||||
|
||||
async def _stop_monitoring(self) -> None:
|
||||
"""Stop the monitoring task."""
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
self._monitoring_task.cancel()
|
||||
try:
|
||||
await self._monitoring_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
# self.logger.debug('[StorageStateWatchdog] Stopped storage monitoring task')
|
||||
|
||||
async def _check_for_cookie_changes_cdp(self, event: dict) -> None:
|
||||
"""Check if a CDP network event indicates cookie changes.
|
||||
|
||||
This would be called by Network.responseReceivedExtraInfo events
|
||||
if we set up CDP event listeners.
|
||||
"""
|
||||
try:
|
||||
# Check for Set-Cookie headers in the response
|
||||
headers = event.get('headers', {})
|
||||
if 'set-cookie' in headers or 'Set-Cookie' in headers:
|
||||
self.logger.debug('[StorageStateWatchdog] Cookie change detected via CDP')
|
||||
|
||||
# If save on change is enabled, trigger save immediately
|
||||
if self.save_on_change:
|
||||
await self._save_storage_state()
|
||||
except Exception as e:
|
||||
self.logger.warning(f'[StorageStateWatchdog] Error checking for cookie changes: {e}')
|
||||
|
||||
async def _monitor_storage_changes(self) -> None:
|
||||
"""Periodically check for storage changes and auto-save."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(self.auto_save_interval)
|
||||
|
||||
# Check if cookies have changed
|
||||
if await self._have_cookies_changed():
|
||||
self.logger.debug('[StorageStateWatchdog] Detected changes to sync with storage_state.json')
|
||||
await self._save_storage_state()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Error in monitoring loop: {e}')
|
||||
|
||||
async def _have_cookies_changed(self) -> bool:
|
||||
"""Check if cookies have changed since last save."""
|
||||
if not self.browser_session.cdp_client:
|
||||
return False
|
||||
|
||||
try:
|
||||
# Get current cookies using CDP
|
||||
current_cookies = await self.browser_session._cdp_get_cookies()
|
||||
|
||||
# Convert to comparable format, using .get() for optional fields
|
||||
current_cookie_set = {
|
||||
(c.get('name', ''), c.get('domain', ''), c.get('path', '')): c.get('value', '') for c in current_cookies
|
||||
}
|
||||
|
||||
last_cookie_set = {
|
||||
(c.get('name', ''), c.get('domain', ''), c.get('path', '')): c.get('value', '') for c in self._last_cookie_state
|
||||
}
|
||||
|
||||
return current_cookie_set != last_cookie_set
|
||||
except Exception as e:
|
||||
self.logger.debug(f'[StorageStateWatchdog] Error comparing cookies: {e}')
|
||||
return False
|
||||
|
||||
async def _save_storage_state(self, path: str | None = None) -> None:
|
||||
"""Save browser storage state to file."""
|
||||
async with self._save_lock:
|
||||
# Check if CDP client is available
|
||||
assert await self.browser_session.get_or_create_cdp_session(target_id=None)
|
||||
|
||||
save_path = path or self.browser_session.browser_profile.storage_state
|
||||
if not save_path:
|
||||
return
|
||||
|
||||
# Skip saving if the storage state is already a dict (indicates it was loaded from memory)
|
||||
# We only save to file if it started as a file path
|
||||
if isinstance(save_path, dict):
|
||||
self.logger.debug('[StorageStateWatchdog] Storage state is already a dict, skipping file save')
|
||||
return
|
||||
|
||||
try:
|
||||
# Get current storage state using CDP
|
||||
storage_state = await self.browser_session._cdp_get_storage_state()
|
||||
|
||||
# Update our last known state
|
||||
self._last_cookie_state = storage_state.get('cookies', []).copy()
|
||||
|
||||
# Convert path to Path object
|
||||
json_path = Path(save_path).expanduser().resolve()
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Merge with existing state if file exists
|
||||
merged_state = storage_state
|
||||
if json_path.exists():
|
||||
try:
|
||||
existing_state = json.loads(json_path.read_text())
|
||||
merged_state = self._merge_storage_states(existing_state, dict(storage_state))
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to merge with existing state: {e}')
|
||||
|
||||
# Write atomically
|
||||
temp_path = json_path.with_suffix('.json.tmp')
|
||||
temp_path.write_text(json.dumps(merged_state, indent=4, ensure_ascii=False), encoding='utf-8')
|
||||
|
||||
# Backup existing file
|
||||
if json_path.exists():
|
||||
backup_path = json_path.with_suffix('.json.bak')
|
||||
json_path.replace(backup_path)
|
||||
|
||||
# Move temp to final
|
||||
temp_path.replace(json_path)
|
||||
|
||||
# Emit success event
|
||||
self.event_bus.dispatch(
|
||||
StorageStateSavedEvent(
|
||||
path=str(json_path),
|
||||
cookies_count=len(merged_state.get('cookies', [])),
|
||||
origins_count=len(merged_state.get('origins', [])),
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.debug(
|
||||
f'[StorageStateWatchdog] Saved storage state to {json_path} '
|
||||
f'({len(merged_state.get("cookies", []))} cookies, '
|
||||
f'{len(merged_state.get("origins", []))} origins)'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to save storage state: {e}')
|
||||
|
||||
async def _load_storage_state(self, path: str | None = None) -> None:
|
||||
"""Load browser storage state from file."""
|
||||
if not self.browser_session.cdp_client:
|
||||
self.logger.warning('[StorageStateWatchdog] No CDP client available for loading')
|
||||
return
|
||||
|
||||
load_path = path or self.browser_session.browser_profile.storage_state
|
||||
if not load_path or not os.path.exists(str(load_path)):
|
||||
return
|
||||
|
||||
try:
|
||||
# Read the storage state file asynchronously
|
||||
import anyio
|
||||
|
||||
content = await anyio.Path(str(load_path)).read_text()
|
||||
storage = json.loads(content)
|
||||
|
||||
# Apply cookies if present
|
||||
if 'cookies' in storage and storage['cookies']:
|
||||
# Playwright exports session cookies with expires=0/-1. CDP treats expires=0 as expired.
|
||||
# Normalize session cookies by omitting expires
|
||||
normalized_cookies: list[Cookie] = []
|
||||
for cookie in storage['cookies']:
|
||||
if not isinstance(cookie, dict):
|
||||
normalized_cookies.append(cookie) # type: ignore[arg-type]
|
||||
continue
|
||||
c = dict(cookie)
|
||||
expires = c.get('expires')
|
||||
if expires in (0, 0.0, -1, -1.0):
|
||||
c.pop('expires', None)
|
||||
normalized_cookies.append(Cookie(**c))
|
||||
|
||||
await self.browser_session._cdp_set_cookies(normalized_cookies)
|
||||
self._last_cookie_state = storage['cookies'].copy()
|
||||
self.logger.debug(f'[StorageStateWatchdog] Added {len(storage["cookies"])} cookies from storage state')
|
||||
|
||||
# Apply origins (localStorage/sessionStorage) if present
|
||||
if 'origins' in storage and storage['origins']:
|
||||
for origin in storage['origins']:
|
||||
origin_value = origin.get('origin')
|
||||
if not origin_value:
|
||||
continue
|
||||
|
||||
# Scope storage restoration to its origin to avoid cross-site pollution.
|
||||
if origin.get('localStorage'):
|
||||
lines = []
|
||||
for item in origin['localStorage']:
|
||||
lines.append(f'window.localStorage.setItem({json.dumps(item["name"])}, {json.dumps(item["value"])});')
|
||||
script = (
|
||||
'(function(){\n'
|
||||
f' if (window.location && window.location.origin !== {json.dumps(origin_value)}) return;\n'
|
||||
' try {\n'
|
||||
f' {" ".join(lines)}\n'
|
||||
' } catch (e) {}\n'
|
||||
'})();'
|
||||
)
|
||||
await self.browser_session._cdp_add_init_script(script)
|
||||
|
||||
if origin.get('sessionStorage'):
|
||||
lines = []
|
||||
for item in origin['sessionStorage']:
|
||||
lines.append(
|
||||
f'window.sessionStorage.setItem({json.dumps(item["name"])}, {json.dumps(item["value"])});'
|
||||
)
|
||||
script = (
|
||||
'(function(){\n'
|
||||
f' if (window.location && window.location.origin !== {json.dumps(origin_value)}) return;\n'
|
||||
' try {\n'
|
||||
f' {" ".join(lines)}\n'
|
||||
' } catch (e) {}\n'
|
||||
'})();'
|
||||
)
|
||||
await self.browser_session._cdp_add_init_script(script)
|
||||
self.logger.debug(
|
||||
f'[StorageStateWatchdog] Applied localStorage/sessionStorage from {len(storage["origins"])} origins'
|
||||
)
|
||||
|
||||
self.event_bus.dispatch(
|
||||
StorageStateLoadedEvent(
|
||||
path=str(load_path),
|
||||
cookies_count=len(storage.get('cookies', [])),
|
||||
origins_count=len(storage.get('origins', [])),
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.debug(f'[StorageStateWatchdog] Loaded storage state from: {load_path}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to load storage state: {e}')
|
||||
|
||||
@staticmethod
|
||||
def _merge_storage_states(existing: dict[str, Any], new: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Merge two storage states, with new values taking precedence."""
|
||||
merged = existing.copy()
|
||||
|
||||
# Merge cookies
|
||||
existing_cookies = {(c['name'], c['domain'], c['path']): c for c in existing.get('cookies', [])}
|
||||
|
||||
for cookie in new.get('cookies', []):
|
||||
key = (cookie['name'], cookie['domain'], cookie['path'])
|
||||
existing_cookies[key] = cookie
|
||||
|
||||
merged['cookies'] = list(existing_cookies.values())
|
||||
|
||||
# Merge origins
|
||||
existing_origins = {origin['origin']: origin for origin in existing.get('origins', [])}
|
||||
|
||||
for origin in new.get('origins', []):
|
||||
existing_origins[origin['origin']] = origin
|
||||
|
||||
merged['origins'] = list(existing_origins.values())
|
||||
|
||||
return merged
|
||||
|
||||
async def get_current_cookies(self) -> list[dict[str, Any]]:
|
||||
"""Get current cookies using CDP."""
|
||||
if not self.browser_session.cdp_client:
|
||||
return []
|
||||
|
||||
try:
|
||||
cookies = await self.browser_session._cdp_get_cookies()
|
||||
# Cookie is a TypedDict, cast to dict for compatibility
|
||||
return [dict(cookie) for cookie in cookies]
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to get cookies: {e}')
|
||||
return []
|
||||
|
||||
async def add_cookies(self, cookies: list[dict[str, Any]]) -> None:
|
||||
"""Add cookies using CDP."""
|
||||
if not self.browser_session.cdp_client:
|
||||
self.logger.warning('[StorageStateWatchdog] No CDP client available for adding cookies')
|
||||
return
|
||||
|
||||
try:
|
||||
# Convert dicts to Cookie objects
|
||||
cookie_objects = [Cookie(**cookie_dict) if isinstance(cookie_dict, dict) else cookie_dict for cookie_dict in cookies]
|
||||
# Set cookies using CDP
|
||||
await self.browser_session._cdp_set_cookies(cookie_objects)
|
||||
self.logger.debug(f'[StorageStateWatchdog] Added {len(cookies)} cookies')
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to add cookies: {e}')
|
||||
@@ -0,0 +1,464 @@
|
||||
"""Browser Use CLI backed by Browser Harness"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
from io import StringIO
|
||||
|
||||
|
||||
def _browser_use_version() -> str:
|
||||
try:
|
||||
return version('browser-use')
|
||||
except PackageNotFoundError:
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def _exit_code(result: int | str | None) -> int:
|
||||
if result is None:
|
||||
return 0
|
||||
if isinstance(result, int):
|
||||
return result
|
||||
return 1
|
||||
|
||||
|
||||
def _set_harness_client_env() -> None:
|
||||
import os
|
||||
|
||||
os.environ['BH_CLIENT'] = 'browser-use-cli'
|
||||
os.environ['BH_CLIENT_VERSION'] = _browser_use_version()
|
||||
|
||||
|
||||
def _capture_via_harness(
|
||||
*,
|
||||
command: str,
|
||||
start_time: float,
|
||||
result: int | str | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
from browser_harness import telemetry as harness_telemetry
|
||||
|
||||
capture_cli_event = getattr(harness_telemetry, 'capture_cli_event', None)
|
||||
if capture_cli_event is None:
|
||||
return
|
||||
_set_harness_client_env()
|
||||
code = _exit_code(result)
|
||||
capture_cli_event(
|
||||
action='error' if code else 'completed',
|
||||
command=command,
|
||||
duration_seconds=time.monotonic() - start_time,
|
||||
exit_code=code,
|
||||
error_message=error_message,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _run_mcp_stdio_server(module_name: str) -> None:
|
||||
"""Silence all logging"""
|
||||
import asyncio
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
os.environ['BROWSER_USE_LOGGING_LEVEL'] = 'critical'
|
||||
os.environ['BROWSER_USE_SETUP_LOGGING'] = 'false'
|
||||
logging.disable(logging.CRITICAL)
|
||||
|
||||
main = importlib.import_module(module_name).main
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
def _run_mcp_server() -> None:
|
||||
_run_mcp_stdio_server('browser_use.mcp.server')
|
||||
|
||||
|
||||
def _run_cli_mcp_server() -> None:
|
||||
_run_mcp_stdio_server('browser_use.mcp.cli_mcp')
|
||||
|
||||
|
||||
def _run_install_command(argv: list[str]) -> int:
|
||||
if any(arg in {'-h', '--help'} for arg in argv):
|
||||
print('usage: browser-use install')
|
||||
print()
|
||||
print('Install Chromium browser and system dependencies.')
|
||||
return 0
|
||||
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
print('Installing Chromium browser + system dependencies...')
|
||||
print('This may take a few minutes...\n')
|
||||
|
||||
cmd = ['uvx', 'playwright', 'install', 'chromium']
|
||||
if platform.system() == 'Linux':
|
||||
cmd.append('--with-deps')
|
||||
cmd.append('--no-shell')
|
||||
|
||||
result = subprocess.run(cmd)
|
||||
if result.returncode == 0:
|
||||
print('\nInstallation complete.')
|
||||
print('Ready to use. Run: uvx browser-use')
|
||||
return 0
|
||||
|
||||
print('\nInstallation failed', file=sys.stderr)
|
||||
return result.returncode or 1
|
||||
|
||||
|
||||
def _run_init_command(argv: list[str]) -> int | None:
|
||||
from browser_use.init_cmd import main as init_main
|
||||
|
||||
original_argv = sys.argv
|
||||
try:
|
||||
sys.argv = [original_argv[0], *argv]
|
||||
init_main()
|
||||
except SystemExit as exc:
|
||||
if exc.code is None:
|
||||
return 0
|
||||
if isinstance(exc.code, int):
|
||||
return exc.code
|
||||
print(exc.code, file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
sys.argv = original_argv
|
||||
return 0
|
||||
|
||||
|
||||
def _as_browser_use_cli_text(text: str) -> str:
|
||||
return text.replace('Browser Harness', 'Browser Use').replace('browser-harness', 'browser-use')
|
||||
|
||||
|
||||
def _normalize_captured_cli_output(func, argv: list[str]) -> int:
|
||||
stdout = StringIO()
|
||||
stderr = StringIO()
|
||||
try:
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
result = func(argv)
|
||||
except SystemExit as exc:
|
||||
result = exc.code
|
||||
|
||||
out = stdout.getvalue()
|
||||
err = stderr.getvalue()
|
||||
if out:
|
||||
print(_as_browser_use_cli_text(out), end='')
|
||||
if err:
|
||||
print(_as_browser_use_cli_text(err), end='', file=sys.stderr)
|
||||
if result is None:
|
||||
return 0
|
||||
if isinstance(result, int):
|
||||
return result
|
||||
if isinstance(result, str):
|
||||
print(_as_browser_use_cli_text(result), file=sys.stderr)
|
||||
return 1
|
||||
return 1
|
||||
|
||||
|
||||
def _patch_browser_harness_cli_text() -> None:
|
||||
from browser_harness import auth, run, telemetry
|
||||
|
||||
run.HELP = _as_browser_use_cli_text(run.HELP)
|
||||
run.USAGE = _as_browser_use_cli_text(run.USAGE)
|
||||
|
||||
original_auth_cli = auth.run_auth_cli
|
||||
original_telemetry_cli = telemetry.run_telemetry_cli
|
||||
|
||||
def run_auth_cli(argv: list[str]) -> int:
|
||||
if any(arg in {'-h', '--help'} for arg in argv):
|
||||
return _normalize_captured_cli_output(original_auth_cli, argv)
|
||||
return original_auth_cli(argv)
|
||||
|
||||
def run_telemetry_cli(argv: list[str]) -> int:
|
||||
if argv and argv != ['status'] and argv != ['enable'] and argv != ['disable']:
|
||||
return _normalize_captured_cli_output(original_telemetry_cli, argv)
|
||||
return original_telemetry_cli(argv)
|
||||
|
||||
auth.run_auth_cli = run_auth_cli
|
||||
telemetry.run_telemetry_cli = run_telemetry_cli
|
||||
|
||||
|
||||
_delegated_to_harness = False
|
||||
|
||||
|
||||
def _run_browser_harness() -> int | None:
|
||||
from browser_harness import run
|
||||
|
||||
global _delegated_to_harness
|
||||
|
||||
_set_harness_client_env()
|
||||
_patch_browser_harness_cli_text()
|
||||
args = sys.argv[1:]
|
||||
if args and args[0] == 'doctor' and args[1:]:
|
||||
if args[1:] in (['--help'], ['-h']):
|
||||
print('usage: browser-use doctor [--fix-snap]')
|
||||
return 0
|
||||
if args[1:] != ['--fix-snap']:
|
||||
print('usage: browser-use doctor [--fix-snap]', file=sys.stderr)
|
||||
sys.exit(2)
|
||||
_delegated_to_harness = True
|
||||
run.main()
|
||||
return None
|
||||
|
||||
|
||||
# Subcommands and flags from the pre-3.0 CLI; maps to hint
|
||||
_LEGACY_HINTS: dict[str, str] = {
|
||||
'open': 'new_tab("https://example.com")',
|
||||
'state': 'print(page_info())',
|
||||
'screenshot': 'print(capture_screenshot())',
|
||||
'eval': 'print(js("document.title"))',
|
||||
'cookies': 'print(cdp("Network.getCookies"))',
|
||||
'python': '# the CLI runs Python directly now — pipe it on stdin as shown below',
|
||||
'run': '# write the steps as Python using the pre-imported helpers shown below',
|
||||
'connect': '# connecting is automatic — the default flow attaches to your running Chrome',
|
||||
'close': '# restart the local daemon with `browser-use --reload`; stop cloud browsers with stop_remote_daemon(name)',
|
||||
'sessions': '# named local sessions were removed — one default daemon; use BU_NAME=<name> for cloud daemons',
|
||||
'profile': '# profiles now come from your real Chrome; see the profile-sync interaction skill',
|
||||
'cloud': '# authenticate with `browser-use auth login`, then start_remote_daemon("<name>")',
|
||||
'daemon': '# the daemon starts automatically on every call; restart it with `browser-use --reload`',
|
||||
'record': '# session recording was removed; use capture_screenshot() per step',
|
||||
'mcp': '# MCP server mode is now the --mcp flag: `browser-use --mcp`',
|
||||
'--session': '# use BU_NAME=<name> as an env var for named cloud daemons; local runs use one default daemon',
|
||||
'--headed': '# local control always attaches to your real, visible Chrome — no flag needed',
|
||||
'--cdp-url': '# use the BU_CDP_URL=<url> env var instead of a flag',
|
||||
'--profile': '# use your real Chrome profile; for cloud cookie sync see the profile-sync interaction skill',
|
||||
'--json': '# output is whatever your Python prints — use print(json.dumps(...))',
|
||||
'-c': "# pipe code on stdin instead: echo 'print(page_info())' | browser-use",
|
||||
'--code': "# pipe code on stdin instead: echo 'print(page_info())' | browser-use",
|
||||
}
|
||||
|
||||
|
||||
def _legacy_command(args: list[str]) -> str | None:
|
||||
if not args:
|
||||
return None
|
||||
first = args[0].split('=', 1)[0]
|
||||
if first in _LEGACY_HINTS:
|
||||
return first
|
||||
return None
|
||||
|
||||
|
||||
_CLI3_GUIDE = """Core helpers: new_tab(url), goto_url(url), page_info(), capture_screenshot(),
|
||||
click_at_xy(x, y), type_text(text), fill_input(selector, text), press_key(key),
|
||||
scroll(x, y), js(code), cdp(method, ...), wait_for_load(), wait_for_element(selector),
|
||||
list_tabs(), switch_tab(target), close_tab(target)
|
||||
|
||||
Read the full interface now: browser-use skill show
|
||||
Install the CLI 3.0 skill (upgrades the CLI and replaces any pre-existing browser-use skill):
|
||||
browser-use skill install
|
||||
Then load the freshly installed browser-use skill, and follow it to use the latest CLI.
|
||||
Skill reference: https://github.com/browser-use/browser-use/blob/main/browser_use/skills/browser-use/SKILL.md
|
||||
Health check: browser-use --doctor"""
|
||||
|
||||
|
||||
def _legacy_migration_message(command: str) -> str:
|
||||
hint = _LEGACY_HINTS[command]
|
||||
if hint.startswith('#'):
|
||||
replacement = f' {hint.lstrip("# ")}\n\nExample:\n browser-use <<\'PY\'\n new_tab("https://example.com")\n print(page_info())\n PY'
|
||||
else:
|
||||
replacement = f" browser-use <<'PY'\n {hint}\n PY"
|
||||
return f"""The browser-use CLI changed in 3.0, and '{command}' was removed.
|
||||
|
||||
The old preset subcommands are gone. To use the CLI, you write raw Python and
|
||||
pipe it on stdin, and it runs in a persistent browser session. Browser management
|
||||
(daemon startup, Chrome/CDP attach, tabs, waiting) is handled for you.
|
||||
|
||||
Replacement for '{command}':
|
||||
{replacement}
|
||||
|
||||
{_CLI3_GUIDE}"""
|
||||
|
||||
|
||||
def _unknown_helper_message(name: str) -> str:
|
||||
return f"""'{name}' is not defined in the browser-use CLI.
|
||||
|
||||
Example:
|
||||
browser-use <<'PY'
|
||||
new_tab("https://example.com")
|
||||
print(page_info())
|
||||
PY
|
||||
|
||||
{_CLI3_GUIDE}"""
|
||||
|
||||
|
||||
def _unknown_exec_name(exc: NameError) -> str | None:
|
||||
import re
|
||||
|
||||
name = getattr(exc, 'name', None)
|
||||
if name:
|
||||
return name
|
||||
m = re.search(r"'([A-Za-z_][A-Za-z0-9_]*)'", str(exc))
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _raised_from_piped_code(exc: BaseException) -> bool:
|
||||
tb = exc.__traceback__
|
||||
last = None
|
||||
while tb is not None:
|
||||
last = tb
|
||||
tb = tb.tb_next
|
||||
return last is not None and last.tb_frame.f_code.co_filename == '<string>'
|
||||
|
||||
|
||||
_QUICKSTART = """Welcome to the Browser Use CLI. Allow your coding agent to reliably control a web browser.
|
||||
|
||||
The CLI allows your agent to control the browser via Python, and it manages the browser in the background.
|
||||
|
||||
browser-use <<'PY'
|
||||
new_tab("https://news.ycombinator.com")
|
||||
print(page_info())
|
||||
PY
|
||||
|
||||
Core helpers: new_tab(url), goto_url(url), page_info(),
|
||||
capture_screenshot(), click_at_xy(x, y), js(code), cdp(method, ...),
|
||||
wait_for_load()
|
||||
|
||||
Recommended: install the skill so your coding agent remembers this:
|
||||
|
||||
browser-use skill install
|
||||
|
||||
You can also paste this into your agent to get started:
|
||||
|
||||
Install or upgrade browser-use to the latest stable version with uv using
|
||||
Python 3.12, register the skill from `browser-use skill`, and connect it to
|
||||
my browser. Follow https://github.com/browser-use/browser-use if setup or
|
||||
connection fails.
|
||||
|
||||
More:
|
||||
browser-use --doctor check install, daemon, and browser health
|
||||
browser-use --help full command list
|
||||
docs: https://github.com/browser-use/browser-use/blob/main/browser_use/skills/browser-use/SKILL.md"""
|
||||
|
||||
_EMPTY_STDIN_MESSAGE = """browser-use received empty stdin. This CLI executes Python piped on stdin:
|
||||
browser-use <<'PY'
|
||||
print(page_info())
|
||||
PY"""
|
||||
|
||||
|
||||
def _command_name(args: list[str]) -> str:
|
||||
if '--cli-mcp' in args:
|
||||
return 'cli-mcp'
|
||||
if '--mcp' in args:
|
||||
return 'mcp'
|
||||
if args and args[0] == 'install':
|
||||
return 'install'
|
||||
if args and args[0] == 'init':
|
||||
return 'init'
|
||||
if '--template' in args or '-t' in args:
|
||||
return 'init'
|
||||
if args and args[0] == 'skill':
|
||||
return 'skill'
|
||||
legacy = _legacy_command(args)
|
||||
if legacy is not None:
|
||||
return f'legacy:{legacy}'
|
||||
return args[0] if args else 'run'
|
||||
|
||||
|
||||
def _dispatch(args: list[str]) -> tuple[int | None, str]:
|
||||
if '--cli-mcp' in args:
|
||||
_run_cli_mcp_server()
|
||||
return 0, 'cli-mcp'
|
||||
if '--mcp' in args:
|
||||
_run_mcp_server()
|
||||
return 0, 'mcp'
|
||||
if args and args[0] == 'install':
|
||||
return _run_install_command(args[1:]), 'install'
|
||||
if args and args[0] == 'init':
|
||||
return _run_init_command(args[1:]), 'init'
|
||||
if '--template' in args or '-t' in args:
|
||||
return _run_init_command(args), 'init'
|
||||
if args and args[0] == 'skill':
|
||||
from browser_use.skills.install import handle as handle_skill_command
|
||||
|
||||
return handle_skill_command(args[1:]), 'skill'
|
||||
|
||||
legacy = _legacy_command(args)
|
||||
if legacy is not None:
|
||||
print(_legacy_migration_message(legacy), file=sys.stderr)
|
||||
return 2, f'legacy:{legacy}'
|
||||
|
||||
if not args:
|
||||
if sys.stdin.isatty():
|
||||
print(_QUICKSTART)
|
||||
return 0, 'quickstart'
|
||||
code = sys.stdin.read()
|
||||
if not code.strip():
|
||||
print(_EMPTY_STDIN_MESSAGE, file=sys.stderr)
|
||||
return 1, 'run'
|
||||
sys.stdin = StringIO(code)
|
||||
|
||||
try:
|
||||
return _run_browser_harness(), args[0] if args else 'run'
|
||||
except NameError as exc:
|
||||
name = _unknown_exec_name(exc)
|
||||
if name is None or not _raised_from_piped_code(exc):
|
||||
raise
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
print(_unknown_helper_message(name), file=sys.stderr)
|
||||
return 2, args[0] if args else 'run'
|
||||
|
||||
|
||||
class _StderrTail:
|
||||
"""Pass-through stderr wrapper that remembers the tail as error context."""
|
||||
|
||||
def __init__(self, wrapped):
|
||||
self._wrapped = wrapped
|
||||
self.tail = ''
|
||||
|
||||
def write(self, text):
|
||||
self.tail = (self.tail + text)[-500:]
|
||||
return self._wrapped.write(text)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._wrapped, name)
|
||||
|
||||
|
||||
def browser_use_tui_main() -> int | None:
|
||||
print('browser-use-tui is deprecated; use browser-use instead.', file=sys.stderr)
|
||||
return main()
|
||||
|
||||
|
||||
def main() -> int | None:
|
||||
global _delegated_to_harness
|
||||
|
||||
_delegated_to_harness = False
|
||||
args = sys.argv[1:]
|
||||
start_time = time.monotonic()
|
||||
command = _command_name(args)
|
||||
stderr_tail = _StderrTail(sys.stderr)
|
||||
sys.stderr = stderr_tail
|
||||
try:
|
||||
result, command = _dispatch(args)
|
||||
except SystemExit as exc:
|
||||
result = exc.code
|
||||
if not _delegated_to_harness:
|
||||
_capture_via_harness(
|
||||
command=command,
|
||||
start_time=start_time,
|
||||
result=result,
|
||||
error_message=str(result) if isinstance(result, str) else stderr_tail.tail.strip() or None,
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
if not _delegated_to_harness:
|
||||
_capture_via_harness(command=command, start_time=start_time, result=1, error_message=str(exc))
|
||||
raise
|
||||
finally:
|
||||
sys.stderr = stderr_tail._wrapped
|
||||
|
||||
if not _delegated_to_harness:
|
||||
_capture_via_harness(
|
||||
command=command,
|
||||
start_time=start_time,
|
||||
result=result,
|
||||
error_message=(stderr_tail.tail.strip() or None) if _exit_code(result) else None,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result = main()
|
||||
if result is not None:
|
||||
sys.exit(result)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user