4cd2d4af2b
Test Browser Use CLI Install / uv pip install (ubuntu-latest) (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use from local wheel (push) Failing after 1s
Test Browser Use CLI Install / uvx browser-use[cli] from PyPI (push) Failing after 1s
package / pip-install-on-macos-latest-py-3.11 (push) Has been skipped
package / pip-install-on-macos-latest-py-3.13 (push) Has been skipped
package / pip-install-on-ubuntu-latest-py-3.11 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.13 (push) Has been skipped
cloud_evals / trigger_cloud_eval_image_build (push) Failing after 1s
docker / build_publish_image (push) Failing after 1s
Test Browser Use CLI Install / browser-use skill sync (push) Failing after 1s
lint / code-style (push) Failing after 0s
lint / type-checker (push) Failing after 1s
package / pip-build (push) Failing after 1s
lint / syntax-errors (push) Failing after 3s
package / pip-install-on-ubuntu-latest-py-3.13 (push) Has been skipped
package / pip-install-on-windows-latest-py-3.11 (push) Has been skipped
test / ${{ matrix.test_filename }} (push) Has been skipped
test / evaluate-tasks (push) Has been skipped
test / setup-chromium (push) Failing after 2s
test / find_tests (push) Failing after 2s
Test Browser Use CLI Install / uv pip install (windows-latest) (push) Has been cancelled
Test Browser Use CLI Install / uv pip install (macos-latest) (push) Has been cancelled
3703 lines
129 KiB
Python
3703 lines
129 KiB
Python
"""Default browser action handlers using CDP."""
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
|
|
from cdp_use.cdp.input.commands import DispatchKeyEventParameters
|
|
|
|
from browser_use.actor.utils import get_key_info
|
|
from browser_use.browser.events import (
|
|
ClickCoordinateEvent,
|
|
ClickElementEvent,
|
|
GetDropdownOptionsEvent,
|
|
GoBackEvent,
|
|
GoForwardEvent,
|
|
RefreshEvent,
|
|
ScrollEvent,
|
|
ScrollToTextEvent,
|
|
SelectDropdownOptionEvent,
|
|
SendKeysEvent,
|
|
TypeTextEvent,
|
|
UploadFileEvent,
|
|
WaitEvent,
|
|
)
|
|
from browser_use.browser.views import BrowserError, URLNotAllowedError
|
|
from browser_use.browser.watchdog_base import BaseWatchdog
|
|
from browser_use.dom.service import EnhancedDOMTreeNode
|
|
from browser_use.observability import observe_debug
|
|
|
|
# Import EnhancedDOMTreeNode and rebuild event models that have forward references to it
|
|
# This must be done after all imports are complete
|
|
ClickCoordinateEvent.model_rebuild()
|
|
ClickElementEvent.model_rebuild()
|
|
GetDropdownOptionsEvent.model_rebuild()
|
|
SelectDropdownOptionEvent.model_rebuild()
|
|
TypeTextEvent.model_rebuild()
|
|
ScrollEvent.model_rebuild()
|
|
UploadFileEvent.model_rebuild()
|
|
|
|
|
|
class DefaultActionWatchdog(BaseWatchdog):
|
|
"""Handles default browser actions like click, type, and scroll using CDP."""
|
|
|
|
async def _execute_click_with_download_detection(
|
|
self,
|
|
click_coro,
|
|
download_start_timeout: float = 0.5,
|
|
download_complete_timeout: float = 30.0,
|
|
) -> dict | None:
|
|
"""Execute a click operation and automatically wait for any triggered download
|
|
|
|
Args:
|
|
click_coro: Coroutine that performs the click (should return click_metadata dict or None)
|
|
download_start_timeout: Time to wait for download to start after click (seconds)
|
|
download_complete_timeout: Time to wait for download to complete once started (seconds)
|
|
|
|
Returns:
|
|
Click metadata dict, potentially with 'download' key containing download info.
|
|
If a download times out but is still in progress, includes 'download_in_progress' with status.
|
|
"""
|
|
import time
|
|
|
|
download_started = asyncio.Event()
|
|
download_completed = asyncio.Event()
|
|
download_info: dict = {}
|
|
progress_info: dict = {'last_update': 0.0, 'received_bytes': 0, 'total_bytes': 0, 'state': ''}
|
|
|
|
def on_download_start(info: dict) -> None:
|
|
"""Direct callback when download starts (called from CDP handler)."""
|
|
if info.get('auto_download'):
|
|
return # ignore auto-downloads
|
|
download_info['guid'] = info.get('guid', '')
|
|
download_info['url'] = info.get('url', '')
|
|
download_info['suggested_filename'] = info.get('suggested_filename', 'download')
|
|
download_started.set()
|
|
self.logger.debug(f'[ClickWithDownload] Download started: {download_info["suggested_filename"]}')
|
|
|
|
def on_download_progress(info: dict) -> None:
|
|
"""Direct callback when download progress updates (called from CDP handler)."""
|
|
# Match by guid if available
|
|
if download_info.get('guid') and info.get('guid') != download_info['guid']:
|
|
return # different download
|
|
progress_info['last_update'] = time.time()
|
|
progress_info['received_bytes'] = info.get('received_bytes', 0)
|
|
progress_info['total_bytes'] = info.get('total_bytes', 0)
|
|
progress_info['state'] = info.get('state', '')
|
|
self.logger.debug(
|
|
f'[ClickWithDownload] Progress: {progress_info["received_bytes"]}/{progress_info["total_bytes"]} bytes ({progress_info["state"]})'
|
|
)
|
|
|
|
def on_download_complete(info: dict) -> None:
|
|
"""Direct callback when download completes (called from CDP handler)."""
|
|
if info.get('auto_download'):
|
|
return # ignore auto-downloads
|
|
# Match by guid if available, otherwise accept any non-auto download
|
|
if download_info.get('guid') and info.get('guid') and info.get('guid') != download_info['guid']:
|
|
return # different download
|
|
download_info['path'] = info.get('path', '')
|
|
download_info['file_name'] = info.get('file_name', '')
|
|
download_info['file_size'] = info.get('file_size', 0)
|
|
download_info['file_type'] = info.get('file_type')
|
|
download_info['mime_type'] = info.get('mime_type')
|
|
download_completed.set()
|
|
self.logger.debug(f'[ClickWithDownload] Download completed: {download_info["file_name"]}')
|
|
|
|
# Get the downloads watchdog and register direct callbacks
|
|
downloads_watchdog = self.browser_session._downloads_watchdog
|
|
self.logger.debug(f'[ClickWithDownload] downloads_watchdog={downloads_watchdog is not None}')
|
|
if downloads_watchdog:
|
|
self.logger.debug('[ClickWithDownload] Registering download callbacks...')
|
|
downloads_watchdog.register_download_callbacks(
|
|
on_start=on_download_start,
|
|
on_progress=on_download_progress,
|
|
on_complete=on_download_complete,
|
|
)
|
|
else:
|
|
self.logger.warning('[ClickWithDownload] No downloads_watchdog available!')
|
|
|
|
try:
|
|
# Perform the click
|
|
click_metadata = await click_coro
|
|
|
|
# Check for validation errors - return them immediately without waiting for downloads
|
|
if isinstance(click_metadata, dict) and 'validation_error' in click_metadata:
|
|
return click_metadata
|
|
|
|
# Wait briefly to see if a download starts
|
|
try:
|
|
await asyncio.wait_for(download_started.wait(), timeout=download_start_timeout)
|
|
|
|
# Download started!
|
|
self.logger.info(f'📥 Download started: {download_info.get("suggested_filename", "unknown")}')
|
|
|
|
# Now wait for it to complete with longer timeout
|
|
try:
|
|
await asyncio.wait_for(download_completed.wait(), timeout=download_complete_timeout)
|
|
|
|
# Download completed successfully
|
|
msg = f'Downloaded file: {download_info["file_name"]} ({download_info["file_size"]} bytes) saved to {download_info["path"]}'
|
|
self.logger.info(f'💾 {msg}')
|
|
|
|
# Merge download info into click_metadata
|
|
if click_metadata is None:
|
|
click_metadata = {}
|
|
click_metadata['download'] = {
|
|
'path': download_info['path'],
|
|
'file_name': download_info['file_name'],
|
|
'file_size': download_info['file_size'],
|
|
'file_type': download_info.get('file_type'),
|
|
'mime_type': download_info.get('mime_type'),
|
|
}
|
|
except TimeoutError:
|
|
# Download timed out - check if it's still in progress
|
|
if click_metadata is None:
|
|
click_metadata = {}
|
|
|
|
filename = download_info.get('suggested_filename', 'unknown')
|
|
received = progress_info.get('received_bytes', 0)
|
|
total = progress_info.get('total_bytes', 0)
|
|
state = progress_info.get('state', 'unknown')
|
|
last_update = progress_info.get('last_update', 0.0)
|
|
time_since_update = time.time() - last_update if last_update > 0 else float('inf')
|
|
|
|
# Check if download is still actively progressing (received update in last 5 seconds)
|
|
is_still_active = time_since_update < 5.0 and state == 'inProgress'
|
|
|
|
if is_still_active:
|
|
# Download is still progressing - suggest waiting
|
|
if total > 0:
|
|
percent = (received / total) * 100
|
|
progress_str = f'{percent:.1f}% ({received:,}/{total:,} bytes)'
|
|
else:
|
|
progress_str = f'{received:,} bytes downloaded (total size unknown)'
|
|
|
|
msg = (
|
|
f'Download timed out after {download_complete_timeout}s but is still in progress: '
|
|
f'{filename} - {progress_str}. '
|
|
f'The download appears to be progressing normally. Consider using the wait action '
|
|
f'to allow more time for the download to complete.'
|
|
)
|
|
self.logger.warning(f'⏱️ {msg}')
|
|
click_metadata['download_in_progress'] = {
|
|
'file_name': filename,
|
|
'received_bytes': received,
|
|
'total_bytes': total,
|
|
'state': state,
|
|
'message': msg,
|
|
}
|
|
else:
|
|
# Download may be stalled or completed
|
|
if received > 0:
|
|
msg = (
|
|
f'Download timed out after {download_complete_timeout}s: {filename}. '
|
|
f'Last progress: {received:,} bytes received. '
|
|
f'The download may have stalled or completed - check the downloads folder.'
|
|
)
|
|
else:
|
|
msg = (
|
|
f'Download timed out after {download_complete_timeout}s: {filename}. '
|
|
f'No progress data received - the download may have failed to start properly.'
|
|
)
|
|
self.logger.warning(f'⏱️ {msg}')
|
|
click_metadata['download_timeout'] = {
|
|
'file_name': filename,
|
|
'received_bytes': received,
|
|
'total_bytes': total,
|
|
'message': msg,
|
|
}
|
|
except TimeoutError:
|
|
# No download started within grace period
|
|
pass
|
|
|
|
return click_metadata if isinstance(click_metadata, dict) else None
|
|
|
|
finally:
|
|
# Unregister download callbacks
|
|
if downloads_watchdog:
|
|
downloads_watchdog.unregister_download_callbacks(
|
|
on_start=on_download_start,
|
|
on_progress=on_download_progress,
|
|
on_complete=on_download_complete,
|
|
)
|
|
|
|
def _is_print_related_element(self, element_node: EnhancedDOMTreeNode) -> bool:
|
|
"""Check if an element is related to printing (print buttons, print dialogs, etc.).
|
|
|
|
Primary check: onclick attribute (most reliable for print detection)
|
|
Fallback: button text/value (for cases without onclick)
|
|
"""
|
|
# Primary: Check onclick attribute for print-related functions (most reliable)
|
|
onclick = element_node.attributes.get('onclick', '').lower() if element_node.attributes else ''
|
|
if onclick and 'print' in onclick:
|
|
# Matches: window.print(), PrintElem(), print(), etc.
|
|
return True
|
|
|
|
return False
|
|
|
|
async def _handle_print_button_click(self, element_node: EnhancedDOMTreeNode) -> dict | None:
|
|
"""Handle print button by directly generating PDF via CDP instead of opening dialog.
|
|
|
|
Returns:
|
|
Metadata dict with download path if successful, None otherwise
|
|
"""
|
|
try:
|
|
import base64
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# Get CDP session
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
|
|
|
|
# Generate PDF using CDP Page.printToPDF
|
|
result = await asyncio.wait_for(
|
|
cdp_session.cdp_client.send.Page.printToPDF(
|
|
params={
|
|
'printBackground': True,
|
|
'preferCSSPageSize': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
),
|
|
timeout=15.0, # 15 second timeout for PDF generation
|
|
)
|
|
|
|
pdf_data = result.get('data')
|
|
if not pdf_data:
|
|
self.logger.warning('⚠️ PDF generation returned no data')
|
|
return None
|
|
|
|
# Decode base64 PDF data
|
|
pdf_bytes = base64.b64decode(pdf_data)
|
|
|
|
# Get downloads path
|
|
downloads_path = self.browser_session.browser_profile.downloads_path
|
|
if not downloads_path:
|
|
self.logger.warning('⚠️ No downloads path configured, cannot save PDF')
|
|
return None
|
|
|
|
# Generate filename from page title or URL
|
|
try:
|
|
page_title = await asyncio.wait_for(self.browser_session.get_current_page_title(), timeout=2.0)
|
|
# Sanitize title for filename
|
|
import re
|
|
|
|
safe_title = re.sub(r'[^\w\s-]', '', page_title)[:50] # Max 50 chars
|
|
filename = f'{safe_title}.pdf' if safe_title else 'print.pdf'
|
|
except Exception:
|
|
filename = 'print.pdf'
|
|
|
|
# Ensure downloads directory exists
|
|
downloads_dir = Path(downloads_path).expanduser().resolve()
|
|
downloads_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Generate unique filename if file exists
|
|
final_path = downloads_dir / filename
|
|
if final_path.exists():
|
|
base, ext = os.path.splitext(filename)
|
|
counter = 1
|
|
while (downloads_dir / f'{base} ({counter}){ext}').exists():
|
|
counter += 1
|
|
final_path = downloads_dir / f'{base} ({counter}){ext}'
|
|
|
|
# Write PDF to file
|
|
import anyio
|
|
|
|
async with await anyio.open_file(final_path, 'wb') as f:
|
|
await f.write(pdf_bytes)
|
|
|
|
file_size = final_path.stat().st_size
|
|
self.logger.info(f'✅ Generated PDF via CDP: {final_path} ({file_size:,} bytes)')
|
|
|
|
# Dispatch FileDownloadedEvent
|
|
from browser_use.browser.events import FileDownloadedEvent
|
|
|
|
page_url = await self.browser_session.get_current_page_url()
|
|
self.browser_session.event_bus.dispatch(
|
|
FileDownloadedEvent(
|
|
url=page_url,
|
|
path=str(final_path),
|
|
file_name=final_path.name,
|
|
file_size=file_size,
|
|
file_type='pdf',
|
|
mime_type='application/pdf',
|
|
auto_download=False, # This was intentional (user clicked print)
|
|
)
|
|
)
|
|
|
|
return {'pdf_generated': True, 'path': str(final_path)}
|
|
|
|
except TimeoutError:
|
|
self.logger.warning('⏱️ PDF generation timed out')
|
|
return None
|
|
except Exception as e:
|
|
self.logger.warning(f'⚠️ Failed to generate PDF via CDP: {type(e).__name__}: {e}')
|
|
return None
|
|
|
|
@observe_debug(ignore_input=True, ignore_output=True, name='click_element_event')
|
|
async def on_ClickElementEvent(self, event: ClickElementEvent) -> dict | None:
|
|
"""Handle click request with CDP. Automatically waits for file downloads if triggered."""
|
|
try:
|
|
# Check if session is alive before attempting any operations
|
|
if not self.browser_session.agent_focus_target_id:
|
|
error_msg = 'Cannot execute click: browser session is corrupted (target_id=None). Session may have crashed.'
|
|
self.logger.error(f'{error_msg}')
|
|
raise BrowserError(error_msg)
|
|
|
|
# Use the provided node
|
|
element_node = event.node
|
|
index_for_logging = element_node.backend_node_id or 'unknown'
|
|
|
|
# Check if element is a file input (should not be clicked)
|
|
if self.browser_session.is_file_input(element_node):
|
|
msg = f'Index {index_for_logging} - has an element which opens file upload dialog. To upload files please use a specific function to upload files'
|
|
self.logger.info(f'{msg}')
|
|
return {'validation_error': msg}
|
|
|
|
# Detect print-related elements and handle them specially
|
|
is_print_element = self._is_print_related_element(element_node)
|
|
if is_print_element:
|
|
self.logger.info(
|
|
f'🖨️ Detected print button (index {index_for_logging}), generating PDF directly instead of opening dialog...'
|
|
)
|
|
click_metadata = await self._handle_print_button_click(element_node)
|
|
if click_metadata and click_metadata.get('pdf_generated'):
|
|
msg = f'Generated PDF: {click_metadata.get("path")}'
|
|
self.logger.info(f'💾 {msg}')
|
|
return click_metadata
|
|
else:
|
|
self.logger.warning('⚠️ PDF generation failed, falling back to regular click')
|
|
|
|
# Execute click with automatic download detection
|
|
click_metadata = await self._execute_click_with_download_detection(self._click_element_node_impl(element_node))
|
|
|
|
# Check for validation errors
|
|
if isinstance(click_metadata, dict) and 'validation_error' in click_metadata:
|
|
self.logger.info(f'{click_metadata["validation_error"]}')
|
|
return click_metadata
|
|
|
|
# Build success message for non-download clicks
|
|
if 'download' not in (click_metadata or {}):
|
|
msg = f'Clicked button {element_node.node_name}: {element_node.get_all_children_text(max_depth=2)}'
|
|
self.logger.debug(f'🖱️ {msg}')
|
|
self.logger.debug(f'Element xpath: {element_node.xpath}')
|
|
|
|
return click_metadata
|
|
|
|
except Exception:
|
|
raise
|
|
|
|
async def on_ClickCoordinateEvent(self, event: ClickCoordinateEvent) -> dict | None:
|
|
"""Handle click at coordinates with CDP. Automatically waits for file downloads if triggered."""
|
|
try:
|
|
# Check if session is alive before attempting any operations
|
|
if not self.browser_session.agent_focus_target_id:
|
|
error_msg = 'Cannot execute click: browser session is corrupted (target_id=None). Session may have crashed.'
|
|
self.logger.error(f'{error_msg}')
|
|
raise BrowserError(error_msg)
|
|
|
|
# If force=True, skip safety checks and click directly (with download detection)
|
|
if event.force:
|
|
self.logger.debug(f'Force clicking at coordinates ({event.coordinate_x}, {event.coordinate_y})')
|
|
return await self._execute_click_with_download_detection(
|
|
self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=True)
|
|
)
|
|
|
|
# Get element at coordinates for safety checks
|
|
element_node = await self.browser_session.get_dom_element_at_coordinates(event.coordinate_x, event.coordinate_y)
|
|
if element_node is None:
|
|
# No element found, click directly (with download detection)
|
|
self.logger.debug(
|
|
f'No element found at coordinates ({event.coordinate_x}, {event.coordinate_y}), proceeding with click anyway'
|
|
)
|
|
return await self._execute_click_with_download_detection(
|
|
self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=False)
|
|
)
|
|
|
|
# Safety check: file input
|
|
if self.browser_session.is_file_input(element_node):
|
|
msg = f'Cannot click at ({event.coordinate_x}, {event.coordinate_y}) - element is a file input. To upload files please use upload_file action'
|
|
self.logger.info(f'{msg}')
|
|
return {'validation_error': msg}
|
|
|
|
# Safety check: select element
|
|
tag_name = element_node.tag_name.lower() if element_node.tag_name else ''
|
|
if tag_name == 'select':
|
|
msg = f'Cannot click at ({event.coordinate_x}, {event.coordinate_y}) - element is a <select>. Use dropdown_options action instead.'
|
|
self.logger.info(f'{msg}')
|
|
return {'validation_error': msg}
|
|
|
|
# Safety check: print-related elements
|
|
is_print_element = self._is_print_related_element(element_node)
|
|
if is_print_element:
|
|
self.logger.info(
|
|
f'🖨️ Detected print button at ({event.coordinate_x}, {event.coordinate_y}), generating PDF directly instead of opening dialog...'
|
|
)
|
|
click_metadata = await self._handle_print_button_click(element_node)
|
|
if click_metadata and click_metadata.get('pdf_generated'):
|
|
msg = f'Generated PDF: {click_metadata.get("path")}'
|
|
self.logger.info(f'💾 {msg}')
|
|
return click_metadata
|
|
else:
|
|
self.logger.warning('⚠️ PDF generation failed, falling back to regular click')
|
|
|
|
# All safety checks passed, click at coordinates (with download detection)
|
|
return await self._execute_click_with_download_detection(
|
|
self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=False)
|
|
)
|
|
|
|
except Exception:
|
|
raise
|
|
|
|
async def on_TypeTextEvent(self, event: TypeTextEvent) -> dict | None:
|
|
"""Handle text input request with CDP."""
|
|
try:
|
|
# Use the provided node
|
|
element_node = event.node
|
|
index_for_logging = element_node.backend_node_id or 'unknown'
|
|
|
|
# Check if this is index 0 or a falsy index - type to the page (whatever has focus)
|
|
if not element_node.backend_node_id or element_node.backend_node_id == 0:
|
|
# Type to the page without focusing any specific element
|
|
await self._type_to_page(event.text)
|
|
# Log with sensitive data protection
|
|
if event.is_sensitive:
|
|
if event.sensitive_key_name:
|
|
self.logger.info(f'⌨️ Typed <{event.sensitive_key_name}> to the page (current focus)')
|
|
else:
|
|
self.logger.info('⌨️ Typed <sensitive> to the page (current focus)')
|
|
else:
|
|
self.logger.info(f'⌨️ Typed "{event.text}" to the page (current focus)')
|
|
return None # No coordinates available for page typing
|
|
else:
|
|
try:
|
|
# Try to type to the specific element
|
|
input_metadata = await self._input_text_element_node_impl(
|
|
element_node,
|
|
event.text,
|
|
clear=event.clear or (not event.text),
|
|
is_sensitive=event.is_sensitive,
|
|
)
|
|
# Log with sensitive data protection
|
|
if event.is_sensitive:
|
|
if event.sensitive_key_name:
|
|
self.logger.info(f'⌨️ Typed <{event.sensitive_key_name}> into element with index {index_for_logging}')
|
|
else:
|
|
self.logger.info(f'⌨️ Typed <sensitive> into element with index {index_for_logging}')
|
|
else:
|
|
self.logger.info(f'⌨️ Typed "{event.text}" into element with index {index_for_logging}')
|
|
self.logger.debug(f'Element xpath: {element_node.xpath}')
|
|
return input_metadata # Return coordinates if available
|
|
except Exception as e:
|
|
# Element not found or error - fall back to typing to the page
|
|
self.logger.warning(f'Failed to type to element {index_for_logging}: {e}. Falling back to page typing.')
|
|
try:
|
|
await asyncio.wait_for(self._click_element_node_impl(element_node), timeout=10.0)
|
|
except Exception as e:
|
|
pass
|
|
await self._type_to_page(event.text)
|
|
# Log with sensitive data protection
|
|
if event.is_sensitive:
|
|
if event.sensitive_key_name:
|
|
self.logger.info(f'⌨️ Typed <{event.sensitive_key_name}> to the page as fallback')
|
|
else:
|
|
self.logger.info('⌨️ Typed <sensitive> to the page as fallback')
|
|
else:
|
|
self.logger.info(f'⌨️ Typed "{event.text}" to the page as fallback')
|
|
return None # No coordinates available for fallback typing
|
|
|
|
# Note: We don't clear cached state here - let multi_act handle DOM change detection
|
|
# by explicitly rebuilding and comparing when needed
|
|
except Exception as e:
|
|
raise
|
|
|
|
async def on_ScrollEvent(self, event: ScrollEvent) -> None:
|
|
"""Handle scroll request with CDP."""
|
|
# Check if we have a current target for scrolling
|
|
if not self.browser_session.agent_focus_target_id:
|
|
error_msg = 'No active target for scrolling'
|
|
raise BrowserError(error_msg)
|
|
|
|
try:
|
|
|
|
def invalidate_dom_cache() -> None:
|
|
if self.browser_session._dom_watchdog:
|
|
self.browser_session._dom_watchdog.clear_cache()
|
|
|
|
# Convert direction and amount to pixels
|
|
# Positive pixels = scroll down, negative = scroll up
|
|
pixels = event.amount if event.direction == 'down' else -event.amount
|
|
|
|
# Element-specific scrolling if node is provided
|
|
if event.node is not None:
|
|
element_node = event.node
|
|
index_for_logging = element_node.backend_node_id or 'unknown'
|
|
|
|
# Check if the element is an iframe
|
|
is_iframe = element_node.tag_name and element_node.tag_name.upper() == 'IFRAME'
|
|
|
|
# Try to scroll the element's container
|
|
success = await self._scroll_element_container(element_node, pixels)
|
|
if success:
|
|
self.logger.debug(
|
|
f'📜 Scrolled element {index_for_logging} container {event.direction} by {event.amount} pixels'
|
|
)
|
|
|
|
# For iframe scrolling, we need to force a full DOM refresh
|
|
# because the iframe's content has changed position
|
|
if is_iframe:
|
|
self.logger.debug('🔄 Forcing DOM refresh after iframe scroll')
|
|
# Note: We don't clear cached state here - let multi_act handle DOM change detection
|
|
# by explicitly rebuilding and comparing when needed
|
|
|
|
# Wait a bit for the scroll to settle and DOM to update
|
|
await asyncio.sleep(0.2)
|
|
|
|
invalidate_dom_cache()
|
|
return None
|
|
|
|
# Perform target-level scroll
|
|
await self._scroll_with_cdp_gesture(pixels)
|
|
|
|
# Note: We don't clear cached state here - let multi_act handle DOM change detection
|
|
# by explicitly rebuilding and comparing when needed
|
|
invalidate_dom_cache()
|
|
|
|
# Log success
|
|
self.logger.debug(f'📜 Scrolled {event.direction} by {event.amount} pixels')
|
|
return None
|
|
except Exception as e:
|
|
raise
|
|
|
|
# ========== Implementation Methods ==========
|
|
|
|
async def _check_element_occlusion(self, backend_node_id: int, x: float, y: float, cdp_session) -> bool:
|
|
"""Check if an element is occluded by other elements at the given coordinates.
|
|
|
|
Args:
|
|
backend_node_id: The backend node ID of the target element
|
|
x: X coordinate to check
|
|
y: Y coordinate to check
|
|
cdp_session: CDP session to use
|
|
|
|
Returns:
|
|
True if element is occluded, False if clickable
|
|
"""
|
|
try:
|
|
session_id = cdp_session.session_id
|
|
|
|
# Get target element info for comparison
|
|
target_result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': backend_node_id}, session_id=session_id
|
|
)
|
|
|
|
if 'object' not in target_result:
|
|
self.logger.debug('Could not resolve target element, assuming occluded')
|
|
return True
|
|
|
|
object_id = target_result['object']['objectId']
|
|
|
|
# Get target element info
|
|
target_info_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'objectId': object_id,
|
|
'functionDeclaration': """
|
|
function() {
|
|
const getElementInfo = (el) => {
|
|
return {
|
|
tagName: el.tagName,
|
|
id: el.id || '',
|
|
className: el.className || '',
|
|
textContent: (el.textContent || '').substring(0, 100)
|
|
};
|
|
};
|
|
|
|
|
|
const elementAtPoint = document.elementFromPoint(arguments[0], arguments[1]);
|
|
if (!elementAtPoint) {
|
|
return { targetInfo: getElementInfo(this), isClickable: false };
|
|
}
|
|
|
|
|
|
// Simple containment-based clickability logic
|
|
let isClickable = this === elementAtPoint ||
|
|
this.contains(elementAtPoint) ||
|
|
elementAtPoint.contains(this);
|
|
|
|
// Check label-input associations when containment check fails
|
|
if (!isClickable) {
|
|
const target = this;
|
|
const atPoint = elementAtPoint;
|
|
|
|
// Case 1: target is <input>, atPoint is its associated <label> (or child of that label)
|
|
if (target.tagName === 'INPUT' && target.id) {
|
|
const escapedId = CSS.escape(target.id);
|
|
const assocLabel = document.querySelector('label[for="' + escapedId + '"]');
|
|
if (assocLabel && (assocLabel === atPoint || assocLabel.contains(atPoint))) {
|
|
isClickable = true;
|
|
}
|
|
}
|
|
|
|
// Case 2: target is <input>, atPoint is inside a <label> ancestor that wraps the target
|
|
if (!isClickable && target.tagName === 'INPUT') {
|
|
let ancestor = atPoint;
|
|
for (let i = 0; i < 3 && ancestor; i++) {
|
|
if (ancestor.tagName === 'LABEL' && ancestor.contains(target)) {
|
|
isClickable = true;
|
|
break;
|
|
}
|
|
ancestor = ancestor.parentElement;
|
|
}
|
|
}
|
|
|
|
// Case 3: target is <label>, atPoint is the associated <input>
|
|
if (!isClickable && target.tagName === 'LABEL') {
|
|
if (target.htmlFor && atPoint.tagName === 'INPUT' && atPoint.id === target.htmlFor) {
|
|
isClickable = true;
|
|
}
|
|
// Also check if atPoint is an input inside the label
|
|
if (!isClickable && atPoint.tagName === 'INPUT' && target.contains(atPoint)) {
|
|
isClickable = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
targetInfo: getElementInfo(this),
|
|
elementAtPointInfo: getElementInfo(elementAtPoint),
|
|
isClickable: isClickable
|
|
};
|
|
}
|
|
""",
|
|
'arguments': [{'value': x}, {'value': y}],
|
|
'returnByValue': True,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
|
|
if 'result' not in target_info_result or 'value' not in target_info_result['result']:
|
|
self.logger.debug('Could not get target element info, assuming occluded')
|
|
return True
|
|
|
|
target_data = target_info_result['result']['value']
|
|
is_clickable = target_data.get('isClickable', False)
|
|
|
|
if is_clickable:
|
|
self.logger.debug('Element is clickable (target, contained, or semantically related)')
|
|
return False
|
|
else:
|
|
target_info = target_data.get('targetInfo', {})
|
|
element_at_point_info = target_data.get('elementAtPointInfo', {})
|
|
self.logger.debug(
|
|
f'Element is occluded. Target: {target_info.get("tagName", "unknown")} '
|
|
f'(id={target_info.get("id", "none")}), '
|
|
f'ElementAtPoint: {element_at_point_info.get("tagName", "unknown")} '
|
|
f'(id={element_at_point_info.get("id", "none")})'
|
|
)
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.debug(f'Occlusion check failed: {e}, assuming not occluded')
|
|
return False
|
|
|
|
async def _click_element_node_impl(self, element_node) -> dict | None:
|
|
"""
|
|
Click an element using pure CDP with multiple fallback methods for getting element geometry.
|
|
|
|
Args:
|
|
element_node: The DOM element to click
|
|
"""
|
|
|
|
try:
|
|
# Check if element is a file input or select dropdown - these should not be clicked
|
|
tag_name = element_node.tag_name.lower() if element_node.tag_name else ''
|
|
element_type = element_node.attributes.get('type', '').lower() if element_node.attributes else ''
|
|
|
|
if tag_name == 'select':
|
|
msg = f'Cannot click on <select> elements. Use dropdown_options(index={element_node.backend_node_id}) action instead.'
|
|
# Return error dict instead of raising to avoid ERROR logs
|
|
return {'validation_error': msg}
|
|
|
|
if tag_name == 'input' and element_type == 'file':
|
|
msg = f'Cannot click on file input element (index={element_node.backend_node_id}). File uploads must be handled using upload_file_to_element action.'
|
|
# Return error dict instead of raising to avoid ERROR logs
|
|
return {'validation_error': msg}
|
|
|
|
# Get CDP client
|
|
cdp_session = await self.browser_session.cdp_client_for_node(element_node)
|
|
|
|
# Get the correct session ID for the element's frame
|
|
session_id = cdp_session.session_id
|
|
|
|
# Get element bounds
|
|
backend_node_id = element_node.backend_node_id
|
|
|
|
# For checkbox/radio: capture pre-click state to verify toggle worked
|
|
is_toggle_element = tag_name == 'input' and element_type in ('checkbox', 'radio')
|
|
pre_click_checked: bool | None = None
|
|
checkbox_object_id: str | None = None
|
|
if is_toggle_element and backend_node_id:
|
|
try:
|
|
resolve_res = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': backend_node_id}, session_id=session_id
|
|
)
|
|
obj_info = resolve_res.get('object', {})
|
|
checkbox_object_id = obj_info.get('objectId') if obj_info else None
|
|
if not checkbox_object_id:
|
|
raise Exception('Failed to resolve checkbox element objectId')
|
|
state_res = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { return this.checked; }',
|
|
'objectId': checkbox_object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
pre_click_checked = state_res.get('result', {}).get('value')
|
|
self.logger.debug(f'Checkbox pre-click state: checked={pre_click_checked}')
|
|
except Exception as e:
|
|
self.logger.debug(f'Could not capture pre-click checkbox state: {e}')
|
|
|
|
# Get viewport dimensions for visibility checks
|
|
layout_metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=session_id)
|
|
viewport_width = layout_metrics['layoutViewport']['clientWidth']
|
|
viewport_height = layout_metrics['layoutViewport']['clientHeight']
|
|
|
|
# Scroll element into view FIRST before getting coordinates
|
|
try:
|
|
await cdp_session.cdp_client.send.DOM.scrollIntoViewIfNeeded(
|
|
params={'backendNodeId': backend_node_id}, session_id=session_id
|
|
)
|
|
await asyncio.sleep(0.05) # Wait for scroll to complete
|
|
self.logger.debug('Scrolled element into view before getting coordinates')
|
|
except Exception as e:
|
|
self.logger.debug(f'Failed to scroll element into view: {e}')
|
|
|
|
# Get element coordinates using the unified method AFTER scrolling
|
|
element_rect = await self.browser_session.get_element_coordinates(backend_node_id, cdp_session)
|
|
|
|
# Convert rect to quads format if we got coordinates
|
|
quads = []
|
|
if element_rect:
|
|
# Convert DOMRect to quad format
|
|
x, y, w, h = element_rect.x, element_rect.y, element_rect.width, element_rect.height
|
|
quads = [
|
|
[
|
|
x,
|
|
y, # top-left
|
|
x + w,
|
|
y, # top-right
|
|
x + w,
|
|
y + h, # bottom-right
|
|
x,
|
|
y + h, # bottom-left
|
|
]
|
|
]
|
|
self.logger.debug(
|
|
f'Got coordinates from unified method: {element_rect.x}, {element_rect.y}, {element_rect.width}x{element_rect.height}'
|
|
)
|
|
|
|
# If we still don't have quads, fall back to JS click
|
|
if not quads:
|
|
self.logger.warning('Could not get element geometry from any method, falling back to JavaScript click')
|
|
try:
|
|
result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': backend_node_id},
|
|
session_id=session_id,
|
|
)
|
|
assert 'object' in result and 'objectId' in result['object'], (
|
|
'Failed to find DOM element based on backendNodeId, maybe page content changed?'
|
|
)
|
|
object_id = result['object']['objectId']
|
|
|
|
await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { this.click(); }',
|
|
'objectId': object_id,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
await asyncio.sleep(0.05)
|
|
# Navigation is handled by BrowserSession via events
|
|
return None
|
|
except Exception as js_e:
|
|
self.logger.warning(f'CDP JavaScript click also failed: {js_e}')
|
|
if 'No node with given id found' in str(js_e):
|
|
raise Exception('Element with given id not found')
|
|
else:
|
|
raise Exception(f'Failed to click element: {js_e}')
|
|
|
|
# Find the largest visible quad within the viewport
|
|
best_quad = None
|
|
best_area = 0
|
|
|
|
for quad in quads:
|
|
if len(quad) < 8:
|
|
continue
|
|
|
|
# Calculate quad bounds
|
|
xs = [quad[i] for i in range(0, 8, 2)]
|
|
ys = [quad[i] for i in range(1, 8, 2)]
|
|
min_x, max_x = min(xs), max(xs)
|
|
min_y, max_y = min(ys), max(ys)
|
|
|
|
# Check if quad intersects with viewport
|
|
if max_x < 0 or max_y < 0 or min_x > viewport_width or min_y > viewport_height:
|
|
continue # Quad is completely outside viewport
|
|
|
|
# Calculate visible area (intersection with viewport)
|
|
visible_min_x = max(0, min_x)
|
|
visible_max_x = min(viewport_width, max_x)
|
|
visible_min_y = max(0, min_y)
|
|
visible_max_y = min(viewport_height, max_y)
|
|
|
|
visible_width = visible_max_x - visible_min_x
|
|
visible_height = visible_max_y - visible_min_y
|
|
visible_area = visible_width * visible_height
|
|
|
|
if visible_area > best_area:
|
|
best_area = visible_area
|
|
best_quad = quad
|
|
|
|
if not best_quad:
|
|
# No visible quad found, use the first quad anyway
|
|
best_quad = quads[0]
|
|
self.logger.warning('No visible quad found, using first quad')
|
|
|
|
# Calculate center point of the best quad
|
|
center_x = sum(best_quad[i] for i in range(0, 8, 2)) / 4
|
|
center_y = sum(best_quad[i] for i in range(1, 8, 2)) / 4
|
|
|
|
# Ensure click point is within viewport bounds
|
|
center_x = max(0, min(viewport_width - 1, center_x))
|
|
center_y = max(0, min(viewport_height - 1, center_y))
|
|
|
|
# Check for occlusion before attempting CDP click
|
|
is_occluded = await self._check_element_occlusion(backend_node_id, center_x, center_y, cdp_session)
|
|
|
|
if is_occluded:
|
|
self.logger.debug('🚫 Element is occluded, falling back to JavaScript click')
|
|
try:
|
|
result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': backend_node_id},
|
|
session_id=session_id,
|
|
)
|
|
assert 'object' in result and 'objectId' in result['object'], (
|
|
'Failed to find DOM element based on backendNodeId'
|
|
)
|
|
object_id = result['object']['objectId']
|
|
|
|
await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { this.click(); }',
|
|
'objectId': object_id,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
await asyncio.sleep(0.05)
|
|
return None
|
|
except Exception as js_e:
|
|
self.logger.error(f'JavaScript click fallback failed: {js_e}')
|
|
raise Exception(f'Failed to click occluded element: {js_e}')
|
|
|
|
# Perform the click using CDP (element is not occluded)
|
|
try:
|
|
self.logger.debug(f'👆 Dragging mouse over element before clicking x: {center_x}px y: {center_y}px ...')
|
|
# Move mouse to element
|
|
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mouseMoved',
|
|
'x': center_x,
|
|
'y': center_y,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
await asyncio.sleep(0.05)
|
|
|
|
# Mouse down
|
|
self.logger.debug(f'👆🏾 Clicking x: {center_x}px y: {center_y}px ...')
|
|
try:
|
|
await asyncio.wait_for(
|
|
cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mousePressed',
|
|
'x': center_x,
|
|
'y': center_y,
|
|
'button': 'left',
|
|
'clickCount': 1,
|
|
},
|
|
session_id=session_id,
|
|
),
|
|
timeout=3.0, # 3 second timeout for mousePressed
|
|
)
|
|
await asyncio.sleep(0.08)
|
|
except TimeoutError:
|
|
self.logger.debug('⏱️ Mouse down timed out (likely due to dialog), continuing...')
|
|
# Don't sleep if we timed out
|
|
|
|
# Mouse up
|
|
try:
|
|
await asyncio.wait_for(
|
|
cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mouseReleased',
|
|
'x': center_x,
|
|
'y': center_y,
|
|
'button': 'left',
|
|
'clickCount': 1,
|
|
},
|
|
session_id=session_id,
|
|
),
|
|
timeout=5.0, # 5 second timeout for mouseReleased
|
|
)
|
|
except TimeoutError:
|
|
self.logger.debug('⏱️ Mouse up timed out (possibly due to lag or dialog popup), continuing...')
|
|
|
|
self.logger.debug('🖱️ Clicked successfully using x,y coordinates')
|
|
|
|
# For checkbox/radio: verify state toggled, fall back to JS element.click() if not
|
|
if is_toggle_element and pre_click_checked is not None and checkbox_object_id:
|
|
try:
|
|
await asyncio.sleep(0.05)
|
|
state_res = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { return this.checked; }',
|
|
'objectId': checkbox_object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
post_click_checked = state_res.get('result', {}).get('value')
|
|
if post_click_checked == pre_click_checked:
|
|
# CDP mouse events didn't toggle the checkbox — try JS element.click()
|
|
self.logger.debug(
|
|
f'Checkbox state unchanged after CDP click (checked={pre_click_checked}), using JS fallback'
|
|
)
|
|
await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={'functionDeclaration': 'function() { this.click(); }', 'objectId': checkbox_object_id},
|
|
session_id=session_id,
|
|
)
|
|
await asyncio.sleep(0.05)
|
|
final_res = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { return this.checked; }',
|
|
'objectId': checkbox_object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
post_click_checked = final_res.get('result', {}).get('value')
|
|
self.logger.debug(f'Checkbox post-click state: checked={post_click_checked}')
|
|
return {'click_x': center_x, 'click_y': center_y, 'checked': post_click_checked}
|
|
except Exception as e:
|
|
self.logger.debug(f'Checkbox state verification failed (non-critical): {e}')
|
|
|
|
# Return coordinates as dict for metadata
|
|
return {'click_x': center_x, 'click_y': center_y}
|
|
|
|
except Exception as e:
|
|
self.logger.warning(f'CDP click failed: {type(e).__name__}: {e}')
|
|
# Fall back to JavaScript click via CDP
|
|
try:
|
|
result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': backend_node_id},
|
|
session_id=session_id,
|
|
)
|
|
assert 'object' in result and 'objectId' in result['object'], (
|
|
'Failed to find DOM element based on backendNodeId, maybe page content changed?'
|
|
)
|
|
object_id = result['object']['objectId']
|
|
|
|
await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { this.click(); }',
|
|
'objectId': object_id,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
|
|
# Small delay for dialog dismissal
|
|
await asyncio.sleep(0.1)
|
|
|
|
return None
|
|
except Exception as js_e:
|
|
self.logger.warning(f'CDP JavaScript click also failed: {js_e}')
|
|
raise Exception(f'Failed to click element: {e}')
|
|
finally:
|
|
# Always re-focus back to original top-level page session context in case click opened a new tab/popup/window/dialog/etc.
|
|
# Use timeout to prevent hanging if dialog is blocking
|
|
try:
|
|
cdp_session = await asyncio.wait_for(self.browser_session.get_or_create_cdp_session(focus=True), timeout=3.0)
|
|
await asyncio.wait_for(
|
|
cdp_session.cdp_client.send.Runtime.runIfWaitingForDebugger(session_id=cdp_session.session_id),
|
|
timeout=2.0,
|
|
)
|
|
except TimeoutError:
|
|
self.logger.debug('⏱️ Refocus after click timed out (page may be blocked by dialog). Continuing...')
|
|
except Exception as e:
|
|
self.logger.debug(f'⚠️ Refocus error (non-critical): {type(e).__name__}: {e}')
|
|
|
|
except URLNotAllowedError as e:
|
|
raise e
|
|
except BrowserError as e:
|
|
raise e
|
|
except Exception as e:
|
|
# Extract key element info for error message
|
|
element_info = f'<{element_node.tag_name or "unknown"}'
|
|
if element_node.backend_node_id:
|
|
element_info += f' index={element_node.backend_node_id}'
|
|
element_info += '>'
|
|
|
|
# Create helpful error message based on context
|
|
error_detail = f'Failed to click element {element_info}. The element may not be interactable or visible.'
|
|
|
|
# Add hint if element has index (common in code-use mode)
|
|
if element_node.backend_node_id:
|
|
error_detail += f' If the page changed after navigation/interaction, the index [{element_node.backend_node_id}] may be stale. Get fresh browser state before retrying.'
|
|
|
|
raise BrowserError(
|
|
message=f'Failed to click element: {str(e)}',
|
|
long_term_memory=error_detail,
|
|
)
|
|
|
|
async def _click_on_coordinate(self, coordinate_x: int, coordinate_y: int, force: bool = False) -> dict | None:
|
|
"""
|
|
Click directly at coordinates using CDP Input.dispatchMouseEvent.
|
|
|
|
Args:
|
|
coordinate_x: X coordinate in viewport
|
|
coordinate_y: Y coordinate in viewport
|
|
force: If True, skip all safety checks (used when force=True in event)
|
|
|
|
Returns:
|
|
Dict with click coordinates or None
|
|
"""
|
|
try:
|
|
# Get CDP session
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
|
session_id = cdp_session.session_id
|
|
|
|
self.logger.debug(f'👆 Moving mouse to ({coordinate_x}, {coordinate_y})...')
|
|
|
|
# Move mouse to coordinates
|
|
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mouseMoved',
|
|
'x': coordinate_x,
|
|
'y': coordinate_y,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
await asyncio.sleep(0.05)
|
|
|
|
# Mouse down
|
|
self.logger.debug(f'👆🏾 Clicking at ({coordinate_x}, {coordinate_y})...')
|
|
try:
|
|
await asyncio.wait_for(
|
|
cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mousePressed',
|
|
'x': coordinate_x,
|
|
'y': coordinate_y,
|
|
'button': 'left',
|
|
'clickCount': 1,
|
|
},
|
|
session_id=session_id,
|
|
),
|
|
timeout=3.0,
|
|
)
|
|
await asyncio.sleep(0.05)
|
|
except TimeoutError:
|
|
self.logger.debug('⏱️ Mouse down timed out (likely due to dialog), continuing...')
|
|
|
|
# Mouse up
|
|
try:
|
|
await asyncio.wait_for(
|
|
cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mouseReleased',
|
|
'x': coordinate_x,
|
|
'y': coordinate_y,
|
|
'button': 'left',
|
|
'clickCount': 1,
|
|
},
|
|
session_id=session_id,
|
|
),
|
|
timeout=5.0,
|
|
)
|
|
except TimeoutError:
|
|
self.logger.debug('⏱️ Mouse up timed out (possibly due to lag or dialog popup), continuing...')
|
|
|
|
self.logger.debug(f'🖱️ Clicked successfully at ({coordinate_x}, {coordinate_y})')
|
|
|
|
# Return coordinates as metadata
|
|
return {'click_x': coordinate_x, 'click_y': coordinate_y}
|
|
|
|
except Exception as e:
|
|
self.logger.error(f'Failed to click at coordinates ({coordinate_x}, {coordinate_y}): {type(e).__name__}: {e}')
|
|
raise BrowserError(
|
|
message=f'Failed to click at coordinates: {e}',
|
|
long_term_memory=f'Failed to click at coordinates ({coordinate_x}, {coordinate_y}). The coordinates may be outside viewport or the page may have changed.',
|
|
)
|
|
|
|
async def _type_to_page(self, text: str):
|
|
"""
|
|
Type text to the page (whatever element currently has focus).
|
|
This is used when index is 0 or when an element can't be found.
|
|
"""
|
|
try:
|
|
# Get CDP client and session
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session(target_id=None, focus=True)
|
|
|
|
# Type the text character by character to the focused element
|
|
for char in text:
|
|
# Handle newline characters as Enter key
|
|
if char == '\n':
|
|
# Send proper Enter key sequence
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': 'Enter',
|
|
'code': 'Enter',
|
|
'windowsVirtualKeyCode': 13,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
# Send char event with carriage return
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'char',
|
|
'text': '\r',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
# Send keyup
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': 'Enter',
|
|
'code': 'Enter',
|
|
'windowsVirtualKeyCode': 13,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
else:
|
|
# Handle regular characters
|
|
# Send keydown
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': char,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
# Send char for actual text input
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'char',
|
|
'text': char,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
# Send keyup
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': char,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
# Add 10ms delay between keystrokes
|
|
await asyncio.sleep(0.010)
|
|
except Exception as e:
|
|
raise Exception(f'Failed to type to page: {str(e)}')
|
|
|
|
def _get_char_modifiers_and_vk(self, char: str) -> tuple[int, int, str]:
|
|
"""Get modifiers, virtual key code, and base key for a character.
|
|
|
|
Returns:
|
|
(modifiers, windowsVirtualKeyCode, base_key)
|
|
"""
|
|
# Characters that require Shift modifier
|
|
shift_chars = {
|
|
'!': ('1', 49),
|
|
'@': ('2', 50),
|
|
'#': ('3', 51),
|
|
'$': ('4', 52),
|
|
'%': ('5', 53),
|
|
'^': ('6', 54),
|
|
'&': ('7', 55),
|
|
'*': ('8', 56),
|
|
'(': ('9', 57),
|
|
')': ('0', 48),
|
|
'_': ('-', 189),
|
|
'+': ('=', 187),
|
|
'{': ('[', 219),
|
|
'}': (']', 221),
|
|
'|': ('\\', 220),
|
|
':': (';', 186),
|
|
'"': ("'", 222),
|
|
'<': (',', 188),
|
|
'>': ('.', 190),
|
|
'?': ('/', 191),
|
|
'~': ('`', 192),
|
|
}
|
|
|
|
# Check if character requires Shift
|
|
if char in shift_chars:
|
|
base_key, vk_code = shift_chars[char]
|
|
return (8, vk_code, base_key) # Shift=8
|
|
|
|
# Some Unicode characters' upper()/lower() expand to multiple code points
|
|
# (e.g. 'ß'.upper() == 'SS', 'ffi'.upper() == 'FFI'). ord() rejects those,
|
|
# so fall back to the original char's code point for the VK code.
|
|
def _vk_from(c: str) -> int:
|
|
up = c.upper()
|
|
return ord(up) if len(up) == 1 else ord(c)
|
|
|
|
# Uppercase letters require Shift
|
|
if char.isupper():
|
|
return (8, ord(char), char.lower()[:1] or char) # Shift=8
|
|
|
|
# Lowercase letters
|
|
if char.islower():
|
|
return (0, _vk_from(char), char)
|
|
|
|
# Numbers
|
|
if char.isdigit():
|
|
return (0, ord(char), char)
|
|
|
|
# Special characters without Shift
|
|
no_shift_chars = {
|
|
' ': 32,
|
|
'-': 189,
|
|
'=': 187,
|
|
'[': 219,
|
|
']': 221,
|
|
'\\': 220,
|
|
';': 186,
|
|
"'": 222,
|
|
',': 188,
|
|
'.': 190,
|
|
'/': 191,
|
|
'`': 192,
|
|
}
|
|
|
|
if char in no_shift_chars:
|
|
return (0, no_shift_chars[char], char)
|
|
|
|
# Fallback
|
|
return (0, _vk_from(char) if char.isalpha() else ord(char), char)
|
|
|
|
def _get_key_code_for_char(self, char: str) -> str:
|
|
"""Get the proper key code for a character (like Playwright does)."""
|
|
# Key code mapping for common characters (using proper base keys + modifiers)
|
|
key_codes = {
|
|
' ': 'Space',
|
|
'.': 'Period',
|
|
',': 'Comma',
|
|
'-': 'Minus',
|
|
'_': 'Minus', # Underscore uses Minus with Shift
|
|
'@': 'Digit2', # @ uses Digit2 with Shift
|
|
'!': 'Digit1', # ! uses Digit1 with Shift (not 'Exclamation')
|
|
'?': 'Slash', # ? uses Slash with Shift
|
|
':': 'Semicolon', # : uses Semicolon with Shift
|
|
';': 'Semicolon',
|
|
'(': 'Digit9', # ( uses Digit9 with Shift
|
|
')': 'Digit0', # ) uses Digit0 with Shift
|
|
'[': 'BracketLeft',
|
|
']': 'BracketRight',
|
|
'{': 'BracketLeft', # { uses BracketLeft with Shift
|
|
'}': 'BracketRight', # } uses BracketRight with Shift
|
|
'/': 'Slash',
|
|
'\\': 'Backslash',
|
|
'=': 'Equal',
|
|
'+': 'Equal', # + uses Equal with Shift
|
|
'*': 'Digit8', # * uses Digit8 with Shift
|
|
'&': 'Digit7', # & uses Digit7 with Shift
|
|
'%': 'Digit5', # % uses Digit5 with Shift
|
|
'$': 'Digit4', # $ uses Digit4 with Shift
|
|
'#': 'Digit3', # # uses Digit3 with Shift
|
|
'^': 'Digit6', # ^ uses Digit6 with Shift
|
|
'~': 'Backquote', # ~ uses Backquote with Shift
|
|
'`': 'Backquote',
|
|
"'": 'Quote',
|
|
'"': 'Quote', # " uses Quote with Shift
|
|
}
|
|
|
|
# Numbers
|
|
if char.isdigit():
|
|
return f'Digit{char}'
|
|
|
|
# Letters
|
|
if char.isalpha():
|
|
return f'Key{char.upper()}'
|
|
|
|
# Special characters
|
|
if char in key_codes:
|
|
return key_codes[char]
|
|
|
|
# Fallback for unknown characters
|
|
return f'Key{char.upper()}'
|
|
|
|
async def _clear_text_field(self, object_id: str, cdp_session) -> bool:
|
|
"""Clear text field using multiple strategies, starting with the most reliable."""
|
|
try:
|
|
# Strategy 1: Direct JavaScript value/content setting (handles both inputs and contenteditable)
|
|
self.logger.debug('🧹 Clearing text field using JavaScript value setting')
|
|
|
|
clear_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': """
|
|
function() {
|
|
// Check if it's a contenteditable element
|
|
const hasContentEditable = this.getAttribute('contenteditable') === 'true' ||
|
|
this.getAttribute('contenteditable') === '' ||
|
|
this.isContentEditable === true;
|
|
|
|
if (hasContentEditable) {
|
|
// For contenteditable elements, clear all content
|
|
while (this.firstChild) {
|
|
this.removeChild(this.firstChild);
|
|
}
|
|
this.textContent = "";
|
|
this.innerHTML = "";
|
|
|
|
// Focus and position cursor at the beginning
|
|
this.focus();
|
|
const selection = window.getSelection();
|
|
const range = document.createRange();
|
|
range.setStart(this, 0);
|
|
range.setEnd(this, 0);
|
|
selection.removeAllRanges();
|
|
selection.addRange(range);
|
|
|
|
// Dispatch events
|
|
this.dispatchEvent(new Event("input", { bubbles: true }));
|
|
this.dispatchEvent(new Event("change", { bubbles: true }));
|
|
|
|
return {cleared: true, method: 'contenteditable', finalText: this.textContent};
|
|
} else if (this.value !== undefined) {
|
|
// For regular inputs with value property
|
|
try {
|
|
this.select();
|
|
} catch (e) {
|
|
// ignore
|
|
}
|
|
this.value = "";
|
|
this.dispatchEvent(new Event("input", { bubbles: true }));
|
|
this.dispatchEvent(new Event("change", { bubbles: true }));
|
|
return {cleared: true, method: 'value', finalText: this.value};
|
|
} else {
|
|
return {cleared: false, method: 'none', error: 'Not a supported input type'};
|
|
}
|
|
}
|
|
""",
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Check the clear result
|
|
clear_info = clear_result.get('result', {}).get('value', {})
|
|
self.logger.debug(f'Clear result: {clear_info}')
|
|
|
|
if clear_info.get('cleared'):
|
|
final_text = clear_info.get('finalText', '')
|
|
if not final_text or not final_text.strip():
|
|
self.logger.debug(f'✅ Text field cleared successfully using {clear_info.get("method")}')
|
|
return True
|
|
else:
|
|
self.logger.debug(f'⚠️ JavaScript clear partially failed, field still contains: "{final_text}"')
|
|
else:
|
|
self.logger.debug(f'❌ JavaScript clear failed: {clear_info.get("error", "Unknown error")}')
|
|
|
|
except Exception as e:
|
|
self.logger.debug(f'JavaScript clear failed with exception: {e}')
|
|
return False
|
|
|
|
# Strategy 2: Triple-click + Delete (fallback for stubborn fields)
|
|
try:
|
|
self.logger.debug('🧹 Fallback: Clearing using triple-click + Delete')
|
|
|
|
# Get element center coordinates for triple-click
|
|
bounds_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { return this.getBoundingClientRect(); }',
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
if bounds_result.get('result', {}).get('value'):
|
|
bounds = bounds_result['result']['value']
|
|
center_x = bounds['x'] + bounds['width'] / 2
|
|
center_y = bounds['y'] + bounds['height'] / 2
|
|
|
|
# Triple-click to select all text
|
|
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mousePressed',
|
|
'x': center_x,
|
|
'y': center_y,
|
|
'button': 'left',
|
|
'clickCount': 3,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mouseReleased',
|
|
'x': center_x,
|
|
'y': center_y,
|
|
'button': 'left',
|
|
'clickCount': 3,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Delete selected text
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': 'Delete',
|
|
'code': 'Delete',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': 'Delete',
|
|
'code': 'Delete',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
self.logger.debug('✅ Text field cleared using triple-click + Delete')
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.debug(f'Triple-click clear failed: {e}')
|
|
|
|
# Strategy 3: Keyboard shortcuts (last resort)
|
|
try:
|
|
import platform
|
|
|
|
is_macos = platform.system() == 'Darwin'
|
|
select_all_modifier = 4 if is_macos else 2 # Meta=4 (Cmd), Ctrl=2
|
|
modifier_name = 'Cmd' if is_macos else 'Ctrl'
|
|
|
|
self.logger.debug(f'🧹 Last resort: Clearing using {modifier_name}+A + Backspace')
|
|
|
|
# Select all text (Ctrl/Cmd+A)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': 'a',
|
|
'code': 'KeyA',
|
|
'modifiers': select_all_modifier,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': 'a',
|
|
'code': 'KeyA',
|
|
'modifiers': select_all_modifier,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Delete selected text (Backspace)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': 'Backspace',
|
|
'code': 'Backspace',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': 'Backspace',
|
|
'code': 'Backspace',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
self.logger.debug('✅ Text field cleared using keyboard shortcuts')
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.debug(f'All clearing strategies failed: {e}')
|
|
return False
|
|
|
|
async def _focus_element_simple(
|
|
self, backend_node_id: int, object_id: str, cdp_session, input_coordinates: dict | None = None
|
|
) -> bool:
|
|
"""Simple focus strategy: CDP first, then click if failed."""
|
|
|
|
# Strategy 1: Try CDP DOM.focus first
|
|
try:
|
|
result = await cdp_session.cdp_client.send.DOM.focus(
|
|
params={'backendNodeId': backend_node_id},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
self.logger.debug(f'Element focused using CDP DOM.focus (result: {result})')
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.debug(f'❌ CDP DOM.focus threw exception: {type(e).__name__}: {e}')
|
|
|
|
# Strategy 2: Try click to focus if CDP failed
|
|
if input_coordinates and 'input_x' in input_coordinates and 'input_y' in input_coordinates:
|
|
try:
|
|
click_x = input_coordinates['input_x']
|
|
click_y = input_coordinates['input_y']
|
|
|
|
self.logger.debug(f'🎯 Attempting click-to-focus at ({click_x:.1f}, {click_y:.1f})')
|
|
|
|
# Click to focus
|
|
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mousePressed',
|
|
'x': click_x,
|
|
'y': click_y,
|
|
'button': 'left',
|
|
'clickCount': 1,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mouseReleased',
|
|
'x': click_x,
|
|
'y': click_y,
|
|
'button': 'left',
|
|
'clickCount': 1,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
self.logger.debug('✅ Element focused using click method')
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.debug(f'Click focus failed: {e}')
|
|
|
|
# Both strategies failed
|
|
self.logger.debug('Focus strategies failed, will attempt typing anyway')
|
|
return False
|
|
|
|
def _requires_direct_value_assignment(self, element_node: EnhancedDOMTreeNode) -> bool:
|
|
"""
|
|
Check if an element requires direct value assignment instead of character-by-character typing.
|
|
|
|
Certain input types have compound components, custom plugins, or special requirements
|
|
that make character-by-character typing unreliable. These need direct .value assignment:
|
|
|
|
Native HTML5:
|
|
- date, time, datetime-local: Have spinbutton components (ISO format required)
|
|
- month, week: Similar compound structure
|
|
- color: Expects hex format #RRGGBB
|
|
- range: Needs numeric value within min/max
|
|
|
|
jQuery/Bootstrap Datepickers:
|
|
- Detected by class names or data attributes
|
|
- Often expect specific date formats (MM/DD/YYYY, DD/MM/YYYY, etc.)
|
|
|
|
Note: We use direct assignment because:
|
|
1. Typing triggers intermediate validation that might reject partial values
|
|
2. Compound components (like date spinbuttons) don't work with sequential typing
|
|
3. It's much faster and more reliable
|
|
4. We dispatch proper input/change events afterward to trigger listeners
|
|
"""
|
|
if not element_node.tag_name or not element_node.attributes:
|
|
return False
|
|
|
|
tag_name = element_node.tag_name.lower()
|
|
|
|
# Check for native HTML5 inputs that need direct assignment
|
|
if tag_name == 'input':
|
|
input_type = element_node.attributes.get('type', '').lower()
|
|
|
|
# Native HTML5 inputs with compound components or strict formats
|
|
if input_type in {'date', 'time', 'datetime-local', 'month', 'week', 'color', 'range'}:
|
|
return True
|
|
|
|
# Detect jQuery/Bootstrap datepickers (text inputs with datepicker plugins)
|
|
if input_type in {'text', ''}:
|
|
# Check for common datepicker indicators
|
|
class_attr = element_node.attributes.get('class', '').lower()
|
|
if any(
|
|
indicator in class_attr
|
|
for indicator in ['datepicker', 'daterangepicker', 'datetimepicker', 'bootstrap-datepicker']
|
|
):
|
|
return True
|
|
|
|
# Check for data attributes indicating datepickers
|
|
if any(attr in element_node.attributes for attr in ['data-datepicker', 'data-date-format', 'data-provide']):
|
|
return True
|
|
|
|
return False
|
|
|
|
async def _set_value_directly(self, element_node: EnhancedDOMTreeNode, text: str, object_id: str, cdp_session) -> None:
|
|
"""
|
|
Set element value directly using JavaScript for inputs that don't support typing.
|
|
|
|
This is used for:
|
|
- Date/time inputs where character-by-character typing doesn't work
|
|
- jQuery datepickers that need direct value assignment
|
|
- Color/range inputs that need specific formats
|
|
- Any input with custom plugins that intercept typing
|
|
|
|
After setting the value, we dispatch comprehensive events to ensure all frameworks
|
|
and plugins recognize the change (React, Vue, Angular, jQuery, etc.)
|
|
"""
|
|
try:
|
|
# Set the value using JavaScript with comprehensive event dispatching
|
|
# callFunctionOn expects a function body (not a self-invoking function)
|
|
set_value_js = f"""
|
|
function() {{
|
|
// Store old value for comparison
|
|
const oldValue = this.value;
|
|
|
|
// REACT-COMPATIBLE VALUE SETTING:
|
|
// React uses Object.getOwnPropertyDescriptor to track input changes
|
|
// We need to use the native setter to bypass React's tracking and then trigger events
|
|
const nativeInputValueSetter = Object.getOwnPropertyDescriptor(
|
|
window.HTMLInputElement.prototype,
|
|
'value'
|
|
).set;
|
|
|
|
// Set the value using the native setter (bypasses React's control)
|
|
nativeInputValueSetter.call(this, {json.dumps(text)});
|
|
|
|
// Dispatch comprehensive events to ensure all frameworks detect the change
|
|
// Order matters: focus -> input -> change -> blur (mimics user interaction)
|
|
|
|
// 1. Focus event (in case element isn't focused)
|
|
this.dispatchEvent(new FocusEvent('focus', {{ bubbles: true }}));
|
|
|
|
// 2. Input event (CRITICAL for React onChange)
|
|
// React listens to 'input' events on the document and checks for value changes
|
|
const inputEvent = new Event('input', {{ bubbles: true, cancelable: true }});
|
|
this.dispatchEvent(inputEvent);
|
|
|
|
// 3. Change event (for form handling, traditional listeners)
|
|
const changeEvent = new Event('change', {{ bubbles: true, cancelable: true }});
|
|
this.dispatchEvent(changeEvent);
|
|
|
|
// 4. Blur event (triggers final validation in some libraries)
|
|
this.dispatchEvent(new FocusEvent('blur', {{ bubbles: true }}));
|
|
|
|
// 5. jQuery-specific events (if jQuery is present)
|
|
if (typeof jQuery !== 'undefined' && jQuery.fn) {{
|
|
try {{
|
|
jQuery(this).trigger('change');
|
|
// Trigger datepicker-specific events if it's a datepicker
|
|
if (jQuery(this).data('datepicker')) {{
|
|
jQuery(this).datepicker('update');
|
|
}}
|
|
}} catch (e) {{
|
|
// jQuery not available or error, continue anyway
|
|
}}
|
|
}}
|
|
|
|
return this.value;
|
|
}}
|
|
"""
|
|
|
|
result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'objectId': object_id,
|
|
'functionDeclaration': set_value_js,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Verify the value was set correctly
|
|
if 'result' in result and 'value' in result['result']:
|
|
actual_value = result['result']['value']
|
|
self.logger.debug(f'✅ Value set directly to: "{actual_value}"')
|
|
else:
|
|
self.logger.warning('⚠️ Could not verify value was set correctly')
|
|
|
|
except Exception as e:
|
|
self.logger.error(f'❌ Failed to set value directly: {e}')
|
|
raise
|
|
|
|
async def _input_text_element_node_impl(
|
|
self, element_node: EnhancedDOMTreeNode, text: str, clear: bool = True, is_sensitive: bool = False
|
|
) -> dict | None:
|
|
"""
|
|
Input text into an element using pure CDP with improved focus fallbacks.
|
|
|
|
For date/time inputs, uses direct value assignment instead of typing.
|
|
"""
|
|
|
|
try:
|
|
# Get CDP client
|
|
cdp_client = self.browser_session.cdp_client
|
|
|
|
# Get the correct session ID for the element's iframe
|
|
# session_id = await self._get_session_id_for_element(element_node)
|
|
|
|
# cdp_session = await self.browser_session.get_or_create_cdp_session(target_id=element_node.target_id, focus=True)
|
|
cdp_session = await self.browser_session.cdp_client_for_node(element_node)
|
|
|
|
# Get element info
|
|
backend_node_id = element_node.backend_node_id
|
|
|
|
# Track coordinates for metadata
|
|
input_coordinates = None
|
|
|
|
# Scroll element into view
|
|
try:
|
|
await cdp_session.cdp_client.send.DOM.scrollIntoViewIfNeeded(
|
|
params={'backendNodeId': backend_node_id}, session_id=cdp_session.session_id
|
|
)
|
|
await asyncio.sleep(0.01)
|
|
except Exception as e:
|
|
# Node detached errors are common with shadow DOM and dynamic content
|
|
# The element can still be interacted with even if scrolling fails
|
|
error_str = str(e)
|
|
if 'Node is detached from document' in error_str or 'detached from document' in error_str:
|
|
self.logger.debug(
|
|
f'Element node temporarily detached during scroll (common with shadow DOM), continuing: {element_node}'
|
|
)
|
|
else:
|
|
self.logger.debug(f'Failed to scroll element {element_node} into view before typing: {type(e).__name__}: {e}')
|
|
|
|
# Get object ID for the element
|
|
result = await cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': backend_node_id},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
assert 'object' in result and 'objectId' in result['object'], (
|
|
'Failed to find DOM element based on backendNodeId, maybe page content changed?'
|
|
)
|
|
object_id = result['object']['objectId']
|
|
|
|
# Get current coordinates using unified method
|
|
coords = await self.browser_session.get_element_coordinates(backend_node_id, cdp_session)
|
|
if coords:
|
|
center_x = coords.x + coords.width / 2
|
|
center_y = coords.y + coords.height / 2
|
|
|
|
# Check for occlusion before using coordinates for focus
|
|
is_occluded = await self._check_element_occlusion(backend_node_id, center_x, center_y, cdp_session)
|
|
|
|
if is_occluded:
|
|
self.logger.debug('🚫 Input element is occluded, skipping coordinate-based focus')
|
|
input_coordinates = None # Force fallback to CDP-only focus
|
|
else:
|
|
input_coordinates = {'input_x': center_x, 'input_y': center_y}
|
|
self.logger.debug(f'Using unified coordinates: x={center_x:.1f}, y={center_y:.1f}')
|
|
else:
|
|
input_coordinates = None
|
|
self.logger.debug('No coordinates found for element')
|
|
|
|
# Ensure we have a valid object_id before proceeding
|
|
if not object_id:
|
|
raise ValueError('Could not get object_id for element')
|
|
|
|
# Step 1: Focus the element using simple strategy
|
|
focused_successfully = await self._focus_element_simple(
|
|
backend_node_id=backend_node_id, object_id=object_id, cdp_session=cdp_session, input_coordinates=input_coordinates
|
|
)
|
|
|
|
# Step 2: Check if this element requires direct value assignment (date/time inputs)
|
|
requires_direct_assignment = self._requires_direct_value_assignment(element_node)
|
|
|
|
if requires_direct_assignment:
|
|
# Date/time inputs: use direct value assignment instead of typing
|
|
self.logger.debug(
|
|
f'🎯 Element type={element_node.attributes.get("type")} requires direct value assignment, setting value directly'
|
|
)
|
|
await self._set_value_directly(element_node, text, object_id, cdp_session)
|
|
|
|
# Return input coordinates for metadata
|
|
return input_coordinates
|
|
|
|
# Step 3: Clear existing text if requested (only for regular inputs that support typing)
|
|
if clear:
|
|
cleared_successfully = await self._clear_text_field(object_id=object_id, cdp_session=cdp_session)
|
|
if not cleared_successfully:
|
|
self.logger.warning('⚠️ Text field clearing failed, typing may append to existing text')
|
|
|
|
# Step 4: Type the text character by character using proper human-like key events
|
|
# This emulates exactly how a human would type, which modern websites expect
|
|
if is_sensitive:
|
|
# Note: sensitive_key_name is not passed to this low-level method,
|
|
# but we could extend the signature if needed for more granular logging
|
|
self.logger.debug('🎯 Typing <sensitive> character by character')
|
|
else:
|
|
self.logger.debug(f'🎯 Typing text character by character: "{text}"')
|
|
|
|
# Detect contenteditable elements (may have leaf-start bug where first char is dropped)
|
|
_attrs = element_node.attributes or {}
|
|
_is_contenteditable = _attrs.get('contenteditable') in ('true', '') or (
|
|
_attrs.get('role') == 'textbox' and element_node.tag_name not in ('input', 'textarea')
|
|
)
|
|
|
|
# For contenteditable: after typing first char, check if dropped and retype if needed
|
|
_check_first_char = _is_contenteditable and len(text) > 0 and clear
|
|
_first_char = text[0] if _check_first_char else None
|
|
|
|
for i, char in enumerate(text):
|
|
# Handle newline characters as Enter key
|
|
if char == '\n':
|
|
# Send proper Enter key sequence
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': 'Enter',
|
|
'code': 'Enter',
|
|
'windowsVirtualKeyCode': 13,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Small delay to emulate human typing speed
|
|
await asyncio.sleep(0.001)
|
|
|
|
# Send char event with carriage return
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'char',
|
|
'text': '\r',
|
|
'key': 'Enter',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Send keyUp event
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': 'Enter',
|
|
'code': 'Enter',
|
|
'windowsVirtualKeyCode': 13,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
else:
|
|
# Handle regular characters
|
|
# Get proper modifiers, VK code, and base key for the character
|
|
modifiers, vk_code, base_key = self._get_char_modifiers_and_vk(char)
|
|
key_code = self._get_key_code_for_char(base_key)
|
|
|
|
# self.logger.debug(f'🎯 Typing character {i + 1}/{len(text)}: "{char}" (base_key: {base_key}, code: {key_code}, modifiers: {modifiers}, vk: {vk_code})')
|
|
|
|
# Step 1: Send keyDown event (NO text parameter)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': base_key,
|
|
'code': key_code,
|
|
'modifiers': modifiers,
|
|
'windowsVirtualKeyCode': vk_code,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Small delay to emulate human typing speed
|
|
await asyncio.sleep(0.005)
|
|
|
|
# Step 2: Send char event (WITH text parameter) - this is crucial for text input
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'char',
|
|
'text': char,
|
|
'key': char,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Step 3: Send keyUp event (NO text parameter)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': base_key,
|
|
'code': key_code,
|
|
'modifiers': modifiers,
|
|
'windowsVirtualKeyCode': vk_code,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# After first char on contenteditable: check if dropped and retype if needed
|
|
if i == 0 and _check_first_char and _first_char:
|
|
check_result = await cdp_session.cdp_client.send.Runtime.evaluate(
|
|
params={'expression': 'document.activeElement.textContent'},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
content = check_result.get('result', {}).get('value', '')
|
|
if _first_char not in content:
|
|
self.logger.debug(f'🎯 First char "{_first_char}" was dropped (leaf-start bug), retyping')
|
|
# Retype the first character - cursor now past leaf-start
|
|
modifiers, vk_code, base_key = self._get_char_modifiers_and_vk(_first_char)
|
|
key_code = self._get_key_code_for_char(base_key)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': base_key,
|
|
'code': key_code,
|
|
'modifiers': modifiers,
|
|
'windowsVirtualKeyCode': vk_code,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await asyncio.sleep(0.005)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={'type': 'char', 'text': _first_char, 'key': _first_char},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': base_key,
|
|
'code': key_code,
|
|
'modifiers': modifiers,
|
|
'windowsVirtualKeyCode': vk_code,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Small delay between characters to look human (realistic typing speed)
|
|
await asyncio.sleep(0.001)
|
|
|
|
# Step 4: Trigger framework-aware DOM events after typing completion
|
|
# Modern JavaScript frameworks (React, Vue, Angular) rely on these events
|
|
# to update their internal state and trigger re-renders
|
|
await self._trigger_framework_events(object_id=object_id, cdp_session=cdp_session)
|
|
|
|
# Step 5: Read back actual value for verification (skip for sensitive data)
|
|
if not is_sensitive:
|
|
try:
|
|
await asyncio.sleep(0.05) # let autocomplete/formatter JS settle
|
|
readback_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'objectId': object_id,
|
|
'functionDeclaration': 'function() { return this.value !== undefined ? this.value : this.textContent; }',
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
actual_value = readback_result.get('result', {}).get('value')
|
|
if actual_value is not None:
|
|
if input_coordinates is None:
|
|
input_coordinates = {}
|
|
input_coordinates['actual_value'] = actual_value
|
|
except Exception as e:
|
|
self.logger.debug(f'Value readback failed (non-critical): {e}')
|
|
|
|
# Step 6: Auto-retry on concatenation mismatch (only when clear was requested)
|
|
# If we asked to clear but the readback value contains the typed text as a substring
|
|
# yet is longer, the field had pre-existing text that wasn't cleared. Set directly.
|
|
if clear and not is_sensitive and input_coordinates and 'actual_value' in input_coordinates:
|
|
actual_value = input_coordinates['actual_value']
|
|
if (
|
|
isinstance(actual_value, str)
|
|
and actual_value != text
|
|
and len(actual_value) > len(text)
|
|
and (actual_value.endswith(text) or actual_value.startswith(text))
|
|
):
|
|
self.logger.info(f'🔄 Concatenation detected: got "{actual_value}", expected "{text}" — auto-retrying')
|
|
try:
|
|
# Clear + set value via native setter in one JS call (works with React/Vue)
|
|
retry_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'objectId': object_id,
|
|
'functionDeclaration': """
|
|
function(newValue) {
|
|
if (this.value !== undefined) {
|
|
var desc = Object.getOwnPropertyDescriptor(
|
|
HTMLInputElement.prototype, 'value'
|
|
) || Object.getOwnPropertyDescriptor(
|
|
HTMLTextAreaElement.prototype, 'value'
|
|
);
|
|
if (desc && desc.set) {
|
|
desc.set.call(this, newValue);
|
|
} else {
|
|
this.value = newValue;
|
|
}
|
|
} else if (this.isContentEditable) {
|
|
this.textContent = newValue;
|
|
}
|
|
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
this.dispatchEvent(new Event('change', { bubbles: true }));
|
|
return this.value !== undefined ? this.value : this.textContent;
|
|
}
|
|
""",
|
|
'arguments': [{'value': text}],
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
retry_value = retry_result.get('result', {}).get('value')
|
|
if retry_value is not None:
|
|
input_coordinates['actual_value'] = retry_value
|
|
if retry_value == text:
|
|
self.logger.info('✅ Auto-retry fixed concatenation')
|
|
else:
|
|
self.logger.warning(f'⚠️ Auto-retry value still differs: "{retry_value}"')
|
|
except Exception as e:
|
|
self.logger.debug(f'Auto-retry failed (non-critical): {e}')
|
|
|
|
# Return coordinates metadata if available
|
|
return input_coordinates
|
|
|
|
except Exception as e:
|
|
self.logger.error(f'Failed to input text via CDP: {type(e).__name__}: {e}')
|
|
raise BrowserError(f'Failed to input text into element: {repr(element_node)}')
|
|
|
|
async def _trigger_framework_events(self, object_id: str, cdp_session) -> None:
|
|
"""
|
|
Trigger framework-aware DOM events after text input completion.
|
|
|
|
This is critical for modern JavaScript frameworks (React, Vue, Angular, etc.)
|
|
that rely on DOM events to update their internal state and trigger re-renders.
|
|
|
|
Args:
|
|
object_id: CDP object ID of the input element
|
|
cdp_session: CDP session for the element's context
|
|
"""
|
|
try:
|
|
# Execute JavaScript to trigger comprehensive event sequence
|
|
framework_events_script = """
|
|
function() {
|
|
// Find the target element (available as 'this' when using objectId)
|
|
const element = this;
|
|
if (!element) return false;
|
|
|
|
// Ensure element is focused
|
|
element.focus();
|
|
|
|
// Comprehensive event sequence for maximum framework compatibility
|
|
const events = [
|
|
// Input event - primary event for React controlled components
|
|
{ type: 'input', bubbles: true, cancelable: true },
|
|
// Change event - important for form validation and Vue v-model
|
|
{ type: 'change', bubbles: true, cancelable: true },
|
|
// Blur event - triggers validation in many frameworks
|
|
{ type: 'blur', bubbles: true, cancelable: true }
|
|
];
|
|
|
|
let success = true;
|
|
|
|
events.forEach(eventConfig => {
|
|
try {
|
|
const event = new Event(eventConfig.type, {
|
|
bubbles: eventConfig.bubbles,
|
|
cancelable: eventConfig.cancelable
|
|
});
|
|
|
|
// Special handling for InputEvent (more specific than Event)
|
|
if (eventConfig.type === 'input') {
|
|
const inputEvent = new InputEvent('input', {
|
|
bubbles: true,
|
|
cancelable: true,
|
|
data: element.value,
|
|
inputType: 'insertText'
|
|
});
|
|
element.dispatchEvent(inputEvent);
|
|
} else {
|
|
element.dispatchEvent(event);
|
|
}
|
|
} catch (e) {
|
|
success = false;
|
|
console.warn('Framework event dispatch failed:', eventConfig.type, e);
|
|
}
|
|
});
|
|
|
|
// Special React synthetic event handling
|
|
// React uses internal fiber properties for event system
|
|
if (element._reactInternalFiber || element._reactInternalInstance || element.__reactInternalInstance) {
|
|
try {
|
|
// Trigger React's synthetic event system
|
|
const syntheticInputEvent = new InputEvent('input', {
|
|
bubbles: true,
|
|
cancelable: true,
|
|
data: element.value
|
|
});
|
|
|
|
// Force React to process this as a synthetic event
|
|
Object.defineProperty(syntheticInputEvent, 'isTrusted', { value: true });
|
|
element.dispatchEvent(syntheticInputEvent);
|
|
} catch (e) {
|
|
console.warn('React synthetic event failed:', e);
|
|
}
|
|
}
|
|
|
|
// Special Vue reactivity trigger
|
|
// Vue uses __vueParentComponent or __vue__ for component access
|
|
if (element.__vue__ || element._vnode || element.__vueParentComponent) {
|
|
try {
|
|
// Vue often needs explicit input event with proper timing
|
|
const vueEvent = new Event('input', { bubbles: true });
|
|
setTimeout(() => element.dispatchEvent(vueEvent), 0);
|
|
} catch (e) {
|
|
console.warn('Vue reactivity trigger failed:', e);
|
|
}
|
|
}
|
|
|
|
return success;
|
|
}
|
|
"""
|
|
|
|
# Execute the framework events script
|
|
result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'objectId': object_id,
|
|
'functionDeclaration': framework_events_script,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
success = result.get('result', {}).get('value', False)
|
|
if success:
|
|
self.logger.debug('✅ Framework events triggered successfully')
|
|
else:
|
|
self.logger.warning('⚠️ Failed to trigger framework events')
|
|
|
|
except Exception as e:
|
|
self.logger.warning(f'⚠️ Failed to trigger framework events: {type(e).__name__}: {e}')
|
|
# Don't raise - framework events are a best-effort enhancement
|
|
|
|
async def _scroll_with_cdp_gesture(self, pixels: int) -> bool:
|
|
"""
|
|
Scroll using CDP Input.synthesizeScrollGesture to simulate realistic scroll gesture.
|
|
|
|
Args:
|
|
pixels: Number of pixels to scroll (positive = down, negative = up)
|
|
|
|
Returns:
|
|
True if successful, False if failed
|
|
"""
|
|
try:
|
|
# Get focused CDP session using public API (validates and waits for recovery if needed)
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
|
cdp_client = cdp_session.cdp_client
|
|
session_id = cdp_session.session_id
|
|
|
|
# Get viewport dimensions from cached value if available
|
|
if self.browser_session._original_viewport_size:
|
|
viewport_width, viewport_height = self.browser_session._original_viewport_size
|
|
else:
|
|
# Fallback: query layout metrics
|
|
layout_metrics = await cdp_client.send.Page.getLayoutMetrics(session_id=session_id)
|
|
viewport_width = layout_metrics['layoutViewport']['clientWidth']
|
|
viewport_height = layout_metrics['layoutViewport']['clientHeight']
|
|
|
|
# Calculate center of viewport
|
|
center_x = viewport_width / 2
|
|
center_y = viewport_height / 2
|
|
|
|
# For scroll gesture, positive yDistance scrolls up, negative scrolls down
|
|
# (opposite of mouseWheel deltaY convention)
|
|
y_distance = -pixels
|
|
|
|
# Synthesize scroll gesture - use very high speed for near-instant scrolling
|
|
await cdp_client.send.Input.synthesizeScrollGesture(
|
|
params={
|
|
'x': center_x,
|
|
'y': center_y,
|
|
'xDistance': 0,
|
|
'yDistance': y_distance,
|
|
'speed': 50000, # pixels per second (high = near-instant scroll)
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
|
|
self.logger.debug(f'📄 Scrolled via CDP gesture: {pixels}px')
|
|
return True
|
|
|
|
except Exception as e:
|
|
# Not critical - JavaScript fallback will handle scrolling
|
|
self.logger.debug(f'CDP gesture scroll failed ({type(e).__name__}: {e}), falling back to JS')
|
|
return False
|
|
|
|
async def _scroll_element_container(self, element_node, pixels: int) -> bool:
|
|
"""Try to scroll an element's container using CDP."""
|
|
try:
|
|
cdp_session = await self.browser_session.cdp_client_for_node(element_node)
|
|
|
|
# Check if this is an iframe - if so, scroll its content directly
|
|
if element_node.tag_name and element_node.tag_name.upper() == 'IFRAME':
|
|
# For iframes, we need to scroll the content document, not the iframe element itself
|
|
# Use JavaScript to directly scroll the iframe's content
|
|
backend_node_id = element_node.backend_node_id
|
|
|
|
# Resolve the node to get an object ID
|
|
result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': backend_node_id},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
if 'object' in result and 'objectId' in result['object']:
|
|
object_id = result['object']['objectId']
|
|
|
|
# Scroll the iframe's content directly
|
|
scroll_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': f"""
|
|
function() {{
|
|
try {{
|
|
const doc = this.contentDocument || this.contentWindow.document;
|
|
if (doc) {{
|
|
const scrollElement = doc.documentElement || doc.body;
|
|
if (scrollElement) {{
|
|
const oldScrollTop = scrollElement.scrollTop;
|
|
scrollElement.scrollTop += {pixels};
|
|
const newScrollTop = scrollElement.scrollTop;
|
|
return {{
|
|
success: true,
|
|
oldScrollTop: oldScrollTop,
|
|
newScrollTop: newScrollTop,
|
|
scrolled: newScrollTop - oldScrollTop
|
|
}};
|
|
}}
|
|
}}
|
|
return {{success: false, error: 'Could not access iframe content'}};
|
|
}} catch (e) {{
|
|
return {{success: false, error: e.toString()}};
|
|
}}
|
|
}}
|
|
""",
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
if scroll_result and 'result' in scroll_result and 'value' in scroll_result['result']:
|
|
result_value = scroll_result['result']['value']
|
|
if result_value.get('success'):
|
|
self.logger.debug(f'Successfully scrolled iframe content by {result_value.get("scrolled", 0)}px')
|
|
return True
|
|
else:
|
|
self.logger.debug(f'Failed to scroll iframe: {result_value.get("error", "Unknown error")}')
|
|
|
|
# For non-iframe elements, use the standard mouse wheel approach
|
|
# Get element bounds to know where to scroll
|
|
backend_node_id = element_node.backend_node_id
|
|
box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(
|
|
params={'backendNodeId': backend_node_id}, session_id=cdp_session.session_id
|
|
)
|
|
content_quad = box_model['model']['content']
|
|
|
|
# Calculate center point
|
|
center_x = (content_quad[0] + content_quad[2] + content_quad[4] + content_quad[6]) / 4
|
|
center_y = (content_quad[1] + content_quad[3] + content_quad[5] + content_quad[7]) / 4
|
|
|
|
# Dispatch mouse wheel event at element location
|
|
await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
|
|
params={
|
|
'type': 'mouseWheel',
|
|
'x': center_x,
|
|
'y': center_y,
|
|
'deltaX': 0,
|
|
'deltaY': pixels,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
return True
|
|
except Exception as e:
|
|
self.logger.debug(f'Failed to scroll element container via CDP: {e}')
|
|
return False
|
|
|
|
async def _get_session_id_for_element(self, element_node: EnhancedDOMTreeNode) -> str | None:
|
|
"""Get the appropriate CDP session ID for an element based on its frame."""
|
|
if element_node.frame_id:
|
|
# Element is in an iframe, need to get session for that frame
|
|
try:
|
|
all_targets = self.browser_session.session_manager.get_all_targets()
|
|
|
|
# Find the target for this frame
|
|
for target_id, target in all_targets.items():
|
|
if target.target_type == 'iframe' and element_node.frame_id in str(target_id):
|
|
# Create temporary session for iframe target without switching focus
|
|
temp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
|
|
return temp_session.session_id
|
|
|
|
# If frame not found in targets, use main target session
|
|
self.logger.debug(f'Frame {element_node.frame_id} not found in targets, using main session')
|
|
except Exception as e:
|
|
self.logger.debug(f'Error getting frame session: {e}, using main session')
|
|
|
|
# Use main target session - get_or_create_cdp_session validates focus automatically
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
|
return cdp_session.session_id
|
|
|
|
async def on_GoBackEvent(self, event: GoBackEvent) -> None:
|
|
"""Handle navigate back request with CDP."""
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
|
try:
|
|
# Get CDP client and session
|
|
|
|
# Get navigation history
|
|
history = await cdp_session.cdp_client.send.Page.getNavigationHistory(session_id=cdp_session.session_id)
|
|
current_index = history['currentIndex']
|
|
entries = history['entries']
|
|
|
|
# Check if we can go back
|
|
if current_index <= 0:
|
|
self.logger.warning('⚠️ Cannot go back - no previous entry in history')
|
|
return
|
|
|
|
# Navigate to the previous entry
|
|
previous_entry_id = entries[current_index - 1]['id']
|
|
await cdp_session.cdp_client.send.Page.navigateToHistoryEntry(
|
|
params={'entryId': previous_entry_id}, session_id=cdp_session.session_id
|
|
)
|
|
|
|
# Wait for navigation
|
|
await asyncio.sleep(0.5)
|
|
# Navigation is handled by BrowserSession via events
|
|
|
|
self.logger.info(f'🔙 Navigated back to {entries[current_index - 1]["url"]}')
|
|
except Exception as e:
|
|
raise
|
|
|
|
async def on_GoForwardEvent(self, event: GoForwardEvent) -> None:
|
|
"""Handle navigate forward request with CDP."""
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
|
try:
|
|
# Get navigation history
|
|
history = await cdp_session.cdp_client.send.Page.getNavigationHistory(session_id=cdp_session.session_id)
|
|
current_index = history['currentIndex']
|
|
entries = history['entries']
|
|
|
|
# Check if we can go forward
|
|
if current_index >= len(entries) - 1:
|
|
self.logger.warning('⚠️ Cannot go forward - no next entry in history')
|
|
return
|
|
|
|
# Navigate to the next entry
|
|
next_entry_id = entries[current_index + 1]['id']
|
|
await cdp_session.cdp_client.send.Page.navigateToHistoryEntry(
|
|
params={'entryId': next_entry_id}, session_id=cdp_session.session_id
|
|
)
|
|
|
|
# Wait for navigation
|
|
await asyncio.sleep(0.5)
|
|
# Navigation is handled by BrowserSession via events
|
|
|
|
self.logger.info(f'🔜 Navigated forward to {entries[current_index + 1]["url"]}')
|
|
except Exception as e:
|
|
raise
|
|
|
|
async def on_RefreshEvent(self, event: RefreshEvent) -> None:
|
|
"""Handle target refresh request with CDP."""
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
|
try:
|
|
# Reload the target
|
|
await cdp_session.cdp_client.send.Page.reload(session_id=cdp_session.session_id)
|
|
|
|
# Wait for reload
|
|
await asyncio.sleep(1.0)
|
|
|
|
# Note: We don't clear cached state here - let the next state fetch rebuild as needed
|
|
|
|
# Navigation is handled by BrowserSession via events
|
|
|
|
self.logger.info('🔄 Target refreshed')
|
|
except Exception as e:
|
|
raise
|
|
|
|
@observe_debug(ignore_input=True, ignore_output=True, name='wait_event_handler')
|
|
async def on_WaitEvent(self, event: WaitEvent) -> None:
|
|
"""Handle wait request."""
|
|
try:
|
|
# Cap wait time at maximum
|
|
actual_seconds = min(max(event.seconds, 0), event.max_seconds)
|
|
if actual_seconds != event.seconds:
|
|
self.logger.info(f'🕒 Waiting for {actual_seconds} seconds (capped from {event.seconds}s)')
|
|
else:
|
|
self.logger.info(f'🕒 Waiting for {actual_seconds} seconds')
|
|
|
|
await asyncio.sleep(actual_seconds)
|
|
except Exception as e:
|
|
raise
|
|
|
|
async def _dispatch_key_event(self, cdp_session, event_type: str, key: str, modifiers: int = 0) -> None:
|
|
"""Helper to dispatch a keyboard event with proper key codes."""
|
|
code, vk_code = get_key_info(key)
|
|
params: DispatchKeyEventParameters = {
|
|
'type': event_type,
|
|
'key': key,
|
|
'code': code,
|
|
}
|
|
if modifiers:
|
|
params['modifiers'] = modifiers
|
|
if vk_code is not None:
|
|
params['windowsVirtualKeyCode'] = vk_code
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(params=params, session_id=cdp_session.session_id)
|
|
|
|
async def on_SendKeysEvent(self, event: SendKeysEvent) -> None:
|
|
"""Handle send keys request with CDP."""
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
|
|
try:
|
|
# Normalize key names from common aliases
|
|
key_aliases = {
|
|
'ctrl': 'Control',
|
|
'control': 'Control',
|
|
'alt': 'Alt',
|
|
'option': 'Alt',
|
|
'meta': 'Meta',
|
|
'cmd': 'Meta',
|
|
'command': 'Meta',
|
|
'shift': 'Shift',
|
|
'enter': 'Enter',
|
|
'return': 'Enter',
|
|
'tab': 'Tab',
|
|
'delete': 'Delete',
|
|
'backspace': 'Backspace',
|
|
'escape': 'Escape',
|
|
'esc': 'Escape',
|
|
'space': ' ',
|
|
'up': 'ArrowUp',
|
|
'down': 'ArrowDown',
|
|
'left': 'ArrowLeft',
|
|
'right': 'ArrowRight',
|
|
'pageup': 'PageUp',
|
|
'pagedown': 'PageDown',
|
|
'home': 'Home',
|
|
'end': 'End',
|
|
}
|
|
|
|
# Parse and normalize the key string
|
|
keys = event.keys
|
|
if '+' in keys:
|
|
# Handle key combinations like "ctrl+a"
|
|
parts = keys.split('+')
|
|
normalized_parts = []
|
|
for part in parts:
|
|
part_lower = part.strip().lower()
|
|
normalized = key_aliases.get(part_lower, part)
|
|
normalized_parts.append(normalized)
|
|
normalized_keys = '+'.join(normalized_parts)
|
|
else:
|
|
# Single key
|
|
keys_lower = keys.strip().lower()
|
|
normalized_keys = key_aliases.get(keys_lower, keys)
|
|
|
|
# Handle key combinations like "Control+A"
|
|
if '+' in normalized_keys:
|
|
parts = normalized_keys.split('+')
|
|
modifiers = parts[:-1]
|
|
main_key = parts[-1]
|
|
|
|
# Calculate modifier bitmask
|
|
modifier_value = 0
|
|
modifier_map = {'Alt': 1, 'Control': 2, 'Meta': 4, 'Shift': 8}
|
|
for mod in modifiers:
|
|
modifier_value |= modifier_map.get(mod, 0)
|
|
|
|
# Press modifier keys
|
|
for mod in modifiers:
|
|
await self._dispatch_key_event(cdp_session, 'keyDown', mod)
|
|
|
|
# Press main key with modifiers bitmask
|
|
await self._dispatch_key_event(cdp_session, 'keyDown', main_key, modifier_value)
|
|
|
|
await self._dispatch_key_event(cdp_session, 'keyUp', main_key, modifier_value)
|
|
|
|
# Release modifier keys
|
|
for mod in reversed(modifiers):
|
|
await self._dispatch_key_event(cdp_session, 'keyUp', mod)
|
|
else:
|
|
# Check if this is a text string or special key
|
|
special_keys = {
|
|
'Enter',
|
|
'Tab',
|
|
'Delete',
|
|
'Backspace',
|
|
'Escape',
|
|
'ArrowUp',
|
|
'ArrowDown',
|
|
'ArrowLeft',
|
|
'ArrowRight',
|
|
'PageUp',
|
|
'PageDown',
|
|
'Home',
|
|
'End',
|
|
'Control',
|
|
'Alt',
|
|
'Meta',
|
|
'Shift',
|
|
'F1',
|
|
'F2',
|
|
'F3',
|
|
'F4',
|
|
'F5',
|
|
'F6',
|
|
'F7',
|
|
'F8',
|
|
'F9',
|
|
'F10',
|
|
'F11',
|
|
'F12',
|
|
}
|
|
|
|
# If it's a special key, use original logic
|
|
if normalized_keys in special_keys:
|
|
await self._dispatch_key_event(cdp_session, 'keyDown', normalized_keys)
|
|
# For Enter key, also dispatch a char event to trigger keypress listeners
|
|
if normalized_keys == 'Enter':
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'char',
|
|
'text': '\r',
|
|
'key': 'Enter',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await self._dispatch_key_event(cdp_session, 'keyUp', normalized_keys)
|
|
else:
|
|
# It's text (single character or string) - send each character as text input
|
|
# This is crucial for text to appear in focused input fields
|
|
for char in normalized_keys:
|
|
# Special-case newline characters to dispatch as Enter
|
|
if char in ('\n', '\r'):
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'rawKeyDown',
|
|
'windowsVirtualKeyCode': 13,
|
|
'unmodifiedText': '\r',
|
|
'text': '\r',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'char',
|
|
'windowsVirtualKeyCode': 13,
|
|
'unmodifiedText': '\r',
|
|
'text': '\r',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'windowsVirtualKeyCode': 13,
|
|
'unmodifiedText': '\r',
|
|
'text': '\r',
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
continue
|
|
|
|
# Get proper modifiers and key info for the character
|
|
modifiers, vk_code, base_key = self._get_char_modifiers_and_vk(char)
|
|
key_code = self._get_key_code_for_char(base_key)
|
|
|
|
# Send keyDown
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyDown',
|
|
'key': base_key,
|
|
'code': key_code,
|
|
'modifiers': modifiers,
|
|
'windowsVirtualKeyCode': vk_code,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Send char event with text - this is what makes text appear in input fields
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'char',
|
|
'text': char,
|
|
'key': char,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Send keyUp
|
|
await cdp_session.cdp_client.send.Input.dispatchKeyEvent(
|
|
params={
|
|
'type': 'keyUp',
|
|
'key': base_key,
|
|
'code': key_code,
|
|
'modifiers': modifiers,
|
|
'windowsVirtualKeyCode': vk_code,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Small delay between characters (10ms)
|
|
await asyncio.sleep(0.010)
|
|
|
|
self.logger.info(f'⌨️ Sent keys: {event.keys}')
|
|
|
|
# Note: We don't clear cached state on Enter; multi_act will detect DOM changes
|
|
# and rebuild explicitly. We still wait briefly for potential navigation.
|
|
if 'enter' in event.keys.lower() or 'return' in event.keys.lower():
|
|
await asyncio.sleep(0.1)
|
|
except Exception as e:
|
|
raise
|
|
|
|
async def on_UploadFileEvent(self, event: UploadFileEvent) -> None:
|
|
"""Handle file upload request with CDP."""
|
|
try:
|
|
# Use the provided node
|
|
element_node = event.node
|
|
index_for_logging = element_node.backend_node_id or 'unknown'
|
|
|
|
# Check if it's a file input
|
|
if not self.browser_session.is_file_input(element_node):
|
|
msg = f'Upload failed - element {index_for_logging} is not a file input.'
|
|
raise BrowserError(message=msg, long_term_memory=msg)
|
|
|
|
# Get CDP client and session
|
|
cdp_client = self.browser_session.cdp_client
|
|
session_id = await self._get_session_id_for_element(element_node)
|
|
|
|
# Validate file before upload
|
|
if os.path.exists(event.file_path):
|
|
file_size = os.path.getsize(event.file_path)
|
|
if file_size == 0:
|
|
msg = f'Upload failed - file {event.file_path} is empty (0 bytes).'
|
|
raise BrowserError(message=msg, long_term_memory=msg)
|
|
self.logger.debug(f'📎 File {event.file_path} validated ({file_size} bytes)')
|
|
|
|
# Set file(s) to upload
|
|
backend_node_id = element_node.backend_node_id
|
|
await cdp_client.send.DOM.setFileInputFiles(
|
|
params={
|
|
'files': [event.file_path],
|
|
'backendNodeId': backend_node_id,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
|
|
self.logger.info(f'📎 Uploaded file {event.file_path} to element {index_for_logging}')
|
|
except Exception as e:
|
|
raise
|
|
|
|
async def on_ScrollToTextEvent(self, event: ScrollToTextEvent) -> None:
|
|
"""Handle scroll to text request with CDP. Raises exception if text not found."""
|
|
|
|
# TODO: handle looking for text inside cross-origin iframes as well
|
|
|
|
# Get focused CDP session using public API (validates and waits for recovery if needed)
|
|
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
|
cdp_client = cdp_session.cdp_client
|
|
session_id = cdp_session.session_id
|
|
|
|
# Enable DOM
|
|
await cdp_client.send.DOM.enable(session_id=session_id)
|
|
|
|
# Get document
|
|
doc = await cdp_client.send.DOM.getDocument(params={'depth': -1}, session_id=session_id)
|
|
root_node_id = doc['root']['nodeId']
|
|
|
|
# Search for text using XPath
|
|
search_queries = [
|
|
f'//*[contains(text(), "{event.text}")]',
|
|
f'//*[contains(., "{event.text}")]',
|
|
f'//*[@*[contains(., "{event.text}")]]',
|
|
]
|
|
|
|
found = False
|
|
for query in search_queries:
|
|
try:
|
|
# Perform search
|
|
search_result = await cdp_client.send.DOM.performSearch(params={'query': query}, session_id=session_id)
|
|
search_id = search_result['searchId']
|
|
result_count = search_result['resultCount']
|
|
|
|
if result_count > 0:
|
|
# Get the first match
|
|
node_ids = await cdp_client.send.DOM.getSearchResults(
|
|
params={'searchId': search_id, 'fromIndex': 0, 'toIndex': 1},
|
|
session_id=session_id,
|
|
)
|
|
|
|
if node_ids['nodeIds']:
|
|
node_id = node_ids['nodeIds'][0]
|
|
|
|
# Scroll the element into view
|
|
await cdp_client.send.DOM.scrollIntoViewIfNeeded(params={'nodeId': node_id}, session_id=session_id)
|
|
|
|
found = True
|
|
self.logger.debug(f'📜 Scrolled to text: "{event.text}"')
|
|
break
|
|
|
|
# Clean up search
|
|
await cdp_client.send.DOM.discardSearchResults(params={'searchId': search_id}, session_id=session_id)
|
|
except Exception as e:
|
|
self.logger.debug(f'Search query failed: {query}, error: {e}')
|
|
continue
|
|
|
|
if not found:
|
|
# Fallback: Try JavaScript search
|
|
js_result = await cdp_client.send.Runtime.evaluate(
|
|
params={
|
|
'expression': f'''
|
|
(() => {{
|
|
const walker = document.createTreeWalker(
|
|
document.body,
|
|
NodeFilter.SHOW_TEXT,
|
|
null,
|
|
false
|
|
);
|
|
let node;
|
|
while (node = walker.nextNode()) {{
|
|
if (node.textContent.includes("{event.text}")) {{
|
|
node.parentElement.scrollIntoView({{behavior: 'smooth', block: 'center'}});
|
|
return true;
|
|
}}
|
|
}}
|
|
return false;
|
|
}})()
|
|
'''
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
|
|
if js_result.get('result', {}).get('value'):
|
|
self.logger.debug(f'📜 Scrolled to text: "{event.text}" (via JS)')
|
|
return None
|
|
else:
|
|
self.logger.warning(f'⚠️ Text not found: "{event.text}"')
|
|
raise BrowserError(f'Text not found: "{event.text}"', details={'text': event.text})
|
|
|
|
# If we got here and found is True, return None (success)
|
|
if found:
|
|
return None
|
|
else:
|
|
raise BrowserError(f'Text not found: "{event.text}"', details={'text': event.text})
|
|
|
|
async def on_GetDropdownOptionsEvent(self, event: GetDropdownOptionsEvent) -> dict[str, str]:
|
|
"""Handle get dropdown options request with CDP."""
|
|
try:
|
|
# Use the provided node
|
|
element_node = event.node
|
|
index_for_logging = element_node.backend_node_id or 'unknown'
|
|
|
|
# Get CDP session for this node
|
|
cdp_session = await self.browser_session.cdp_client_for_node(element_node)
|
|
|
|
# Convert node to object ID for CDP operations
|
|
try:
|
|
object_result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': element_node.backend_node_id}, session_id=cdp_session.session_id
|
|
)
|
|
remote_object = object_result.get('object', {})
|
|
object_id = remote_object.get('objectId')
|
|
if not object_id:
|
|
raise ValueError('Could not get object ID from resolved node')
|
|
except Exception as e:
|
|
raise ValueError(f'Failed to resolve node to object: {e}') from e
|
|
|
|
# Check if this is an ARIA combobox that needs expansion
|
|
# ARIA comboboxes have options in a separate element referenced by aria-controls
|
|
check_combobox_script = """
|
|
function() {
|
|
const element = this;
|
|
const role = element.getAttribute('role');
|
|
const ariaControls = element.getAttribute('aria-controls');
|
|
const ariaExpanded = element.getAttribute('aria-expanded');
|
|
|
|
if (role === 'combobox' && ariaControls) {
|
|
return {
|
|
isCombobox: true,
|
|
ariaControls: ariaControls,
|
|
isExpanded: ariaExpanded === 'true',
|
|
tagName: element.tagName.toLowerCase()
|
|
};
|
|
}
|
|
return { isCombobox: false };
|
|
}
|
|
"""
|
|
|
|
combobox_check = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': check_combobox_script,
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
combobox_info = combobox_check.get('result', {}).get('value', {})
|
|
|
|
# If it's an ARIA combobox with aria-controls, handle it specially
|
|
if combobox_info.get('isCombobox'):
|
|
return await self._handle_aria_combobox_options(cdp_session, object_id, combobox_info, index_for_logging)
|
|
|
|
# Use JavaScript to extract dropdown options (existing logic for non-combobox elements)
|
|
options_script = """
|
|
function() {
|
|
const startElement = this;
|
|
|
|
// Function to check if an element is a dropdown and extract options
|
|
function checkDropdownElement(element) {
|
|
// Check if it's a native select element
|
|
if (element.tagName.toLowerCase() === 'select') {
|
|
return {
|
|
type: 'select',
|
|
options: Array.from(element.options).map((opt, idx) => ({
|
|
text: opt.text.trim(),
|
|
value: opt.value,
|
|
index: idx,
|
|
selected: opt.selected
|
|
})),
|
|
id: element.id || '',
|
|
name: element.name || '',
|
|
source: 'target'
|
|
};
|
|
}
|
|
|
|
// Check if it's an ARIA dropdown/menu (not combobox - handled separately)
|
|
const role = element.getAttribute('role');
|
|
if (role === 'menu' || role === 'listbox') {
|
|
// Find all menu items/options
|
|
const menuItems = element.querySelectorAll('[role="menuitem"], [role="option"]');
|
|
const options = [];
|
|
|
|
menuItems.forEach((item, idx) => {
|
|
const text = item.textContent ? item.textContent.trim() : '';
|
|
if (text) {
|
|
options.push({
|
|
text: text,
|
|
value: item.getAttribute('data-value') || text,
|
|
index: idx,
|
|
selected: item.getAttribute('aria-selected') === 'true' || item.classList.contains('selected')
|
|
});
|
|
}
|
|
});
|
|
|
|
return {
|
|
type: 'aria',
|
|
options: options,
|
|
id: element.id || '',
|
|
name: element.getAttribute('aria-label') || '',
|
|
source: 'target'
|
|
};
|
|
}
|
|
|
|
// Check if it's a Semantic UI dropdown or similar
|
|
if (element.classList.contains('dropdown') || element.classList.contains('ui')) {
|
|
const menuItems = element.querySelectorAll('.item, .option, [data-value]');
|
|
const options = [];
|
|
|
|
menuItems.forEach((item, idx) => {
|
|
const text = item.textContent ? item.textContent.trim() : '';
|
|
if (text) {
|
|
options.push({
|
|
text: text,
|
|
value: item.getAttribute('data-value') || text,
|
|
index: idx,
|
|
selected: item.classList.contains('selected') || item.classList.contains('active')
|
|
});
|
|
}
|
|
});
|
|
|
|
if (options.length > 0) {
|
|
return {
|
|
type: 'custom',
|
|
options: options,
|
|
id: element.id || '',
|
|
name: element.getAttribute('aria-label') || '',
|
|
source: 'target'
|
|
};
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
// Function to recursively search children up to specified depth
|
|
function searchChildrenForDropdowns(element, maxDepth, currentDepth = 0) {
|
|
if (currentDepth >= maxDepth) return null;
|
|
|
|
// Check all direct children
|
|
for (let child of element.children) {
|
|
// Check if this child is a dropdown
|
|
const result = checkDropdownElement(child);
|
|
if (result) {
|
|
result.source = `child-depth-${currentDepth + 1}`;
|
|
return result;
|
|
}
|
|
|
|
// Recursively check this child's children
|
|
const childResult = searchChildrenForDropdowns(child, maxDepth, currentDepth + 1);
|
|
if (childResult) {
|
|
return childResult;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
// First check the target element itself
|
|
let dropdownResult = checkDropdownElement(startElement);
|
|
if (dropdownResult) {
|
|
return dropdownResult;
|
|
}
|
|
|
|
// If target element is not a dropdown, search children up to depth 4
|
|
dropdownResult = searchChildrenForDropdowns(startElement, 4);
|
|
if (dropdownResult) {
|
|
return dropdownResult;
|
|
}
|
|
|
|
return {
|
|
error: `Element and its children (depth 4) are not recognizable dropdown types (tag: ${startElement.tagName}, role: ${startElement.getAttribute('role')}, classes: ${startElement.className})`
|
|
};
|
|
}
|
|
"""
|
|
|
|
result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': options_script,
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
dropdown_data = result.get('result', {}).get('value', {})
|
|
|
|
if dropdown_data.get('error'):
|
|
raise BrowserError(message=dropdown_data['error'], long_term_memory=dropdown_data['error'])
|
|
|
|
if not dropdown_data.get('options'):
|
|
msg = f'No options found in dropdown at index {index_for_logging}'
|
|
return {
|
|
'error': msg,
|
|
'short_term_memory': msg,
|
|
'long_term_memory': msg,
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
|
|
# Format options for display
|
|
formatted_options = []
|
|
for opt in dropdown_data['options']:
|
|
# Use JSON encoding to ensure exact string matching
|
|
encoded_text = json.dumps(opt['text'])
|
|
status = ' (selected)' if opt.get('selected') else ''
|
|
formatted_options.append(f'{opt["index"]}: text={encoded_text}, value={json.dumps(opt["value"])}{status}')
|
|
|
|
dropdown_type = dropdown_data.get('type', 'select')
|
|
element_info = f'Index: {index_for_logging}, Type: {dropdown_type}, ID: {dropdown_data.get("id", "none")}, Name: {dropdown_data.get("name", "none")}'
|
|
source_info = dropdown_data.get('source', 'unknown')
|
|
|
|
if source_info == 'target':
|
|
msg = f'Found {dropdown_type} dropdown ({element_info}):\n' + '\n'.join(formatted_options)
|
|
else:
|
|
msg = f'Found {dropdown_type} dropdown in {source_info} ({element_info}):\n' + '\n'.join(formatted_options)
|
|
msg += (
|
|
f'\n\nUse the exact text or value string (without quotes) in select_dropdown(index={index_for_logging}, text=...)'
|
|
)
|
|
|
|
if source_info == 'target':
|
|
self.logger.info(f'📋 Found {len(dropdown_data["options"])} dropdown options for index {index_for_logging}')
|
|
else:
|
|
self.logger.info(
|
|
f'📋 Found {len(dropdown_data["options"])} dropdown options for index {index_for_logging} in {source_info}'
|
|
)
|
|
|
|
# Create structured memory for the response
|
|
short_term_memory = msg
|
|
long_term_memory = f'Got dropdown options for index {index_for_logging}'
|
|
|
|
# Return the dropdown data as a dict with structured memory
|
|
return {
|
|
'type': dropdown_type,
|
|
'options': json.dumps(dropdown_data['options']), # Convert list to JSON string for dict[str, str] type
|
|
'element_info': element_info,
|
|
'source': source_info,
|
|
'formatted_options': '\n'.join(formatted_options),
|
|
'message': msg,
|
|
'short_term_memory': short_term_memory,
|
|
'long_term_memory': long_term_memory,
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
|
|
except BrowserError:
|
|
# Re-raise BrowserError as-is to preserve structured memory
|
|
raise
|
|
except TimeoutError:
|
|
msg = f'Failed to get dropdown options for index {index_for_logging} due to timeout.'
|
|
self.logger.error(msg)
|
|
raise BrowserError(message=msg, long_term_memory=msg)
|
|
except Exception as e:
|
|
msg = 'Failed to get dropdown options'
|
|
error_msg = f'{msg}: {str(e)}'
|
|
self.logger.error(error_msg)
|
|
raise BrowserError(
|
|
message=error_msg, long_term_memory=f'Failed to get dropdown options for index {index_for_logging}.'
|
|
)
|
|
|
|
async def _handle_aria_combobox_options(
|
|
self,
|
|
cdp_session,
|
|
object_id: str,
|
|
combobox_info: dict,
|
|
index_for_logging: int | str,
|
|
) -> dict[str, str]:
|
|
"""Handle ARIA combobox elements with options in a separate listbox element.
|
|
|
|
ARIA comboboxes (role="combobox") have options in a separate element referenced
|
|
by aria-controls. Options may only be rendered when the combobox is expanded.
|
|
|
|
This method:
|
|
1. Expands the combobox if collapsed (by clicking/focusing it)
|
|
2. Waits for options to render
|
|
3. Finds options in the aria-controls referenced element
|
|
4. Collapses the combobox after extracting options
|
|
"""
|
|
aria_controls_id = combobox_info.get('ariaControls')
|
|
was_expanded = combobox_info.get('isExpanded', False)
|
|
|
|
# If combobox is collapsed, expand it first to trigger option rendering
|
|
if not was_expanded:
|
|
# Use more robust expansion: dispatch proper DOM events that trigger event listeners
|
|
expand_script = """
|
|
function() {
|
|
const element = this;
|
|
|
|
// Dispatch focus event properly
|
|
const focusEvent = new FocusEvent('focus', { bubbles: true, cancelable: true });
|
|
element.dispatchEvent(focusEvent);
|
|
|
|
// Also call native focus
|
|
element.focus();
|
|
|
|
// Dispatch focusin event (bubbles, unlike focus)
|
|
const focusInEvent = new FocusEvent('focusin', { bubbles: true, cancelable: true });
|
|
element.dispatchEvent(focusInEvent);
|
|
|
|
// For some comboboxes, a click is needed
|
|
const clickEvent = new MouseEvent('click', {
|
|
bubbles: true,
|
|
cancelable: true,
|
|
view: window
|
|
});
|
|
element.dispatchEvent(clickEvent);
|
|
|
|
// Some comboboxes respond to mousedown
|
|
const mousedownEvent = new MouseEvent('mousedown', {
|
|
bubbles: true,
|
|
cancelable: true,
|
|
view: window
|
|
});
|
|
element.dispatchEvent(mousedownEvent);
|
|
|
|
return {
|
|
success: true,
|
|
ariaExpanded: element.getAttribute('aria-expanded')
|
|
};
|
|
}
|
|
"""
|
|
await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': expand_script,
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
await asyncio.sleep(0.5)
|
|
|
|
# Now extract options from the aria-controls referenced element
|
|
extract_options_script = """
|
|
function(ariaControlsId) {
|
|
const combobox = this;
|
|
|
|
// Find the listbox element referenced by aria-controls
|
|
const listbox = document.getElementById(ariaControlsId);
|
|
|
|
if (!listbox) {
|
|
return {
|
|
error: `Could not find listbox element with id "${ariaControlsId}" referenced by aria-controls`,
|
|
ariaControlsId: ariaControlsId
|
|
};
|
|
}
|
|
|
|
// Find all option elements in the listbox
|
|
const optionElements = listbox.querySelectorAll('[role="option"]');
|
|
const options = [];
|
|
|
|
optionElements.forEach((item, idx) => {
|
|
const text = item.textContent ? item.textContent.trim() : '';
|
|
if (text) {
|
|
options.push({
|
|
text: text,
|
|
value: item.getAttribute('data-value') || item.getAttribute('value') || text,
|
|
index: idx,
|
|
selected: item.getAttribute('aria-selected') === 'true' || item.classList.contains('selected')
|
|
});
|
|
}
|
|
});
|
|
|
|
// If no options with role="option", try other common patterns
|
|
if (options.length === 0) {
|
|
// Try li elements inside
|
|
const liElements = listbox.querySelectorAll('li');
|
|
liElements.forEach((item, idx) => {
|
|
const text = item.textContent ? item.textContent.trim() : '';
|
|
if (text) {
|
|
options.push({
|
|
text: text,
|
|
value: item.getAttribute('data-value') || item.getAttribute('value') || text,
|
|
index: idx,
|
|
selected: item.getAttribute('aria-selected') === 'true' || item.classList.contains('selected')
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
return {
|
|
type: 'aria-combobox',
|
|
options: options,
|
|
id: combobox.id || '',
|
|
name: combobox.getAttribute('aria-label') || combobox.getAttribute('name') || '',
|
|
listboxId: ariaControlsId,
|
|
source: 'aria-controls'
|
|
};
|
|
}
|
|
"""
|
|
|
|
result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': extract_options_script,
|
|
'objectId': object_id,
|
|
'arguments': [{'value': aria_controls_id}],
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
dropdown_data = result.get('result', {}).get('value', {})
|
|
|
|
# Collapse the combobox if we expanded it (blur to close)
|
|
if not was_expanded:
|
|
collapse_script = """
|
|
function() {
|
|
this.blur();
|
|
// Also dispatch escape key to close dropdowns
|
|
const escEvent = new KeyboardEvent('keydown', { key: 'Escape', bubbles: true });
|
|
this.dispatchEvent(escEvent);
|
|
return true;
|
|
}
|
|
"""
|
|
await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': collapse_script,
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
# Handle errors
|
|
if dropdown_data.get('error'):
|
|
raise BrowserError(message=dropdown_data['error'], long_term_memory=dropdown_data['error'])
|
|
|
|
if not dropdown_data.get('options'):
|
|
msg = f'No options found in ARIA combobox at index {index_for_logging} (listbox: {aria_controls_id})'
|
|
return {
|
|
'error': msg,
|
|
'short_term_memory': msg,
|
|
'long_term_memory': msg,
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
|
|
# Format options for display
|
|
formatted_options = []
|
|
for opt in dropdown_data['options']:
|
|
encoded_text = json.dumps(opt['text'])
|
|
status = ' (selected)' if opt.get('selected') else ''
|
|
formatted_options.append(f'{opt["index"]}: text={encoded_text}, value={json.dumps(opt["value"])}{status}')
|
|
|
|
dropdown_type = dropdown_data.get('type', 'aria-combobox')
|
|
element_info = f'Index: {index_for_logging}, Type: {dropdown_type}, ID: {dropdown_data.get("id", "none")}, Name: {dropdown_data.get("name", "none")}'
|
|
source_info = f'aria-controls → {aria_controls_id}'
|
|
|
|
msg = f'Found {dropdown_type} dropdown ({element_info}):\n' + '\n'.join(formatted_options)
|
|
msg += f'\n\nUse the exact text or value string (without quotes) in select_dropdown(index={index_for_logging}, text=...)'
|
|
|
|
self.logger.info(f'📋 Found {len(dropdown_data["options"])} options in ARIA combobox at index {index_for_logging}')
|
|
|
|
return {
|
|
'type': dropdown_type,
|
|
'options': json.dumps(dropdown_data['options']),
|
|
'element_info': element_info,
|
|
'source': source_info,
|
|
'formatted_options': '\n'.join(formatted_options),
|
|
'message': msg,
|
|
'short_term_memory': msg,
|
|
'long_term_memory': f'Got dropdown options for ARIA combobox at index {index_for_logging}',
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
|
|
async def on_SelectDropdownOptionEvent(self, event: SelectDropdownOptionEvent) -> dict[str, str]:
|
|
"""Handle select dropdown option request with CDP."""
|
|
try:
|
|
# Use the provided node
|
|
element_node = event.node
|
|
index_for_logging = element_node.backend_node_id or 'unknown'
|
|
target_text = event.text
|
|
|
|
# Get CDP session for this node
|
|
cdp_session = await self.browser_session.cdp_client_for_node(element_node)
|
|
|
|
# Convert node to object ID for CDP operations
|
|
try:
|
|
object_result = await cdp_session.cdp_client.send.DOM.resolveNode(
|
|
params={'backendNodeId': element_node.backend_node_id}, session_id=cdp_session.session_id
|
|
)
|
|
remote_object = object_result.get('object', {})
|
|
object_id = remote_object.get('objectId')
|
|
if not object_id:
|
|
raise ValueError('Could not get object ID from resolved node')
|
|
except Exception as e:
|
|
raise ValueError(f'Failed to resolve node to object: {e}') from e
|
|
|
|
try:
|
|
# Use JavaScript to select the option
|
|
selection_script = """
|
|
function(targetText) {
|
|
const startElement = this;
|
|
|
|
// Function to attempt selection on a dropdown element
|
|
function attemptSelection(element) {
|
|
// Handle native select elements
|
|
if (element.tagName.toLowerCase() === 'select') {
|
|
const options = Array.from(element.options);
|
|
const targetTextLower = targetText.toLowerCase();
|
|
|
|
for (const option of options) {
|
|
const optionTextLower = option.text.trim().toLowerCase();
|
|
const optionValueLower = option.value.toLowerCase();
|
|
|
|
// Match against both text and value (case-insensitive)
|
|
if (optionTextLower === targetTextLower || optionValueLower === targetTextLower) {
|
|
const expectedValue = option.value;
|
|
|
|
// Focus the element FIRST (important for Svelte/Vue/React and other reactive frameworks)
|
|
// This simulates the user focusing on the dropdown before changing it
|
|
element.focus();
|
|
|
|
// Then set the value using multiple methods for maximum compatibility
|
|
element.value = expectedValue;
|
|
option.selected = true;
|
|
element.selectedIndex = option.index;
|
|
|
|
// Trigger all necessary events for reactive frameworks
|
|
// 1. input event - critical for Vue's v-model and Svelte's bind:value
|
|
const inputEvent = new Event('input', { bubbles: true, cancelable: true });
|
|
element.dispatchEvent(inputEvent);
|
|
|
|
// 2. change event - traditional form validation and framework reactivity
|
|
const changeEvent = new Event('change', { bubbles: true, cancelable: true });
|
|
element.dispatchEvent(changeEvent);
|
|
|
|
// 3. blur event - completes the interaction, triggers validation
|
|
element.blur();
|
|
|
|
// Verification: Check if the selection actually stuck (avoid intercepting and resetting the value)
|
|
if (element.value !== expectedValue) {
|
|
// Selection was reverted - need to try clicking instead
|
|
return {
|
|
success: false,
|
|
error: `Selection was set but reverted by page framework. The dropdown may require clicking.`,
|
|
selectionReverted: true,
|
|
targetOption: {
|
|
text: option.text.trim(),
|
|
value: expectedValue,
|
|
index: option.index
|
|
},
|
|
availableOptions: Array.from(element.options).map(opt => ({
|
|
text: opt.text.trim(),
|
|
value: opt.value
|
|
}))
|
|
};
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
message: `Selected option: ${option.text.trim()} (value: ${option.value})`,
|
|
value: option.value
|
|
};
|
|
}
|
|
}
|
|
|
|
// Return available options as separate field
|
|
const availableOptions = options.map(opt => ({
|
|
text: opt.text.trim(),
|
|
value: opt.value
|
|
}));
|
|
|
|
return {
|
|
success: false,
|
|
error: `Option with text or value '${targetText}' not found in select element`,
|
|
availableOptions: availableOptions
|
|
};
|
|
}
|
|
|
|
// Handle ARIA dropdowns/menus
|
|
const role = element.getAttribute('role');
|
|
if (role === 'menu' || role === 'listbox' || role === 'combobox') {
|
|
const menuItems = element.querySelectorAll('[role="menuitem"], [role="option"]');
|
|
const targetTextLower = targetText.toLowerCase();
|
|
|
|
for (const item of menuItems) {
|
|
if (item.textContent) {
|
|
const itemTextLower = item.textContent.trim().toLowerCase();
|
|
const itemValueLower = (item.getAttribute('data-value') || '').toLowerCase();
|
|
|
|
// Match against both text and data-value (case-insensitive)
|
|
if (itemTextLower === targetTextLower || itemValueLower === targetTextLower) {
|
|
// Clear previous selections
|
|
menuItems.forEach(mi => {
|
|
mi.setAttribute('aria-selected', 'false');
|
|
mi.classList.remove('selected');
|
|
});
|
|
|
|
// Select this item
|
|
item.setAttribute('aria-selected', 'true');
|
|
item.classList.add('selected');
|
|
|
|
// Trigger click and change events
|
|
item.click();
|
|
const clickEvent = new MouseEvent('click', { view: window, bubbles: true, cancelable: true });
|
|
item.dispatchEvent(clickEvent);
|
|
|
|
return {
|
|
success: true,
|
|
message: `Selected ARIA menu item: ${item.textContent.trim()}`
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
// Return available options as separate field
|
|
const availableOptions = Array.from(menuItems).map(item => ({
|
|
text: item.textContent ? item.textContent.trim() : '',
|
|
value: item.getAttribute('data-value') || ''
|
|
})).filter(opt => opt.text || opt.value);
|
|
|
|
return {
|
|
success: false,
|
|
error: `Menu item with text or value '${targetText}' not found`,
|
|
availableOptions: availableOptions
|
|
};
|
|
}
|
|
|
|
// Handle Semantic UI or custom dropdowns
|
|
if (element.classList.contains('dropdown') || element.classList.contains('ui')) {
|
|
const menuItems = element.querySelectorAll('.item, .option, [data-value]');
|
|
const targetTextLower = targetText.toLowerCase();
|
|
|
|
for (const item of menuItems) {
|
|
if (item.textContent) {
|
|
const itemTextLower = item.textContent.trim().toLowerCase();
|
|
const itemValueLower = (item.getAttribute('data-value') || '').toLowerCase();
|
|
|
|
// Match against both text and data-value (case-insensitive)
|
|
if (itemTextLower === targetTextLower || itemValueLower === targetTextLower) {
|
|
// Clear previous selections
|
|
menuItems.forEach(mi => {
|
|
mi.classList.remove('selected', 'active');
|
|
});
|
|
|
|
// Select this item
|
|
item.classList.add('selected', 'active');
|
|
|
|
// Update dropdown text if there's a text element
|
|
const textElement = element.querySelector('.text');
|
|
if (textElement) {
|
|
textElement.textContent = item.textContent.trim();
|
|
}
|
|
|
|
// Trigger click and change events
|
|
item.click();
|
|
const clickEvent = new MouseEvent('click', { view: window, bubbles: true, cancelable: true });
|
|
item.dispatchEvent(clickEvent);
|
|
|
|
// Also dispatch on the main dropdown element
|
|
const dropdownChangeEvent = new Event('change', { bubbles: true });
|
|
element.dispatchEvent(dropdownChangeEvent);
|
|
|
|
return {
|
|
success: true,
|
|
message: `Selected custom dropdown item: ${item.textContent.trim()}`
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
// Return available options as separate field
|
|
const availableOptions = Array.from(menuItems).map(item => ({
|
|
text: item.textContent ? item.textContent.trim() : '',
|
|
value: item.getAttribute('data-value') || ''
|
|
})).filter(opt => opt.text || opt.value);
|
|
|
|
return {
|
|
success: false,
|
|
error: `Custom dropdown item with text or value '${targetText}' not found`,
|
|
availableOptions: availableOptions
|
|
};
|
|
}
|
|
|
|
return null; // Not a dropdown element
|
|
}
|
|
|
|
// Function to recursively search children for dropdowns
|
|
function searchChildrenForSelection(element, maxDepth, currentDepth = 0) {
|
|
if (currentDepth >= maxDepth) return null;
|
|
|
|
// Check all direct children
|
|
for (let child of element.children) {
|
|
// Try selection on this child
|
|
const result = attemptSelection(child);
|
|
if (result && result.success) {
|
|
return result;
|
|
}
|
|
|
|
// Recursively check this child's children
|
|
const childResult = searchChildrenForSelection(child, maxDepth, currentDepth + 1);
|
|
if (childResult && childResult.success) {
|
|
return childResult;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
// First try the target element itself
|
|
let selectionResult = attemptSelection(startElement);
|
|
if (selectionResult) {
|
|
// If attemptSelection returned a result (success or failure), use it
|
|
// Don't search children if we found a dropdown element but selection failed
|
|
return selectionResult;
|
|
}
|
|
|
|
// Only search children if target element is not a dropdown element
|
|
selectionResult = searchChildrenForSelection(startElement, 4);
|
|
if (selectionResult && selectionResult.success) {
|
|
return selectionResult;
|
|
}
|
|
|
|
return {
|
|
success: false,
|
|
error: `Element and its children (depth 4) do not contain a dropdown with option '${targetText}' (tag: ${startElement.tagName}, role: ${startElement.getAttribute('role')}, classes: ${startElement.className})`
|
|
};
|
|
}
|
|
"""
|
|
|
|
result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': selection_script,
|
|
'arguments': [{'value': target_text}],
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
selection_result = result.get('result', {}).get('value', {})
|
|
|
|
# If selection failed and all options are empty, the dropdown may be lazily populated.
|
|
# Focus the element (triggers lazy loaders) and retry once after a wait.
|
|
if not selection_result.get('success'):
|
|
available_options = selection_result.get('availableOptions', [])
|
|
all_empty = available_options and all(
|
|
(not opt.get('text', '').strip() and not opt.get('value', '').strip())
|
|
if isinstance(opt, dict)
|
|
else not str(opt).strip()
|
|
for opt in available_options
|
|
)
|
|
if all_empty:
|
|
self.logger.info(
|
|
'⚠️ All dropdown options are empty — options may be lazily loaded. Focusing element and retrying...'
|
|
)
|
|
|
|
# Use element.focus() only — no synthetic mouse events that leak isTrusted=false
|
|
try:
|
|
await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': 'function() { this.focus(); }',
|
|
'objectId': object_id,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
except Exception:
|
|
pass # non-fatal, best-effort
|
|
|
|
await asyncio.sleep(1.0)
|
|
|
|
retry_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': selection_script,
|
|
'arguments': [{'value': target_text}],
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
selection_result = retry_result.get('result', {}).get('value', {})
|
|
|
|
# Check if selection was reverted by framework - try clicking as fallback
|
|
if selection_result.get('selectionReverted'):
|
|
self.logger.info('⚠️ Selection was reverted by page framework, trying click fallback...')
|
|
target_option = selection_result.get('targetOption', {})
|
|
option_index = target_option.get('index', 0)
|
|
|
|
# Try clicking on the option element directly
|
|
click_fallback_script = """
|
|
function(optionIndex) {
|
|
const select = this;
|
|
if (select.tagName.toLowerCase() !== 'select') return { success: false, error: 'Not a select element' };
|
|
|
|
const option = select.options[optionIndex];
|
|
if (!option) return { success: false, error: 'Option not found at index ' + optionIndex };
|
|
|
|
// Method 1: Try using the native selectedIndex setter with a small delay
|
|
const originalValue = select.value;
|
|
|
|
// Simulate opening the dropdown (some frameworks need this)
|
|
select.focus();
|
|
const mouseDown = new MouseEvent('mousedown', { bubbles: true, cancelable: true, view: window });
|
|
select.dispatchEvent(mouseDown);
|
|
|
|
// Set using selectedIndex (more reliable for some frameworks)
|
|
select.selectedIndex = optionIndex;
|
|
|
|
// Click the option
|
|
option.selected = true;
|
|
const optionClick = new MouseEvent('click', { bubbles: true, cancelable: true, view: window });
|
|
option.dispatchEvent(optionClick);
|
|
|
|
// Close dropdown
|
|
const mouseUp = new MouseEvent('mouseup', { bubbles: true, cancelable: true, view: window });
|
|
select.dispatchEvent(mouseUp);
|
|
|
|
// Fire change event
|
|
const changeEvent = new Event('change', { bubbles: true, cancelable: true });
|
|
select.dispatchEvent(changeEvent);
|
|
|
|
// Blur to finalize
|
|
select.blur();
|
|
|
|
// Verify
|
|
if (select.value === option.value || select.selectedIndex === optionIndex) {
|
|
return {
|
|
success: true,
|
|
message: 'Selected via click fallback: ' + option.text.trim(),
|
|
value: option.value
|
|
};
|
|
}
|
|
|
|
return {
|
|
success: false,
|
|
error: 'Click fallback also failed - framework may block all programmatic selection',
|
|
finalValue: select.value,
|
|
expectedValue: option.value
|
|
};
|
|
}
|
|
"""
|
|
|
|
fallback_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
|
|
params={
|
|
'functionDeclaration': click_fallback_script,
|
|
'arguments': [{'value': option_index}],
|
|
'objectId': object_id,
|
|
'returnByValue': True,
|
|
},
|
|
session_id=cdp_session.session_id,
|
|
)
|
|
|
|
fallback_data = fallback_result.get('result', {}).get('value', {})
|
|
if fallback_data.get('success'):
|
|
msg = fallback_data.get('message', f'Selected option via click: {target_text}')
|
|
self.logger.info(f'✅ {msg}')
|
|
return {
|
|
'success': 'true',
|
|
'message': msg,
|
|
'value': fallback_data.get('value', target_text),
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
else:
|
|
self.logger.warning(f'⚠️ Click fallback also failed: {fallback_data.get("error", "unknown")}')
|
|
# Continue to error handling below
|
|
|
|
if selection_result.get('success'):
|
|
msg = selection_result.get('message', f'Selected option: {target_text}')
|
|
self.logger.debug(f'{msg}')
|
|
|
|
# Return the result as a dict
|
|
return {
|
|
'success': 'true',
|
|
'message': msg,
|
|
'value': selection_result.get('value', target_text),
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
else:
|
|
error_msg = selection_result.get('error', f'Failed to select option: {target_text}')
|
|
available_options = selection_result.get('availableOptions', [])
|
|
self.logger.error(f'❌ {error_msg}')
|
|
self.logger.debug(f'Available options from JavaScript: {available_options}')
|
|
|
|
# If we have available options, return structured error data
|
|
if available_options:
|
|
# Format options for short_term_memory (simple bulleted list)
|
|
short_term_options = []
|
|
for opt in available_options:
|
|
if isinstance(opt, dict):
|
|
text = opt.get('text', '').strip()
|
|
value = opt.get('value', '').strip()
|
|
if text:
|
|
short_term_options.append(f'- {text}')
|
|
elif value:
|
|
short_term_options.append(f'- {value}')
|
|
elif isinstance(opt, str):
|
|
short_term_options.append(f'- {opt}')
|
|
|
|
if short_term_options:
|
|
short_term_memory = 'Available dropdown options are:\n' + '\n'.join(short_term_options)
|
|
long_term_memory = (
|
|
f"Couldn't select the dropdown option as '{target_text}' is not one of the available options."
|
|
)
|
|
|
|
# Return error result with structured memory instead of raising exception
|
|
return {
|
|
'success': 'false',
|
|
'error': error_msg,
|
|
'short_term_memory': short_term_memory,
|
|
'long_term_memory': long_term_memory,
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
|
|
# Fallback to regular error result if no available options
|
|
return {
|
|
'success': 'false',
|
|
'error': error_msg,
|
|
'backend_node_id': str(index_for_logging),
|
|
}
|
|
|
|
except Exception as e:
|
|
error_msg = f'Failed to select dropdown option: {str(e)}'
|
|
self.logger.error(error_msg)
|
|
raise ValueError(error_msg) from e
|
|
|
|
except Exception as e:
|
|
error_msg = f'Failed to select dropdown option "{target_text}" for element {index_for_logging}: {str(e)}'
|
|
self.logger.error(error_msg)
|
|
raise ValueError(error_msg) from e
|