Files
vercel-labs--agent-browser/cli/src/native/interaction.rs
T
wehub-resource-sync cb15c5e0d8
Release / Check for new version (push) Has been cancelled
Release / Build macOS ARM64 (push) Has been cancelled
Release / Build macOS x64 (push) Has been cancelled
Release / Build Linux ARM64 (push) Has been cancelled
Release / Build Linux musl ARM64 (push) Has been cancelled
Release / Build Linux musl x64 (push) Has been cancelled
Release / Build Linux x64 (push) Has been cancelled
Release / Build Windows x64 (push) Has been cancelled
Release / Publish to npm (push) Has been cancelled
Release / Publish sandbox package to npm (push) Has been cancelled
Release / Create GitHub Release (push) Has been cancelled
CI / Rust (windows-latest - x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Native E2E Tests (push) Has been cancelled
CI / Windows Integration Test (push) Has been cancelled
CI / Global Install (macos-latest) (push) Has been cancelled
CI / Global Install (ubuntu-latest) (push) Has been cancelled
CI / Version Sync Check (push) Has been cancelled
CI / Rust (push) Has been cancelled
CI / Dashboard (push) Has been cancelled
CI / Sandbox Package (push) Has been cancelled
CI / Rust (macos-latest - aarch64-apple-darwin) (push) Has been cancelled
CI / Rust (macos-latest - x86_64-apple-darwin) (push) Has been cancelled
CI / Global Install (windows-latest) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:35:58 +08:00

1338 lines
41 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use std::collections::HashMap;
use serde_json::Value;
use super::cdp::client::CdpClient;
use super::cdp::types::*;
use super::element::{resolve_element_center, resolve_element_object_id, RefMap};
/// Outcome of a click. `dialog_opened` is true if a JavaScript dialog opened
/// mid-sequence (the page is then blocked until `dialog accept`/`dismiss`).
/// `pending_release` is set only when the dialog opened after mousePressed but
/// before mouseReleased: the button is logically held until the caller
/// dispatches the release (done once the dialog is resolved), otherwise the
/// next click would register as a drag or double-click.
#[derive(Default)]
pub struct ClickResult {
pub dialog_opened: bool,
pub pending_release: Option<PendingRelease>,
}
pub struct PendingRelease {
pub session_id: String,
pub x: f64,
pub y: f64,
pub button: String,
}
pub async fn click(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
button: &str,
click_count: i32,
iframe_sessions: &HashMap<String, String>,
) -> Result<ClickResult, String> {
let (x, y, effective_session_id) = resolve_element_center(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// A click-triggered dialog can fire on the frame's own session (OOPIF) or
// on the top-level page session; both count as "ours". A dialog on any
// other session belongs to a background tab and must not abort this click.
dispatch_click(
client,
&effective_session_id,
&[effective_session_id.as_str(), session_id],
x,
y,
button,
click_count,
)
.await
}
pub async fn dblclick(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<ClickResult, String> {
click(
client,
session_id,
ref_map,
selector_or_ref,
"left",
2,
iframe_sessions,
)
.await
}
pub async fn hover(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (x, y, effective_session_id) = resolve_element_center(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
"Input.dispatchMouseEvent",
&DispatchMouseEventParams {
event_type: "mouseMoved".to_string(),
x,
y,
button: None,
buttons: None,
click_count: None,
delta_x: None,
delta_y: None,
modifiers: None,
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn fill(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
value: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// Focus the element
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: "function() { this.focus(); }".to_string(),
object_id: Some(object_id.clone()),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
// Select all + delete to clear
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: r#"function() {
this.select && this.select();
this.value = '';
this.dispatchEvent(new Event('input', { bubbles: true }));
}"#
.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
// Insert text (keyboard input dispatched at page level, use parent session_id)
client
.send_command_typed::<_, Value>(
"Input.insertText",
&InsertTextParams {
text: value.to_string(),
},
Some(session_id),
)
.await?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub async fn type_text(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
text: &str,
clear: bool,
delay_ms: Option<u64>,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// Focus
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: "function() { this.focus(); }".to_string(),
object_id: Some(object_id.clone()),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
if clear {
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: r#"function() {
this.select && this.select();
this.value = '';
this.dispatchEvent(new Event('input', { bubbles: true }));
}"#
.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
}
type_text_into_active_context(client, session_id, text, delay_ms).await
}
pub async fn type_text_into_active_context(
client: &CdpClient,
session_id: &str,
text: &str,
delay_ms: Option<u64>,
) -> Result<(), String> {
let delay = delay_ms.unwrap_or(0);
for ch in text.chars() {
if matches!(ch, '\n' | '\r' | '\t') {
let (key, code, key_code) = char_to_key_info(ch);
let text_str = key_text(&key);
client
.send_command_typed::<_, Value>(
"Input.dispatchKeyEvent",
&DispatchKeyEventParams {
event_type: "keyDown".to_string(),
key: Some(key.clone()),
code: Some(code.clone()),
text: text_str.clone(),
unmodified_text: text_str,
windows_virtual_key_code: Some(key_code),
native_virtual_key_code: Some(key_code),
modifiers: None,
},
Some(session_id),
)
.await?;
client
.send_command_typed::<_, Value>(
"Input.dispatchKeyEvent",
&DispatchKeyEventParams {
event_type: "keyUp".to_string(),
key: Some(key),
code: Some(code),
text: None,
unmodified_text: None,
windows_virtual_key_code: Some(key_code),
native_virtual_key_code: Some(key_code),
modifiers: None,
},
Some(session_id),
)
.await?;
} else {
// VS Code/Electron webviews reject repeated dispatchKeyEvent calls
// carrying printable `text`. Insert printable characters directly
// and reserve key events for controls like Enter and Tab.
client
.send_command_typed::<_, Value>(
"Input.insertText",
&InsertTextParams {
text: ch.to_string(),
},
Some(session_id),
)
.await?;
}
if delay > 0 {
tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;
}
}
Ok(())
}
pub async fn press_key(client: &CdpClient, session_id: &str, key: &str) -> Result<(), String> {
press_key_with_modifiers(client, session_id, key, None).await
}
/// Dispatch a keyDown+keyUp sequence for `key` with an optional CDP modifier bitmask.
///
/// Modifier values follow the CDP `Input.dispatchKeyEvent` spec:
/// 1 = Alt, 2 = Control, 4 = Meta (Cmd), 8 = Shift.
///
/// Callers that need a platform-appropriate modifier (e.g. Cmd on macOS,
/// Ctrl elsewhere) must choose the value themselves -- see `cfg!(target_os)`.
pub async fn press_key_with_modifiers(
client: &CdpClient,
session_id: &str,
key: &str,
modifiers: Option<i32>,
) -> Result<(), String> {
let (key_name, code, key_code) = named_key_info(key);
// Suppress text insertion when Control (2) or Meta (4) modifiers are active,
// since these are command chords (e.g. Ctrl+A = select-all), not text input.
let has_command_modifier = modifiers.is_some_and(|m| m & (2 | 4) != 0);
let text = if has_command_modifier {
None
} else {
key_text(&key_name)
};
client
.send_command_typed::<_, Value>(
"Input.dispatchKeyEvent",
&DispatchKeyEventParams {
event_type: "keyDown".to_string(),
key: Some(key_name.clone()),
code: Some(code.clone()),
text: text.clone(),
unmodified_text: text.clone(),
windows_virtual_key_code: Some(key_code),
native_virtual_key_code: Some(key_code),
modifiers,
},
Some(session_id),
)
.await?;
client
.send_command_typed::<_, Value>(
"Input.dispatchKeyEvent",
&DispatchKeyEventParams {
event_type: "keyUp".to_string(),
key: Some(key_name),
code: Some(code),
text: None,
unmodified_text: None,
windows_virtual_key_code: Some(key_code),
native_virtual_key_code: Some(key_code),
modifiers,
},
Some(session_id),
)
.await?;
Ok(())
}
pub async fn scroll(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: Option<&str>,
delta_x: f64,
delta_y: f64,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
if let Some(sel) = selector_or_ref {
let (object_id, effective_session_id) =
resolve_element_object_id(client, session_id, ref_map, sel, iframe_sessions).await?;
let js = "function(dx, dy) { this.scrollBy(dx, dy); }".to_string();
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: js,
object_id: Some(object_id),
arguments: Some(vec![
CallArgument {
value: Some(serde_json::json!(delta_x)),
object_id: None,
},
CallArgument {
value: Some(serde_json::json!(delta_y)),
object_id: None,
},
]),
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
} else {
let js = format!("window.scrollBy({}, {})", delta_x, delta_y);
client
.send_command_typed::<_, Value>(
"Runtime.evaluate",
&EvaluateParams {
expression: js,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(session_id),
)
.await?;
}
Ok(())
}
pub async fn select_option(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
values: &[String],
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
// Matching nothing must be an error, not a silent success: an agent that
// selects a misspelled option otherwise sees "Done", and only discovers
// the page state is wrong after more commands. List what was available.
let js = r#"function(vals) {
const options = Array.from(this.options);
let matched = 0;
for (const opt of options) {
opt.selected = vals.includes(opt.value) || vals.includes(opt.textContent.trim());
if (opt.selected) matched += 1;
}
if (matched === 0) {
const available = options.map(o => o.value + ' ("' + o.textContent.trim() + '")').join(', ');
return { error: 'No option matched ' + JSON.stringify(vals) + '. Available options: ' + available };
}
this.dispatchEvent(new Event('change', { bubbles: true }));
return { matched };
}"#
.to_string();
let result = client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: js,
object_id: Some(object_id),
arguments: Some(vec![CallArgument {
value: Some(serde_json::json!(values)),
object_id: None,
}]),
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
if let Some(error) = result
.get("result")
.and_then(|r| r.get("value"))
.and_then(|v| v.get("error"))
.and_then(|e| e.as_str())
{
return Err(error.to_string());
}
Ok(())
}
pub async fn check(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let is_checked = super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
if !is_checked {
click(
client,
session_id,
ref_map,
selector_or_ref,
"left",
1,
iframe_sessions,
)
.await?;
// Verify the click changed the state (Playwright parity: _setChecked re-checks).
// If the coordinate-based click missed (e.g. hidden input, overlay), retry
// with a JS .click() on the element and its associated input.
if !super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?
{
js_click_checkbox(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
}
}
Ok(())
}
pub async fn uncheck(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let is_checked = super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
if is_checked {
click(
client,
session_id,
ref_map,
selector_or_ref,
"left",
1,
iframe_sessions,
)
.await?;
// Same verify-and-retry as check().
if super::element::is_element_checked(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?
{
js_click_checkbox(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
}
}
Ok(())
}
/// Fallback for when the coordinate-based CDP click did not toggle the
/// checkbox/radio state. This mirrors how Playwright dispatches clicks
/// through the DOM rather than via raw Input.dispatchMouseEvent coordinates.
///
/// Uses the same follow-label resolution as `is_element_checked`:
/// 1. If the element is a native input → `.click()` it directly.
/// 2. If the element is inside a `<label>` → `.click()` the label's `.control`.
/// 3. If the element has a nested `<input>` → `.click()` that input.
/// 4. Otherwise → `.click()` the element itself (handles ARIA role controls).
async fn js_click_checkbox(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let js = r#"function() {
var el = this;
var tag = el.tagName && el.tagName.toUpperCase();
// 1. Native input — click it directly
if (tag === 'INPUT' && (el.type === 'checkbox' || el.type === 'radio')) {
el.click();
return;
}
// 2. Follow label → control association
var label = tag === 'LABEL' ? el : (el.closest && el.closest('label'));
if (label && label.tagName && label.tagName.toUpperCase() === 'LABEL' && label.control) {
label.control.click();
return;
}
// 3. Nested native input
var input = el.querySelector && el.querySelector('input[type="checkbox"], input[type="radio"]');
if (input) {
input.click();
return;
}
// 4. ARIA role control — click the element itself
el.click();
}"#;
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: js.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn focus(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: "function() { this.focus(); }".to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn clear(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: r#"function() {
this.focus();
this.value = '';
this.dispatchEvent(new Event('input', { bubbles: true }));
this.dispatchEvent(new Event('change', { bubbles: true }));
}"#
.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn select_all(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: r#"function() {
this.focus();
if (typeof this.select === 'function') {
this.select();
} else {
const range = document.createRange();
range.selectNodeContents(this);
const sel = window.getSelection();
sel.removeAllRanges();
sel.addRange(range);
}
}"#
.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn scroll_into_view(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration:
"function() { this.scrollIntoView({ block: 'center', inline: 'center' }); }"
.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn dispatch_event(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
event_type: &str,
event_init: Option<&Value>,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
let init_json = event_init
.map(|v| serde_json::to_string(v).unwrap_or("{}".to_string()))
.unwrap_or_else(|| "{ bubbles: true }".to_string());
let js = format!(
"function() {{ this.dispatchEvent(new Event({}, {})); }}",
serde_json::to_string(event_type).unwrap_or_default(),
init_json
);
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: js,
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn highlight(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (object_id, effective_session_id) = resolve_element_object_id(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command_typed::<_, Value>(
"Runtime.callFunctionOn",
&CallFunctionOnParams {
function_declaration: r#"function() {
this.style.outline = '2px solid red';
this.style.outlineOffset = '2px';
const el = this;
setTimeout(() => {
el.style.outline = '';
el.style.outlineOffset = '';
}, 3000);
}"#
.to_string(),
object_id: Some(object_id),
arguments: None,
return_by_value: Some(true),
await_promise: Some(false),
},
Some(&effective_session_id),
)
.await?;
Ok(())
}
pub async fn tap_touch(
client: &CdpClient,
session_id: &str,
ref_map: &RefMap,
selector_or_ref: &str,
iframe_sessions: &HashMap<String, String>,
) -> Result<(), String> {
let (x, y, effective_session_id) = resolve_element_center(
client,
session_id,
ref_map,
selector_or_ref,
iframe_sessions,
)
.await?;
client
.send_command(
"Input.dispatchTouchEvent",
Some(serde_json::json!({
"type": "touchStart",
"touchPoints": [{ "x": x, "y": y }],
})),
Some(&effective_session_id),
)
.await?;
client
.send_command(
"Input.dispatchTouchEvent",
Some(serde_json::json!({
"type": "touchEnd",
"touchPoints": [],
})),
Some(&effective_session_id),
)
.await?;
Ok(())
}
/// Dispatches one mouse event and waits for the browser to ack it, but
/// returns Ok(true) if a JavaScript dialog opens first. A synchronous dialog
/// (confirm/prompt/alert in the event handler) blocks the renderer's main
/// thread, so the input ack cannot arrive until the dialog is resolved;
/// without this the command hangs until the client read timeout and the agent
/// never sees the pending-dialog warning.
async fn dispatch_mouse_or_dialog(
client: &CdpClient,
session_id: &str,
accept_sessions: &[&str],
params: &DispatchMouseEventParams,
) -> Result<bool, String> {
use tokio::sync::broadcast::error::RecvError;
// Subscribe before sending so the dialog event cannot slip past us.
let mut events = client.subscribe();
let send =
client.send_command_typed::<_, Value>("Input.dispatchMouseEvent", params, Some(session_id));
tokio::pin!(send);
loop {
tokio::select! {
res = &mut send => {
res?;
return Ok(false);
}
event = events.recv() => {
match event {
Ok(e) if e.method == "Page.javascriptDialogOpening" => {
// Only a dialog on this click's frame/page session
// aborts it; a background-tab dialog must not. A
// session-less event has no flat session and is
// treated as the top-level page (i.e. ours).
let ours = match e.session_id.as_deref() {
Some(sid) => accept_sessions.contains(&sid),
None => true,
};
if ours {
return Ok(true);
}
continue;
}
Ok(_) => continue,
Err(RecvError::Lagged(_)) => continue,
Err(RecvError::Closed) => {
(&mut send).await?;
return Ok(false);
}
}
}
}
}
}
async fn dispatch_click(
client: &CdpClient,
session_id: &str,
accept_sessions: &[&str],
x: f64,
y: f64,
button: &str,
click_count: i32,
) -> Result<ClickResult, String> {
// Move
if dispatch_mouse_or_dialog(
client,
session_id,
accept_sessions,
&DispatchMouseEventParams {
event_type: "mouseMoved".to_string(),
x,
y,
button: None,
buttons: None,
click_count: None,
delta_x: None,
delta_y: None,
modifiers: None,
},
)
.await?
{
// No button was pressed yet, nothing to release.
return Ok(ClickResult {
dialog_opened: true,
pending_release: None,
});
}
let button_value = match button {
"right" => 2,
"middle" => 4,
_ => 1,
};
// Press
if dispatch_mouse_or_dialog(
client,
session_id,
accept_sessions,
&DispatchMouseEventParams {
event_type: "mousePressed".to_string(),
x,
y,
button: Some(button.to_string()),
buttons: Some(button_value),
click_count: Some(click_count),
delta_x: None,
delta_y: None,
modifiers: None,
},
)
.await?
{
// Dialog opened from the mousedown handler: the button is held and the
// release will never arrive on its own. Hand the caller what it needs
// to release once the dialog is resolved.
return Ok(ClickResult {
dialog_opened: true,
pending_release: Some(PendingRelease {
session_id: session_id.to_string(),
x,
y,
button: button.to_string(),
}),
});
}
// Release. A dialog here fired from the click/mouseup handler, which runs
// after the button is already up, so there is nothing left to release.
let dialog_opened = dispatch_mouse_or_dialog(
client,
session_id,
accept_sessions,
&DispatchMouseEventParams {
event_type: "mouseReleased".to_string(),
x,
y,
button: Some(button.to_string()),
buttons: Some(0),
click_count: Some(click_count),
delta_x: None,
delta_y: None,
modifiers: None,
},
)
.await?;
Ok(ClickResult {
dialog_opened,
pending_release: None,
})
}
/// Best-effort mouseReleased to clear a button left logically down when a
/// dialog opened mid-click. Called after the dialog is resolved.
pub async fn dispatch_pending_release(
client: &CdpClient,
release: &PendingRelease,
) -> Result<(), String> {
client
.send_command_typed::<_, Value>(
"Input.dispatchMouseEvent",
&DispatchMouseEventParams {
event_type: "mouseReleased".to_string(),
x: release.x,
y: release.y,
button: Some(release.button.clone()),
buttons: Some(0),
click_count: Some(1),
delta_x: None,
delta_y: None,
modifiers: None,
},
Some(&release.session_id),
)
.await?;
Ok(())
}
fn char_to_key_info(ch: char) -> (String, String, i32) {
match ch {
'\n' | '\r' => ("Enter".to_string(), "Enter".to_string(), 13),
'\t' => ("Tab".to_string(), "Tab".to_string(), 9),
' ' => (" ".to_string(), "Space".to_string(), 32),
_ => {
let key = ch.to_string();
if ch.is_ascii_alphabetic() {
// For letters the Windows VK code equals the uppercase ASCII value.
let upper = ch.to_ascii_uppercase();
let code = format!("Key{}", upper);
let key_code = upper as i32;
(key, code, key_code)
} else if ch.is_ascii_digit() {
let code = format!("Digit{}", ch);
let key_code = ch as i32;
(key, code, key_code)
} else {
let (code, key_code) = punctuation_key_info(ch);
(key, code.to_string(), key_code)
}
}
}
}
/// Return the DOM `KeyboardEvent.code` value and Windows virtual-key code for
/// a punctuation / symbol character assuming a US keyboard layout.
///
/// The Windows virtual-key codes (VK_OEM_*) differ from ASCII values for
/// punctuation. Using the raw ASCII code would misidentify characters e.g.
/// '.' (ASCII 46) collides with VK_DELETE (0x2E = 46), causing the period to
/// be swallowed.
fn punctuation_key_info(ch: char) -> (&'static str, i32) {
match ch {
// VK_OEM_1 (0xBA = 186) — ";:" key on US layout
';' | ':' => ("Semicolon", 186),
// VK_OEM_PLUS (0xBB = 187) — "=+" key
'=' | '+' => ("Equal", 187),
// VK_OEM_COMMA (0xBC = 188) — ",<" key
',' | '<' => ("Comma", 188),
// VK_OEM_MINUS (0xBD = 189) — "-_" key
'-' | '_' => ("Minus", 189),
// VK_OEM_PERIOD (0xBE = 190) — ".>" key
'.' | '>' => ("Period", 190),
// VK_OEM_2 (0xBF = 191) — "/?" key
'/' | '?' => ("Slash", 191),
// VK_OEM_3 (0xC0 = 192) — "`~" key
'`' | '~' => ("Backquote", 192),
// VK_OEM_4 (0xDB = 219) — "[{" key
'[' | '{' => ("BracketLeft", 219),
// VK_OEM_5 (0xDC = 220) — "\\|" key
'\\' | '|' => ("Backslash", 220),
// VK_OEM_6 (0xDD = 221) — "]}" key
']' | '}' => ("BracketRight", 221),
// VK_OEM_7 (0xDE = 222) — "'\""" key
'\'' | '"' => ("Quote", 222),
_ => ("", 0),
}
}
/// Return the `text` value that CDP `Input.dispatchKeyEvent` needs on the
/// `keyDown` event so that Chrome performs the default action for the key.
/// For example Enter needs `"\r"` to actually submit a form, and Tab needs
/// `"\t"` to move focus. Non-printable / navigation keys return `None`.
fn key_text(key_name: &str) -> Option<String> {
match key_name {
"Enter" => Some("\r".to_string()),
"Tab" => Some("\t".to_string()),
" " => Some(" ".to_string()),
_ => {
// Single printable characters carry themselves as text.
if key_name.len() == 1 {
Some(key_name.to_string())
} else {
None
}
}
}
}
fn named_key_info(key: &str) -> (String, String, i32) {
match key.to_lowercase().as_str() {
"enter" | "return" => ("Enter".to_string(), "Enter".to_string(), 13),
"tab" => ("Tab".to_string(), "Tab".to_string(), 9),
"escape" | "esc" => ("Escape".to_string(), "Escape".to_string(), 27),
"backspace" => ("Backspace".to_string(), "Backspace".to_string(), 8),
"delete" => ("Delete".to_string(), "Delete".to_string(), 46),
"arrowup" | "up" => ("ArrowUp".to_string(), "ArrowUp".to_string(), 38),
"arrowdown" | "down" => ("ArrowDown".to_string(), "ArrowDown".to_string(), 40),
"arrowleft" | "left" => ("ArrowLeft".to_string(), "ArrowLeft".to_string(), 37),
"arrowright" | "right" => ("ArrowRight".to_string(), "ArrowRight".to_string(), 39),
"home" => ("Home".to_string(), "Home".to_string(), 36),
"end" => ("End".to_string(), "End".to_string(), 35),
"pageup" => ("PageUp".to_string(), "PageUp".to_string(), 33),
"pagedown" => ("PageDown".to_string(), "PageDown".to_string(), 34),
"space" | " " => (" ".to_string(), "Space".to_string(), 32),
_ => {
if key.len() == 1 {
let ch = key.chars().next().unwrap();
char_to_key_info(ch)
} else {
(key.to_string(), key.to_string(), 0)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Verify that `char_to_key_info` returns the correct (key, code,
/// windowsVirtualKeyCode) triple for every character in Playwright's
/// USKeyboardLayout. The expected values below are taken verbatim from
/// playwright-core/lib/server/usKeyboardLayout.js so that any drift from
/// Playwright's behaviour is caught immediately.
#[test]
fn test_char_to_key_info_matches_playwright_layout() {
// (character, expected_code, expected_vk_code)
let cases: &[(char, &str, i32)] = &[
// Letters VK code must equal the uppercase ASCII value.
('a', "KeyA", 65),
('z', "KeyZ", 90),
('A', "KeyA", 65),
// Digits
('0', "Digit0", 48),
('9', "Digit9", 57),
// Punctuation these are the values from Playwright's layout.
// The bug that prompted this test sent '.' as VK 46 (= VK_DELETE).
('.', "Period", 190),
(',', "Comma", 188),
('/', "Slash", 191),
(';', "Semicolon", 186),
('\'', "Quote", 222),
('[', "BracketLeft", 219),
(']', "BracketRight", 221),
('\\', "Backslash", 220),
('`', "Backquote", 192),
('-', "Minus", 189),
('=', "Equal", 187),
// Shifted variants produced by the same physical keys.
('>', "Period", 190),
('<', "Comma", 188),
('?', "Slash", 191),
(':', "Semicolon", 186),
('"', "Quote", 222),
('{', "BracketLeft", 219),
('}', "BracketRight", 221),
('|', "Backslash", 220),
('~', "Backquote", 192),
('_', "Minus", 189),
('+', "Equal", 187),
// Whitespace / control
(' ', "Space", 32),
('\n', "Enter", 13),
('\t', "Tab", 9),
];
for &(ch, expected_code, expected_vk) in cases {
let (key, code, vk) = char_to_key_info(ch);
assert_eq!(
code, expected_code,
"char {:?}: expected code {:?}, got {:?}",
ch, expected_code, code
);
assert_eq!(
vk, expected_vk,
"char {:?}: expected VK {}, got {} (ASCII would be {})",
ch, expected_vk, vk, ch as i32
);
// key should be the character itself (except control chars).
if !ch.is_control() {
assert_eq!(key, ch.to_string(), "char {:?}: key mismatch", ch);
}
}
}
/// Regression test: period must NEVER map to VK 46 (VK_DELETE).
#[test]
fn test_period_is_not_vk_delete() {
let (_, _, vk) = char_to_key_info('.');
assert_ne!(
vk, 46,
"Period must not use VK code 46 (VK_DELETE); expected 190 (VK_OEM_PERIOD)"
);
assert_eq!(vk, 190);
}
/// Characters outside the US keyboard layout should return (key, "", 0)
/// so that `type_text` falls back to `Input.insertText`.
#[test]
fn test_unmapped_chars_return_zero_keycode() {
for ch in ['@', '#', '$', '%', '^', '&', '*', '(', ')', '€', '£', '你'] {
let (key, code, vk) = char_to_key_info(ch);
assert_eq!(
code, "",
"char {:?}: unmapped char should have empty code, got {:?}",
ch, code
);
assert_eq!(
vk, 0,
"char {:?}: unmapped char should have VK 0, got {}",
ch, vk
);
assert_eq!(key, ch.to_string());
}
}
#[test]
fn test_key_text_returns_correct_text_for_special_keys() {
assert_eq!(key_text("Enter"), Some("\r".to_string()));
assert_eq!(key_text("Tab"), Some("\t".to_string()));
assert_eq!(key_text(" "), Some(" ".to_string()));
// Single printable characters carry themselves.
assert_eq!(key_text("a"), Some("a".to_string()));
assert_eq!(key_text("Z"), Some("Z".to_string()));
// Non-printable named keys return None.
assert_eq!(key_text("Escape"), None);
assert_eq!(key_text("ArrowUp"), None);
assert_eq!(key_text("Backspace"), None);
assert_eq!(key_text("Delete"), None);
}
}