wehub-resource-sync
91e75e620b
CI: cua-driver distro-compat matrix / debian:12 (glibc 2.36) (push) Has been cancelled
CI: SPDX Headers / Check SPDX headers (warn-only) (push) Has been cancelled
CD: Docs MCP Server / build (linux/amd64) (push) Has been cancelled
CD: Docs MCP Server / build (linux/arm64) (push) Has been cancelled
CD: Docs MCP Server / merge (push) Has been cancelled
CI: cua-driver distro-compat matrix / Resolve release version (push) Has been cancelled
CI: cua-driver distro-compat matrix / fedora:41 (glibc 2.40) (push) Has been cancelled
CI: cua-driver distro-compat matrix / rockylinux:9 (glibc 2.34) (push) Has been cancelled
CI: cua-driver distro-compat matrix / ubuntu:22.04 (glibc 2.35) (push) Has been cancelled
CI: cua-driver distro-compat matrix / ubuntu:24.04 (glibc 2.39) (push) Has been cancelled
CI: cua-driver distro-compat matrix / Distro compat summary (push) Has been cancelled
CI: Rust Linux unit / Rust Linux unit and compile (push) Has been cancelled
CI: Rust Windows unit / Rust Windows unit and compile (push) Has been cancelled
CI: Nix Linux Rust source / Nix / compositor build (push) Has been cancelled
CI: Nix Linux Rust source / Nix / driver package (push) Has been cancelled
CI: Nix Linux Rust source / Nix / Rust unit tests (push) Has been cancelled
554 lines
20 KiB
Python
554 lines
20 KiB
Python
"""
|
|
Linux implementation of automation and accessibility handlers.
|
|
|
|
This implementation uses pynput for GUI automation. For screenshots and screen size,
|
|
it uses PIL's ImageGrab (works with X11/Xvfb) and provides simulated fallbacks where needed.
|
|
To use GUI automation in a headless environment:
|
|
1. Install Xvfb: sudo apt-get install xvfb
|
|
2. Run with virtual display: xvfb-run python -m computer_server
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
from io import BytesIO
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from PIL import Image, ImageGrab
|
|
|
|
# Configure logger
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
from pynput.keyboard import Controller as KeyboardController
|
|
from pynput.keyboard import Key
|
|
from pynput.mouse import Button
|
|
from pynput.mouse import Controller as MouseController
|
|
|
|
from .base import (
|
|
BaseAccessibilityHandler,
|
|
BaseAutomationHandler,
|
|
normalize_screenshot_format,
|
|
)
|
|
|
|
|
|
class LinuxAccessibilityHandler(BaseAccessibilityHandler):
|
|
"""Linux implementation of accessibility handler."""
|
|
|
|
async def get_accessibility_tree(self) -> Dict[str, Any]:
|
|
"""Get the accessibility tree of the current window.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary containing success status and a simulated tree structure
|
|
since Linux doesn't have equivalent accessibility API like macOS.
|
|
"""
|
|
# Linux doesn't have equivalent accessibility API like macOS
|
|
# Return a minimal dummy tree
|
|
logger.info(
|
|
"Getting accessibility tree (simulated, no accessibility API available on Linux)"
|
|
)
|
|
return {
|
|
"success": True,
|
|
"tree": {
|
|
"role": "Window",
|
|
"title": "Linux Window",
|
|
"position": {"x": 0, "y": 0},
|
|
"size": {"width": 1920, "height": 1080},
|
|
"children": [],
|
|
},
|
|
}
|
|
|
|
async def find_element(
|
|
self, role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None
|
|
) -> Dict[str, Any]:
|
|
"""Find an element in the accessibility tree by criteria.
|
|
|
|
Args:
|
|
role: The role of the element to find.
|
|
title: The title of the element to find.
|
|
value: The value of the element to find.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary indicating that element search is not supported on Linux.
|
|
"""
|
|
logger.info(
|
|
f"Finding element with role={role}, title={title}, value={value} (not supported on Linux)"
|
|
)
|
|
return {"success": False, "message": "Element search not supported on Linux"}
|
|
|
|
def get_cursor_position(self) -> Tuple[int, int]:
|
|
"""Get the current cursor position.
|
|
|
|
Returns:
|
|
Tuple[int, int]: The x and y coordinates of the cursor position.
|
|
Returns (0, 0) if cursor position cannot be determined.
|
|
"""
|
|
try:
|
|
# Use pynput mouse controller
|
|
from pynput.mouse import Controller as MouseController
|
|
|
|
m = MouseController()
|
|
x, y = m.position
|
|
return int(x), int(y)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to get cursor position: {e}")
|
|
logger.info("Getting cursor position (simulated)")
|
|
return 0, 0
|
|
|
|
def get_screen_size(self) -> Tuple[int, int]:
|
|
"""Get the screen size.
|
|
|
|
Returns:
|
|
Tuple[int, int]: The width and height of the screen in pixels.
|
|
Returns (1920, 1080) if screen size cannot be determined.
|
|
"""
|
|
try:
|
|
img = ImageGrab.grab()
|
|
return img.width, img.height
|
|
except Exception as e:
|
|
logger.warning(f"Failed to get screen size via ImageGrab: {e}")
|
|
logger.info("Getting screen size (simulated)")
|
|
return 1920, 1080
|
|
|
|
|
|
class LinuxAutomationHandler(BaseAutomationHandler):
|
|
"""Linux implementation of automation handler using pynput."""
|
|
|
|
keyboard = KeyboardController()
|
|
mouse = MouseController()
|
|
|
|
# Mouse Actions
|
|
async def mouse_down(
|
|
self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
|
|
) -> Dict[str, Any]:
|
|
"""Press and hold a mouse button at the specified coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to move to before pressing. If None, uses current position.
|
|
y: The y coordinate to move to before pressing. If None, uses current position.
|
|
button: The mouse button to press ("left", "right", or "middle").
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
if x is not None and y is not None:
|
|
self.mouse.position = (x, y)
|
|
from pynput.mouse import Button
|
|
|
|
btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
|
|
self.mouse.press(btn)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def mouse_up(
|
|
self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
|
|
) -> Dict[str, Any]:
|
|
"""Release a mouse button at the specified coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to move to before releasing. If None, uses current position.
|
|
y: The y coordinate to move to before releasing. If None, uses current position.
|
|
button: The mouse button to release ("left", "right", or "middle").
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
if x is not None and y is not None:
|
|
self.mouse.position = (x, y)
|
|
from pynput.mouse import Button
|
|
|
|
btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
|
|
self.mouse.release(btn)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
|
|
"""Move the cursor to the specified coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to move to.
|
|
y: The y coordinate to move to.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
self.mouse.position = (x, y)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
|
|
"""Perform a left mouse click at the specified coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to click at. If None, clicks at current position.
|
|
y: The y coordinate to click at. If None, clicks at current position.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.mouse import Button
|
|
|
|
if x is not None and y is not None:
|
|
self.mouse.position = (x, y)
|
|
self.mouse.click(Button.left, 1)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
|
|
"""Perform a right mouse click at the specified coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to click at. If None, clicks at current position.
|
|
y: The y coordinate to click at. If None, clicks at current position.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.mouse import Button
|
|
|
|
if x is not None and y is not None:
|
|
self.mouse.position = (x, y)
|
|
self.mouse.click(Button.right, 1)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def middle_click(
|
|
self, x: Optional[int] = None, y: Optional[int] = None
|
|
) -> Dict[str, Any]:
|
|
"""Perform a middle mouse click at the specified coordinates."""
|
|
try:
|
|
from pynput.mouse import Button
|
|
|
|
if x is not None and y is not None:
|
|
self.mouse.position = (x, y)
|
|
self.mouse.click(Button.middle, 1)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def double_click(
|
|
self, x: Optional[int] = None, y: Optional[int] = None
|
|
) -> Dict[str, Any]:
|
|
"""Perform a double click at the specified coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to double click at. If None, clicks at current position.
|
|
y: The y coordinate to double click at. If None, clicks at current position.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.mouse import Button
|
|
|
|
if x is not None and y is not None:
|
|
self.mouse.position = (x, y)
|
|
self.mouse.click(Button.left, 2)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def click(
|
|
self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
|
|
) -> Dict[str, Any]:
|
|
"""Perform a mouse click with the specified button at the given coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to click at. If None, clicks at current position.
|
|
y: The y coordinate to click at. If None, clicks at current position.
|
|
button: The mouse button to click ("left", "right", or "middle").
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.mouse import Button
|
|
|
|
if x is not None and y is not None:
|
|
self.mouse.position = (x, y)
|
|
btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
|
|
self.mouse.click(btn, 1)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def drag_to(
|
|
self, x: int, y: int, button: str = "left", duration: float = 0.5
|
|
) -> Dict[str, Any]:
|
|
"""Drag from the current position to the specified coordinates.
|
|
|
|
Args:
|
|
x: The x coordinate to drag to.
|
|
y: The y coordinate to drag to.
|
|
button: The mouse button to use for dragging.
|
|
duration: The time in seconds to take for the drag operation.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.mouse import Button
|
|
|
|
btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
|
|
self.mouse.press(btn)
|
|
self.mouse.position = (x, y)
|
|
self.mouse.release(btn)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def drag(
|
|
self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5
|
|
) -> Dict[str, Any]:
|
|
"""Drag along a path defined by a list of coordinates.
|
|
|
|
Args:
|
|
path: A list of (x, y) coordinate tuples defining the drag path.
|
|
button: The mouse button to use for dragging.
|
|
duration: The time in seconds to take for each segment of the drag.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.mouse import Button
|
|
|
|
if not path:
|
|
return {"success": False, "error": "Path is empty"}
|
|
btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
|
|
self.mouse.position = path[0]
|
|
for x, y in path[1:]:
|
|
self.mouse.press(btn)
|
|
self.mouse.position = (x, y)
|
|
self.mouse.release(btn)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
# Keyboard Actions
|
|
async def key_down(self, key: str) -> Dict[str, Any]:
|
|
"""Press and hold a key.
|
|
|
|
Args:
|
|
key: The key to press down.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.keyboard import Key
|
|
|
|
k = getattr(Key, key) if hasattr(Key, key) else (key if len(key) == 1 else None)
|
|
if k is None:
|
|
return {"success": False, "error": f"Unknown key: {key}"}
|
|
self.keyboard.press(k)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def key_up(self, key: str) -> Dict[str, Any]:
|
|
"""Release a key.
|
|
|
|
Args:
|
|
key: The key to release.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.keyboard import Key
|
|
|
|
k = getattr(Key, key) if hasattr(Key, key) else (key if len(key) == 1 else None)
|
|
if k is None:
|
|
return {"success": False, "error": f"Unknown key: {key}"}
|
|
self.keyboard.release(k)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def type_text(self, text: str) -> Dict[str, Any]:
|
|
"""Type the specified text using the keyboard.
|
|
|
|
Args:
|
|
text: The text to type.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
# use pynput for Unicode support
|
|
self.keyboard.type(text)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def press_key(self, key: str) -> Dict[str, Any]:
|
|
"""Press and release a key.
|
|
|
|
Args:
|
|
key: The key to press.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.keyboard import Key
|
|
|
|
k = getattr(Key, key) if hasattr(Key, key) else (key if len(key) == 1 else None)
|
|
if k is None:
|
|
return {"success": False, "error": f"Unknown key: {key}"}
|
|
self.keyboard.press(k)
|
|
self.keyboard.release(k)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
|
|
"""Press a combination of keys simultaneously.
|
|
|
|
Args:
|
|
keys: A list of keys to press together as a hotkey combination.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.keyboard import Key
|
|
|
|
seq = []
|
|
for k in keys:
|
|
kk = getattr(Key, k) if hasattr(Key, k) else (k if len(k) == 1 else None)
|
|
if kk is None:
|
|
return {"success": False, "error": f"Unknown key in hotkey: {k}"}
|
|
seq.append(kk)
|
|
for k in seq[:-1]:
|
|
self.keyboard.press(k)
|
|
last = seq[-1]
|
|
self.keyboard.press(last)
|
|
self.keyboard.release(last)
|
|
for k in reversed(seq[:-1]):
|
|
self.keyboard.release(k)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
# Scrolling Actions
|
|
async def scroll(self, x: int, y: int) -> Dict[str, Any]:
|
|
"""Scroll the mouse wheel.
|
|
|
|
Args:
|
|
x: The horizontal scroll amount.
|
|
y: The vertical scroll amount.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
self.mouse.scroll(x, y)
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
|
|
"""Scroll down by the specified number of clicks.
|
|
|
|
Args:
|
|
clicks: The number of scroll clicks to perform downward.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
self.mouse.scroll(0, -abs(clicks))
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
|
|
"""Scroll up by the specified number of clicks.
|
|
|
|
Args:
|
|
clicks: The number of scroll clicks to perform upward.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary with success status and error message if failed.
|
|
"""
|
|
try:
|
|
self.mouse.scroll(0, abs(clicks))
|
|
return {"success": True}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
# Screen Actions
|
|
async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
|
|
"""Take a screenshot of the current screen.
|
|
|
|
Args:
|
|
format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).
|
|
quality: JPEG quality 1-95 (clamped); ignored for PNG.
|
|
"""
|
|
try:
|
|
fmt, quality = normalize_screenshot_format(format, quality)
|
|
except ValueError as e:
|
|
return {"success": False, "error": str(e)}
|
|
try:
|
|
screenshot = ImageGrab.grab()
|
|
if not isinstance(screenshot, Image.Image):
|
|
return {"success": False, "error": "Failed to capture screenshot"}
|
|
buffered = BytesIO()
|
|
if fmt == "jpeg":
|
|
screenshot.convert("RGB").save(
|
|
buffered, format="JPEG", quality=quality, optimize=True
|
|
)
|
|
else:
|
|
screenshot.save(buffered, format="PNG", optimize=True)
|
|
buffered.seek(0)
|
|
image_data = base64.b64encode(buffered.getvalue()).decode()
|
|
return {"success": True, "image_data": image_data, "format": fmt}
|
|
except Exception as e:
|
|
return {"success": False, "error": f"Screenshot error: {str(e)}"}
|
|
|
|
async def get_screen_size(self) -> Dict[str, Any]:
|
|
"""Get the size of the screen.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary containing success status and screen dimensions,
|
|
or error message if failed.
|
|
"""
|
|
try:
|
|
img = ImageGrab.grab()
|
|
return {"success": True, "size": {"width": img.width, "height": img.height}}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def get_cursor_position(self) -> Dict[str, Any]:
|
|
"""Get the current position of the cursor.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary containing success status and cursor coordinates,
|
|
or error message if failed.
|
|
"""
|
|
try:
|
|
from pynput.mouse import Controller as MouseController
|
|
|
|
m = MouseController()
|
|
x, y = m.position
|
|
return {"success": True, "position": {"x": int(x), "y": int(y)}}
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
# Clipboard and run_command inherited from BaseAutomationHandler
|