chore: import upstream snapshot with attribution

libs/python/computer-server/computer_server/__init__.py

@@ -0,0 +1,20 @@
+"""
+Computer API package.
+Provides a server interface for the Computer API.
+"""
+
+from __future__ import annotations
+
+__version__: str = "0.1.0"
+
+# Explicitly export Server for static type checkers
+from .server import Server as Server  # noqa: F401
+
+__all__ = ["Server", "run_cli"]
+
+
+def run_cli() -> None:
+    """Entry point for CLI"""
+    from .cli import main
+
+    main()

libs/python/computer-server/computer_server/__main__.py

@@ -0,0 +1,11 @@
+"""
+Main entry point for running the Computer Server as a module.
+This allows the server to be started with `python -m computer_server`.
+"""
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

libs/python/computer-server/computer_server/browser.py

@@ -0,0 +1,374 @@
+"""
+Browser manager using Playwright for programmatic browser control.
+This allows agents to control a browser that runs visibly on the XFCE desktop.
+"""
+
+import asyncio
+import logging
+import os
+from typing import Any, Dict, Optional
+
+try:
+    from playwright.async_api import Browser, BrowserContext, Page, async_playwright
+except ImportError:
+    async_playwright = None
+    Browser = None
+    BrowserContext = None
+    Page = None
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserManager:
+    """
+    Manages a Playwright browser instance that runs visibly on the XFCE desktop.
+    Uses persistent context to maintain cookies and sessions.
+    """
+
+    def __init__(self):
+        """Initialize the BrowserManager."""
+        self.playwright = None
+        self.browser: Optional[Browser] = None
+        self.context: Optional[BrowserContext] = None
+        self.page: Optional[Page] = None
+        self._initialized = False
+        self._initialization_error: Optional[str] = None
+        self._lock = asyncio.Lock()
+
+    async def _ensure_initialized(self):
+        """Ensure the browser is initialized."""
+        # Check if browser was closed and needs reinitialization
+        if self._initialized:
+            try:
+                # Check if context is still valid by trying to access it
+                if self.context:
+                    # Try to get pages - this will raise if context is closed
+                    _ = self.context.pages
+                    # If we get here, context is still alive
+                    return
+                else:
+                    # Context was closed, need to reinitialize
+                    self._initialized = False
+                    logger.warning("Browser context was closed, will reinitialize...")
+            except Exception as e:
+                # Context is dead, need to reinitialize
+                logger.warning(f"Browser context is dead ({e}), will reinitialize...")
+                self._initialized = False
+                self.context = None
+                self.page = None
+                # Clean up playwright if it exists
+                if self.playwright:
+                    try:
+                        await self.playwright.stop()
+                    except Exception:
+                        pass
+                    self.playwright = None
+
+        async with self._lock:
+            # Double-check after acquiring lock (another thread might have initialized it)
+            if self._initialized:
+                try:
+                    if self.context:
+                        _ = self.context.pages
+                        return
+                except Exception:
+                    self._initialized = False
+                    self.context = None
+                    self.page = None
+                    if self.playwright:
+                        try:
+                            await self.playwright.stop()
+                        except Exception:
+                            pass
+                        self.playwright = None
+
+            if async_playwright is None:
+                raise RuntimeError(
+                    "playwright is not installed. Please install it with: pip install playwright && playwright install --with-deps firefox"
+                )
+
+            try:
+                # Get display from environment or default to :1
+                display = os.environ.get("DISPLAY", ":1")
+                logger.info(f"Initializing browser with DISPLAY={display}")
+
+                # Start playwright
+                self.playwright = await async_playwright().start()
+
+                # Launch Firefox with persistent context (keeps cookies/sessions)
+                # headless=False is CRITICAL so the visual agent can see it
+                user_data_dir = os.path.join(os.path.expanduser("~"), ".playwright-firefox")
+                os.makedirs(user_data_dir, exist_ok=True)
+
+                # launch_persistent_context returns a BrowserContext, not a Browser
+                # Note: Removed --kiosk mode so the desktop remains visible
+                self.context = await self.playwright.firefox.launch_persistent_context(
+                    user_data_dir=user_data_dir,
+                    headless=False,  # CRITICAL: visible for visual agent
+                    viewport={"width": 1024, "height": 768},
+                    # Removed --kiosk to allow desktop visibility
+                )
+
+                # Add init script to make the browser less detectable
+                await self.context.add_init_script(
+                    """const defaultGetter = Object.getOwnPropertyDescriptor(
+      Navigator.prototype,
+      "webdriver"
+    ).get;
+    defaultGetter.apply(navigator);
+    defaultGetter.toString();
+    Object.defineProperty(Navigator.prototype, "webdriver", {
+      set: undefined,
+      enumerable: true,
+      configurable: true,
+      get: new Proxy(defaultGetter, {
+        apply: (target, thisArg, args) => {
+          Reflect.apply(target, thisArg, args);
+          return false;
+        },
+      }),
+    });
+    const patchedGetter = Object.getOwnPropertyDescriptor(
+      Navigator.prototype,
+      "webdriver"
+    ).get;
+    patchedGetter.apply(navigator);
+    patchedGetter.toString();"""
+                )
+
+                # Get the first page or create one
+                pages = self.context.pages
+                if pages:
+                    self.page = pages[0]
+                else:
+                    self.page = await self.context.new_page()
+
+                self._initialized = True
+                logger.info("Browser initialized successfully")
+
+            except Exception as e:
+                logger.error(f"Failed to initialize browser: {e}")
+                import traceback
+
+                logger.error(traceback.format_exc())
+                # Don't raise - return error in execute_command instead
+                self._initialization_error = str(e)
+                raise
+
+    async def _execute_command_impl(self, cmd: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        """Internal implementation of command execution."""
+        if cmd == "visit_url":
+            url = params.get("url")
+            if not url:
+                return {"success": False, "error": "url parameter is required"}
+            await self.page.goto(url, wait_until="domcontentloaded", timeout=30000)
+            return {"success": True, "url": self.page.url}
+
+        elif cmd == "click":
+            x = params.get("x")
+            y = params.get("y")
+            if x is None or y is None:
+                return {"success": False, "error": "x and y parameters are required"}
+            await self.page.mouse.click(x, y)
+            return {"success": True}
+
+        elif cmd == "type":
+            text = params.get("text")
+            if text is None:
+                return {"success": False, "error": "text parameter is required"}
+            await self.page.keyboard.type(text)
+            return {"success": True}
+
+        elif cmd == "scroll":
+            delta_x = params.get("delta_x", 0)
+            delta_y = params.get("delta_y", 0)
+            await self.page.mouse.wheel(delta_x, delta_y)
+            return {"success": True}
+
+        elif cmd == "web_search":
+            query = params.get("query")
+            if not query:
+                return {"success": False, "error": "query parameter is required"}
+            # Navigate to Google search
+            search_url = f"https://www.google.com/search?q={query}"
+            await self.page.goto(search_url, wait_until="domcontentloaded", timeout=30000)
+            return {"success": True, "url": self.page.url}
+
+        elif cmd == "screenshot":
+            # Take a screenshot and return as base64
+            import base64
+
+            screenshot_bytes = await self.page.screenshot(type="png")
+            screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
+            return {"success": True, "screenshot": screenshot_b64}
+
+        elif cmd == "get_current_url":
+            # Get the current URL
+            current_url = self.page.url
+            return {"success": True, "url": current_url}
+
+        elif cmd == "go_back":
+            await self.page.go_back()
+            return {"success": True, "url": self.page.url}
+
+        elif cmd == "go_forward":
+            await self.page.go_forward()
+            return {"success": True, "url": self.page.url}
+
+        else:
+            return {"success": False, "error": f"Unknown command: {cmd}"}
+
+    async def execute_command(self, cmd: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Execute a browser command with automatic recovery.
+
+        Args:
+            cmd: Command name (visit_url, click, type, scroll, web_search)
+            params: Command parameters
+
+        Returns:
+            Result dictionary with success status and any data
+        """
+        max_retries = 2
+        for attempt in range(max_retries):
+            try:
+                await self._ensure_initialized()
+            except Exception as e:
+                error_msg = getattr(self, "_initialization_error", None) or str(e)
+                logger.error(f"Browser initialization failed: {error_msg}")
+                return {
+                    "success": False,
+                    "error": f"Browser initialization failed: {error_msg}. "
+                    f"Make sure Playwright and Firefox are installed, and DISPLAY is set correctly.",
+                }
+
+            # Check if page is still valid and get a new one if needed
+            page_valid = False
+            try:
+                if self.page is not None and not self.page.is_closed():
+                    # Try to access page.url to check if it's still valid
+                    _ = self.page.url
+                    page_valid = True
+            except Exception as e:
+                logger.warning(f"Page is invalid: {e}, will get a new page...")
+                self.page = None
+
+            # Get a valid page if we don't have one
+            if not page_valid or self.page is None:
+                try:
+                    if self.context:
+                        pages = self.context.pages
+                        if pages:
+                            # Find first non-closed page
+                            for p in pages:
+                                try:
+                                    if not p.is_closed():
+                                        self.page = p
+                                        logger.info("Reusing existing open page")
+                                        page_valid = True
+                                        break
+                                except Exception:
+                                    continue
+
+                        # If no valid page found, create a new one
+                        if not page_valid:
+                            self.page = await self.context.new_page()
+                            logger.info("Created new page")
+                except Exception as e:
+                    logger.error(f"Failed to get new page: {e}, browser may be closed")
+                    # Browser was closed - force reinitialization
+                    self._initialized = False
+                    self.context = None
+                    self.page = None
+                    if self.playwright:
+                        try:
+                            await self.playwright.stop()
+                        except Exception:
+                            pass
+                        self.playwright = None
+
+                    # If this isn't the last attempt, continue to retry
+                    if attempt < max_retries - 1:
+                        logger.info("Browser was closed, retrying with fresh initialization...")
+                        continue
+                    else:
+                        return {
+                            "success": False,
+                            "error": f"Browser was closed and cannot be recovered: {e}",
+                        }
+
+            # Try to execute the command
+            try:
+                return await self._execute_command_impl(cmd, params)
+            except Exception as e:
+                error_str = str(e)
+                logger.error(f"Error executing command {cmd}: {e}")
+
+                # Check if this is a "browser/page/context closed" error
+                if any(keyword in error_str.lower() for keyword in ["closed", "target", "context"]):
+                    logger.warning(
+                        f"Browser/page was closed during command execution (attempt {attempt + 1}/{max_retries})"
+                    )
+
+                    # Force reinitialization
+                    self._initialized = False
+                    self.context = None
+                    self.page = None
+                    if self.playwright:
+                        try:
+                            await self.playwright.stop()
+                        except Exception:
+                            pass
+                        self.playwright = None
+
+                    # If this isn't the last attempt, retry
+                    if attempt < max_retries - 1:
+                        logger.info("Retrying command after browser reinitialization...")
+                        continue
+                    else:
+                        return {
+                            "success": False,
+                            "error": f"Command failed after {max_retries} attempts: {error_str}",
+                        }
+                else:
+                    # Not a browser closed error, return immediately
+                    import traceback
+
+                    logger.error(traceback.format_exc())
+                    return {"success": False, "error": error_str}
+
+        # Should never reach here, but just in case
+        return {"success": False, "error": "Command failed after all retries"}
+
+    async def close(self):
+        """Close the browser and cleanup resources."""
+        async with self._lock:
+            try:
+                if self.context:
+                    await self.context.close()
+                    self.context = None
+                if self.browser:
+                    await self.browser.close()
+                    self.browser = None
+
+                if self.playwright:
+                    await self.playwright.stop()
+                    self.playwright = None
+
+                self.page = None
+                self._initialized = False
+                logger.info("Browser closed successfully")
+            except Exception as e:
+                logger.error(f"Error closing browser: {e}")
+
+
+# Global instance
+_browser_manager: Optional[BrowserManager] = None
+
+
+def get_browser_manager() -> BrowserManager:
+    """Get or create the global BrowserManager instance."""
+    global _browser_manager
+    if _browser_manager is None:
+        _browser_manager = BrowserManager()
+    return _browser_manager

libs/python/computer-server/computer_server/cli.py

@@ -0,0 +1,151 @@
+"""
+Command-line interface for the Computer API server.
+"""
+
+import argparse
+import logging
+import os
+import sys
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(description="Start the Computer API server")
+    parser.add_argument(
+        "--width",
+        type=int,
+        help="Target width for screenshots (coordinates will be scaled accordingly)",
+    )
+    parser.add_argument(
+        "--height",
+        type=int,
+        help="Target height for screenshots (coordinates will be scaled accordingly)",
+    )
+    parser.add_argument(
+        "--detect-resolution",
+        action="store_true",
+        help="Auto-detect and log the actual screen resolution at startup",
+    )
+    parser.add_argument(
+        "--host",
+        default="127.0.0.1",
+        help="Host to bind the server to (default: 127.0.0.1; use 0.0.0.0 for external access)",
+    )
+    parser.add_argument(
+        "--port", type=int, default=8000, help="Port to bind the server to (default: 8000)"
+    )
+    parser.add_argument(
+        "--log-level",
+        choices=["debug", "info", "warning", "error", "critical"],
+        default="info",
+        help="Logging level (default: info)",
+    )
+    parser.add_argument(
+        "--ssl-keyfile",
+        type=str,
+        help="Path to SSL private key file (enables HTTPS)",
+    )
+    parser.add_argument(
+        "--ssl-certfile",
+        type=str,
+        help="Path to SSL certificate file (enables HTTPS)",
+    )
+    # Backend selection
+    parser.add_argument(
+        "--backend",
+        choices=["native", "vnc"],
+        default="native",
+        help="Handler backend: 'native' uses OS-specific handlers, 'vnc' uses VNC (default: native)",
+    )
+
+    # VNC backend options
+    parser.add_argument(
+        "--vnc-host",
+        type=str,
+        help="VNC server host (required when --backend=vnc)",
+    )
+    parser.add_argument(
+        "--vnc-port",
+        type=int,
+        default=5900,
+        help="VNC server port (default: 5900)",
+    )
+    parser.add_argument(
+        "--vnc-password",
+        type=str,
+        default="",
+        help="VNC server password",
+    )
+
+    return parser.parse_args(args)
+
+
+def main() -> None:
+    """Main entry point for the CLI."""
+    args = parse_args()
+
+    # Configure logging
+    logging.basicConfig(
+        level=getattr(logging, args.log_level.upper()),
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        stream=sys.stderr,  # Use stderr for MCP compatibility
+    )
+
+    # Set backend env vars from CLI args before Server import triggers handler creation
+    if args.backend == "vnc" or args.vnc_host:
+        if not args.vnc_host and not os.environ.get("CUA_VNC_HOST"):
+            parser_err = "--vnc-host is required when using --backend=vnc"
+            logger.error(parser_err)
+            sys.exit(1)
+        os.environ["CUA_BACKEND"] = "vnc"
+        if args.vnc_host:
+            os.environ["CUA_VNC_HOST"] = args.vnc_host
+        # Only override env vars from CLI args when explicitly provided
+        if args.vnc_port != 5900 or "CUA_VNC_PORT" not in os.environ:
+            os.environ["CUA_VNC_PORT"] = str(args.vnc_port)
+        if args.vnc_password:
+            os.environ["CUA_VNC_PASSWORD"] = args.vnc_password
+        vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")
+        logger.info(f"VNC backend enabled → {vnc_host}:{args.vnc_port}")
+
+    # Create and start the server
+    logger.info(f"Starting Cua Computer API server on {args.host}:{args.port}...")
+    logger.info("HTTP API available at /ws, /cmd, /status endpoints")
+    logger.info("MCP server available at /mcp endpoint (if fastmcp installed)")
+
+    # Handle SSL configuration
+    ssl_args = {}
+    if args.ssl_keyfile and args.ssl_certfile:
+        ssl_args = {
+            "ssl_keyfile": args.ssl_keyfile,
+            "ssl_certfile": args.ssl_certfile,
+        }
+        logger.info("HTTPS mode enabled with SSL certificates")
+    elif args.ssl_keyfile or args.ssl_certfile:
+        logger.warning(
+            "Both --ssl-keyfile and --ssl-certfile are required for HTTPS. Running in HTTP mode."
+        )
+    else:
+        logger.info("HTTP mode (no SSL certificates provided)")
+
+    # Import Server lazily so env vars (e.g. CUA_VNC_HOST) are set before
+    # the module-level handler factory runs in main.py.
+    from .server import Server
+
+    server = Server(host=args.host, port=args.port, log_level=args.log_level, **ssl_args)
+
+    try:
+        server.start()
+    except KeyboardInterrupt:
+        logger.info("Server stopped by user")
+        sys.exit(0)
+    except Exception as e:
+        logger.error(f"Error starting server: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

libs/python/computer-server/computer_server/diorama/base.py

@@ -0,0 +1,5 @@
+class BaseDioramaHandler:
+    """Base Diorama handler for unsupported OSes."""
+
+    async def diorama_cmd(self, action: str, arguments: dict = None) -> dict:
+        return {"success": False, "error": "Diorama is not supported on this OS yet."}

libs/python/computer-server/computer_server/diorama/diorama.py

@@ -0,0 +1,570 @@
+#!/usr/bin/env python3
+"""Diorama: A virtual desktop manager for macOS"""
+
+import asyncio
+import io
+import logging
+import os
+import sys
+from typing import Union
+
+from computer_server.diorama.diorama_computer import DioramaComputer
+from computer_server.diorama.draw import (
+    AppActivationContext,
+    capture_all_apps,
+    get_all_windows,
+    get_frontmost_and_active_app,
+    get_running_apps,
+)
+from computer_server.handlers.macos import *
+from PIL import Image, ImageDraw
+
+# simple, nicely formatted logging
+logger = logging.getLogger(__name__)
+
+automation_handler = MacOSAutomationHandler()
+
+
+class Diorama:
+    """Virtual desktop manager that provides automation capabilities for macOS applications.
+
+    Manages application windows and provides an interface for taking screenshots,
+    mouse interactions, keyboard input, and coordinate transformations between
+    screenshot space and screen space.
+    """
+
+    _scheduler_queue = None
+    _scheduler_task = None
+    _loop = None
+    _scheduler_started = False
+
+    @classmethod
+    def create_from_apps(cls, *args) -> DioramaComputer:
+        """Create a DioramaComputer instance from a list of application names.
+
+        Args:
+            *args: Variable number of application names to include in the desktop
+
+        Returns:
+            DioramaComputer: A computer interface for the specified applications
+        """
+        cls._ensure_scheduler()
+        return cls(args).computer
+
+    # Dictionary to store cursor positions for each unique app_list hash
+    _cursor_positions = {}
+
+    def __init__(self, app_list):
+        """Initialize a Diorama instance for the specified applications.
+
+        Args:
+            app_list: List of application names to manage
+        """
+        self.app_list = app_list
+        self.interface = self.Interface(self)
+        self.computer = DioramaComputer(self)
+        self.focus_context = None
+
+        # Create a hash for this app_list to use as a key
+        self.app_list_hash = hash(tuple(sorted(app_list)))
+
+        # Initialize cursor position for this app_list if it doesn't exist
+        if self.app_list_hash not in Diorama._cursor_positions:
+            Diorama._cursor_positions[self.app_list_hash] = (0, 0)
+
+    @classmethod
+    def _ensure_scheduler(cls):
+        """Ensure the async scheduler loop is running.
+
+        Creates and starts the scheduler task if it hasn't been started yet.
+        """
+        if not cls._scheduler_started:
+            logger.info("Starting Diorama scheduler loop…")
+            cls._scheduler_queue = asyncio.Queue()
+            cls._loop = asyncio.get_event_loop()
+            cls._scheduler_task = cls._loop.create_task(cls._scheduler_loop())
+            cls._scheduler_started = True
+
+    @classmethod
+    async def _scheduler_loop(cls):
+        """Main scheduler loop that processes automation commands.
+
+        Continuously processes commands from the scheduler queue, handling
+        screenshots, mouse actions, keyboard input, and scrolling operations.
+        """
+        while True:
+            cmd = await cls._scheduler_queue.get()
+            action = cmd.get("action")
+            args = cmd.get("arguments", {})
+            future = cmd.get("future")
+            logger.info(f"Processing command: {action} | args={args}")
+
+            app_whitelist = args.get("app_list", [])
+
+            all_windows = get_all_windows()
+            running_apps = get_running_apps()
+            frontmost_app, active_app_to_use, active_app_pid = get_frontmost_and_active_app(
+                all_windows, running_apps, app_whitelist
+            )
+            focus_context = AppActivationContext(active_app_pid, active_app_to_use, logger)
+
+            with focus_context:
+                try:
+                    if action == "screenshot":
+                        logger.info(f"Taking screenshot for apps: {app_whitelist}")
+                        result, img = capture_all_apps(
+                            app_whitelist=app_whitelist, save_to_disk=False, take_focus=False
+                        )
+                        logger.info("Screenshot complete.")
+                        if future:
+                            future.set_result((result, img))
+                    # Mouse actions
+                    elif action in [
+                        "left_click",
+                        "right_click",
+                        "double_click",
+                        "move_cursor",
+                        "drag_to",
+                    ]:
+                        x = args.get("x")
+                        y = args.get("y")
+
+                        duration = args.get("duration", 0.5)
+                        if action == "left_click":
+                            await automation_handler.left_click(x, y)
+                        elif action == "right_click":
+                            await automation_handler.right_click(x, y)
+                        elif action == "double_click":
+                            await automation_handler.double_click(x, y)
+                        elif action == "move_cursor":
+                            await automation_handler.move_cursor(x, y)
+                        elif action == "drag_to":
+                            await automation_handler.drag_to(x, y, duration=duration)
+                        if future:
+                            future.set_result(None)
+                    elif action in ["scroll_up", "scroll_down"]:
+                        x = args.get("x")
+                        y = args.get("y")
+                        if x is not None and y is not None:
+                            await automation_handler.move_cursor(x, y)
+
+                        clicks = args.get("clicks", 1)
+                        if action == "scroll_up":
+                            await automation_handler.scroll_up(clicks)
+                        else:
+                            await automation_handler.scroll_down(clicks)
+                        if future:
+                            future.set_result(None)
+                    # Keyboard actions
+                    elif action == "type_text":
+                        text = args.get("text")
+                        await automation_handler.type_text(text)
+                        if future:
+                            future.set_result(None)
+                    elif action == "press_key":
+                        key = args.get("key")
+                        await automation_handler.press_key(key)
+                        if future:
+                            future.set_result(None)
+                    elif action == "hotkey":
+                        keys = args.get("keys", [])
+                        await automation_handler.hotkey(keys)
+                        if future:
+                            future.set_result(None)
+                    elif action == "get_cursor_position":
+                        pos = await automation_handler.get_cursor_position()
+                        if future:
+                            future.set_result(pos)
+                    else:
+                        logger.warning(f"Unknown action: {action}")
+                        if future:
+                            future.set_exception(ValueError(f"Unknown action: {action}"))
+                except Exception as e:
+                    logger.error(f"Exception during {action}: {e}", exc_info=True)
+                    if future:
+                        future.set_exception(e)
+
+    class Interface:
+        """Interface for interacting with the virtual desktop.
+
+        Provides methods for taking screenshots, mouse interactions, keyboard input,
+        and coordinate transformations between screenshot and screen coordinates.
+        """
+
+        def __init__(self, diorama):
+            """Initialize the interface with a reference to the parent Diorama instance.
+
+            Args:
+                diorama: The parent Diorama instance
+            """
+            self._diorama = diorama
+
+            self._scene_hitboxes = []
+            self._scene_size = None
+
+        async def _send_cmd(self, action, arguments=None):
+            """Send a command to the scheduler queue.
+
+            Args:
+                action (str): The action to perform
+                arguments (dict, optional): Arguments for the action
+
+            Returns:
+                The result of the command execution
+            """
+            Diorama._ensure_scheduler()
+            loop = asyncio.get_event_loop()
+            future = loop.create_future()
+            logger.info(f"Enqueuing {action} command for apps: {self._diorama.app_list}")
+            await Diorama._scheduler_queue.put(
+                {
+                    "action": action,
+                    "arguments": {"app_list": self._diorama.app_list, **(arguments or {})},
+                    "future": future,
+                }
+            )
+            try:
+                return await future
+            except asyncio.CancelledError:
+                logger.warning(f"Command was cancelled: {action}")
+                return None
+
+        async def screenshot(self, as_bytes: bool = True) -> Union[str, Image.Image]:
+            """Take a screenshot of the managed applications.
+
+            Args:
+                as_bytes (bool): If True, return base64-encoded bytes; if False, return PIL Image
+
+            Returns:
+                Union[str, Image.Image]: Base64-encoded PNG bytes or PIL Image object
+            """
+            import base64
+
+            result, img = await self._send_cmd("screenshot")
+            self._scene_hitboxes = result.get("hitboxes", [])
+            self._scene_size = img.size
+
+            if as_bytes:
+                # PIL Image to bytes, then base64 encode for JSON
+                import io
+
+                img_byte_arr = io.BytesIO()
+                img.save(img_byte_arr, format="PNG")
+                img_bytes = img_byte_arr.getvalue()
+                img_b64 = base64.b64encode(img_bytes).decode("ascii")
+                return img_b64
+            else:
+                return img
+
+        async def left_click(self, x, y):
+            """Perform a left mouse click at the specified coordinates.
+
+            Args:
+                x (int): X coordinate in screenshot space (or None to use last position)
+                y (int): Y coordinate in screenshot space (or None to use last position)
+            """
+            # Get last cursor position for this app_list hash
+            app_list_hash = hash(tuple(sorted(self._diorama.app_list)))
+            last_pos = Diorama._cursor_positions.get(app_list_hash, (0, 0))
+            x, y = x or last_pos[0], y or last_pos[1]
+            # Update cursor position for this app_list hash
+            Diorama._cursor_positions[app_list_hash] = (x, y)
+
+            sx, sy = await self.to_screen_coordinates(x, y)
+            await self._send_cmd("left_click", {"x": sx, "y": sy})
+
+        async def right_click(self, x, y):
+            """Perform a right mouse click at the specified coordinates.
+
+            Args:
+                x (int): X coordinate in screenshot space (or None to use last position)
+                y (int): Y coordinate in screenshot space (or None to use last position)
+            """
+            # Get last cursor position for this app_list hash
+            app_list_hash = hash(tuple(sorted(self._diorama.app_list)))
+            last_pos = Diorama._cursor_positions.get(app_list_hash, (0, 0))
+            x, y = x or last_pos[0], y or last_pos[1]
+            # Update cursor position for this app_list hash
+            Diorama._cursor_positions[app_list_hash] = (x, y)
+
+            sx, sy = await self.to_screen_coordinates(x, y)
+            await self._send_cmd("right_click", {"x": sx, "y": sy})
+
+        async def double_click(self, x, y):
+            """Perform a double mouse click at the specified coordinates.
+
+            Args:
+                x (int): X coordinate in screenshot space (or None to use last position)
+                y (int): Y coordinate in screenshot space (or None to use last position)
+            """
+            # Get last cursor position for this app_list hash
+            app_list_hash = hash(tuple(sorted(self._diorama.app_list)))
+            last_pos = Diorama._cursor_positions.get(app_list_hash, (0, 0))
+            x, y = x or last_pos[0], y or last_pos[1]
+            # Update cursor position for this app_list hash
+            Diorama._cursor_positions[app_list_hash] = (x, y)
+
+            sx, sy = await self.to_screen_coordinates(x, y)
+            await self._send_cmd("double_click", {"x": sx, "y": sy})
+
+        async def move_cursor(self, x, y):
+            """Move the mouse cursor to the specified coordinates.
+
+            Args:
+                x (int): X coordinate in screenshot space (or None to use last position)
+                y (int): Y coordinate in screenshot space (or None to use last position)
+            """
+            # Get last cursor position for this app_list hash
+            app_list_hash = hash(tuple(sorted(self._diorama.app_list)))
+            last_pos = Diorama._cursor_positions.get(app_list_hash, (0, 0))
+            x, y = x or last_pos[0], y or last_pos[1]
+            # Update cursor position for this app_list hash
+            Diorama._cursor_positions[app_list_hash] = (x, y)
+
+            sx, sy = await self.to_screen_coordinates(x, y)
+            await self._send_cmd("move_cursor", {"x": sx, "y": sy})
+
+        async def drag_to(self, x, y, duration=0.5):
+            """Drag the mouse from current position to the specified coordinates.
+
+            Args:
+                x (int): X coordinate in screenshot space (or None to use last position)
+                y (int): Y coordinate in screenshot space (or None to use last position)
+                duration (float): Duration of the drag operation in seconds
+            """
+            # Get last cursor position for this app_list hash
+            app_list_hash = hash(tuple(sorted(self._diorama.app_list)))
+            last_pos = Diorama._cursor_positions.get(app_list_hash, (0, 0))
+            x, y = x or last_pos[0], y or last_pos[1]
+            # Update cursor position for this app_list hash
+            Diorama._cursor_positions[app_list_hash] = (x, y)
+
+            sx, sy = await self.to_screen_coordinates(x, y)
+            await self._send_cmd("drag_to", {"x": sx, "y": sy, "duration": duration})
+
+        async def get_cursor_position(self):
+            """Get the current cursor position in screen coordinates.
+
+            Returns:
+                tuple: (x, y) coordinates of the cursor in screen space
+            """
+            return await self._send_cmd("get_cursor_position")
+
+        async def type_text(self, text):
+            """Type the specified text using the keyboard.
+
+            Args:
+                text (str): The text to type
+            """
+            await self._send_cmd("type_text", {"text": text})
+
+        async def press_key(self, key):
+            """Press a single key on the keyboard.
+
+            Args:
+                key (str): The key to press
+            """
+            await self._send_cmd("press_key", {"key": key})
+
+        async def hotkey(self, keys):
+            """Press a combination of keys simultaneously.
+
+            Args:
+                keys (list): List of keys to press together
+            """
+            await self._send_cmd("hotkey", {"keys": list(keys)})
+
+        async def scroll_up(self, clicks: int = 1):
+            """Scroll up at the current cursor position.
+
+            Args:
+                clicks (int): Number of scroll clicks to perform
+            """
+            # Get last cursor position for this app_list hash
+            app_list_hash = hash(tuple(sorted(self._diorama.app_list)))
+            last_pos = Diorama._cursor_positions.get(app_list_hash, (0, 0))
+            x, y = last_pos[0], last_pos[1]
+
+            await self._send_cmd("scroll_up", {"clicks": clicks, "x": x, "y": y})
+
+        async def scroll_down(self, clicks: int = 1):
+            """Scroll down at the current cursor position.
+
+            Args:
+                clicks (int): Number of scroll clicks to perform
+            """
+            # Get last cursor position for this app_list hash
+            app_list_hash = hash(tuple(sorted(self._diorama.app_list)))
+            last_pos = Diorama._cursor_positions.get(app_list_hash, (0, 0))
+            x, y = last_pos[0], last_pos[1]
+
+            await self._send_cmd("scroll_down", {"clicks": clicks, "x": x, "y": y})
+
+        async def get_screen_size(self) -> dict[str, int]:
+            """Get the size of the screenshot area.
+
+            Returns:
+                dict[str, int]: Dictionary with 'width' and 'height' keys
+            """
+            if not self._scene_size:
+                await self.screenshot()
+            return {"width": self._scene_size[0], "height": self._scene_size[1]}
+
+        async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:
+            """Convert screenshot coordinates to screen coordinates.
+
+            Args:
+                x: X absolute coordinate in screenshot space
+                y: Y absolute coordinate in screenshot space
+
+            Returns:
+                tuple[float, float]: (x, y) absolute coordinates in screen space
+            """
+            if not self._scene_hitboxes:
+                await self.screenshot()  # get hitboxes
+            # Try all hitboxes
+            for h in self._scene_hitboxes[::-1]:
+                rect_from = h.get("hitbox")
+                rect_to = h.get("target")
+                if not rect_from or len(rect_from) != 4:
+                    continue
+
+                # check if (x, y) is inside rect_from
+                x0, y0, x1, y1 = rect_from
+                if x0 <= x <= x1 and y0 <= y <= y1:
+                    logger.info(f"Found hitbox: {h}")
+                    # remap (x, y) to rect_to
+                    tx0, ty0, tx1, ty1 = rect_to
+
+                    # calculate offset from x0, y0
+                    offset_x = x - x0
+                    offset_y = y - y0
+
+                    # remap offset to rect_to
+                    tx = tx0 + offset_x
+                    ty = ty0 + offset_y
+
+                    return tx, ty
+            return x, y
+
+        async def to_screenshot_coordinates(self, x: float, y: float) -> tuple[float, float]:
+            """Convert screen coordinates to screenshot coordinates.
+
+            Args:
+                x: X absolute coordinate in screen space
+                y: Y absolute coordinate in screen space
+
+            Returns:
+                tuple[float, float]: (x, y) absolute coordinates in screenshot space
+            """
+            if not self._scene_hitboxes:
+                await self.screenshot()  # get hitboxes
+            # Try all hitboxes
+            for h in self._scene_hitboxes[::-1]:
+                rect_from = h.get("target")
+                rect_to = h.get("hitbox")
+                if not rect_from or len(rect_from) != 4:
+                    continue
+
+                # check if (x, y) is inside rect_from
+                x0, y0, x1, y1 = rect_from
+                if x0 <= x <= x1 and y0 <= y <= y1:
+                    # remap (x, y) to rect_to
+                    tx0, ty0, tx1, ty1 = rect_to
+
+                    # calculate offset from x0, y0
+                    offset_x = x - x0
+                    offset_y = y - y0
+
+                    # remap offset to rect_to
+                    tx = tx0 + offset_x
+                    ty = ty0 + offset_y
+
+                    return tx, ty
+            return x, y
+
+
+import time
+
+from pynput.mouse import Controller as MouseController
+
+
+async def main():
+    """Main function demonstrating Diorama usage with multiple desktops and mouse tracking."""
+    desktop1 = Diorama.create_from_apps(["Discord", "Notes"])
+    desktop2 = Diorama.create_from_apps(["Terminal"])
+
+    img1 = await desktop1.interface.screenshot(as_bytes=False)
+    img2 = await desktop2.interface.screenshot(as_bytes=False)
+
+    img1.save("app_screenshots/desktop1.png")
+    img2.save("app_screenshots/desktop2.png")
+    # Initialize Diorama desktop
+    desktop3 = Diorama.create_from_apps("Safari")
+    screen_size = await desktop3.interface.get_screen_size()
+    print(screen_size)
+
+    # Take initial screenshot
+    img = await desktop3.interface.screenshot(as_bytes=False)
+    img.save("app_screenshots/desktop3.png")
+
+    # Prepare hitboxes and draw on the single screenshot
+    hitboxes = desktop3.interface._scene_hitboxes[::-1]
+    base_img = img.copy()
+    draw = ImageDraw.Draw(base_img)
+    for h in hitboxes:
+        rect = h.get("hitbox")
+        if not rect or len(rect) != 4:
+            continue
+        draw.rectangle(rect, outline="red", width=2)
+
+    # Track and draw mouse position in real time (single screenshot size)
+    last_mouse_pos = None
+    print("Tracking mouse... Press Ctrl+C to stop.")
+    try:
+        mouse = MouseController()
+        while True:
+            mx, my = mouse.position
+            mouse_x, mouse_y = int(mx), int(my)
+            if last_mouse_pos != (mouse_x, mouse_y):
+                last_mouse_pos = (mouse_x, mouse_y)
+                # Map to screenshot coordinates
+                sx, sy = await desktop3.interface.to_screenshot_coordinates(mouse_x, mouse_y)
+                # Draw on a copy of the screenshot
+                frame = base_img.copy()
+                frame_draw = ImageDraw.Draw(frame)
+                frame_draw.ellipse((sx - 5, sy - 5, sx + 5, sy + 5), fill="blue", outline="blue")
+                # Save the frame
+                frame.save("app_screenshots/desktop3_mouse.png")
+                print(f"Mouse at screen ({mouse_x}, {mouse_y}) -> screenshot ({sx:.1f}, {sy:.1f})")
+            time.sleep(0.05)  # Throttle updates to ~20 FPS
+    except KeyboardInterrupt:
+        print("Stopped tracking.")
+
+        draw.text((rect[0], rect[1]), str(idx), fill="red")
+
+    canvas.save("app_screenshots/desktop3_hitboxes.png")
+
+    # move mouse in a square spiral around the screen
+    import math
+    import random
+
+    step = 20  # pixels per move
+    dot_radius = 10
+    width = screen_size["width"]
+    height = screen_size["height"]
+    x, y = 0, 10
+
+    while x < width and y < height:
+        await desktop3.interface.move_cursor(x, y)
+        img = await desktop3.interface.screenshot(as_bytes=False)
+        draw = ImageDraw.Draw(img)
+        draw.ellipse((x - dot_radius, y - dot_radius, x + dot_radius, y + dot_radius), fill="red")
+        img.save("current.png")
+        await asyncio.sleep(0.03)
+        x += step
+        y = math.sin(x / width * math.pi * 2) * 50 + 25
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

libs/python/computer-server/computer_server/diorama/diorama_computer.py

@@ -0,0 +1,52 @@
+import asyncio
+
+
+class DioramaComputer:
+    """
+    A minimal Computer-like interface for Diorama, compatible with ComputerAgent.
+    Implements _initialized, run(), and __aenter__ for agent compatibility.
+    """
+
+    def __init__(self, diorama):
+        """
+        Initialize the DioramaComputer with a diorama instance.
+
+        Args:
+            diorama: The diorama instance to wrap with a computer-like interface.
+        """
+        self.diorama = diorama
+        self.interface = self.diorama.interface
+        self._initialized = False
+
+    async def __aenter__(self):
+        """
+        Async context manager entry method for compatibility with ComputerAgent.
+
+        Ensures an event loop is running and marks the instance as initialized.
+        Creates a new event loop if none is currently running.
+
+        Returns:
+            DioramaComputer: The initialized instance.
+        """
+        # Ensure the event loop is running (for compatibility)
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            asyncio.set_event_loop(asyncio.new_event_loop())
+        self._initialized = True
+        return self
+
+    async def run(self):
+        """
+        Run method stub for compatibility with ComputerAgent interface.
+
+        Ensures the instance is initialized before returning. If not already
+        initialized, calls __aenter__ to perform initialization.
+
+        Returns:
+            DioramaComputer: The initialized instance.
+        """
+        # This is a stub for compatibility
+        if not self._initialized:
+            await self.__aenter__()
+        return self

libs/python/computer-server/computer_server/diorama/macos.py

@@ -0,0 +1,36 @@
+import inspect
+import platform
+import sys
+from typing import Optional
+
+from computer_server.diorama.base import BaseDioramaHandler
+from computer_server.diorama.diorama import Diorama
+
+
+class MacOSDioramaHandler(BaseDioramaHandler):
+    """Handler for Diorama commands on macOS, using local diorama module."""
+
+    async def diorama_cmd(self, action: str, arguments: Optional[dict] = None) -> dict:
+        if platform.system().lower() != "darwin":
+            return {"success": False, "error": "Diorama is only supported on macOS."}
+        try:
+            app_list = arguments.get("app_list") if arguments else None
+            if not app_list:
+                return {"success": False, "error": "Missing 'app_list' in arguments"}
+            diorama = Diorama(app_list)
+            interface = diorama.interface
+            if not hasattr(interface, action):
+                return {"success": False, "error": f"Unknown diorama action: {action}"}
+            method = getattr(interface, action)
+            # Remove app_list from arguments before calling the method
+            filtered_arguments = dict(arguments)
+            filtered_arguments.pop("app_list", None)
+            if inspect.iscoroutinefunction(method):
+                result = await method(**(filtered_arguments or {}))
+            else:
+                result = method(**(filtered_arguments or {}))
+            return {"success": True, "result": result}
+        except Exception as e:
+            import traceback
+
+            return {"success": False, "error": str(e), "trace": traceback.format_exc()}

libs/python/computer-server/computer_server/diorama/safezone.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""
+UI Safezone Helper - A utility to get accurate bounds for macOS UI elements
+
+This module provides helper functions to get accurate bounds for macOS UI elements
+like the menubar and dock, which are needed for proper screenshot composition.
+"""
+
+import sys
+import time
+from typing import Any, Dict, Optional, Tuple
+
+# Import Objective-C bridge libraries
+try:
+    import AppKit
+    import Foundation
+    from AppKit import NSRunningApplication, NSWorkspace
+    from ApplicationServices import (
+        AXUIElementCopyAttributeValue,
+        AXUIElementCopyAttributeValues,
+        AXUIElementCreateApplication,
+        AXUIElementCreateSystemWide,
+        AXUIElementGetTypeID,
+        AXValueGetType,
+        AXValueGetValue,
+        kAXChildrenAttribute,
+        kAXErrorSuccess,
+        kAXMenuBarAttribute,
+        kAXPositionAttribute,
+        kAXRoleAttribute,
+        kAXSizeAttribute,
+        kAXTitleAttribute,
+        kAXValueCGPointType,
+        kAXValueCGSizeType,
+    )
+except ImportError:
+    print("Error: This script requires PyObjC to be installed.")
+    print("Please install it with: pip install pyobjc")
+    sys.exit(1)
+
+# Constants for accessibility API
+kAXErrorSuccess = 0
+kAXRoleAttribute = "AXRole"
+kAXSubroleAttribute = "AXSubrole"
+kAXTitleAttribute = "AXTitle"
+kAXPositionAttribute = "AXPosition"
+kAXSizeAttribute = "AXSize"
+kAXChildrenAttribute = "AXChildren"
+kAXMenuBarAttribute = "AXMenuBar"
+
+
+def element_attribute(element, attribute):
+    """Get an attribute from an accessibility element"""
+    if attribute == kAXChildrenAttribute:
+        err, value = AXUIElementCopyAttributeValues(element, attribute, 0, 999, None)
+        if err == kAXErrorSuccess:
+            if isinstance(value, Foundation.NSArray):
+                return list(value)
+            else:
+                return value
+    err, value = AXUIElementCopyAttributeValue(element, attribute, None)
+    if err == kAXErrorSuccess:
+        return value
+    return None
+
+
+def element_value(element, type):
+    """Get a value from an accessibility element"""
+    err, value = AXValueGetValue(element, type, None)
+    if err == True:
+        return value
+    return None
+
+
+def get_element_bounds(element):
+    """Get the bounds of an accessibility element"""
+    bounds = {"x": 0, "y": 0, "width": 0, "height": 0}
+
+    # Get position
+    position_value = element_attribute(element, kAXPositionAttribute)
+    if position_value:
+        position_value = element_value(position_value, kAXValueCGPointType)
+        if position_value:
+            bounds["x"] = position_value.x
+            bounds["y"] = position_value.y
+
+    # Get size
+    size_value = element_attribute(element, kAXSizeAttribute)
+    if size_value:
+        size_value = element_value(size_value, kAXValueCGSizeType)
+        if size_value:
+            bounds["width"] = size_value.width
+            bounds["height"] = size_value.height
+
+    return bounds
+
+
+def find_dock_process():
+    """Find the Dock process"""
+    running_apps = NSWorkspace.sharedWorkspace().runningApplications()
+    for app in running_apps:
+        if app.localizedName() == "Dock" and app.bundleIdentifier() == "com.apple.dock":
+            return app.processIdentifier()
+    return None
+
+
+def get_menubar_bounds():
+    """Get the bounds of the macOS menubar
+
+    Returns:
+        Dictionary with x, y, width, height of the menubar
+    """
+    # Get the system-wide accessibility element
+    system_element = AXUIElementCreateSystemWide()
+
+    # Try to find the menubar
+    menubar = element_attribute(system_element, kAXMenuBarAttribute)
+    if menubar is None:
+        # If we can't get it directly, try through the frontmost app
+        frontmost_app = NSWorkspace.sharedWorkspace().frontmostApplication()
+        if frontmost_app:
+            app_pid = frontmost_app.processIdentifier()
+            app_element = AXUIElementCreateApplication(app_pid)
+            menubar = element_attribute(app_element, kAXMenuBarAttribute)
+
+    if menubar is None:
+        print("Error: Could not get menubar")
+        # Return default menubar bounds as fallback
+        return {"x": 0, "y": 0, "width": 1800, "height": 24}
+
+    # Get menubar bounds
+    return get_element_bounds(menubar)
+
+
+def get_dock_bounds():
+    """Get the bounds of the macOS Dock
+
+    Returns:
+        Dictionary with x, y, width, height of the Dock
+    """
+    dock_pid = find_dock_process()
+    if dock_pid is None:
+        print("Error: Could not find Dock process")
+        # Return empty bounds as fallback
+        return {"x": 0, "y": 0, "width": 0, "height": 0}
+
+    # Create an accessibility element for the Dock
+    dock_element = AXUIElementCreateApplication(dock_pid)
+    if dock_element is None:
+        print(f"Error: Could not create accessibility element for Dock (PID {dock_pid})")
+        return {"x": 0, "y": 0, "width": 0, "height": 0}
+
+    # Get the Dock's children
+    children = element_attribute(dock_element, kAXChildrenAttribute)
+    if not children or len(children) == 0:
+        print("Error: Could not get Dock children")
+        return {"x": 0, "y": 0, "width": 0, "height": 0}
+
+    # Find the Dock's list (first child is usually the main dock list)
+    dock_list = None
+    for child in children:
+        role = element_attribute(child, kAXRoleAttribute)
+        if role == "AXList":
+            dock_list = child
+            break
+
+    if dock_list is None:
+        print("Error: Could not find Dock list")
+        return {"x": 0, "y": 0, "width": 0, "height": 0}
+
+    # Get the bounds of the dock list
+    return get_element_bounds(dock_list)
+
+
+def get_ui_element_bounds():
+    """Get the bounds of important UI elements like menubar and dock
+
+    Returns:
+        Dictionary with menubar and dock bounds
+    """
+    menubar_bounds = get_menubar_bounds()
+    dock_bounds = get_dock_bounds()
+
+    return {"menubar": menubar_bounds, "dock": dock_bounds}
+
+
+if __name__ == "__main__":
+    # Example usage
+    bounds = get_ui_element_bounds()
+    print("Menubar bounds:", bounds["menubar"])
+    print("Dock bounds:", bounds["dock"])

libs/python/computer-server/computer_server/handlers/base.py

@@ -0,0 +1,431 @@
+import asyncio
+import logging
+import sys
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAccessibilityHandler(ABC):
+    """Abstract base class for OS-specific accessibility handlers."""
+
+    @abstractmethod
+    async def get_accessibility_tree(self) -> Dict[str, Any]:
+        """Get the accessibility tree of the current window."""
+        pass
+
+    @abstractmethod
+    async def find_element(
+        self, role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Find an element in the accessibility tree by criteria."""
+        pass
+
+
+class BaseFileHandler(ABC):
+    """Abstract base class for OS-specific file handlers."""
+
+    @abstractmethod
+    async def file_exists(self, path: str) -> Dict[str, Any]:
+        """Check if a file exists at the specified path."""
+        pass
+
+    @abstractmethod
+    async def directory_exists(self, path: str) -> Dict[str, Any]:
+        """Check if a directory exists at the specified path."""
+        pass
+
+    @abstractmethod
+    async def list_dir(self, path: str) -> Dict[str, Any]:
+        """List the contents of a directory."""
+        pass
+
+    @abstractmethod
+    async def read_text(self, path: str) -> Dict[str, Any]:
+        """Read the text contents of a file."""
+        pass
+
+    @abstractmethod
+    async def write_text(self, path: str, content: str) -> Dict[str, Any]:
+        """Write text content to a file."""
+        pass
+
+    @abstractmethod
+    async def write_bytes(self, path: str, content_b64: str) -> Dict[str, Any]:
+        """Write binary content to a file. Sent over the websocket as a base64 string."""
+        pass
+
+    @abstractmethod
+    async def delete_file(self, path: str) -> Dict[str, Any]:
+        """Delete a file."""
+        pass
+
+    @abstractmethod
+    async def create_dir(self, path: str) -> Dict[str, Any]:
+        """Create a directory."""
+        pass
+
+    @abstractmethod
+    async def delete_dir(self, path: str) -> Dict[str, Any]:
+        """Delete a directory."""
+        pass
+
+    @abstractmethod
+    async def read_bytes(
+        self, path: str, offset: int = 0, length: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Read the binary contents of a file. Sent over the websocket as a base64 string.
+
+        Args:
+            path: Path to the file
+            offset: Byte offset to start reading from (default: 0)
+            length: Number of bytes to read (default: None for entire file)
+        """
+        pass
+
+    @abstractmethod
+    async def get_file_size(self, path: str) -> Dict[str, Any]:
+        """Get the size of a file in bytes."""
+        pass
+
+
+class BaseDesktopHandler(ABC):
+    """Abstract base class for OS-specific desktop handlers.
+
+    Categories:
+    - Wallpaper Actions: Methods for wallpaper operations
+    - Desktop shortcut actions: Methods for managing desktop shortcuts
+    """
+
+    # Wallpaper Actions
+    @abstractmethod
+    async def get_desktop_environment(self) -> Dict[str, Any]:
+        """Get the current desktop environment name."""
+        pass
+
+    @abstractmethod
+    async def set_wallpaper(self, path: str) -> Dict[str, Any]:
+        """Set the desktop wallpaper to the file at path."""
+        pass
+
+
+class BaseWindowHandler(ABC):
+    """Abstract class for OS-specific window management handlers.
+
+    Categories:
+    - Window Management: Methods for application/window control
+    """
+
+    # Window Management
+    @abstractmethod
+    async def open(self, target: str) -> Dict[str, Any]:
+        """Open a file or URL with the default application."""
+        pass
+
+    @abstractmethod
+    async def launch(self, app: str, args: Optional[List[str]] = None) -> Dict[str, Any]:
+        """Launch an application with optional arguments."""
+        pass
+
+    @abstractmethod
+    async def get_current_window_id(self) -> Dict[str, Any]:
+        """Get the currently active window ID."""
+        pass
+
+    @abstractmethod
+    async def get_application_windows(self, app: str) -> Dict[str, Any]:
+        """Get windows belonging to an application (by name or bundle)."""
+        pass
+
+    @abstractmethod
+    async def get_window_name(self, window_id: str) -> Dict[str, Any]:
+        """Get the title/name of a window by ID."""
+        pass
+
+    @abstractmethod
+    async def get_window_size(self, window_id: str | int) -> Dict[str, Any]:
+        """Get the size of a window by ID as {width, height}."""
+        pass
+
+    @abstractmethod
+    async def activate_window(self, window_id: str | int) -> Dict[str, Any]:
+        """Bring a window to the foreground by ID."""
+        pass
+
+    @abstractmethod
+    async def close_window(self, window_id: str | int) -> Dict[str, Any]:
+        """Close a window by ID."""
+        pass
+
+    @abstractmethod
+    async def get_window_position(self, window_id: str | int) -> Dict[str, Any]:
+        """Get the top-left position of a window as {x, y}."""
+        pass
+
+    @abstractmethod
+    async def set_window_size(
+        self, window_id: str | int, width: int, height: int
+    ) -> Dict[str, Any]:
+        """Set the size of a window by ID."""
+        pass
+
+    @abstractmethod
+    async def set_window_position(self, window_id: str | int, x: int, y: int) -> Dict[str, Any]:
+        """Set the position of a window by ID."""
+        pass
+
+    @abstractmethod
+    async def maximize_window(self, window_id: str | int) -> Dict[str, Any]:
+        """Maximize a window by ID."""
+        pass
+
+    @abstractmethod
+    async def minimize_window(self, window_id: str | int) -> Dict[str, Any]:
+        """Minimize a window by ID."""
+        pass
+
+
+_SUPPORTED_FORMATS = {"png", "jpeg"}
+_FORMAT_ALIASES = {"jpg": "jpeg"}
+
+
+def normalize_screenshot_format(format: str, quality: int) -> tuple[str, int]:
+    """Normalize and validate screenshot format/quality.
+
+    Returns ``(normalized_format, clamped_quality)`` or raises ``ValueError``.
+
+    - Lowercases the format string.
+    - Maps "jpg" → "jpeg".
+    - Rejects anything not in ``{"png", "jpeg"}``.
+    - Clamps JPEG quality to 1–95 (silently caps values above 95).
+    """
+    fmt = _FORMAT_ALIASES.get(format.lower(), format.lower())
+    if fmt not in _SUPPORTED_FORMATS:
+        raise ValueError(
+            f"Unsupported screenshot format {format!r}. " f"Supported: {sorted(_SUPPORTED_FORMATS)}"
+        )
+    if fmt == "jpeg":
+        quality = max(1, min(95, quality))
+    return fmt, quality
+
+
+class BaseAutomationHandler(ABC):
+    """Abstract base class for OS-specific automation handlers.
+
+    Categories:
+    - Mouse Actions: Methods for mouse control
+    - Keyboard Actions: Methods for keyboard input
+    - Scrolling Actions: Methods for scrolling
+    - Screen Actions: Methods for screen interaction
+    - Clipboard Actions: Methods for clipboard operations
+    """
+
+    # Mouse Actions
+    @abstractmethod
+    async def mouse_down(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """Perform a mouse down at the current or specified position."""
+        pass
+
+    @abstractmethod
+    async def mouse_up(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """Perform a mouse up at the current or specified position."""
+        pass
+
+    @abstractmethod
+    async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        """Perform a left click at the current or specified position."""
+        pass
+
+    @abstractmethod
+    async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        """Perform a right click at the current or specified position."""
+        pass
+
+    @abstractmethod
+    async def middle_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Perform a middle click at the current or specified position."""
+        pass
+
+    @abstractmethod
+    async def double_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Perform a double click at the current or specified position."""
+        pass
+
+    @abstractmethod
+    async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
+        """Move the cursor to the specified position."""
+        pass
+
+    @abstractmethod
+    async def drag_to(
+        self, x: int, y: int, button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        """Drag the cursor from current position to specified coordinates.
+
+        Args:
+            x: The x coordinate to drag to
+            y: The y coordinate to drag to
+            button: The mouse button to use ('left', 'middle', 'right')
+            duration: How long the drag should take in seconds
+        """
+        pass
+
+    @abstractmethod
+    async def drag(
+        self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        """Drag the cursor from current position to specified coordinates.
+
+        Args:
+            path: A list of tuples of x and y coordinates to drag to
+            button: The mouse button to use ('left', 'middle', 'right')
+            duration: How long the drag should take in seconds
+        """
+        pass
+
+    # Keyboard Actions
+    @abstractmethod
+    async def key_down(self, key: str) -> Dict[str, Any]:
+        """Press and hold the specified key."""
+        pass
+
+    @abstractmethod
+    async def key_up(self, key: str) -> Dict[str, Any]:
+        """Release the specified key."""
+        pass
+
+    @abstractmethod
+    async def type_text(self, text: str) -> Dict[str, Any]:
+        """Type the specified text."""
+        pass
+
+    @abstractmethod
+    async def press_key(self, key: str) -> Dict[str, Any]:
+        """Press the specified key."""
+        pass
+
+    @abstractmethod
+    async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
+        """Press a combination of keys together."""
+        pass
+
+    # Scrolling Actions
+    @abstractmethod
+    async def scroll(self, x: int, y: int) -> Dict[str, Any]:
+        """Scroll the specified amount."""
+        pass
+
+    @abstractmethod
+    async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
+        """Scroll down by the specified number of clicks."""
+        pass
+
+    @abstractmethod
+    async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
+        """Scroll up by the specified number of clicks."""
+        pass
+
+    # Screen Actions
+    @abstractmethod
+    async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
+        """Take a screenshot and return base64 encoded image data.
+
+        Args:
+            format: Image format - "png" (lossless, default) or "jpeg" (lossy, smaller).
+            quality: JPEG quality 1-95, ignored for PNG.
+        """
+        pass
+
+    @abstractmethod
+    async def get_screen_size(self) -> Dict[str, Any]:
+        """Get the screen size of the VM."""
+        pass
+
+    @abstractmethod
+    async def get_cursor_position(self) -> Dict[str, Any]:
+        """Get the current cursor position."""
+        pass
+
+    # Clipboard Actions
+    async def copy_to_clipboard(self) -> Dict[str, Any]:
+        """Get the current clipboard content using pyperclip."""
+        try:
+            import pyperclip
+
+            content = pyperclip.paste()
+            return {"success": True, "content": content}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def set_clipboard(self, text: str) -> Dict[str, Any]:
+        """Set the clipboard content using pyperclip."""
+        try:
+            import pyperclip
+
+            pyperclip.copy(text)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Command Execution
+    async def run_command(self, command: str, timeout: Optional[float] = None) -> Dict[str, Any]:
+        """Run a shell command locally and return its output.
+
+        When IS_CUA_ANDROID env var is set, routes the command through
+        ``adb shell`` so execution runs inside the Android emulator.
+
+        Args:
+            command: The shell command to run.
+            timeout: Optional timeout in seconds.  When ``None`` (default),
+                the command runs until completion with no upper bound.  SDK
+                callers set this via ``sb.shell.run(cmd, timeout=...)``
+                (see ``cua_sandbox.interfaces.shell.Shell.run``).  On
+                expiry the subprocess is killed and the result is
+                ``{"success": False, "stderr": "Command timed out after <t>s",
+                "return_code": -1}``.
+        """
+        import os
+
+        try:
+            if os.environ.get("IS_CUA_ANDROID") == "true":
+                process = await asyncio.create_subprocess_exec(
+                    "adb",
+                    "shell",
+                    command,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                )
+            else:
+                process = await asyncio.create_subprocess_shell(
+                    command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+                )
+            try:
+                if timeout is None:
+                    stdout, stderr = await process.communicate()
+                else:
+                    stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
+            except asyncio.TimeoutError:
+                process.kill()
+                return {
+                    "success": False,
+                    "stdout": "",
+                    "stderr": f"Command timed out after {timeout}s",
+                    "return_code": -1,
+                }
+            return {
+                "success": True,
+                "stdout": stdout.decode() if stdout else "",
+                "stderr": stderr.decode() if stderr else "",
+                "return_code": process.returncode,
+            }
+        except Exception as e:
+            return {"success": False, "error": str(e)}

libs/python/computer-server/computer_server/handlers/factory.py

@@ -0,0 +1,120 @@
+import logging
+import os
+from typing import Tuple
+
+from computer_server.diorama.base import BaseDioramaHandler
+
+from ..utils.helpers import get_current_os
+from .base import (
+    BaseAccessibilityHandler,
+    BaseAutomationHandler,
+    BaseDesktopHandler,
+    BaseFileHandler,
+    BaseWindowHandler,
+)
+
+logger = logging.getLogger(__name__)
+
+OS_TYPE = get_current_os()
+
+if OS_TYPE == "android":
+    from .android import (
+        AndroidAccessibilityHandler,
+        AndroidAutomationHandler,
+        AndroidDesktopHandler,
+        AndroidFileHandler,
+        AndroidWindowHandler,
+    )
+elif OS_TYPE == "darwin":
+    from computer_server.diorama.macos import MacOSDioramaHandler
+
+    from .macos import MacOSAccessibilityHandler, MacOSAutomationHandler
+elif OS_TYPE == "linux":
+    from .linux import LinuxAccessibilityHandler, LinuxAutomationHandler
+elif OS_TYPE == "windows":
+    from .windows import WindowsAccessibilityHandler, WindowsAutomationHandler
+
+from .generic import GenericDesktopHandler, GenericFileHandler, GenericWindowHandler
+
+
+class HandlerFactory:
+    """Factory for creating OS-specific handlers."""
+
+    @staticmethod
+    def create_handlers() -> Tuple[
+        BaseAccessibilityHandler,
+        BaseAutomationHandler,
+        BaseDioramaHandler,
+        BaseFileHandler,
+        BaseDesktopHandler,
+        BaseWindowHandler,
+    ]:
+        """Create and return appropriate handlers for the current OS.
+
+        Returns:
+            Tuple[BaseAccessibilityHandler, BaseAutomationHandler, BaseDioramaHandler, BaseFileHandler]: A tuple containing
+            the appropriate accessibility, automation, diorama, and file handlers for the current OS.
+
+        Raises:
+            NotImplementedError: If the current OS is not supported
+            RuntimeError: If unable to determine the current OS
+        """
+        backend = os.environ.get("CUA_BACKEND", "native")
+        vnc_host = os.environ.get("CUA_VNC_HOST")
+        if backend == "vnc" or vnc_host:
+            if not vnc_host:
+                raise RuntimeError(
+                    "CUA_VNC_HOST must be set when using VNC backend "
+                    "(--backend=vnc requires --vnc-host)"
+                )
+            from .vnc import VNCAccessibilityHandler, VNCAutomationHandler
+
+            vnc_port = int(os.environ.get("CUA_VNC_PORT", "5900"))
+            vnc_password = os.environ.get("CUA_VNC_PASSWORD", "")
+            logger.info(f"Using VNC backend → {vnc_host}:{vnc_port}")
+            return (
+                VNCAccessibilityHandler(),
+                VNCAutomationHandler(host=vnc_host, port=vnc_port, password=vnc_password),
+                BaseDioramaHandler(),
+                GenericFileHandler(),
+                GenericDesktopHandler(),
+                GenericWindowHandler(),
+            )
+        elif OS_TYPE == "android":
+            return (
+                AndroidAccessibilityHandler(),
+                AndroidAutomationHandler(),
+                BaseDioramaHandler(),
+                AndroidFileHandler(),
+                AndroidDesktopHandler(),
+                AndroidWindowHandler(),
+            )
+        elif OS_TYPE == "darwin":
+            return (
+                MacOSAccessibilityHandler(),
+                MacOSAutomationHandler(),
+                MacOSDioramaHandler(),
+                GenericFileHandler(),
+                GenericDesktopHandler(),
+                GenericWindowHandler(),
+            )
+        elif OS_TYPE == "linux":
+            return (
+                LinuxAccessibilityHandler(),
+                LinuxAutomationHandler(),
+                BaseDioramaHandler(),
+                GenericFileHandler(),
+                GenericDesktopHandler(),
+                GenericWindowHandler(),
+            )
+        elif OS_TYPE == "windows":
+            return (
+                WindowsAccessibilityHandler(),
+                WindowsAutomationHandler(),
+                BaseDioramaHandler(),
+                GenericFileHandler(),
+                GenericDesktopHandler(),
+                GenericWindowHandler(),
+            )
+        else:
+            raise NotImplementedError(f"OS '{OS_TYPE}' is not supported")

libs/python/computer-server/computer_server/handlers/generic.py

@@ -0,0 +1,474 @@
+"""
+Generic handlers for all OSes.
+
+Includes:
+- DesktopHandler
+- FileHandler
+
+"""
+
+import base64
+import os
+import platform
+import subprocess
+import webbrowser
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from ..utils import wallpaper
+from .base import BaseDesktopHandler, BaseFileHandler, BaseWindowHandler
+
+try:
+    import pywinctl as pwc
+except Exception:  # pragma: no cover
+    pwc = None  # type: ignore
+
+
+def resolve_path(path: str) -> Path:
+    """Resolve a path to its absolute path. Expand ~ to the user's home directory.
+
+    Args:
+        path: The file or directory path to resolve
+
+    Returns:
+        Path: The resolved absolute path
+    """
+    return Path(path).expanduser().resolve()
+
+
+# ===== Cross-platform Desktop command handlers =====
+
+
+class GenericDesktopHandler(BaseDesktopHandler):
+    """
+    Generic desktop handler providing desktop-related operations.
+
+    Implements:
+    - get_desktop_environment: detect current desktop environment
+    - set_wallpaper: set desktop wallpaper path
+    """
+
+    async def get_desktop_environment(self) -> Dict[str, Any]:
+        """
+        Get the current desktop environment.
+
+        Returns:
+            Dict containing 'success' boolean and either 'environment' string or 'error' string
+        """
+        try:
+            env = wallpaper.get_desktop_environment()
+            return {"success": True, "environment": env}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def set_wallpaper(self, path: str) -> Dict[str, Any]:
+        """
+        Set the desktop wallpaper to the specified path.
+
+        Args:
+            path: The file path to set as wallpaper
+
+        Returns:
+            Dict containing 'success' boolean and optionally 'error' string
+        """
+        try:
+            file_path = resolve_path(path)
+            ok = wallpaper.set_wallpaper(str(file_path))
+            return {"success": bool(ok)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+
+# ===== Cross-platform window control command handlers =====
+
+
+class GenericWindowHandler(BaseWindowHandler):
+    """
+    Cross-platform window management using pywinctl where possible.
+    """
+
+    async def open(self, target: str) -> Dict[str, Any]:
+        try:
+            if target.startswith("http://") or target.startswith("https://"):
+                ok = webbrowser.open(target)
+                return {"success": bool(ok)}
+            path = str(resolve_path(target))
+            sys = platform.system().lower()
+            if sys == "darwin":
+                subprocess.Popen(["open", path])
+            elif sys == "linux":
+                subprocess.Popen(["xdg-open", path])
+            elif sys == "windows":
+                os.startfile(path)  # type: ignore[attr-defined]
+            else:
+                return {"success": False, "error": f"Unsupported OS: {sys}"}
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def launch(self, app: str, args: Optional[list[str]] = None) -> Dict[str, Any]:
+        try:
+            if args:
+                proc = subprocess.Popen([app, *args])
+            else:
+                # allow shell command like "libreoffice --writer"
+                proc = subprocess.Popen(app, shell=True)
+            return {"success": True, "pid": proc.pid}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    def _get_window_by_id(self, window_id: int | str) -> Optional[Any]:
+        if pwc is None:
+            raise RuntimeError("pywinctl not available")
+        # Find by native handle among Window objects; getAllWindowsDict keys are titles
+        try:
+            for w in pwc.getAllWindows():
+                if str(w.getHandle()) == str(window_id):
+                    return w
+            return None
+        except Exception:
+            return None
+
+    async def get_current_window_id(self) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            win = pwc.getActiveWindow()
+            if not win:
+                return {"success": False, "error": "No active window"}
+            return {"success": True, "window_id": win.getHandle()}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_application_windows(self, app: str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            wins = pwc.getWindowsWithTitle(app, condition=pwc.Re.CONTAINS, flags=pwc.Re.IGNORECASE)
+            ids = [w.getHandle() for w in wins]
+            return {"success": True, "windows": ids}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_window_name(self, window_id: int | str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            return {"success": True, "name": w.title}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_window_size(self, window_id: int | str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            width, height = w.size
+            return {"success": True, "width": int(width), "height": int(height)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_window_position(self, window_id: int | str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            x, y = w.position
+            return {"success": True, "x": int(x), "y": int(y)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def set_window_size(
+        self, window_id: int | str, width: int, height: int
+    ) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            ok = w.resizeTo(int(width), int(height))
+            return {"success": bool(ok)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def set_window_position(self, window_id: int | str, x: int, y: int) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            ok = w.moveTo(int(x), int(y))
+            return {"success": bool(ok)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def maximize_window(self, window_id: int | str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            ok = w.maximize()
+            return {"success": bool(ok)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def minimize_window(self, window_id: int | str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            ok = w.minimize()
+            return {"success": bool(ok)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def activate_window(self, window_id: int | str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            ok = w.activate()
+            return {"success": bool(ok)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def close_window(self, window_id: int | str) -> Dict[str, Any]:
+        try:
+            if pwc is None:
+                return {"success": False, "error": "pywinctl not available"}
+            w = self._get_window_by_id(window_id)
+            if not w:
+                return {"success": False, "error": "Window not found"}
+            ok = w.close()
+            return {"success": bool(ok)}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+
+# ===== Cross-platform file system command handlers =====
+
+
+class GenericFileHandler(BaseFileHandler):
+    """
+    Generic file handler that provides file system operations for all operating systems.
+
+    This class implements the BaseFileHandler interface and provides methods for
+    file and directory operations including reading, writing, creating, and deleting
+    files and directories.
+    """
+
+    async def file_exists(self, path: str) -> Dict[str, Any]:
+        """
+        Check if a file exists at the specified path.
+
+        Args:
+            path: The file path to check
+
+        Returns:
+            Dict containing 'success' boolean and either 'exists' boolean or 'error' string
+        """
+        try:
+            return {"success": True, "exists": resolve_path(path).is_file()}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def directory_exists(self, path: str) -> Dict[str, Any]:
+        """
+        Check if a directory exists at the specified path.
+
+        Args:
+            path: The directory path to check
+
+        Returns:
+            Dict containing 'success' boolean and either 'exists' boolean or 'error' string
+        """
+        try:
+            return {"success": True, "exists": resolve_path(path).is_dir()}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def list_dir(self, path: str) -> Dict[str, Any]:
+        """
+        List all files and directories in the specified directory.
+
+        Args:
+            path: The directory path to list
+
+        Returns:
+            Dict containing 'success' boolean and either 'files' list of names or 'error' string
+        """
+        try:
+            return {
+                "success": True,
+                "files": [
+                    p.name for p in resolve_path(path).iterdir() if p.is_file() or p.is_dir()
+                ],
+            }
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def read_text(self, path: str) -> Dict[str, Any]:
+        """
+        Read the contents of a text file.
+
+        Args:
+            path: The file path to read from
+
+        Returns:
+            Dict containing 'success' boolean and either 'content' string or 'error' string
+        """
+        try:
+            return {"success": True, "content": resolve_path(path).read_text()}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def write_text(self, path: str, content: str) -> Dict[str, Any]:
+        """
+        Write text content to a file.
+
+        Args:
+            path: The file path to write to
+            content: The text content to write
+
+        Returns:
+            Dict containing 'success' boolean and optionally 'error' string
+        """
+        try:
+            resolve_path(path).write_text(content)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def write_bytes(
+        self, path: str, content_b64: str, append: bool = False
+    ) -> Dict[str, Any]:
+        """
+        Write binary content to a file from base64 encoded string.
+
+        Args:
+            path: The file path to write to
+            content_b64: Base64 encoded binary content
+            append: If True, append to existing file; if False, overwrite
+
+        Returns:
+            Dict containing 'success' boolean and optionally 'error' string
+        """
+        try:
+            mode = "ab" if append else "wb"
+            with open(resolve_path(path), mode) as f:
+                f.write(base64.b64decode(content_b64))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def read_bytes(
+        self, path: str, offset: int = 0, length: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """
+        Read binary content from a file and return as base64 encoded string.
+
+        Args:
+            path: The file path to read from
+            offset: Byte offset to start reading from
+            length: Number of bytes to read; if None, read entire file from offset
+
+        Returns:
+            Dict containing 'success' boolean and either 'content_b64' string or 'error' string
+        """
+        try:
+            file_path = resolve_path(path)
+            with open(file_path, "rb") as f:
+                if offset > 0:
+                    f.seek(offset)
+
+                if length is not None:
+                    content = f.read(length)
+                else:
+                    content = f.read()
+
+            return {"success": True, "content_b64": base64.b64encode(content).decode("utf-8")}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_file_size(self, path: str) -> Dict[str, Any]:
+        """
+        Get the size of a file in bytes.
+
+        Args:
+            path: The file path to get size for
+
+        Returns:
+            Dict containing 'success' boolean and either 'size' integer or 'error' string
+        """
+        try:
+            file_path = resolve_path(path)
+            size = file_path.stat().st_size
+            return {"success": True, "size": size}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def delete_file(self, path: str) -> Dict[str, Any]:
+        """
+        Delete a file at the specified path.
+
+        Args:
+            path: The file path to delete
+
+        Returns:
+            Dict containing 'success' boolean and optionally 'error' string
+        """
+        try:
+            resolve_path(path).unlink()
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def create_dir(self, path: str) -> Dict[str, Any]:
+        """
+        Create a directory at the specified path.
+
+        Creates parent directories if they don't exist and doesn't raise an error
+        if the directory already exists.
+
+        Args:
+            path: The directory path to create
+
+        Returns:
+            Dict containing 'success' boolean and optionally 'error' string
+        """
+        try:
+            resolve_path(path).mkdir(parents=True, exist_ok=True)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def delete_dir(self, path: str) -> Dict[str, Any]:
+        """
+        Delete an empty directory at the specified path.
+
+        Args:
+            path: The directory path to delete
+
+        Returns:
+            Dict containing 'success' boolean and optionally 'error' string
+        """
+        try:
+            resolve_path(path).rmdir()
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}

libs/python/computer-server/computer_server/handlers/linux.py

@@ -0,0 +1,553 @@
+"""
+Linux implementation of automation and accessibility handlers.
+
+This implementation uses pynput for GUI automation. For screenshots and screen size,
+it uses PIL's ImageGrab (works with X11/Xvfb) and provides simulated fallbacks where needed.
+To use GUI automation in a headless environment:
+1. Install Xvfb: sudo apt-get install xvfb
+2. Run with virtual display: xvfb-run python -m computer_server
+"""
+
+import asyncio
+import base64
+import json
+import logging
+import os
+import subprocess
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+from PIL import Image, ImageGrab
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+
+from pynput.keyboard import Controller as KeyboardController
+from pynput.keyboard import Key
+from pynput.mouse import Button
+from pynput.mouse import Controller as MouseController
+
+from .base import (
+    BaseAccessibilityHandler,
+    BaseAutomationHandler,
+    normalize_screenshot_format,
+)
+
+
+class LinuxAccessibilityHandler(BaseAccessibilityHandler):
+    """Linux implementation of accessibility handler."""
+
+    async def get_accessibility_tree(self) -> Dict[str, Any]:
+        """Get the accessibility tree of the current window.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing success status and a simulated tree structure
+                           since Linux doesn't have equivalent accessibility API like macOS.
+        """
+        # Linux doesn't have equivalent accessibility API like macOS
+        # Return a minimal dummy tree
+        logger.info(
+            "Getting accessibility tree (simulated, no accessibility API available on Linux)"
+        )
+        return {
+            "success": True,
+            "tree": {
+                "role": "Window",
+                "title": "Linux Window",
+                "position": {"x": 0, "y": 0},
+                "size": {"width": 1920, "height": 1080},
+                "children": [],
+            },
+        }
+
+    async def find_element(
+        self, role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Find an element in the accessibility tree by criteria.
+
+        Args:
+            role: The role of the element to find.
+            title: The title of the element to find.
+            value: The value of the element to find.
+
+        Returns:
+            Dict[str, Any]: A dictionary indicating that element search is not supported on Linux.
+        """
+        logger.info(
+            f"Finding element with role={role}, title={title}, value={value} (not supported on Linux)"
+        )
+        return {"success": False, "message": "Element search not supported on Linux"}
+
+    def get_cursor_position(self) -> Tuple[int, int]:
+        """Get the current cursor position.
+
+        Returns:
+            Tuple[int, int]: The x and y coordinates of the cursor position.
+                           Returns (0, 0) if cursor position cannot be determined.
+        """
+        try:
+            # Use pynput mouse controller
+            from pynput.mouse import Controller as MouseController
+
+            m = MouseController()
+            x, y = m.position
+            return int(x), int(y)
+        except Exception as e:
+            logger.warning(f"Failed to get cursor position: {e}")
+            logger.info("Getting cursor position (simulated)")
+            return 0, 0
+
+    def get_screen_size(self) -> Tuple[int, int]:
+        """Get the screen size.
+
+        Returns:
+            Tuple[int, int]: The width and height of the screen in pixels.
+                           Returns (1920, 1080) if screen size cannot be determined.
+        """
+        try:
+            img = ImageGrab.grab()
+            return img.width, img.height
+        except Exception as e:
+            logger.warning(f"Failed to get screen size via ImageGrab: {e}")
+            logger.info("Getting screen size (simulated)")
+            return 1920, 1080
+
+
+class LinuxAutomationHandler(BaseAutomationHandler):
+    """Linux implementation of automation handler using pynput."""
+
+    keyboard = KeyboardController()
+    mouse = MouseController()
+
+    # Mouse Actions
+    async def mouse_down(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """Press and hold a mouse button at the specified coordinates.
+
+        Args:
+            x: The x coordinate to move to before pressing. If None, uses current position.
+            y: The y coordinate to move to before pressing. If None, uses current position.
+            button: The mouse button to press ("left", "right", or "middle").
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            from pynput.mouse import Button
+
+            btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
+            self.mouse.press(btn)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def mouse_up(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """Release a mouse button at the specified coordinates.
+
+        Args:
+            x: The x coordinate to move to before releasing. If None, uses current position.
+            y: The y coordinate to move to before releasing. If None, uses current position.
+            button: The mouse button to release ("left", "right", or "middle").
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            from pynput.mouse import Button
+
+            btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
+            self.mouse.release(btn)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
+        """Move the cursor to the specified coordinates.
+
+        Args:
+            x: The x coordinate to move to.
+            y: The y coordinate to move to.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            self.mouse.position = (x, y)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        """Perform a left mouse click at the specified coordinates.
+
+        Args:
+            x: The x coordinate to click at. If None, clicks at current position.
+            y: The y coordinate to click at. If None, clicks at current position.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.mouse import Button
+
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(Button.left, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        """Perform a right mouse click at the specified coordinates.
+
+        Args:
+            x: The x coordinate to click at. If None, clicks at current position.
+            y: The y coordinate to click at. If None, clicks at current position.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.mouse import Button
+
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(Button.right, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def middle_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Perform a middle mouse click at the specified coordinates."""
+        try:
+            from pynput.mouse import Button
+
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(Button.middle, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def double_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Perform a double click at the specified coordinates.
+
+        Args:
+            x: The x coordinate to double click at. If None, clicks at current position.
+            y: The y coordinate to double click at. If None, clicks at current position.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.mouse import Button
+
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(Button.left, 2)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def click(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """Perform a mouse click with the specified button at the given coordinates.
+
+        Args:
+            x: The x coordinate to click at. If None, clicks at current position.
+            y: The y coordinate to click at. If None, clicks at current position.
+            button: The mouse button to click ("left", "right", or "middle").
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.mouse import Button
+
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
+            self.mouse.click(btn, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def drag_to(
+        self, x: int, y: int, button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        """Drag from the current position to the specified coordinates.
+
+        Args:
+            x: The x coordinate to drag to.
+            y: The y coordinate to drag to.
+            button: The mouse button to use for dragging.
+            duration: The time in seconds to take for the drag operation.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.mouse import Button
+
+            btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
+            self.mouse.press(btn)
+            self.mouse.position = (x, y)
+            self.mouse.release(btn)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def drag(
+        self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        """Drag along a path defined by a list of coordinates.
+
+        Args:
+            path: A list of (x, y) coordinate tuples defining the drag path.
+            button: The mouse button to use for dragging.
+            duration: The time in seconds to take for each segment of the drag.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.mouse import Button
+
+            if not path:
+                return {"success": False, "error": "Path is empty"}
+            btn = getattr(Button, button if button in ["left", "right", "middle"] else "left")
+            self.mouse.position = path[0]
+            for x, y in path[1:]:
+                self.mouse.press(btn)
+                self.mouse.position = (x, y)
+                self.mouse.release(btn)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Keyboard Actions
+    async def key_down(self, key: str) -> Dict[str, Any]:
+        """Press and hold a key.
+
+        Args:
+            key: The key to press down.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.keyboard import Key
+
+            k = getattr(Key, key) if hasattr(Key, key) else (key if len(key) == 1 else None)
+            if k is None:
+                return {"success": False, "error": f"Unknown key: {key}"}
+            self.keyboard.press(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def key_up(self, key: str) -> Dict[str, Any]:
+        """Release a key.
+
+        Args:
+            key: The key to release.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.keyboard import Key
+
+            k = getattr(Key, key) if hasattr(Key, key) else (key if len(key) == 1 else None)
+            if k is None:
+                return {"success": False, "error": f"Unknown key: {key}"}
+            self.keyboard.release(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def type_text(self, text: str) -> Dict[str, Any]:
+        """Type the specified text using the keyboard.
+
+        Args:
+            text: The text to type.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            # use pynput for Unicode support
+            self.keyboard.type(text)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def press_key(self, key: str) -> Dict[str, Any]:
+        """Press and release a key.
+
+        Args:
+            key: The key to press.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.keyboard import Key
+
+            k = getattr(Key, key) if hasattr(Key, key) else (key if len(key) == 1 else None)
+            if k is None:
+                return {"success": False, "error": f"Unknown key: {key}"}
+            self.keyboard.press(k)
+            self.keyboard.release(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
+        """Press a combination of keys simultaneously.
+
+        Args:
+            keys: A list of keys to press together as a hotkey combination.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            from pynput.keyboard import Key
+
+            seq = []
+            for k in keys:
+                kk = getattr(Key, k) if hasattr(Key, k) else (k if len(k) == 1 else None)
+                if kk is None:
+                    return {"success": False, "error": f"Unknown key in hotkey: {k}"}
+                seq.append(kk)
+            for k in seq[:-1]:
+                self.keyboard.press(k)
+            last = seq[-1]
+            self.keyboard.press(last)
+            self.keyboard.release(last)
+            for k in reversed(seq[:-1]):
+                self.keyboard.release(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Scrolling Actions
+    async def scroll(self, x: int, y: int) -> Dict[str, Any]:
+        """Scroll the mouse wheel.
+
+        Args:
+            x: The horizontal scroll amount.
+            y: The vertical scroll amount.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            self.mouse.scroll(x, y)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
+        """Scroll down by the specified number of clicks.
+
+        Args:
+            clicks: The number of scroll clicks to perform downward.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            self.mouse.scroll(0, -abs(clicks))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
+        """Scroll up by the specified number of clicks.
+
+        Args:
+            clicks: The number of scroll clicks to perform upward.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and error message if failed.
+        """
+        try:
+            self.mouse.scroll(0, abs(clicks))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Screen Actions
+    async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
+        """Take a screenshot of the current screen.
+
+        Args:
+            format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).
+            quality: JPEG quality 1-95 (clamped); ignored for PNG.
+        """
+        try:
+            fmt, quality = normalize_screenshot_format(format, quality)
+        except ValueError as e:
+            return {"success": False, "error": str(e)}
+        try:
+            screenshot = ImageGrab.grab()
+            if not isinstance(screenshot, Image.Image):
+                return {"success": False, "error": "Failed to capture screenshot"}
+            buffered = BytesIO()
+            if fmt == "jpeg":
+                screenshot.convert("RGB").save(
+                    buffered, format="JPEG", quality=quality, optimize=True
+                )
+            else:
+                screenshot.save(buffered, format="PNG", optimize=True)
+            buffered.seek(0)
+            image_data = base64.b64encode(buffered.getvalue()).decode()
+            return {"success": True, "image_data": image_data, "format": fmt}
+        except Exception as e:
+            return {"success": False, "error": f"Screenshot error: {str(e)}"}
+
+    async def get_screen_size(self) -> Dict[str, Any]:
+        """Get the size of the screen.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing success status and screen dimensions,
+                           or error message if failed.
+        """
+        try:
+            img = ImageGrab.grab()
+            return {"success": True, "size": {"width": img.width, "height": img.height}}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_cursor_position(self) -> Dict[str, Any]:
+        """Get the current position of the cursor.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing success status and cursor coordinates,
+                           or error message if failed.
+        """
+        try:
+            from pynput.mouse import Controller as MouseController
+
+            m = MouseController()
+            x, y = m.position
+            return {"success": True, "position": {"x": int(x), "y": int(y)}}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Clipboard and run_command inherited from BaseAutomationHandler

libs/python/computer-server/computer_server/handlers/vnc.py

@@ -0,0 +1,598 @@
+"""
+VNC backend for automation and accessibility handlers.
+
+A cross-platform alternative to the native (OS-specific) handlers. Connects to
+any VNC server to perform screenshots, mouse, keyboard, and scroll operations
+over the RFB protocol. Works with Linux, macOS, and Windows targets — the only
+requirement is a reachable VNC server.
+
+Built on vncdotool (Twisted-based RFB client). The Twisted reactor runs in a
+background daemon thread; all handler methods bridge from asyncio via
+`asyncio.to_thread`.
+
+Usage:
+    # Via CLI
+    python -m computer_server --vnc-host 192.168.64.1 --vnc-port 5900 --vnc-password secret
+
+    # Via env vars
+    CUA_VNC_HOST=192.168.64.1 CUA_VNC_PORT=5900 CUA_VNC_PASSWORD=secret python -m computer_server
+"""
+
+import asyncio
+import base64
+import logging
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+from .base import BaseAccessibilityHandler, BaseAutomationHandler
+
+logger = logging.getLogger(__name__)
+
+# Key name aliases: map cua-computer-server key names → vncdotool key names.
+# vncdotool's KEYMAP already covers most X11 keysym names; these bridge
+# the naming differences used by the rest of the computer-server API.
+_KEY_ALIASES = {
+    "return": "enter",
+    "escape": "esc",
+    "backspace": "bsp",
+    "delete": "del",
+    "page_up": "pgup",
+    "pageup": "pgup",
+    "page_down": "pgdn",
+    "pagedown": "pgdn",
+    "insert": "ins",
+    "caps_lock": "caplk",
+    "num_lock": "numlk",
+    "shift_l": "lshift",
+    "shift_r": "rshift",
+    "ctrl_l": "lctrl",
+    "ctrl_r": "rctrl",
+    "control": "ctrl",
+    "control_l": "lctrl",
+    "control_r": "rctrl",
+    "alt_l": "lalt",
+    "alt_r": "ralt",
+    "meta_l": "lmeta",
+    "meta_r": "rmeta",
+    "super_l": "lsuper",
+    "super_r": "rsuper",
+    # macOS-specific names
+    "command": "alt",
+    "cmd": "alt",
+    "option": "meta",
+    "option_l": "lmeta",
+    "option_r": "rmeta",
+}
+
+# Button name → vncdotool button number (1-indexed)
+_BUTTON_MAP = {"left": 1, "middle": 2, "right": 3}
+
+
+def _translate_key(key: str) -> str:
+    """Translate a key name to vncdotool's KEYMAP name."""
+    lower = key.lower()
+    return _KEY_ALIASES.get(lower, lower)
+
+
+# ---------------------------------------------------------------------------
+# Lazy vncdotool client wrapper
+# ---------------------------------------------------------------------------
+
+
+class _VNCConnection:
+    """Manages vncdotool client connections to a VNC server.
+
+    Uses fresh connections per operation to avoid a bug in vncdotool's
+    ThreadedVNCClientProxy where the Twisted deferred chain corrupts
+    coordinate state after operations like refreshScreen.  Each public
+    method creates its own connection, executes, and disconnects.
+
+    All methods are synchronous (blocking) — the handler calls them via
+    asyncio.to_thread to avoid blocking the event loop.
+    """
+
+    def __init__(self, host: str, port: int, password: str = ""):
+        self._host = host
+        self._port = port
+        self._password = password
+        # Track cursor position locally
+        self._cursor_x: int = 0
+        self._cursor_y: int = 0
+
+    def _with_client(self, fn):
+        """Execute fn(client) with a fresh VNC connection via the global Twisted reactor.
+
+        Starts the reactor in a background thread on first use, then reuses it.
+        ``fn(client)`` receives a raw ``VNCDoToolClient`` whose methods return
+        Twisted Deferreds.
+        """
+        import threading
+
+        from twisted.internet import defer, reactor
+        from vncdotool.client import VNCDoToolFactory
+
+        result_holder: list = [None]
+        error_holder: list = [None]
+        done_event = threading.Event()
+
+        def _work():
+            factory = VNCDoToolFactory()
+            factory.password = self._password or None
+
+            @defer.inlineCallbacks
+            def _do():
+                client = None
+                try:
+                    reactor.connectTCP(self._host, self._port, factory)
+                    client = yield factory.deferred
+                    res = yield defer.maybeDeferred(fn, client)
+                    result_holder[0] = res
+                except Exception as e:
+                    error_holder[0] = e
+                finally:
+                    if client is not None and hasattr(client, "transport") and client.transport:
+                        client.transport.loseConnection()
+                    done_event.set()
+
+            _do()
+
+        # Ensure the reactor is running in a background thread
+        if not reactor.running:
+            t = threading.Thread(
+                target=reactor.run,
+                kwargs={"installSignalHandlers": False},
+                daemon=True,
+            )
+            t.start()
+
+        reactor.callFromThread(_work)
+        done_event.wait(timeout=30)
+        if not done_event.is_set():
+            raise TimeoutError("VNC operation timed out")
+        if error_holder[0] is not None:
+            raise error_holder[0]
+        return result_holder[0]
+
+    def disconnect(self):
+        pass  # No persistent connection to close
+
+    def _reset_on_error(self):
+        pass  # No persistent connection to reset
+
+    # -- Screenshot ---------------------------------------------------------
+
+    def capture_screenshot(self) -> bytes:
+        """Capture the screen and return PNG bytes."""
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            yield client.refreshScreen(incremental=False)
+            screen = client.screen
+            buf = BytesIO()
+            screen.save(buf, format="PNG")
+            defer.returnValue(buf.getvalue())
+
+        return self._with_client(_do)
+
+    # -- Mouse --------------------------------------------------------------
+
+    def mouse_move(self, x: int, y: int):
+        self._with_client(lambda c: c.mouseMove(x, y))
+        self._cursor_x = x
+        self._cursor_y = y
+
+    def mouse_click(self, x: int, y: int, button: int = 1, clicks: int = 1):
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            yield client.mouseMove(x, y)
+            for _ in range(clicks):
+                yield client.mousePress(button)
+
+        self._with_client(_do)
+        self._cursor_x = x
+        self._cursor_y = y
+
+    def mouse_down(self, x: int, y: int, button: int = 1):
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            yield client.mouseMove(x, y)
+            yield client.mouseDown(button)
+
+        self._with_client(_do)
+        self._cursor_x = x
+        self._cursor_y = y
+
+    def mouse_up(self, x: int, y: int, button: int = 1):
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            yield client.mouseMove(x, y)
+            yield client.mouseUp(button)
+
+        self._with_client(_do)
+        self._cursor_x = x
+        self._cursor_y = y
+
+    def mouse_drag(self, x: int, y: int, step: int = 1):
+        from twisted.internet import defer
+
+        sx, sy = self._cursor_x, self._cursor_y
+
+        @defer.inlineCallbacks
+        def _do(client):
+            # Move in increments from current position to target
+            dx, dy = x - sx, y - sy
+            steps = max(abs(dx), abs(dy)) // max(step, 1)
+            steps = max(steps, 1)
+            yield client.mouseDown(1)
+            for i in range(1, steps + 1):
+                ix = sx + dx * i // steps
+                iy = sy + dy * i // steps
+                yield client.mouseMove(ix, iy)
+            yield client.mouseUp(1)
+
+        self._with_client(_do)
+        self._cursor_x = x
+        self._cursor_y = y
+
+    def drag_to(
+        self, start_x: int, start_y: int, end_x: int, end_y: int, button: int = 1, step: int = 5
+    ):
+        """Drag from start to end on a single connection."""
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            yield client.mouseMove(start_x, start_y)
+            yield client.mouseDown(button)
+            # Manual drag in increments instead of mouseDrag (avoids doPoll)
+            dx, dy = end_x - start_x, end_y - start_y
+            steps = max(abs(dx), abs(dy)) // max(step, 1)
+            steps = max(steps, 1)
+            for i in range(1, steps + 1):
+                ix = start_x + dx * i // steps
+                iy = start_y + dy * i // steps
+                yield client.mouseMove(ix, iy)
+            yield client.mouseUp(button)
+
+        self._with_client(_do)
+        self._cursor_x = end_x
+        self._cursor_y = end_y
+
+    def drag_path(self, path: List[Tuple[int, int]], button: int = 1):
+        """Drag along a path of points on a single connection."""
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            yield client.mouseMove(path[0][0], path[0][1])
+            yield client.mouseDown(button)
+            for px, py in path[1:]:
+                yield client.mouseMove(px, py)
+            yield client.mouseUp(button)
+
+        self._with_client(_do)
+        self._cursor_x = path[-1][0]
+        self._cursor_y = path[-1][1]
+
+    def scroll(self, x: int, y: int):
+        """Scroll. y>0 = up, y<0 = down (matches macOS native handler convention).
+
+        Uses arrow key presses instead of mouse buttons 4/5 because Apple's
+        _VZVNCServer (used by Lume) does not translate RFB mouse buttons 4-7
+        into scroll wheel events.
+        """
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            if y > 0:
+                for _ in range(y):
+                    yield client.keyPress("up")
+            elif y < 0:
+                for _ in range(abs(y)):
+                    yield client.keyPress("down")
+            if x > 0:
+                for _ in range(x):
+                    yield client.keyPress("right")
+            elif x < 0:
+                for _ in range(abs(x)):
+                    yield client.keyPress("left")
+
+        self._with_client(_do)
+
+    # -- Keyboard -----------------------------------------------------------
+
+    def key_press(self, key: str):
+        self._with_client(lambda c: c.keyPress(_translate_key(key)))
+
+    def key_down(self, key: str):
+        self._with_client(lambda c: c.keyDown(_translate_key(key)))
+
+    def key_up(self, key: str):
+        self._with_client(lambda c: c.keyUp(_translate_key(key)))
+
+    def type_text(self, text: str):
+        """Type text character by character, handling shift for uppercase/symbols."""
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            for ch in text:
+                if ch == " ":
+                    yield client.keyPress("space")
+                elif ch == "\n":
+                    yield client.keyPress("enter")
+                elif ch == "\t":
+                    yield client.keyPress("tab")
+                else:
+                    yield client.keyPress(ch)
+
+        self._with_client(_do)
+
+    def hotkey(self, keys: List[str]):
+        """Press a key combination (e.g. ['command', 'a'])."""
+        translated = [_translate_key(k) for k in keys]
+        combo = "-".join(translated)
+        self._with_client(lambda c: c.keyPress(combo))
+
+    # -- Clipboard ----------------------------------------------------------
+
+    def paste_text(self, text: str):
+        self._with_client(lambda c: c.paste(text))
+
+    # -- Info ---------------------------------------------------------------
+
+    @property
+    def screen_size(self) -> Tuple[int, int]:
+        from twisted.internet import defer
+
+        @defer.inlineCallbacks
+        def _do(client):
+            yield client.refreshScreen(incremental=False)
+            if client.screen is not None:
+                defer.returnValue(client.screen.size)
+            defer.returnValue((0, 0))
+
+        return self._with_client(_do)
+
+    @property
+    def cursor_position(self) -> Tuple[int, int]:
+        return self._cursor_x, self._cursor_y
+
+
+# ---------------------------------------------------------------------------
+# Automation Handler
+# ---------------------------------------------------------------------------
+
+
+class VNCAutomationHandler(BaseAutomationHandler):
+    """Cross-platform automation handler that operates via VNC/RFB protocol.
+
+    Works with any OS target (Linux, macOS, Windows) that has a VNC server.
+    All operations go through the network, bypassing local permission systems.
+    """
+
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 5900,
+        password: str = "",
+    ):
+        self._conn = _VNCConnection(host, port, password)
+
+    def _resolve_coords(self, x: Optional[int], y: Optional[int]) -> Tuple[int, int]:
+        if x is not None and y is not None:
+            return x, y
+        return self._conn.cursor_position
+
+    # -- Screenshot ---------------------------------------------------------
+
+    async def screenshot(self) -> Dict[str, Any]:
+        try:
+            png_bytes = await asyncio.to_thread(self._conn.capture_screenshot)
+            image_data = base64.b64encode(png_bytes).decode()
+            return {"success": True, "image_data": image_data}
+        except Exception as e:
+            logger.error(f"VNC screenshot error: {e}")
+            self._conn._reset_on_error()
+            return {"success": False, "error": f"VNC screenshot error: {e}"}
+
+    # -- Mouse actions ------------------------------------------------------
+
+    async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        try:
+            cx, cy = self._resolve_coords(x, y)
+            await asyncio.to_thread(self._conn.mouse_click, cx, cy, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        try:
+            cx, cy = self._resolve_coords(x, y)
+            await asyncio.to_thread(self._conn.mouse_click, cx, cy, 3)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def middle_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        try:
+            cx, cy = self._resolve_coords(x, y)
+            await asyncio.to_thread(self._conn.mouse_click, cx, cy, 2)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def double_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        try:
+            cx, cy = self._resolve_coords(x, y)
+            await asyncio.to_thread(self._conn.mouse_click, cx, cy, 1, 2)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
+        try:
+            await asyncio.to_thread(self._conn.mouse_move, x, y)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def mouse_down(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        try:
+            cx, cy = self._resolve_coords(x, y)
+            btn = _BUTTON_MAP.get(button, 1)
+            await asyncio.to_thread(self._conn.mouse_down, cx, cy, btn)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def mouse_up(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        try:
+            cx, cy = self._resolve_coords(x, y)
+            btn = _BUTTON_MAP.get(button, 1)
+            await asyncio.to_thread(self._conn.mouse_up, cx, cy, btn)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def drag_to(
+        self, x: int, y: int, button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        try:
+            btn = _BUTTON_MAP.get(button, 1)
+            cx, cy = self._conn.cursor_position
+            step = max(1, int(1 / max(duration, 0.01) * 5))
+            await asyncio.to_thread(self._conn.drag_to, cx, cy, x, y, btn, step)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def drag(
+        self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        try:
+            if not path:
+                return {"success": False, "error": "Empty path"}
+            btn = _BUTTON_MAP.get(button, 1)
+            await asyncio.to_thread(self._conn.drag_path, path, btn)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # -- Keyboard actions ---------------------------------------------------
+
+    async def type_text(self, text: str) -> Dict[str, Any]:
+        try:
+            await asyncio.to_thread(self._conn.type_text, text)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def press_key(self, key: str) -> Dict[str, Any]:
+        try:
+            await asyncio.to_thread(self._conn.key_press, key)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def key_down(self, key: str) -> Dict[str, Any]:
+        try:
+            await asyncio.to_thread(self._conn.key_down, key)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def key_up(self, key: str) -> Dict[str, Any]:
+        try:
+            await asyncio.to_thread(self._conn.key_up, key)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
+        try:
+            await asyncio.to_thread(self._conn.hotkey, keys)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # -- Scrolling ----------------------------------------------------------
+
+    async def scroll(self, x: int, y: int) -> Dict[str, Any]:
+        try:
+            await asyncio.to_thread(self._conn.scroll, x, y)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
+        return await self.scroll(0, -clicks)
+
+    async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
+        return await self.scroll(0, clicks)
+
+    # -- Screen info --------------------------------------------------------
+
+    async def get_screen_size(self) -> Dict[str, Any]:
+        try:
+            w, h = await asyncio.to_thread(lambda: self._conn.screen_size)
+            return {"success": True, "size": {"width": w, "height": h}}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_cursor_position(self) -> Dict[str, Any]:
+        try:
+            x, y = self._conn.cursor_position
+            return {"success": True, "position": {"x": x, "y": y}}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Clipboard and run_command inherited from BaseAutomationHandler
+
+
+# ---------------------------------------------------------------------------
+# Accessibility Handler (stub)
+# ---------------------------------------------------------------------------
+
+
+class VNCAccessibilityHandler(BaseAccessibilityHandler):
+    """Stub accessibility handler for VNC — no accessibility tree available."""
+
+    async def get_accessibility_tree(self) -> Dict[str, Any]:
+        return {
+            "success": True,
+            "tree": {
+                "role": "Window",
+                "title": "VNC Remote Desktop",
+                "position": {"x": 0, "y": 0},
+                "size": {"width": 0, "height": 0},
+                "children": [],
+            },
+        }
+
+    async def find_element(
+        self,
+        role: Optional[str] = None,
+        title: Optional[str] = None,
+        value: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        return {
+            "success": False,
+            "error": "Accessibility tree not available over VNC",
+        }

libs/python/computer-server/computer_server/handlers/windows.py

@@ -0,0 +1,772 @@
+"""
+Windows implementation of automation and accessibility handlers.
+
+This implementation uses pynput for GUI automation and Windows-specific APIs
+for accessibility and system operations.
+"""
+
+import asyncio
+import base64
+import functools
+import logging
+import os
+import subprocess
+from io import BytesIO
+from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def require_unlocked_desktop(func: F) -> F:
+    """Decorator that checks if the Windows desktop is locked before executing.
+
+    Returns an error response if the desktop is locked, preventing automation
+    actions that would silently fail on the Windows Secure Desktop.
+    """
+
+    @functools.wraps(func)
+    async def wrapper(
+        self: "WindowsAutomationHandler", *args: Any, **kwargs: Any
+    ) -> Dict[str, Any]:
+        if self.is_desktop_locked():
+            return {
+                "success": False,
+                "error": "Windows desktop is locked. Automation input is blocked by OS security.",
+            }
+        return await func(self, *args, **kwargs)
+
+    return wrapper  # type: ignore[return-value]
+
+
+from PIL import Image, ImageGrab
+from pynput.keyboard import Controller as KeyboardController
+from pynput.keyboard import Key as KBKey
+from pynput.mouse import Button as MouseButton
+from pynput.mouse import Controller as MouseController
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+# pyautogui removed in favor of pynput
+
+# Try to import Windows-specific modules
+try:
+    import win32api
+    import win32con
+    import win32gui
+
+    logger.info("Windows API modules successfully imported")
+    WINDOWS_API_AVAILABLE = True
+except Exception as e:
+    logger.error(
+        f"Windows API modules import failed: {str(e)}. Some Windows-specific features will be unavailable."
+    )
+    WINDOWS_API_AVAILABLE = False
+
+from .base import (
+    BaseAccessibilityHandler,
+    BaseAutomationHandler,
+    normalize_screenshot_format,
+)
+
+
+class WindowsAccessibilityHandler(BaseAccessibilityHandler):
+    """Windows implementation of accessibility handler."""
+
+    async def get_accessibility_tree(self) -> Dict[str, Any]:
+        """Get the accessibility tree of the current window.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the success status and either
+                           the accessibility tree or an error message.
+                           Structure: {"success": bool, "tree": dict} or
+                                    {"success": bool, "error": str}
+        """
+        if not WINDOWS_API_AVAILABLE:
+            return {"success": False, "error": "Windows API not available"}
+
+        try:
+            # Get the foreground window
+            hwnd = win32gui.GetForegroundWindow()
+            if not hwnd:
+                return {"success": False, "error": "No foreground window found"}
+
+            # Get window information
+            window_text = win32gui.GetWindowText(hwnd)
+            rect = win32gui.GetWindowRect(hwnd)
+
+            tree = {
+                "role": "Window",
+                "title": window_text,
+                "position": {"x": rect[0], "y": rect[1]},
+                "size": {"width": rect[2] - rect[0], "height": rect[3] - rect[1]},
+                "children": [],
+            }
+
+            # Enumerate child windows
+            def enum_child_proc(hwnd_child, children_list):
+                """Callback function to enumerate child windows and collect their information.
+
+                Args:
+                    hwnd_child: Handle to the child window being enumerated.
+                    children_list: List to append child window information to.
+
+                Returns:
+                    bool: True to continue enumeration, False to stop.
+                """
+                try:
+                    child_text = win32gui.GetWindowText(hwnd_child)
+                    child_rect = win32gui.GetWindowRect(hwnd_child)
+                    child_class = win32gui.GetClassName(hwnd_child)
+
+                    child_info = {
+                        "role": child_class,
+                        "title": child_text,
+                        "position": {"x": child_rect[0], "y": child_rect[1]},
+                        "size": {
+                            "width": child_rect[2] - child_rect[0],
+                            "height": child_rect[3] - child_rect[1],
+                        },
+                        "children": [],
+                    }
+                    children_list.append(child_info)
+                except Exception as e:
+                    logger.debug(f"Error getting child window info: {e}")
+                return True
+
+            win32gui.EnumChildWindows(hwnd, enum_child_proc, tree["children"])
+
+            return {"success": True, "tree": tree}
+
+        except Exception as e:
+            logger.error(f"Error getting accessibility tree: {e}")
+            return {"success": False, "error": str(e)}
+
+    async def find_element(
+        self, role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Find an element in the accessibility tree by criteria.
+
+        Args:
+            role (Optional[str]): The role or class name of the element to find.
+            title (Optional[str]): The title or text of the element to find.
+            value (Optional[str]): The value of the element (not used in Windows implementation).
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the success status and either
+                           the found element or an error message.
+                           Structure: {"success": bool, "element": dict} or
+                                    {"success": bool, "error": str}
+        """
+        if not WINDOWS_API_AVAILABLE:
+            return {"success": False, "error": "Windows API not available"}
+
+        try:
+            # Find window by title if specified
+            if title:
+                hwnd = win32gui.FindWindow(None, title)
+                if hwnd:
+                    rect = win32gui.GetWindowRect(hwnd)
+                    return {
+                        "success": True,
+                        "element": {
+                            "role": "Window",
+                            "title": title,
+                            "position": {"x": rect[0], "y": rect[1]},
+                            "size": {"width": rect[2] - rect[0], "height": rect[3] - rect[1]},
+                        },
+                    }
+
+            # Find window by class name if role is specified
+            if role:
+                hwnd = win32gui.FindWindow(role, None)
+                if hwnd:
+                    window_text = win32gui.GetWindowText(hwnd)
+                    rect = win32gui.GetWindowRect(hwnd)
+                    return {
+                        "success": True,
+                        "element": {
+                            "role": role,
+                            "title": window_text,
+                            "position": {"x": rect[0], "y": rect[1]},
+                            "size": {"width": rect[2] - rect[0], "height": rect[3] - rect[1]},
+                        },
+                    }
+
+            return {"success": False, "error": "Element not found"}
+
+        except Exception as e:
+            logger.error(f"Error finding element: {e}")
+            return {"success": False, "error": str(e)}
+
+
+class WindowsAutomationHandler(BaseAutomationHandler):
+    """Windows implementation of automation handler using pynput and Windows APIs."""
+
+    mouse = MouseController()
+    keyboard = KeyboardController()
+
+    def is_desktop_locked(self) -> bool:
+        try:
+            import ctypes
+
+            user32 = ctypes.windll.user32
+            hwnd = user32.GetForegroundWindow()
+            return hwnd == 0
+        except Exception:
+            return False
+
+    def _map_button(self, button: str) -> MouseButton:
+        """Map a string button name to pynput MouseButton."""
+        b = (button or "left").lower()
+        if b == "left":
+            return MouseButton.left
+        if b == "right":
+            return MouseButton.right
+        if b == "middle":
+            return MouseButton.middle
+        # default to left
+        return MouseButton.left
+
+    def _key_from_string(self, key: str):
+        """Convert a key string (e.g., 'enter', 'ctrl', 'a') to pynput Key or char."""
+        if not key:
+            return None
+        lk = key.lower()
+        special = {
+            "enter": KBKey.enter,
+            "return": KBKey.enter,
+            "esc": KBKey.esc,
+            "escape": KBKey.esc,
+            "space": KBKey.space,
+            "tab": KBKey.tab,
+            "backspace": KBKey.backspace,
+            "delete": KBKey.delete,
+            "home": KBKey.home,
+            "end": KBKey.end,
+            "pageup": KBKey.page_up,
+            "pagedown": KBKey.page_down,
+            "up": KBKey.up,
+            "down": KBKey.down,
+            "left": KBKey.left,
+            "right": KBKey.right,
+            "shift": KBKey.shift,
+            "ctrl": KBKey.ctrl,
+            "control": KBKey.ctrl,
+            "alt": KBKey.alt,
+            "cmd": KBKey.cmd,
+            "win": KBKey.cmd,
+            "meta": KBKey.cmd,
+            "capslock": KBKey.caps_lock,
+            "f1": KBKey.f1,
+            "f2": KBKey.f2,
+            "f3": KBKey.f3,
+            "f4": KBKey.f4,
+            "f5": KBKey.f5,
+            "f6": KBKey.f6,
+            "f7": KBKey.f7,
+            "f8": KBKey.f8,
+            "f9": KBKey.f9,
+            "f10": KBKey.f10,
+            "f11": KBKey.f11,
+            "f12": KBKey.f12,
+        }
+        if lk in special:
+            return special[lk]
+        # single character
+        if len(key) == 1:
+            return key
+        return None
+
+    # Mouse Actions
+    @require_unlocked_desktop
+    async def mouse_down(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """Press and hold a mouse button at the specified coordinates.
+
+        Args:
+            x (Optional[int]): The x-coordinate to move to before pressing. If None, uses current position.
+            y (Optional[int]): The y-coordinate to move to before pressing. If None, uses current position.
+            button (str): The mouse button to press ("left", "right", or "middle").
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.press(self._map_button(button))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def mouse_up(
+        self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """Release a mouse button at the specified coordinates.
+
+        Args:
+            x (Optional[int]): The x-coordinate to move to before releasing. If None, uses current position.
+            y (Optional[int]): The y-coordinate to move to before releasing. If None, uses current position.
+            button (str): The mouse button to release ("left", "right", or "middle").
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.release(self._map_button(button))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
+        """Move the mouse cursor to the specified coordinates.
+
+        Args:
+            x (int): The x-coordinate to move to.
+            y (int): The y-coordinate to move to.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            self.mouse.position = (x, y)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        """Perform a left mouse click at the specified coordinates.
+
+        Args:
+            x (Optional[int]): The x-coordinate to click at. If None, clicks at current position.
+            y (Optional[int]): The y-coordinate to click at. If None, clicks at current position.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(MouseButton.left, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+        """Perform a right mouse click at the specified coordinates.
+
+        Args:
+            x (Optional[int]): The x-coordinate to click at. If None, clicks at current position.
+            y (Optional[int]): The y-coordinate to click at. If None, clicks at current position.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(MouseButton.right, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def middle_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Perform a middle mouse click at the specified coordinates.
+
+        Args:
+            x (Optional[int]): The x-coordinate to click at. If None, clicks at current position.
+            y (Optional[int]): The y-coordinate to click at. If None, clicks at current position.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(MouseButton.middle, 1)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def double_click(
+        self, x: Optional[int] = None, y: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Perform a double left mouse click at the specified coordinates.
+
+        Args:
+            x (Optional[int]): The x-coordinate to double-click at. If None, clicks at current position.
+            y (Optional[int]): The y-coordinate to double-click at. If None, clicks at current position.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            if x is not None and y is not None:
+                self.mouse.position = (x, y)
+            self.mouse.click(MouseButton.left, 2)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def drag_to(
+        self, x: int, y: int, button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        """Drag from the current position to the specified coordinates.
+
+        Args:
+            x (int): The x-coordinate to drag to.
+            y (int): The y-coordinate to drag to.
+            button (str): The mouse button to use for dragging ("left", "right", or "middle").
+            duration (float): The time in seconds to take for the drag operation.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            # simple drag implementation
+            self.mouse.press(self._map_button(button))
+            self.mouse.position = (x, y)
+            self.mouse.release(self._map_button(button))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def drag(
+        self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5
+    ) -> Dict[str, Any]:
+        """Drag the mouse through a series of coordinates.
+
+        Args:
+            path (List[Tuple[int, int]]): A list of (x, y) coordinate tuples to drag through.
+            button (str): The mouse button to use for dragging ("left", "right", or "middle").
+            duration (float): The total time in seconds for the entire drag operation.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            if not path:
+                return {"success": False, "error": "Path is empty"}
+
+            # Move to first position
+            self.mouse.position = path[0]
+
+            # Drag through all positions
+            for x, y in path[1:]:
+                self.mouse.press(self._map_button(button))
+                self.mouse.position = (x, y)
+                self.mouse.release(self._map_button(button))
+
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Keyboard Actions
+    @require_unlocked_desktop
+    async def key_down(self, key: str) -> Dict[str, Any]:
+        """Press and hold a keyboard key.
+
+        Args:
+            key (str): The key to press down (e.g., 'ctrl', 'shift', 'a').
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            k = self._key_from_string(key)
+            if k is None:
+                return {"success": False, "error": f"Unknown key: {key}"}
+            self.keyboard.press(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def key_up(self, key: str) -> Dict[str, Any]:
+        """Release a keyboard key.
+
+        Args:
+            key (str): The key to release (e.g., 'ctrl', 'shift', 'a').
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            k = self._key_from_string(key)
+            if k is None:
+                return {"success": False, "error": f"Unknown key: {key}"}
+            self.keyboard.release(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def type_text(self, text: str) -> Dict[str, Any]:
+        """Type the specified text.
+
+        Args:
+            text (str): The text to type.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            # use pynput for Unicode support
+            self.keyboard.type(text)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def press_key(self, key: str) -> Dict[str, Any]:
+        """Press and release a keyboard key.
+
+        Args:
+            key (str): The key to press (e.g., 'enter', 'space', 'tab').
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            k = self._key_from_string(key)
+            if k is None:
+                return {"success": False, "error": f"Unknown key: {key}"}
+            self.keyboard.press(k)
+            self.keyboard.release(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
+        """Press a combination of keys simultaneously.
+
+        Args:
+            keys (List[str]): The keys to press together (e.g., ['ctrl', 'c'], ['alt', 'tab']).
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            # press keys sequentially while holding modifiers
+            resolved = [self._key_from_string(k) for k in keys]
+            if any(k is None for k in resolved):
+                return {"success": False, "error": "Unknown key in hotkey sequence"}
+            seq: List[Union[str, KBKey]] = [k for k in resolved if k is not None]  # type: ignore[assignment]
+            if not seq:
+                return {"success": False, "error": "Empty hotkey sequence"}
+            # hold all except the last
+            for k in seq[:-1]:
+                self.keyboard.press(k)
+            # tap last
+            last = seq[-1]
+            self.keyboard.press(last)
+            self.keyboard.release(last)
+            # release modifiers
+            for k in reversed(seq[:-1]):
+                self.keyboard.release(k)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Scrolling Actions
+    @require_unlocked_desktop
+    async def scroll(self, x: int, y: int) -> Dict[str, Any]:
+        """Scroll vertically at the current cursor position.
+
+        Args:
+            x (int): Horizontal scroll amount.
+            y (int): Vertical scroll amount. Positive values scroll up, negative values scroll down.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            self.mouse.scroll(x, y)
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
+        """Scroll down by the specified number of clicks.
+
+        Args:
+            clicks (int): The number of scroll clicks to perform downward.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            # negative y to scroll down
+            self.mouse.scroll(0, -abs(clicks))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    @require_unlocked_desktop
+    async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
+        """Scroll up by the specified number of clicks.
+
+        Args:
+            clicks (int): The number of scroll clicks to perform upward.
+
+        Returns:
+            Dict[str, Any]: A dictionary with success status and optional error message.
+        """
+        try:
+            self.mouse.scroll(0, abs(clicks))
+            return {"success": True}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Screen Actions
+    @require_unlocked_desktop
+    async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
+        """Capture a screenshot of the entire screen.
+
+        Args:
+            format: "png" (lossless, default), "jpeg" or "jpg" (lossy, smaller).
+            quality: JPEG quality 1-95 (clamped); ignored for PNG.
+        """
+        try:
+            fmt, quality = normalize_screenshot_format(format, quality)
+        except ValueError as e:
+            return {"success": False, "error": str(e)}
+        try:
+            screenshot = ImageGrab.grab()
+            if not isinstance(screenshot, Image.Image):
+                return {"success": False, "error": "Failed to capture screenshot"}
+
+            buffered = BytesIO()
+            if fmt == "jpeg":
+                screenshot.convert("RGB").save(
+                    buffered, format="JPEG", quality=quality, optimize=True
+                )
+            else:
+                screenshot.save(buffered, format="PNG", optimize=True)
+            buffered.seek(0)
+            image_data = base64.b64encode(buffered.getvalue()).decode()
+            return {"success": True, "image_data": image_data, "format": fmt}
+        except Exception as e:
+            return {"success": False, "error": f"Screenshot error: {str(e)}"}
+
+    async def get_screen_size(self) -> Dict[str, Any]:
+        """Get the size of the screen in pixels.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the success status and either
+                           screen size information or an error message.
+                           Structure: {"success": bool, "size": {"width": int, "height": int}} or
+                                    {"success": bool, "error": str}
+        """
+        try:
+            if WINDOWS_API_AVAILABLE:
+                width = win32api.GetSystemMetrics(win32con.SM_CXSCREEN)
+                height = win32api.GetSystemMetrics(win32con.SM_CYSCREEN)
+                return {"success": True, "size": {"width": width, "height": height}}
+            else:
+                # Fallback: use ImageGrab
+                img = ImageGrab.grab()
+                return {"success": True, "size": {"width": img.width, "height": img.height}}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_cursor_position(self) -> Dict[str, Any]:
+        """Get the current position of the mouse cursor.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the success status and either
+                           cursor position or an error message.
+                           Structure: {"success": bool, "position": {"x": int, "y": int}} or
+                                    {"success": bool, "error": str}
+        """
+        try:
+            if WINDOWS_API_AVAILABLE:
+                pos = win32gui.GetCursorPos()
+                return {"success": True, "position": {"x": pos[0], "y": pos[1]}}
+            else:
+                # Fallback: use pynput controller
+                x, y = self.mouse.position
+                return {"success": True, "position": {"x": int(x), "y": int(y)}}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # Clipboard inherited from BaseAutomationHandler
+
+    # Command Execution (Windows override for multi-encoding support)
+    async def run_command(self, command: str, timeout: Optional[float] = None) -> Dict[str, Any]:
+        """Execute a shell command asynchronously.
+
+        Args:
+            command (str): The shell command to execute.
+            timeout (Optional[float]): Optional timeout in seconds.  When
+                ``None`` (default), waits indefinitely.  SDK callers set
+                this via ``sb.shell.run(cmd, timeout=...)``.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the success status and either
+                           command output or an error message.
+                           Structure: {"success": bool, "stdout": str, "stderr": str, "return_code": int} or
+                                    {"success": bool, "error": str}
+        """
+
+        def decode_output(data: bytes) -> str:
+            if not data:
+                return ""
+            encodings = ["utf-8", "gbk", "gb2312", "cp936", "latin1"]
+            for enc in encodings:
+                try:
+                    return data.decode(enc)
+                except (UnicodeDecodeError, LookupError):
+                    continue
+            return data.decode("utf-8", errors="replace")
+
+        try:
+            if os.environ.get("IS_CUA_ANDROID") == "true":
+                process = await asyncio.create_subprocess_exec(
+                    "adb",
+                    "shell",
+                    command,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                )
+            else:
+                process = await asyncio.create_subprocess_shell(
+                    command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+                )
+            try:
+                if timeout is None:
+                    stdout, stderr = await process.communicate()
+                else:
+                    stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
+            except asyncio.TimeoutError:
+                process.kill()
+                return {
+                    "success": False,
+                    "stdout": "",
+                    "stderr": f"Command timed out after {timeout}s",
+                    "return_code": -1,
+                }
+            return {
+                "success": True,
+                "stdout": decode_output(stdout),
+                "stderr": decode_output(stderr),
+                "return_code": process.returncode,
+            }
+        except Exception as e:
+            return {"success": False, "error": str(e)}

libs/python/computer-server/computer_server/mcp_server.py

@@ -0,0 +1,846 @@
+"""
+MCP (Model Context Protocol) server interface for Computer Server.
+
+This module provides an MCP interface to the Computer Server's functionality,
+enabling Claude Code and other MCP-compatible tools to directly control computers.
+
+Usage:
+    python -m computer_server --mcp
+    python -m computer_server --mcp --width 1512 --height 982
+    python -m computer_server --mcp --detect-resolution
+"""
+
+import base64
+import logging
+import sys
+from typing import Any, Dict, List, Optional, Tuple
+
+from fastmcp import FastMCP
+from fastmcp.utilities.types import Image
+
+logger = logging.getLogger(__name__)
+
+
+# Lazy handler initialization to avoid crashes during import
+_handlers = None
+
+# Resolution scaling configuration
+_scale_x: float = 1.0
+_scale_y: float = 1.0
+_target_width: Optional[int] = None
+_target_height: Optional[int] = None
+_actual_width: Optional[int] = None
+_actual_height: Optional[int] = None
+
+
+def _get_handlers():
+    """Lazily initialize handlers on first use."""
+    global _handlers
+    if _handlers is None:
+        from .handlers.factory import HandlerFactory
+
+        _handlers = HandlerFactory.create_handlers()
+    return _handlers
+
+
+async def _detect_actual_resolution_async() -> Tuple[int, int]:
+    """Detect the actual screen resolution using the automation handler."""
+    try:
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        result = await automation_handler.get_screen_size()
+        return result["width"], result["height"]
+    except Exception as e:
+        logger.warning(f"Failed to detect resolution via handler: {e}")
+        return 1920, 1080  # Default fallback
+
+
+def _detect_actual_resolution() -> Tuple[int, int]:
+    """Detect the actual screen resolution (sync wrapper)."""
+    import asyncio
+
+    try:
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # If called from async context, can't use run_until_complete
+            # Return default and let async init handle it
+            return 1920, 1080
+        return loop.run_until_complete(_detect_actual_resolution_async())
+    except Exception as e:
+        logger.warning(f"Failed to detect resolution: {e}")
+        return 1920, 1080  # Default fallback
+
+
+def _configure_scaling(
+    target_width: Optional[int] = None,
+    target_height: Optional[int] = None,
+    detect_resolution: bool = False,
+) -> None:
+    """Configure resolution scaling based on target and actual resolution."""
+    global _scale_x, _scale_y, _target_width, _target_height, _actual_width, _actual_height
+
+    # Detect actual resolution
+    _actual_width, _actual_height = _detect_actual_resolution()
+    logger.info(f"Detected actual screen resolution: {_actual_width}x{_actual_height}")
+
+    if target_width and target_height:
+        _target_width = target_width
+        _target_height = target_height
+        _scale_x = _actual_width / target_width
+        _scale_y = _actual_height / target_height
+        logger.info(
+            f"Target resolution: {target_width}x{target_height}, "
+            f"Scale factors: x={_scale_x:.3f}, y={_scale_y:.3f}"
+        )
+    else:
+        # No target specified or detect-resolution mode, use actual resolution
+        _target_width = _actual_width
+        _target_height = _actual_height
+        _scale_x = 1.0
+        _scale_y = 1.0
+        if detect_resolution:
+            logger.info("Resolution detection enabled - coordinates will use actual resolution")
+
+
+def _scale_to_actual(x: int, y: int) -> Tuple[int, int]:
+    """Scale coordinates from target resolution to actual resolution."""
+    return int(x * _scale_x), int(y * _scale_y)
+
+
+def _scale_to_target(x: int, y: int) -> Tuple[int, int]:
+    """Scale coordinates from actual resolution to target resolution."""
+    if _scale_x == 0 or _scale_y == 0:
+        return x, y
+    return int(x / _scale_x), int(y / _scale_y)
+
+
+def create_mcp_server() -> FastMCP:
+    """
+    Create and configure the MCP server with all computer control tools.
+
+    Returns:
+        FastMCP: Configured MCP server instance
+    """
+    mcp = FastMCP(
+        name="cua-computer-server",
+        instructions="""You are connected to a computer control server that provides low-level
+        primitives for interacting with a desktop computer. You can take screenshots, click,
+        type text, press keys, scroll, manage windows, read/write files, and run commands.
+
+        Always take a screenshot first to see the current state before performing actions.
+        After performing actions, take another screenshot to verify the result.""",
+    )
+
+    # ============================================================
+    # SCREEN & MOUSE ACTIONS
+    # ============================================================
+
+    @mcp.tool
+    async def computer_screenshot() -> Image:
+        """
+        Capture a screenshot of the current screen.
+
+        Returns the current screen state as an image. Always call this first
+        to see what's on screen before performing any actions.
+        """
+        from io import BytesIO
+
+        from PIL import Image as PILImage
+
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        result = await automation_handler.screenshot()
+        image_data = base64.b64decode(result["image_data"])
+
+        # Decode the image
+        img = PILImage.open(BytesIO(image_data))
+
+        # If target resolution is set, resize to that
+        if _target_width and _target_height and (_scale_x != 1.0 or _scale_y != 1.0):
+            img = img.resize((_target_width, _target_height), PILImage.Resampling.LANCZOS)
+        else:
+            # Resize large images to keep under 1MB limit
+            # Max dimension of 1280 is a good balance for visibility and size
+            max_dimension = 1280
+            width, height = img.size
+            if width > max_dimension or height > max_dimension:
+                if width > height:
+                    new_width = max_dimension
+                    new_height = int(height * max_dimension / width)
+                else:
+                    new_height = max_dimension
+                    new_width = int(width * max_dimension / height)
+                img = img.resize((new_width, new_height), PILImage.Resampling.LANCZOS)
+
+                # Initialize coordinate scaling if not already configured
+                # This ensures clicks are properly scaled from resized space to actual device space
+                if not _target_width:
+                    _configure_scaling(target_width=new_width, target_height=new_height)
+
+        # Convert to RGB if necessary (for JPEG compatibility)
+        if img.mode in ("RGBA", "P"):
+            img = img.convert("RGB")
+
+        # Use JPEG with quality setting to ensure small file size
+        buffered = BytesIO()
+        img.save(buffered, format="JPEG", quality=85, optimize=True)
+        buffered.seek(0)
+        image_data = buffered.getvalue()
+
+        # If still too large, reduce quality further
+        if len(image_data) > 900000:  # 900KB threshold
+            buffered = BytesIO()
+            img.save(buffered, format="JPEG", quality=70, optimize=True)
+            buffered.seek(0)
+            image_data = buffered.getvalue()
+
+        return Image(data=image_data, format="jpeg")
+
+    @mcp.tool
+    async def computer_get_screen_size() -> Dict[str, Any]:
+        """
+        Get the screen dimensions.
+
+        Returns:
+            Dictionary with 'width' and 'height' keys in pixels.
+        """
+        # Return target resolution if scaling is configured
+        if _target_width is not None and _target_height is not None:
+            return {"width": int(_target_width), "height": int(_target_height)}
+        # Use handler to get screen size
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.get_screen_size()
+
+    @mcp.tool
+    async def computer_get_cursor_position() -> Dict[str, Any]:
+        """
+        Get the current cursor position.
+
+        Returns:
+            Dictionary with 'x' and 'y' coordinates.
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        result = await automation_handler.get_cursor_position()
+        # Scale to target coordinates if scaling is configured
+        if "x" in result and "y" in result:
+            scaled_x, scaled_y = _scale_to_target(int(result["x"]), int(result["y"]))
+            return {"x": scaled_x, "y": scaled_y}
+        return result
+
+    @mcp.tool
+    async def computer_click(x: int, y: int, button: str = "left") -> Dict[str, Any]:
+        """
+        Click at the specified screen coordinates.
+
+        Args:
+            x: X coordinate in pixels
+            y: Y coordinate in pixels
+            button: Mouse button - 'left', 'right', or 'middle' (default: 'left')
+        """
+        # Scale from target to actual coordinates
+        actual_x, actual_y = _scale_to_actual(x, y)
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        if button == "left":
+            return await automation_handler.left_click(actual_x, actual_y)
+        elif button == "right":
+            return await automation_handler.right_click(actual_x, actual_y)
+        else:
+            return await automation_handler.left_click(actual_x, actual_y)
+
+    @mcp.tool
+    async def computer_double_click(x: int, y: int) -> Dict[str, Any]:
+        """
+        Double-click at the specified screen coordinates.
+
+        Args:
+            x: X coordinate in pixels
+            y: Y coordinate in pixels
+        """
+        # Scale from target to actual coordinates
+        actual_x, actual_y = _scale_to_actual(x, y)
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.double_click(actual_x, actual_y)
+
+    @mcp.tool
+    async def computer_move(x: int, y: int) -> Dict[str, Any]:
+        """
+        Move the cursor to the specified screen coordinates.
+
+        Args:
+            x: X coordinate in pixels
+            y: Y coordinate in pixels
+        """
+        # Scale from target to actual coordinates
+        actual_x, actual_y = _scale_to_actual(x, y)
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.move_cursor(actual_x, actual_y)
+
+    @mcp.tool
+    async def computer_drag(
+        start_x: int,
+        start_y: int,
+        end_x: int,
+        end_y: int,
+        button: str = "left",
+        duration: float = 0.5,
+    ) -> Dict[str, Any]:
+        """
+        Drag from start coordinates to end coordinates.
+
+        Args:
+            start_x: Starting X coordinate
+            start_y: Starting Y coordinate
+            end_x: Ending X coordinate
+            end_y: Ending Y coordinate
+            button: Mouse button to use (default: 'left')
+            duration: Duration of the drag in seconds (default: 0.5)
+        """
+        # Scale from target to actual coordinates
+        actual_start_x, actual_start_y = _scale_to_actual(start_x, start_y)
+        actual_end_x, actual_end_y = _scale_to_actual(end_x, end_y)
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        # Move to start position first
+        await automation_handler.move_cursor(actual_start_x, actual_start_y)
+        return await automation_handler.drag_to(
+            actual_end_x, actual_end_y, button=button, duration=duration
+        )
+
+    @mcp.tool
+    async def computer_scroll(
+        x: int, y: int, scroll_x: int = 0, scroll_y: int = 0
+    ) -> Dict[str, Any]:
+        """
+        Scroll at the specified position.
+
+        Args:
+            x: X coordinate where to scroll
+            y: Y coordinate where to scroll
+            scroll_x: Horizontal scroll amount (positive = right, negative = left)
+            scroll_y: Vertical scroll amount (positive = down, negative = up)
+        """
+        # Scale from target to actual coordinates
+        actual_x, actual_y = _scale_to_actual(x, y)
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        await automation_handler.move_cursor(actual_x, actual_y)
+        return await automation_handler.scroll(scroll_x, scroll_y)
+
+    @mcp.tool
+    async def computer_mouse_down(
+        x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """
+        Press and hold a mouse button.
+
+        Args:
+            x: Optional X coordinate (uses current position if not specified)
+            y: Optional Y coordinate (uses current position if not specified)
+            button: Mouse button - 'left', 'right', or 'middle' (default: 'left')
+        """
+        # Scale from target to actual coordinates if provided
+        actual_x, actual_y = None, None
+        if x is not None and y is not None:
+            actual_x, actual_y = _scale_to_actual(x, y)
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.mouse_down(actual_x, actual_y, button=button)
+
+    @mcp.tool
+    async def computer_mouse_up(
+        x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+    ) -> Dict[str, Any]:
+        """
+        Release a mouse button.
+
+        Args:
+            x: Optional X coordinate (uses current position if not specified)
+            y: Optional Y coordinate (uses current position if not specified)
+            button: Mouse button - 'left', 'right', or 'middle' (default: 'left')
+        """
+        # Scale from target to actual coordinates if provided
+        actual_x, actual_y = None, None
+        if x is not None and y is not None:
+            actual_x, actual_y = _scale_to_actual(x, y)
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.mouse_up(actual_x, actual_y, button=button)
+
+    # ============================================================
+    # KEYBOARD ACTIONS
+    # ============================================================
+
+    @mcp.tool
+    async def computer_type(text: str) -> Dict[str, Any]:
+        """
+        Type text at the current cursor position.
+
+        Args:
+            text: The text to type
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.type_text(text)
+
+    @mcp.tool
+    async def computer_press_key(key: str) -> Dict[str, Any]:
+        """
+        Press a single key.
+
+        Args:
+            key: The key to press (e.g., 'enter', 'tab', 'escape', 'a', 'b', etc.)
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.press_key(key)
+
+    @mcp.tool
+    async def computer_hotkey(keys: List[str]) -> Dict[str, Any]:
+        """
+        Press a key combination (hotkey).
+
+        Args:
+            keys: List of keys to press together (e.g., ['ctrl', 'c'] for copy,
+                  ['cmd', 'v'] for paste on Mac, ['alt', 'tab'] for window switch)
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.hotkey(keys)
+
+    @mcp.tool
+    async def computer_key_down(key: str) -> Dict[str, Any]:
+        """
+        Press and hold a key.
+
+        Args:
+            key: The key to hold down
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.key_down(key)
+
+    @mcp.tool
+    async def computer_key_up(key: str) -> Dict[str, Any]:
+        """
+        Release a held key.
+
+        Args:
+            key: The key to release
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.key_up(key)
+
+    # ============================================================
+    # CLIPBOARD ACTIONS
+    # ============================================================
+
+    @mcp.tool
+    async def computer_clipboard_get() -> Dict[str, Any]:
+        """
+        Get the current clipboard content.
+
+        Returns:
+            Dictionary with clipboard content.
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.copy_to_clipboard()
+
+    @mcp.tool
+    async def computer_clipboard_set(text: str) -> Dict[str, Any]:
+        """
+        Set the clipboard content.
+
+        Args:
+            text: Text to copy to clipboard
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.set_clipboard(text)
+
+    # ============================================================
+    # SHELL COMMANDS
+    # ============================================================
+
+    @mcp.tool
+    async def computer_run_command(command: str) -> Dict[str, Any]:
+        """
+        Execute a shell command and return the output.
+
+        Args:
+            command: The shell command to execute
+
+        Returns:
+            Dictionary with 'stdout', 'stderr', and 'returncode' keys.
+        """
+        _, automation_handler, _, _, _, _ = _get_handlers()
+        return await automation_handler.run_command(command)
+
+    # ============================================================
+    # FILE SYSTEM OPERATIONS
+    # ============================================================
+
+    @mcp.tool
+    async def computer_file_read(path: str) -> Dict[str, Any]:
+        """
+        Read the text content of a file.
+
+        Args:
+            path: Path to the file to read
+
+        Returns:
+            Dictionary with 'content' key containing the file contents.
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.read_text(path)
+
+    @mcp.tool
+    async def computer_file_write(path: str, content: str) -> Dict[str, Any]:
+        """
+        Write text content to a file.
+
+        Args:
+            path: Path to the file to write
+            content: Text content to write
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.write_text(path, content)
+
+    @mcp.tool
+    async def computer_file_exists(path: str) -> Dict[str, Any]:
+        """
+        Check if a file exists.
+
+        Args:
+            path: Path to check
+
+        Returns:
+            Dictionary with 'exists' boolean key.
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.file_exists(path)
+
+    @mcp.tool
+    async def computer_directory_exists(path: str) -> Dict[str, Any]:
+        """
+        Check if a directory exists.
+
+        Args:
+            path: Path to check
+
+        Returns:
+            Dictionary with 'exists' boolean key.
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.directory_exists(path)
+
+    @mcp.tool
+    async def computer_list_directory(path: str) -> Dict[str, Any]:
+        """
+        List the contents of a directory.
+
+        Args:
+            path: Path to the directory
+
+        Returns:
+            Dictionary with 'entries' list containing file/folder names.
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.list_dir(path)
+
+    @mcp.tool
+    async def computer_create_directory(path: str) -> Dict[str, Any]:
+        """
+        Create a directory.
+
+        Args:
+            path: Path of the directory to create
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.create_dir(path)
+
+    @mcp.tool
+    async def computer_delete_file(path: str) -> Dict[str, Any]:
+        """
+        Delete a file.
+
+        Args:
+            path: Path to the file to delete
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.delete_file(path)
+
+    @mcp.tool
+    async def computer_delete_directory(path: str) -> Dict[str, Any]:
+        """
+        Delete a directory.
+
+        Args:
+            path: Path to the directory to delete
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.delete_dir(path)
+
+    @mcp.tool
+    async def computer_get_file_size(path: str) -> Dict[str, Any]:
+        """
+        Get the size of a file in bytes.
+
+        Args:
+            path: Path to the file
+
+        Returns:
+            Dictionary with 'size' key in bytes.
+        """
+        _, _, _, file_handler, _, _ = _get_handlers()
+        return await file_handler.get_file_size(path)
+
+    # ============================================================
+    # WINDOW MANAGEMENT
+    # ============================================================
+
+    @mcp.tool
+    async def computer_open(target: str) -> Dict[str, Any]:
+        """
+        Open a file or URL with the default application.
+
+        Args:
+            target: File path or URL to open
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.open(target)
+
+    @mcp.tool
+    async def computer_launch_app(app: str, args: Optional[List[str]] = None) -> Dict[str, Any]:
+        """
+        Launch an application.
+
+        Args:
+            app: Application name or path
+            args: Optional list of arguments to pass to the application
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.launch(app, args)
+
+    @mcp.tool
+    async def computer_get_active_window() -> Dict[str, Any]:
+        """
+        Get the currently active window ID.
+
+        Returns:
+            Dictionary with 'window_id' key.
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.get_current_window_id()
+
+    @mcp.tool
+    async def computer_get_window_name(window_id: str) -> Dict[str, Any]:
+        """
+        Get the title/name of a window.
+
+        Args:
+            window_id: Window identifier
+
+        Returns:
+            Dictionary with 'name' key.
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.get_window_name(window_id)
+
+    @mcp.tool
+    async def computer_get_window_size(window_id: str) -> Dict[str, Any]:
+        """
+        Get the size of a window.
+
+        Args:
+            window_id: Window identifier
+
+        Returns:
+            Dictionary with 'width' and 'height' keys.
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.get_window_size(window_id)
+
+    @mcp.tool
+    async def computer_get_window_position(window_id: str) -> Dict[str, Any]:
+        """
+        Get the position of a window.
+
+        Args:
+            window_id: Window identifier
+
+        Returns:
+            Dictionary with 'x' and 'y' keys.
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.get_window_position(window_id)
+
+    @mcp.tool
+    async def computer_set_window_size(window_id: str, width: int, height: int) -> Dict[str, Any]:
+        """
+        Set the size of a window.
+
+        Args:
+            window_id: Window identifier
+            width: New width in pixels
+            height: New height in pixels
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.set_window_size(window_id, width, height)
+
+    @mcp.tool
+    async def computer_set_window_position(window_id: str, x: int, y: int) -> Dict[str, Any]:
+        """
+        Set the position of a window.
+
+        Args:
+            window_id: Window identifier
+            x: New X position
+            y: New Y position
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.set_window_position(window_id, x, y)
+
+    @mcp.tool
+    async def computer_activate_window(window_id: str) -> Dict[str, Any]:
+        """
+        Bring a window to the foreground and focus it.
+
+        Args:
+            window_id: Window identifier
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.activate_window(window_id)
+
+    @mcp.tool
+    async def computer_minimize_window(window_id: str) -> Dict[str, Any]:
+        """
+        Minimize a window.
+
+        Args:
+            window_id: Window identifier
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.minimize_window(window_id)
+
+    @mcp.tool
+    async def computer_maximize_window(window_id: str) -> Dict[str, Any]:
+        """
+        Maximize a window.
+
+        Args:
+            window_id: Window identifier
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.maximize_window(window_id)
+
+    @mcp.tool
+    async def computer_close_window(window_id: str) -> Dict[str, Any]:
+        """
+        Close a window.
+
+        Args:
+            window_id: Window identifier
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.close_window(window_id)
+
+    @mcp.tool
+    async def computer_get_app_windows(app: str) -> Dict[str, Any]:
+        """
+        Get all windows belonging to an application.
+
+        Args:
+            app: Application name or bundle identifier
+
+        Returns:
+            Dictionary with 'windows' list.
+        """
+        _, _, _, _, _, window_handler = _get_handlers()
+        return await window_handler.get_application_windows(app)
+
+    # ============================================================
+    # DESKTOP ENVIRONMENT
+    # ============================================================
+
+    @mcp.tool
+    async def computer_get_desktop_environment() -> Dict[str, Any]:
+        """
+        Get information about the current desktop environment.
+
+        Returns:
+            Dictionary with desktop environment details.
+        """
+        _, _, _, _, desktop_handler, _ = _get_handlers()
+        return await desktop_handler.get_desktop_environment()
+
+    @mcp.tool
+    async def computer_set_wallpaper(path: str) -> Dict[str, Any]:
+        """
+        Set the desktop wallpaper.
+
+        Args:
+            path: Path to the image file to use as wallpaper
+        """
+        _, _, _, _, desktop_handler, _ = _get_handlers()
+        return await desktop_handler.set_wallpaper(path)
+
+    # ============================================================
+    # ACCESSIBILITY
+    # ============================================================
+
+    @mcp.tool
+    async def computer_get_accessibility_tree() -> Dict[str, Any]:
+        """
+        Get the accessibility tree of the current window.
+
+        This provides detailed information about UI elements that can be
+        useful for finding clickable elements or understanding the UI structure.
+
+        Returns:
+            Dictionary with the accessibility tree structure.
+        """
+        accessibility_handler, _, _, _, _, _ = _get_handlers()
+        return await accessibility_handler.get_accessibility_tree()
+
+    @mcp.tool
+    async def computer_find_element(
+        role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Find a UI element in the accessibility tree.
+
+        Args:
+            role: Element role/type to search for (e.g., 'button', 'textfield')
+            title: Element title/label to search for
+            value: Element value to search for
+
+        Returns:
+            Dictionary with matching element information including position.
+        """
+        accessibility_handler, _, _, _, _, _ = _get_handlers()
+        return await accessibility_handler.find_element(role=role, title=title, value=value)
+
+    return mcp
+
+
+def run_mcp_server(
+    target_width: Optional[int] = None,
+    target_height: Optional[int] = None,
+    detect_resolution: bool = False,
+) -> None:
+    """Run the MCP server.
+
+    Args:
+        target_width: Target width for screenshots (coordinates will be scaled accordingly)
+        target_height: Target height for screenshots (coordinates will be scaled accordingly)
+        detect_resolution: If True, auto-detect and log the actual screen resolution
+    """
+    logger.info("Starting CUA Computer MCP server...")
+
+    # Configure resolution scaling
+    _configure_scaling(
+        target_width=target_width,
+        target_height=target_height,
+        detect_resolution=detect_resolution,
+    )
+
+    mcp = create_mcp_server()
+    mcp.run()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        stream=sys.stderr,
+    )
+    run_mcp_server()

libs/python/computer-server/computer_server/pty_manager.py

@@ -0,0 +1,203 @@
+"""Async PTY session manager for computer-server.
+
+Wraps :class:`cua_auto.terminal.Terminal` and adds asyncio-compatible
+output broadcasting via per-consumer :class:`asyncio.Queue` objects.
+
+Queue items are dicts:
+  - ``{"type": "output", "data": <bytes>}``  — terminal output chunk
+  - ``{"type": "exit", "code": <int>}``      — session terminated (sentinel)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import threading
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class PtyManager:
+    """Manage PTY sessions with async broadcast queues.
+
+    Usage::
+
+        mgr = PtyManager()
+
+        # Create a session
+        info = await mgr.create(command="bash", cols=80, rows=24)
+        pid = info["pid"]
+
+        # Subscribe to output
+        q = mgr.subscribe(pid)
+        async for msg in _drain(q):
+            ...  # msg is {"type": "output", "data": b"..."} or {"type": "exit", "code": 0}
+
+        # Write to stdin
+        await mgr.send_stdin(pid, b"echo hello\\n")
+
+        # Resize
+        await mgr.resize(pid, 120, 40)
+
+        # Kill
+        await mgr.kill(pid)
+    """
+
+    def __init__(self) -> None:
+        # pid → list of subscriber queues
+        self._queues: Dict[int, List[asyncio.Queue]] = {}
+        # pid → {"cols": int, "rows": int}
+        self._sessions: Dict[int, dict] = {}
+        self._terminal = None
+
+    # ------------------------------------------------------------------
+    # Lazy terminal access
+    # ------------------------------------------------------------------
+
+    def _get_terminal(self):
+        if self._terminal is None:
+            from cua_auto.terminal import Terminal
+
+            self._terminal = Terminal()
+        return self._terminal
+
+    # ------------------------------------------------------------------
+    # Public async API
+    # ------------------------------------------------------------------
+
+    async def create(
+        self,
+        command: Optional[str] = None,
+        cols: int = 80,
+        rows: int = 24,
+        cwd: Optional[str] = None,
+        envs: Optional[dict] = None,
+    ) -> dict:
+        """Spawn a new PTY session.
+
+        Returns:
+            ``{"pid": int, "cols": int, "rows": int}``
+        """
+        loop = asyncio.get_running_loop()
+        terminal = self._get_terminal()
+
+        # We need pid to route output, but we only know it after create().
+        # Use a mutable cell so the callback can look up the pid after creation.
+        # early_buffer holds chunks that arrive before pid_cell[0] is set (the
+        # reader thread can fire before asyncio.to_thread returns the session).
+        pid_cell: List[Optional[int]] = [None]
+        early_buffer: List[bytes] = []
+
+        def _on_data(data: bytes) -> None:
+            pid = pid_cell[0]
+            if pid is None:
+                early_buffer.append(data)
+                return
+            msg = {"type": "output", "data": data}
+            for q in list(self._queues.get(pid, [])):
+                try:
+                    loop.call_soon_threadsafe(q.put_nowait, msg)
+                except Exception:
+                    pass
+
+        session = await asyncio.to_thread(
+            terminal.create,
+            command=command,
+            cols=cols,
+            rows=rows,
+            on_data=_on_data,
+            cwd=cwd,
+            envs=envs,
+        )
+
+        pid_cell[0] = session.pid
+        self._queues[session.pid] = []
+        self._sessions[session.pid] = {"cols": session.cols, "rows": session.rows}
+
+        # Flush any output that arrived before pid_cell[0] was set.
+        for chunk in early_buffer:
+            msg = {"type": "output", "data": chunk}
+            for q in list(self._queues[session.pid]):
+                try:
+                    loop.call_soon_threadsafe(q.put_nowait, msg)
+                except Exception:
+                    pass
+
+        # Watch for process exit in a daemon thread, then broadcast sentinel.
+        def _watch_exit() -> None:
+            exit_code = terminal.wait(session.pid) or 0
+            sentinel = {"type": "exit", "code": exit_code}
+            for q in list(self._queues.get(session.pid, [])):
+                try:
+                    loop.call_soon_threadsafe(q.put_nowait, sentinel)
+                except Exception:
+                    pass
+
+        threading.Thread(target=_watch_exit, daemon=True, name=f"pty-exit-{session.pid}").start()
+
+        logger.info(
+            "PTY session created: pid=%d cmd=%r cols=%d rows=%d", session.pid, command, cols, rows
+        )
+        return {"pid": session.pid, "cols": session.cols, "rows": session.rows}
+
+    async def send_stdin(self, pid: int, data: bytes) -> None:
+        """Write *data* to the stdin of session *pid*."""
+        terminal = self._get_terminal()
+        await asyncio.to_thread(terminal.send_stdin, pid, data)
+
+    async def resize(self, pid: int, cols: int, rows: int) -> None:
+        """Resize the terminal for session *pid*."""
+        terminal = self._get_terminal()
+        await asyncio.to_thread(terminal.resize, pid, cols, rows)
+        if pid in self._sessions:
+            self._sessions[pid]["cols"] = cols
+            self._sessions[pid]["rows"] = rows
+
+    def get_info(self, pid: int) -> Optional[dict]:
+        """Return ``{"pid": int, "cols": int, "rows": int}`` for *pid*, or ``None`` if unknown."""
+        info = self._sessions.get(pid)
+        if info is None:
+            return None
+        return {"pid": pid, "cols": info["cols"], "rows": info["rows"]}
+
+    async def kill(self, pid: int) -> bool:
+        """Kill session *pid*.
+
+        Also broadcasts the exit sentinel to all subscribers.
+        Returns:
+            ``True`` if the signal was delivered.
+        """
+        terminal = self._get_terminal()
+        result = await asyncio.to_thread(terminal.kill, pid)
+        # Broadcast sentinel immediately (the exit watcher will also fire,
+        # but duplicate sentinels are harmless — consumers stop after the first).
+        sentinel = {"type": "exit", "code": -1}
+        for q in list(self._queues.get(pid, [])):
+            try:
+                q.put_nowait(sentinel)
+            except Exception:
+                pass
+        return result
+
+    # ------------------------------------------------------------------
+    # Queue-based pub/sub
+    # ------------------------------------------------------------------
+
+    def subscribe(self, pid: int) -> asyncio.Queue:
+        """Return a new :class:`asyncio.Queue` that will receive output for *pid*.
+
+        The queue receives dicts with keys ``type`` (``"output"`` or ``"exit"``)
+        and either ``data`` (bytes) or ``code`` (int).
+        """
+        q: asyncio.Queue = asyncio.Queue()
+        self._queues.setdefault(pid, []).append(q)
+        return q
+
+    def unsubscribe(self, pid: int, queue: asyncio.Queue) -> None:
+        """Remove *queue* from the subscriber list for *pid*."""
+        qs = self._queues.get(pid, [])
+        try:
+            qs.remove(queue)
+        except ValueError:
+            pass

libs/python/computer-server/computer_server/server.py

@@ -0,0 +1,119 @@
+"""
+Server interface for Computer API.
+Provides a clean API for starting and stopping the server.
+"""
+
+import asyncio
+import logging
+from typing import Optional
+
+import uvicorn
+from fastapi import FastAPI
+
+from .main import app as fastapi_app
+
+logger = logging.getLogger(__name__)
+
+
+class Server:
+    """
+    Server interface for Computer API.
+
+    Usage:
+        from computer_api import Server
+
+        # Synchronous usage
+        server = Server()
+        server.start()  # Blocks until server is stopped
+
+        # Asynchronous usage
+        server = Server()
+        await server.start_async()  # Starts server in background
+        # Do other things
+        await server.stop()  # Stop the server
+    """
+
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 8000,
+        log_level: str = "info",
+        ssl_keyfile: Optional[str] = None,
+        ssl_certfile: Optional[str] = None,
+    ):
+        """
+        Initialize the server.
+
+        Args:
+            host: Host to bind the server to. Defaults to localhost; pass
+                "0.0.0.0" explicitly for external access.
+            port: Port to bind the server to
+            log_level: Logging level (debug, info, warning, error, critical)
+            ssl_keyfile: Path to SSL private key file (for HTTPS)
+            ssl_certfile: Path to SSL certificate file (for HTTPS)
+        """
+        self.host = host
+        self.port = port
+        self.log_level = log_level
+        self.ssl_keyfile = ssl_keyfile
+        self.ssl_certfile = ssl_certfile
+        self.app = fastapi_app
+        self._server_task: Optional[asyncio.Task] = None
+        self._should_exit = asyncio.Event()
+
+    def start(self) -> None:
+        """
+        Start the server synchronously. This will block until the server is stopped.
+        """
+        uvicorn.run(
+            self.app,
+            host=self.host,
+            port=self.port,
+            log_level=self.log_level,
+            ssl_keyfile=self.ssl_keyfile,
+            ssl_certfile=self.ssl_certfile,
+        )
+
+    async def start_async(self) -> None:
+        """
+        Start the server asynchronously. This will return immediately and the server
+        will run in the background.
+        """
+        server_config = uvicorn.Config(
+            self.app,
+            host=self.host,
+            port=self.port,
+            log_level=self.log_level,
+            ssl_keyfile=self.ssl_keyfile,
+            ssl_certfile=self.ssl_certfile,
+        )
+
+        self._should_exit.clear()
+        server = uvicorn.Server(server_config)
+
+        # Create a task to run the server
+        self._server_task = asyncio.create_task(server.serve())
+
+        # Wait a short time to ensure the server starts
+        await asyncio.sleep(0.5)
+
+        protocol = "https" if self.ssl_certfile else "http"
+        logger.info(f"Server started at {protocol}://{self.host}:{self.port}")
+
+    async def stop(self) -> None:
+        """
+        Stop the server if it's running asynchronously.
+        """
+        if self._server_task and not self._server_task.done():
+            # Signal the server to exit
+            self._should_exit.set()
+
+            # Cancel the server task
+            self._server_task.cancel()
+
+            try:
+                await self._server_task
+            except asyncio.CancelledError:
+                logger.info("Server stopped")
+
+            self._server_task = None

libs/python/computer-server/computer_server/utils/__init__.py

@@ -0,0 +1,3 @@
+from . import helpers, wallpaper
+
+__all__ = ["helpers", "wallpaper"]

libs/python/computer-server/computer_server/utils/helpers.py

@@ -0,0 +1,81 @@
+import asyncio
+import os
+import platform
+import subprocess
+from typing import overload
+
+
+def get_current_os() -> str:
+    """Determine the current OS.
+
+    Returns:
+        str: The OS type ('android', 'darwin' for macOS, 'linux' for Linux, or 'windows' for Windows)
+
+    Raises:
+        RuntimeError: If unable to determine the current OS
+    """
+    try:
+        if os.environ.get("IS_CUA_ANDROID") == "true":
+            # Verify emulator is actually running by checking adb devices
+            try:
+                result = subprocess.run(
+                    ["adb", "devices"], capture_output=True, text=True, timeout=5
+                )
+                if result.returncode == 0 and "emulator-5554" in result.stdout:
+                    return "android"
+                else:
+                    raise RuntimeError(
+                        "IS_CUA_ANDROID is set but no emulator found. "
+                        "Ensure Android emulator is running and accessible via adb."
+                    )
+            except subprocess.TimeoutExpired:
+                raise RuntimeError(
+                    "IS_CUA_ANDROID is set but adb command timed out. "
+                    "Emulator may be starting up or unresponsive."
+                )
+
+        system = platform.system().lower()
+        if system in ["darwin", "linux", "windows"]:
+            return system
+
+        # Fallback to uname if platform.system() doesn't return expected values (Unix-like systems only)
+        result = subprocess.run(["uname", "-s"], capture_output=True, text=True)
+        if result.returncode == 0:
+            return result.stdout.strip().lower()
+
+        raise RuntimeError(f"Unsupported OS: {system}")
+    except Exception as e:
+        raise RuntimeError(f"Failed to determine current OS: {str(e)}")
+
+
+class CommandExecutor:
+    def __init__(self, *base_cmd: str) -> None:
+        """Initialize with a base command.
+
+        Args:
+            base_cmd: The base command and its initial arguments.
+        """
+        self.__base_cmd = list(base_cmd)
+
+    @overload
+    async def run(self, *args: str, timeout: int = 10) -> tuple[bool, bytes]: ...
+    @overload
+    async def run(self, *args: str, decode: bool = True, timeout: int = 10) -> tuple[bool, str]: ...
+
+    async def run(
+        self, *args: str, decode: bool = False, timeout: int = 10
+    ) -> tuple[bool, bytes | str]:
+        cmd = self.__base_cmd + list(args)
+        try:
+            result = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await asyncio.wait_for(result.communicate(), timeout=timeout)
+            output = stdout or stderr
+            if decode:
+                output = output.decode("utf-8")
+            return result.returncode == 0, output
+        except asyncio.TimeoutError:
+            return False, f"Command timed out after {timeout}s".encode("utf-8")
+        except Exception as e:
+            return False, str(e).encode("utf-8")

libs/python/computer-server/computer_server/utils/wallpaper.py

@@ -0,0 +1,321 @@
+"""Set the desktop wallpaper."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def get_desktop_environment() -> str:
+    """
+    Returns the name of the current desktop environment.
+    """
+    # From https://stackoverflow.com/a/21213358/2624876
+    # which takes from:
+    # http://stackoverflow.com/questions/2035657/what-is-my-current-desktop-environment
+    # and http://ubuntuforums.org/showthread.php?t=652320
+    # and http://ubuntuforums.org/showthread.php?t=1139057
+    if sys.platform in ["win32", "cygwin"]:
+        return "windows"
+    elif sys.platform == "darwin":
+        return "mac"
+    else:  # Most likely either a POSIX system or something not much common
+        desktop_session = os.environ.get("DESKTOP_SESSION")
+        if (
+            desktop_session is not None
+        ):  # easier to match if we doesn't have to deal with character cases
+            desktop_session = desktop_session.lower()
+            if desktop_session in [
+                "gnome",
+                "unity",
+                "cinnamon",
+                "mate",
+                "xfce4",
+                "lxde",
+                "fluxbox",
+                "blackbox",
+                "openbox",
+                "icewm",
+                "jwm",
+                "afterstep",
+                "trinity",
+                "kde",
+            ]:
+                return desktop_session
+            ## Special cases ##
+            # Canonical sets $DESKTOP_SESSION to Lubuntu rather than LXDE if using LXDE.
+            # There is no guarantee that they will not do the same with the other desktop environments.
+            elif "xfce" in desktop_session or desktop_session.startswith("xubuntu"):
+                return "xfce4"
+            elif desktop_session.startswith("ubuntustudio"):
+                return "kde"
+            elif desktop_session.startswith("ubuntu"):
+                return "gnome"
+            elif desktop_session.startswith("lubuntu"):
+                return "lxde"
+            elif desktop_session.startswith("kubuntu"):
+                return "kde"
+            elif desktop_session.startswith("razor"):  # e.g. razorkwin
+                return "razor-qt"
+            elif desktop_session.startswith("wmaker"):  # e.g. wmaker-common
+                return "windowmaker"
+        gnome_desktop_session_id = os.environ.get("GNOME_DESKTOP_SESSION_ID")
+        if os.environ.get("KDE_FULL_SESSION") == "true":
+            return "kde"
+        elif gnome_desktop_session_id:
+            if "deprecated" not in gnome_desktop_session_id:
+                return "gnome2"
+        # From http://ubuntuforums.org/showthread.php?t=652320
+        elif is_running("xfce-mcs-manage"):
+            return "xfce4"
+        elif is_running("ksmserver"):
+            return "kde"
+    return "unknown"
+
+
+def is_running(process: str) -> bool:
+    """Returns whether a process with the given name is (likely) currently running.
+
+    Uses a basic text search, and so may have false positives.
+    """
+    # From http://www.bloggerpolis.com/2011/05/how-to-check-if-a-process-is-running-using-python/
+    # and http://richarddingwall.name/2009/06/18/windows-equivalents-of-ps-and-kill-commands/
+    try:  # Linux/Unix
+        s = subprocess.Popen(["ps", "axw"], stdout=subprocess.PIPE)
+    except:  # Windows
+        s = subprocess.Popen(["tasklist", "/v"], stdout=subprocess.PIPE)
+    assert s.stdout is not None
+    for x in s.stdout:
+        # if re.search(process, x):
+        if process in str(x):
+            return True
+    return False
+
+
+def set_wallpaper(file_loc: str, first_run: bool = True):
+    """Sets the wallpaper to the given file location."""
+    # From https://stackoverflow.com/a/21213504/2624876
+    # I have not personally tested most of this. -- @1j01
+    # -----------------------------------------
+
+    # Note: There are two common Linux desktop environments where
+    # I have not been able to set the desktop background from
+    # command line: KDE, Enlightenment
+    desktop_env = get_desktop_environment()
+    if desktop_env in ["gnome", "unity", "cinnamon"]:
+        # Tested on Ubuntu 22 -- @1j01
+        uri = Path(file_loc).as_uri()
+        SCHEMA = "org.gnome.desktop.background"
+        KEY = "picture-uri"
+        # Needed for Ubuntu 22 in dark mode
+        # Might be better to set only one or the other, depending on the current theme
+        # In the settings it will say "This background selection only applies to the dark style"
+        # even if it's set for both, arguably referring to the selection that you can make on that page.
+        # -- @1j01
+        KEY_DARK = "picture-uri-dark"
+        try:
+            from gi.repository import Gio  # type: ignore
+
+            gsettings = Gio.Settings.new(SCHEMA)  # type: ignore
+            gsettings.set_string(KEY, uri)
+            gsettings.set_string(KEY_DARK, uri)
+        except Exception:
+            # Fallback tested on Ubuntu 22 -- @1j01
+            args = ["gsettings", "set", SCHEMA, KEY, uri]
+            subprocess.Popen(args)
+            args = ["gsettings", "set", SCHEMA, KEY_DARK, uri]
+            subprocess.Popen(args)
+    elif desktop_env == "mate":
+        try:  # MATE >= 1.6
+            # info from http://wiki.mate-desktop.org/docs:gsettings
+            args = ["gsettings", "set", "org.mate.background", "picture-filename", file_loc]
+            subprocess.Popen(args)
+        except Exception:  # MATE < 1.6
+            # From https://bugs.launchpad.net/variety/+bug/1033918
+            args = [
+                "mateconftool-2",
+                "-t",
+                "string",
+                "--set",
+                "/desktop/mate/background/picture_filename",
+                file_loc,
+            ]
+            subprocess.Popen(args)
+    elif desktop_env == "gnome2":  # Not tested
+        # From https://bugs.launchpad.net/variety/+bug/1033918
+        args = [
+            "gconftool-2",
+            "-t",
+            "string",
+            "--set",
+            "/desktop/gnome/background/picture_filename",
+            file_loc,
+        ]
+        subprocess.Popen(args)
+    ## KDE4 is difficult
+    ## see http://blog.zx2c4.com/699 for a solution that might work
+    elif desktop_env in ["kde3", "trinity"]:
+        # From http://ubuntuforums.org/archive/index.php/t-803417.html
+        args = ["dcop", "kdesktop", "KBackgroundIface", "setWallpaper", "0", file_loc, "6"]
+        subprocess.Popen(args)
+    elif desktop_env == "xfce4":
+        # Iterate over all wallpaper-related keys and set to file_loc
+        try:
+            list_proc = subprocess.run(
+                ["xfconf-query", "-c", "xfce4-desktop", "-l"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=False,
+            )
+            keys = []
+            if list_proc.stdout:
+                for line in list_proc.stdout.splitlines():
+                    line = line.strip()
+                    if not line:
+                        continue
+                    # Common keys: .../last-image and .../image-path
+                    if "/last-image" in line or "/image-path" in line:
+                        keys.append(line)
+            # Fallback: known defaults if none were listed
+            if not keys:
+                keys = [
+                    "/backdrop/screen0/monitorVNC-0/workspace0/last-image",
+                    "/backdrop/screen0/monitor0/image-path",
+                ]
+            for key in keys:
+                subprocess.run(
+                    [
+                        "xfconf-query",
+                        "-c",
+                        "xfce4-desktop",
+                        "-p",
+                        key,
+                        "-s",
+                        file_loc,
+                    ],
+                    check=False,
+                )
+        except Exception:
+            pass
+        # Reload xfdesktop to apply changes
+        subprocess.Popen(["xfdesktop", "--reload"])
+    elif desktop_env == "razor-qt":  # TODO: implement reload of desktop when possible
+        if first_run:
+            import configparser
+
+            desktop_conf = configparser.ConfigParser()
+            # Development version
+            desktop_conf_file = os.path.join(get_config_dir("razor"), "desktop.conf")
+            if os.path.isfile(desktop_conf_file):
+                config_option = R"screens\1\desktops\1\wallpaper"
+            else:
+                desktop_conf_file = os.path.join(get_home_dir(), ".razor/desktop.conf")
+                config_option = R"desktops\1\wallpaper"
+            desktop_conf.read(os.path.join(desktop_conf_file))
+            try:
+                if desktop_conf.has_option("razor", config_option):  # only replacing a value
+                    desktop_conf.set("razor", config_option, file_loc)
+                    with open(desktop_conf_file, "w", encoding="utf-8", errors="replace") as f:
+                        desktop_conf.write(f)
+            except Exception:
+                pass
+        else:
+            # TODO: reload desktop when possible
+            pass
+    elif desktop_env in ["fluxbox", "jwm", "openbox", "afterstep"]:
+        # http://fluxbox-wiki.org/index.php/Howto_set_the_background
+        # used fbsetbg on jwm too since I am too lazy to edit the XML configuration
+        # now where fbsetbg does the job excellent anyway.
+        # and I have not figured out how else it can be set on Openbox and AfterSTep
+        # but fbsetbg works excellent here too.
+        try:
+            args = ["fbsetbg", file_loc]
+            subprocess.Popen(args)
+        except Exception:
+            sys.stderr.write("ERROR: Failed to set wallpaper with fbsetbg!\n")
+            sys.stderr.write("Please make sre that You have fbsetbg installed.\n")
+    elif desktop_env == "icewm":
+        # command found at http://urukrama.wordpress.com/2007/12/05/desktop-backgrounds-in-window-managers/
+        args = ["icewmbg", file_loc]
+        subprocess.Popen(args)
+    elif desktop_env == "blackbox":
+        # command found at http://blackboxwm.sourceforge.net/BlackboxDocumentation/BlackboxBackground
+        args = ["bsetbg", "-full", file_loc]
+        subprocess.Popen(args)
+    elif desktop_env == "lxde":
+        args = ["pcmanfm", "--set-wallpaper", file_loc, "--wallpaper-mode=scaled"]
+        subprocess.Popen(args)
+    elif desktop_env == "windowmaker":
+        # From http://www.commandlinefu.com/commands/view/3857/set-wallpaper-on-windowmaker-in-one-line
+        args = ["wmsetbg", "-s", "-u", file_loc]
+        subprocess.Popen(args)
+    # elif desktop_env == "enlightenment": # I have not been able to make it work on e17. On e16 it would have been something in this direction
+    #     args = ["enlightenment_remote", "-desktop-bg-add", "0", "0", "0", "0", file_loc]
+    #     subprocess.Popen(args)
+    elif desktop_env == "windows":
+        # From https://stackoverflow.com/questions/1977694/change-desktop-background
+        # Tested on Windows 10. -- @1j01
+        import ctypes
+
+        SPI_SETDESKWALLPAPER = 20
+        ctypes.windll.user32.SystemParametersInfoW(SPI_SETDESKWALLPAPER, 0, file_loc, 0)  # type: ignore
+    elif desktop_env == "mac":
+        # From https://stackoverflow.com/questions/431205/how-can-i-programatically-change-the-background-in-mac-os-x
+        try:
+            # Tested on macOS 10.14.6 (Mojave) -- @1j01
+            assert (
+                sys.platform == "darwin"
+            )  # ignore `Import "appscript" could not be resolved` for other platforms
+            from appscript import app, mactypes
+
+            app("Finder").desktop_picture.set(mactypes.File(file_loc))
+        except ImportError:
+            # Tested on macOS 10.14.6 (Mojave) -- @1j01
+            # import subprocess
+            # SCRIPT = f"""/usr/bin/osascript<<END
+            # tell application "Finder" to set desktop picture to POSIX file "{file_loc}"
+            # END"""
+            # subprocess.Popen(SCRIPT, shell=True)
+
+            # Safer version, avoiding string interpolation,
+            # to protect against command injection (both in the shell and in AppleScript):
+            OSASCRIPT = """
+            on run (clp)
+                if clp's length is not 1 then error "Incorrect Parameters"
+                local file_loc
+                set file_loc to clp's item 1
+                tell application "Finder" to set desktop picture to POSIX file file_loc
+            end run
+            """
+            subprocess.Popen(["osascript", "-e", OSASCRIPT, "--", file_loc])
+    else:
+        if first_run:  # don't spam the user with the same message over and over again
+            sys.stderr.write(
+                "Warning: Failed to set wallpaper. Your desktop environment is not supported."
+            )
+            sys.stderr.write(f"You can try manually to set your wallpaper to {file_loc}")
+        return False
+    return True
+
+
+def get_config_dir(app_name: str) -> str:
+    """Returns the configuration directory for the given application name."""
+    if "XDG_CONFIG_HOME" in os.environ:
+        config_home = os.environ["XDG_CONFIG_HOME"]
+    elif "APPDATA" in os.environ:  # On Windows
+        config_home = os.environ["APPDATA"]
+    else:
+        try:
+            from xdg import BaseDirectory
+
+            config_home = BaseDirectory.xdg_config_home
+        except ImportError:  # Most likely a Linux/Unix system anyway
+            config_home = os.path.join(get_home_dir(), ".config")
+    config_dir = os.path.join(config_home, app_name)
+    return config_dir
+
+
+def get_home_dir() -> str:
+    """Returns the home directory of the current user."""
+    return os.path.expanduser("~")