chore: import upstream snapshot with attribution

libs/python/agent/cua_agent/computers/__init__.py

@@ -0,0 +1,61 @@
+"""
+Computer handler factory and interface definitions.
+
+This module provides a factory function to create computer handlers from different
+computer interface types, supporting both the ComputerHandler protocol and the
+Computer library interface.
+"""
+
+try:
+    from computer import Computer as cuaComputer
+
+    from .cua import cuaComputerHandler
+except ImportError:
+    cuaComputer = None  # type: ignore[assignment,misc]
+    cuaComputerHandler = None  # type: ignore[assignment]
+
+try:
+    from cua_sandbox import Sandbox as cuaSandbox
+except ImportError:
+    cuaSandbox = None  # type: ignore[assignment,misc]
+
+from .base import AsyncComputerHandler
+from .custom import CustomComputerHandler
+from .sandbox import SandboxComputerHandler
+
+
+def is_agent_computer(computer):
+    """Check if the given computer is a ComputerHandler or Cua Computer."""
+    return (
+        isinstance(computer, AsyncComputerHandler)
+        or (cuaComputer is not None and isinstance(computer, cuaComputer))
+        or (cuaSandbox is not None and isinstance(computer, cuaSandbox))
+        or (isinstance(computer, dict))
+    )  # and "screenshot" in computer)
+
+
+async def make_computer_handler(computer):
+    """
+    Create a computer handler from a computer interface.
+
+    Args:
+        computer: Either a ComputerHandler instance, Computer instance,
+                  Sandbox instance, or dict of functions
+
+    Returns:
+        ComputerHandler: A computer handler instance
+
+    Raises:
+        ValueError: If the computer type is not supported
+    """
+    if isinstance(computer, AsyncComputerHandler):
+        return computer
+    if cuaComputer is not None and isinstance(computer, cuaComputer):
+        computer_handler = cuaComputerHandler(computer)
+        await computer_handler._initialize()
+        return computer_handler
+    if cuaSandbox is not None and isinstance(computer, cuaSandbox):
+        return SandboxComputerHandler(computer)
+    if isinstance(computer, dict):
+        return CustomComputerHandler(computer)
+    raise ValueError(f"Unsupported computer type: {type(computer)}")

libs/python/agent/cua_agent/computers/base.py

@@ -0,0 +1,83 @@
+"""
+Base computer interface protocol for agent interactions.
+"""
+
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Protocol,
+    Union,
+    runtime_checkable,
+)
+
+
+@runtime_checkable
+class AsyncComputerHandler(Protocol):
+    """Protocol defining the interface for computer interactions."""
+
+    # ==== Computer-Use-Preview Action Space ====
+
+    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
+        """Get the current environment type."""
+        ...
+
+    async def get_dimensions(self) -> tuple[int, int]:
+        """Get screen dimensions as (width, height)."""
+        ...
+
+    async def screenshot(self, text: Optional[str] = None) -> str:
+        """Take a screenshot and return as base64 string.
+
+        Args:
+            text: Optional descriptive text (for compatibility with GPT-4o models, ignored)
+        """
+        ...
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        """Click at coordinates with specified button."""
+        ...
+
+    async def double_click(self, x: int, y: int) -> None:
+        """Double click at coordinates."""
+        ...
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """Scroll at coordinates with specified scroll amounts."""
+        ...
+
+    async def type(self, text: str) -> None:
+        """Type text."""
+        ...
+
+    async def wait(self, ms: int = 1000) -> None:
+        """Wait for specified milliseconds."""
+        ...
+
+    async def move(self, x: int, y: int) -> None:
+        """Move cursor to coordinates."""
+        ...
+
+    async def keypress(self, keys: Union[List[str], str]) -> None:
+        """Press key combination."""
+        ...
+
+    async def drag(self, path: List[Dict[str, int]]) -> None:
+        """Drag along specified path."""
+        ...
+
+    async def get_current_url(self) -> str:
+        """Get current URL (for browser environments)."""
+        ...
+
+    # ==== Anthropic Action Space ====
+
+    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse down at coordinates."""
+        ...
+
+    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse up at coordinates."""
+        ...

libs/python/agent/cua_agent/computers/cua.py

@@ -0,0 +1,184 @@
+"""
+Computer handler implementation for OpenAI computer-use-preview protocol.
+"""
+
+import base64
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from computer import Computer
+
+from .base import AsyncComputerHandler
+
+
+class cuaComputerHandler(AsyncComputerHandler):
+    """Computer handler that implements the Computer protocol using the computer interface."""
+
+    def __init__(self, cua_computer: Computer):
+        """Initialize with a computer interface (from tool schema)."""
+        self.cua_computer = cua_computer
+        self.interface = None
+
+    async def _initialize(self):
+        if hasattr(self.cua_computer, "_initialized") and not self.cua_computer._initialized:
+            await self.cua_computer.run()
+        self.interface = self.cua_computer.interface
+
+    # ==== Computer-Use-Preview Action Space ====
+
+    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
+        """Get the current environment type."""
+        # TODO: detect actual environment
+        return "linux"
+
+    async def get_dimensions(self) -> tuple[int, int]:
+        """Get screen dimensions as (width, height)."""
+        assert self.interface is not None
+        screen_size = await self.interface.get_screen_size()
+        return screen_size["width"], screen_size["height"]
+
+    async def screenshot(self, text: Optional[str] = None) -> str:
+        """Take a screenshot and return as base64 string.
+
+        Args:
+            text: Optional descriptive text (for compatibility with GPT-4o models, ignored)
+        """
+        assert self.interface is not None
+        screenshot_bytes = await self.interface.screenshot()
+        return base64.b64encode(screenshot_bytes).decode("utf-8")
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        """Click at coordinates with specified button."""
+        assert self.interface is not None
+        if button == "left":
+            await self.interface.left_click(x, y)
+        elif button == "right":
+            await self.interface.right_click(x, y)
+        else:
+            # Default to left click for unknown buttons
+            await self.interface.left_click(x, y)
+
+    async def double_click(self, x: int, y: int) -> None:
+        """Double click at coordinates."""
+        assert self.interface is not None
+        await self.interface.double_click(x, y)
+
+    async def right_click(self, x: int, y: int) -> None:
+        """Right click at coordinates."""
+        assert self.interface is not None
+        await self.interface.right_click(x, y)
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """Scroll at coordinates with specified scroll amounts."""
+        assert self.interface is not None
+        await self.interface.move_cursor(x, y)
+        await self.interface.scroll(scroll_x, scroll_y)
+
+    async def type(self, text: str) -> None:
+        """Type text."""
+        assert self.interface is not None
+        await self.interface.type_text(text)
+
+    async def wait(self, ms: int = 1000) -> None:
+        """Wait for specified milliseconds."""
+        assert self.interface is not None
+        import asyncio
+
+        await asyncio.sleep(ms / 1000.0)
+
+    async def move(self, x: int, y: int) -> None:
+        """Move cursor to coordinates."""
+        assert self.interface is not None
+        await self.interface.move_cursor(x, y)
+
+    async def keypress(self, keys: Union[List[str], str]) -> None:
+        """Press key combination."""
+        assert self.interface is not None
+        if isinstance(keys, str):
+            keys = keys.replace("-", "+").split("+")
+        if len(keys) == 1:
+            await self.interface.press_key(keys[0])
+        else:
+            # Handle key combinations
+            await self.interface.hotkey(*keys)
+
+    async def drag(
+        self,
+        path: Optional[List[Dict[str, int]]] = None,
+        start_x: Optional[int] = None,
+        start_y: Optional[int] = None,
+        end_x: Optional[int] = None,
+        end_y: Optional[int] = None,
+    ) -> None:
+        """Drag along specified path or from start to end coordinates.
+
+        Supports two formats:
+        - path: List of {x, y} points to drag through
+        - start_x, start_y, end_x, end_y: Simple drag from start to end
+        """
+        assert self.interface is not None
+
+        # If start/end coordinates provided, convert to path format
+        if start_x is not None and start_y is not None and end_x is not None and end_y is not None:
+            path = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]
+
+        if not path:
+            return
+
+        # Start drag from first point
+        start = path[0]
+        await self.interface.mouse_down(start["x"], start["y"])
+
+        # Move through path
+        for point in path[1:]:
+            await self.interface.move_cursor(point["x"], point["y"])
+
+        # End drag at last point
+        end = path[-1]
+        await self.interface.mouse_up(end["x"], end["y"])
+
+    async def terminate(self, status: str = "success") -> Dict[str, Any]:
+        """Terminate the current task and report its completion status.
+
+        Args:
+            status: Status of the task ("success" or "failure")
+
+        Returns:
+            Dict with terminated flag and status
+        """
+        return {"success": True, "status": status, "terminated": True}
+
+    async def get_current_url(self) -> str:
+        """Get current URL (for browser environments)."""
+        # This would need to be implemented based on the specific browser interface
+        # For now, return empty string
+        return ""
+
+    # ==== Anthropic Computer Action Space ====
+    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse down at coordinates."""
+        assert self.interface is not None
+        await self.interface.mouse_down(x, y, button="left")
+
+    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse up at coordinates."""
+        assert self.interface is not None
+        await self.interface.mouse_up(x, y, button="left")
+
+    # ==== Browser Control Methods (via Playwright) ====
+    async def playwright_exec(
+        self, command: str, params: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """Execute a Playwright browser command.
+
+        Supports: visit_url, click, type, scroll, web_search, screenshot,
+                  get_current_url, go_back, go_forward
+
+        Args:
+            command: The browser command to execute
+            params: Command parameters
+
+        Returns:
+            Dict containing the command result
+        """
+        assert self.interface is not None
+        return await self.interface.playwright_exec(command, params or {})

libs/python/agent/cua_agent/computers/custom.py

@@ -0,0 +1,216 @@
+"""
+Custom computer handler implementation that accepts a dictionary of functions.
+"""
+
+import base64
+import io
+from typing import Any, Callable, Dict, List, Literal, Optional, Union
+
+from PIL import Image
+
+from .base import AsyncComputerHandler
+
+
+class CustomComputerHandler(AsyncComputerHandler):
+    """Computer handler that implements the Computer protocol using a dictionary of custom functions."""
+
+    def __init__(self, functions: Dict[str, Callable]):
+        """
+        Initialize with a dictionary of functions.
+
+        Args:
+            functions: Dictionary where keys are method names and values are callable functions.
+                      Only 'screenshot' is required, all others are optional.
+
+        Raises:
+            ValueError: If required 'screenshot' function is not provided.
+        """
+        if "screenshot" not in functions:
+            raise ValueError("'screenshot' function is required in functions dictionary")
+
+        self.functions = functions
+        self._last_screenshot_size: Optional[tuple[int, int]] = None
+
+    async def _call_function(self, func, *args, **kwargs):
+        """
+        Call a function, handling both async and sync functions.
+
+        Args:
+            func: The function to call
+            *args: Positional arguments to pass to the function
+            **kwargs: Keyword arguments to pass to the function
+
+        Returns:
+            The result of the function call
+        """
+        import asyncio
+        import inspect
+
+        if callable(func):
+            if inspect.iscoroutinefunction(func):
+                return await func(*args, **kwargs)
+            else:
+                return func(*args, **kwargs)
+        else:
+            return func
+
+    async def _get_value(self, attribute: str):
+        """
+        Get value for an attribute, checking both 'get_{attribute}' and '{attribute}' keys.
+
+        Args:
+            attribute: The attribute name to look for
+
+        Returns:
+            The value from the functions dict, called if callable, returned directly if not
+        """
+        # Check for 'get_{attribute}' first
+        get_key = f"get_{attribute}"
+        if get_key in self.functions:
+            return await self._call_function(self.functions[get_key])
+
+        # Check for '{attribute}'
+        if attribute in self.functions:
+            return await self._call_function(self.functions[attribute])
+
+        return None
+
+    def _to_b64_str(self, img: Union[bytes, Image.Image, str]) -> str:
+        """
+        Convert image to base64 string.
+
+        Args:
+            img: Image as bytes, PIL Image, or base64 string
+
+        Returns:
+            str: Base64 encoded image string
+        """
+        if isinstance(img, str):
+            # Already a base64 string
+            return img
+        elif isinstance(img, bytes):
+            # Raw bytes
+            return base64.b64encode(img).decode("utf-8")
+        elif isinstance(img, Image.Image):
+            # PIL Image
+            buffer = io.BytesIO()
+            img.save(buffer, format="PNG")
+            return base64.b64encode(buffer.getvalue()).decode("utf-8")
+        else:
+            raise ValueError(f"Unsupported image type: {type(img)}")
+
+    # ==== Computer-Use-Preview Action Space ====
+
+    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
+        """Get the current environment type."""
+        result = await self._get_value("environment")
+        if result is None:
+            return "linux"
+        assert result in ["windows", "mac", "linux", "browser"]
+        return result  # type: ignore
+
+    async def get_dimensions(self) -> tuple[int, int]:
+        """Get screen dimensions as (width, height)."""
+        result = await self._get_value("dimensions")
+        if result is not None:
+            return result  # type: ignore
+
+        # Fallback: use last screenshot size if available
+        if not self._last_screenshot_size:
+            await self.screenshot()
+        assert self._last_screenshot_size is not None, "Failed to get screenshot size"
+
+        return self._last_screenshot_size
+
+    async def screenshot(self, text: Optional[str] = None) -> str:
+        """Take a screenshot and return as base64 string.
+
+        Args:
+            text: Optional descriptive text (for compatibility with GPT-4o models, ignored)
+        """
+        result = await self._call_function(self.functions["screenshot"])
+        b64_str = self._to_b64_str(result)  # type: ignore
+
+        # Try to extract dimensions for fallback use
+        try:
+            if isinstance(result, Image.Image):
+                self._last_screenshot_size = result.size
+            elif isinstance(result, bytes):
+                # Try to decode bytes to get dimensions
+                img = Image.open(io.BytesIO(result))
+                self._last_screenshot_size = img.size
+        except Exception:
+            # If we can't get dimensions, that's okay
+            pass
+
+        return b64_str
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        """Click at coordinates with specified button."""
+        if "click" in self.functions:
+            await self._call_function(self.functions["click"], x, y, button)
+        # No-op if not implemented
+
+    async def double_click(self, x: int, y: int) -> None:
+        """Double click at coordinates."""
+        if "double_click" in self.functions:
+            await self._call_function(self.functions["double_click"], x, y)
+        # No-op if not implemented
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """Scroll at coordinates with specified scroll amounts."""
+        if "scroll" in self.functions:
+            await self._call_function(self.functions["scroll"], x, y, scroll_x, scroll_y)
+        # No-op if not implemented
+
+    async def type(self, text: str) -> None:
+        """Type text."""
+        if "type" in self.functions:
+            await self._call_function(self.functions["type"], text)
+        # No-op if not implemented
+
+    async def wait(self, ms: int = 1000) -> None:
+        """Wait for specified milliseconds."""
+        if "wait" in self.functions:
+            await self._call_function(self.functions["wait"], ms)
+        else:
+            # Default implementation
+            import asyncio
+
+            await asyncio.sleep(ms / 1000.0)
+
+    async def move(self, x: int, y: int) -> None:
+        """Move cursor to coordinates."""
+        if "move" in self.functions:
+            await self._call_function(self.functions["move"], x, y)
+        # No-op if not implemented
+
+    async def keypress(self, keys: Union[List[str], str]) -> None:
+        """Press key combination."""
+        if "keypress" in self.functions:
+            await self._call_function(self.functions["keypress"], keys)
+        # No-op if not implemented
+
+    async def drag(self, path: List[Dict[str, int]]) -> None:
+        """Drag along specified path."""
+        if "drag" in self.functions:
+            await self._call_function(self.functions["drag"], path)
+        # No-op if not implemented
+
+    async def get_current_url(self) -> str:
+        """Get current URL (for browser environments)."""
+        if "get_current_url" in self.functions:
+            return await self._get_value("current_url")  # type: ignore
+        return ""  # Default fallback
+
+    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse down at coordinates."""
+        if "left_mouse_down" in self.functions:
+            await self._call_function(self.functions["left_mouse_down"], x, y)
+        # No-op if not implemented
+
+    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse up at coordinates."""
+        if "left_mouse_up" in self.functions:
+            await self._call_function(self.functions["left_mouse_up"], x, y)
+        # No-op if not implemented

libs/python/agent/cua_agent/computers/sandbox.py

@@ -0,0 +1,125 @@
+"""
+Computer handler implementation wrapping a cua_sandbox.Sandbox instance.
+"""
+
+import asyncio
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from .base import AsyncComputerHandler
+
+
+class SandboxComputerHandler(AsyncComputerHandler):
+    """Computer handler that adapts a cua_sandbox.Sandbox to the AsyncComputerHandler protocol."""
+
+    def __init__(self, sandbox: Any):
+        self._sandbox = sandbox
+
+    # ==== Computer-Use-Preview Action Space ====
+
+    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
+        return await self._sandbox.get_environment()
+
+    async def get_dimensions(self) -> tuple[int, int]:
+        return await self._sandbox.get_dimensions()
+
+    async def screenshot(self, text: Optional[str] = None) -> str:
+        return await self._sandbox.screenshot_base64()
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        if button == "right":
+            await self._sandbox.mouse.right_click(x, y)
+        else:
+            await self._sandbox.mouse.click(x, y, button=button)
+
+    async def double_click(self, x: int, y: int) -> None:
+        await self._sandbox.mouse.double_click(x, y)
+
+    async def right_click(self, x: int, y: int) -> None:
+        await self._sandbox.mouse.right_click(x, y)
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        await self._sandbox.mouse.scroll(x, y, scroll_x=scroll_x, scroll_y=scroll_y)
+
+    async def type(self, text: str) -> None:
+        await self._sandbox.keyboard.type(text)
+
+    async def wait(self, ms: int = 1000) -> None:
+        await asyncio.sleep(ms / 1000.0)
+
+    async def move(self, x: int, y: int) -> None:
+        await self._sandbox.mouse.move(x, y)
+
+    # Maps Anthropic/X11 key names → pynput Key attribute names used by computer-server
+    _KEY_NAME_MAP = {
+        "return": "enter",
+        "backspace": "backspace",
+        "delete": "delete",
+        "del": "delete",
+        "escape": "esc",
+        "esc": "esc",
+        "tab": "tab",
+        "space": "space",
+        " ": "space",
+        "ctrl": "ctrl",
+        "control": "ctrl",
+        "shift": "shift",
+        "alt": "alt",
+        "super": "cmd",
+        "meta": "cmd",
+        "cmd": "cmd",
+        "command": "cmd",
+        "win": "cmd",
+        "up": "up",
+        "down": "down",
+        "left": "left",
+        "right": "right",
+        "home": "home",
+        "end": "end",
+        "pageup": "page_up",
+        "page_up": "page_up",
+        "pgup": "page_up",
+        "pagedown": "page_down",
+        "page_down": "page_down",
+        "pgdn": "page_down",
+        "insert": "insert",
+        "ins": "insert",
+        "caps_lock": "caps_lock",
+        **{f"f{i}": f"f{i}" for i in range(1, 21)},
+    }
+
+    async def keypress(self, keys: Union[List[str], str]) -> None:
+        if isinstance(keys, str):
+            keys = [keys]
+        normalized = []
+        for k in keys:
+            mapped = self._KEY_NAME_MAP.get(k.lower(), k.lower() if len(k) > 1 else k)
+            normalized.append(mapped)
+        await self._sandbox.keyboard.keypress(normalized)
+
+    async def drag(
+        self,
+        path: Optional[List[Dict[str, int]]] = None,
+        start_x: Optional[int] = None,
+        start_y: Optional[int] = None,
+        end_x: Optional[int] = None,
+        end_y: Optional[int] = None,
+    ) -> None:
+        if start_x is not None and start_y is not None and end_x is not None and end_y is not None:
+            path = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]
+        if not path:
+            return
+        await self._sandbox.mouse.drag(path)
+
+    async def get_current_url(self) -> str:
+        return ""
+
+    async def terminate(self, status: str = "success") -> Dict[str, Any]:
+        return {"success": True, "status": status, "terminated": True}
+
+    # ==== Anthropic Action Space ====
+
+    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._sandbox.mouse.mouse_down(x, y, button="left")
+
+    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._sandbox.mouse.mouse_up(x, y, button="left")