chore: import upstream snapshot with attribution

tools/computer_use/__init__.py

@@ -0,0 +1,43 @@
+"""Computer use toolset — universal (any-model) macOS desktop control.
+
+Architecture
+------------
+This toolset drives macOS apps through cua-driver's background computer-use
+primitive (SkyLight private SPIs for focus-without-raise + pid-scoped event
+posting). Unlike #4562's pyautogui backend, it does NOT steal the user's
+cursor, keyboard focus, or Space — the agent and the user can co-work on the
+same machine.
+
+Unlike #4562's Anthropic-native `computer_20251124` tool, the schema here is
+a plain OpenAI function-calling schema that every tool-capable model can
+drive. Vision models get SOM (set-of-mark) captures — a screenshot with
+numbered overlays on every interactable element plus the AX tree — so they
+click by element index instead of pixel coordinates. Non-vision models can
+drive via the AX tree alone.
+
+Wiring
+------
+* `tool.py`       — registers the `computer_use` tool via tools.registry.
+* `backend.py`    — abstract `ComputerUseBackend`; swappable implementation.
+* `cua_backend.py`— default backend; speaks MCP over stdio to `cua-driver`.
+* `schema.py`     — shared schema + docstring for the generic `computer_use`
+                    tool. Model-agnostic.
+* `capture.py`    — screenshot post-processing (PNG coercion, sizing, SOM
+                    overlay if the backend did not).
+
+The outer integration points (multimodal tool-result plumbing, screenshot
+eviction in the Anthropic adapter, image-aware token estimation, the
+COMPUTER_USE_GUIDANCE prompt block, approval hook, and the skill) live
+alongside this package. See agent/anthropic_adapter.py and
+agent/prompt_builder.py for the salvaged hunks from PR #4562.
+"""
+
+from __future__ import annotations
+
+# Re-export the public surface so `from tools.computer_use import ...` works.
+from tools.computer_use.tool import (  # noqa: F401
+    handle_computer_use,
+    set_approval_callback,
+    check_computer_use_requirements,
+    get_computer_use_schema,
+)

tools/computer_use/backend.py

@@ -0,0 +1,171 @@
+"""Abstract backend interface for computer use.
+
+Any implementation (cua-driver over MCP, pyautogui, noop, future Linux/Windows)
+must return the shape described below. All methods synchronous; async is
+handled inside the backend implementation if needed.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+
+@dataclass
+class UIElement:
+    """One interactable element on the current screen."""
+
+    index: int                       # 1-based SOM index
+    role: str                        # AX role (AXButton, AXTextField, ...)
+    label: str = ""                  # AXTitle / AXDescription / AXValue snippet
+    bounds: Tuple[int, int, int, int] = (0, 0, 0, 0)  # x, y, w, h (logical px)
+    app: str = ""                    # owning bundle ID or app name
+    pid: int = 0                     # owning process PID
+    window_id: int = 0               # SkyLight / CG window ID
+    attributes: Dict[str, Any] = field(default_factory=dict)
+    # Opaque per-snapshot element handle from cua-driver
+    # (trycua/cua#1961 — Surface 6 of NousResearch/hermes-agent#47072).
+    # When set, downstream calls can pass it alongside `index` for
+    # explicit stale-detection: a stale token returns an error from
+    # cua-driver rather than silently re-resolving to a different
+    # element. None for pre-#1961 drivers that didn't carry the field.
+    element_token: Optional[str] = None
+
+    def center(self) -> Tuple[int, int]:
+        x, y, w, h = self.bounds
+        return x + w // 2, y + h // 2
+
+
+@dataclass
+class CaptureResult:
+    """Result of a screen capture call.
+
+    At least one of png_b64 / elements is populated depending on capture mode:
+      * mode="vision" → png_b64 only
+      * mode="ax"     → elements only
+      * mode="som"    → both (default): PNG already has numbered overlays
+                         drawn by the backend, and `elements` holds the
+                         matching index → element mapping.
+    """
+
+    mode: str
+    width: int                      # screenshot width (logical px, pre-Anthropic-scale)
+    height: int
+    png_b64: Optional[str] = None
+    elements: List[UIElement] = field(default_factory=list)
+    # Optional: the target app/window the elements were captured for.
+    app: str = ""
+    window_title: str = ""
+    # Raw bytes we sent to Anthropic, for token estimation.
+    png_bytes_len: int = 0
+    # Explicit MIME type for `png_b64` when the backend supplied it
+    # (cua-driver-rs emits `mimeType` on every image part as of
+    # trycua/cua#1961 — Surface 7 of NousResearch/hermes-agent#47072).
+    # When None, downstream consumers fall back to base64-prefix
+    # sniffing for back-compat with older drivers.
+    image_mime_type: Optional[str] = None
+
+
+@dataclass
+class ActionResult:
+    """Result of any action (click / type / scroll / drag / key / wait)."""
+
+    ok: bool
+    action: str
+    message: str = ""                # human-readable summary
+    # Optional trailing screenshot — set when the caller asked for a
+    # post-action capture or the backend always returns one.
+    capture: Optional[CaptureResult] = None
+    # Arbitrary extra fields for debugging / telemetry.
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+
+class ComputerUseBackend(ABC):
+    """Lifecycle: `start()` before first use, `stop()` at shutdown."""
+
+    @abstractmethod
+    def start(self) -> None: ...
+
+    @abstractmethod
+    def stop(self) -> None: ...
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Return True if the backend can be used on this host right now.
+
+        Used by check_fn gating and by the post-setup wizard.
+        """
+
+    # ── Capture ─────────────────────────────────────────────────────
+    @abstractmethod
+    def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult: ...
+
+    # ── Pointer actions ─────────────────────────────────────────────
+    @abstractmethod
+    def click(
+        self,
+        *,
+        element: Optional[int] = None,
+        x: Optional[int] = None,
+        y: Optional[int] = None,
+        button: str = "left",           # left | right | middle
+        click_count: int = 1,
+        modifiers: Optional[List[str]] = None,
+    ) -> ActionResult: ...
+
+    @abstractmethod
+    def drag(
+        self,
+        *,
+        from_element: Optional[int] = None,
+        to_element: Optional[int] = None,
+        from_xy: Optional[Tuple[int, int]] = None,
+        to_xy: Optional[Tuple[int, int]] = None,
+        button: str = "left",
+        modifiers: Optional[List[str]] = None,
+    ) -> ActionResult: ...
+
+    @abstractmethod
+    def scroll(
+        self,
+        *,
+        direction: str,                 # up | down | left | right
+        amount: int = 3,                # wheel ticks
+        element: Optional[int] = None,
+        x: Optional[int] = None,
+        y: Optional[int] = None,
+        modifiers: Optional[List[str]] = None,
+    ) -> ActionResult: ...
+
+    # ── Keyboard ────────────────────────────────────────────────────
+    @abstractmethod
+    def type_text(self, text: str) -> ActionResult: ...
+
+    @abstractmethod
+    def key(self, keys: str) -> ActionResult:
+        """Send a key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return'."""
+
+    # ── Introspection ───────────────────────────────────────────────
+    @abstractmethod
+    def list_apps(self) -> List[Dict[str, Any]]:
+        """Return running apps with bundle IDs, PIDs, window counts."""
+
+    @abstractmethod
+    def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
+        """Route input to `app` (by name or bundle ID). Default: focus without raise."""
+
+    # ── Native-value mutation ────────────────────────────────────────
+    @abstractmethod
+    def set_value(self, value: str, element: Optional[int] = None) -> ActionResult:
+        """Set a native value on an element (e.g. AXPopUpButton selection).
+
+        `element` is the 1-based SOM index returned by a prior capture call.
+        """
+
+    # ── Timing ──────────────────────────────────────────────────────
+    def wait(self, seconds: float) -> ActionResult:
+        """Default implementation: time.sleep."""
+        import time
+        time.sleep(max(0.0, min(seconds, 30.0)))
+        return ActionResult(ok=True, action="wait", message=f"waited {seconds:.2f}s")

tools/computer_use/doctor.py

@@ -0,0 +1,288 @@
+"""
+`hermes computer-use doctor` — thin client for cua-driver's `health_report` MCP tool.
+
+cua-driver owns the health model (#1908 / be761fac on `main`). This module
+just drives the stdio JSON-RPC handshake, calls `health_report`, and
+renders the structured response. When the driver gets new checks, they
+flow through here without code changes on the Hermes side — the only
+contract is the stable `schema_version="1"` payload shape.
+
+Exit code conventions:
+- 0: overall == "ok"
+- 1: overall in ("degraded", "failed")
+- 2: driver binary missing / unreachable / protocol error
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+from typing import Any, Dict, List, Optional, Sequence
+
+
+# Match the ALLOWED_STATUS_VALUES + ALLOWED_OVERALL_VALUES the cua-driver
+# integration test pins. If health_report widens its vocabulary, add here.
+_STATUS_GLYPH = {
+    "pass": "✅",
+    "fail": "❌",
+    "skip": "⏭️",
+}
+_OVERALL_GLYPH = {
+    "ok":       "✅",
+    "degraded": "⚠️",
+    "failed":   "❌",
+}
+
+
+def _cua_child_env() -> Dict[str, str]:
+    """cua-driver child env with the Hermes telemetry policy applied.
+
+    Delegates to ``cua_backend.cua_driver_child_env`` (telemetry disabled by
+    default unless the user opts in). Falls back to the current environment
+    if that import fails, so doctor never breaks on a telemetry-helper error.
+    """
+    try:
+        from tools.computer_use.cua_backend import cua_driver_child_env
+
+        return cua_driver_child_env()
+    except Exception:
+        return dict(os.environ)
+
+
+def _sanitized_cua_env() -> Dict[str, str]:
+    """Telemetry-policy env with Hermes provider secrets stripped.
+
+    cua-driver is a third-party binary — it must never inherit provider
+    API keys (#53503/#55709/#58889 lineage). Falls back to the unsanitized
+    telemetry env if the sanitizer can't be imported, so doctor keeps
+    working in stripped-down environments.
+    """
+    env = _cua_child_env()
+    try:
+        from tools.environments.local import _sanitize_subprocess_env
+
+        return _sanitize_subprocess_env(env)
+    except Exception:
+        return env
+
+
+def _drive_health_report(
+    binary: str,
+    *,
+    include: Sequence[str] = (),
+    skip: Sequence[str] = (),
+    timeout: float = 12.0,
+) -> Dict[str, Any]:
+    """Spawn `<binary> mcp`, perform the JSON-RPC handshake, call
+    `health_report`, and return the parsed `structuredContent` dict.
+
+    Raises `RuntimeError` on a protocol-level failure (binary crash,
+    malformed response, JSON-RPC error). Never raises on a `health_report`
+    that has failing checks — the tool's contract is to always return a
+    well-formed report with `overall` set, never to set `isError`.
+    """
+    args: Dict[str, Any] = {}
+    if include:
+        args["include"] = list(include)
+    if skip:
+        args["skip"] = list(skip)
+
+    # cua-driver emits UTF-8 (containing emoji in check messages on macOS
+    # and arbitrary file paths on Windows). The Python default
+    # text-mode encoding follows the system locale — `cp1252` on a
+    # default Windows install — which raises UnicodeDecodeError on the
+    # first non-ASCII byte. Pin the codec.
+    proc = subprocess.Popen(
+        [binary, "mcp"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        bufsize=1,
+        env=_sanitized_cua_env(),
+    )
+    try:
+        # 1. initialize
+        proc.stdin.write(json.dumps({
+            "jsonrpc": "2.0", "id": 1,
+            "method": "initialize", "params": {},
+        }) + "\n")
+        proc.stdin.flush()
+        init_line = proc.stdout.readline()
+        if not init_line:
+            stderr_tail = (proc.stderr.read() or "").strip().splitlines()[-3:]
+            raise RuntimeError(
+                f"cua-driver mcp produced no initialize response. "
+                f"stderr tail: {stderr_tail or '(empty)'}"
+            )
+
+        # 2. tools/call health_report
+        proc.stdin.write(json.dumps({
+            "jsonrpc": "2.0", "id": 2,
+            "method": "tools/call",
+            "params": {"name": "health_report", "arguments": args},
+        }) + "\n")
+        proc.stdin.flush()
+        call_line = proc.stdout.readline()
+        if not call_line:
+            raise RuntimeError("cua-driver mcp closed stdout without responding to health_report.")
+    finally:
+        try:
+            proc.stdin.close()
+        except Exception:
+            pass
+        try:
+            proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+
+    try:
+        resp = json.loads(call_line)
+    except (ValueError, TypeError) as e:
+        raise RuntimeError(f"health_report response was not valid JSON: {e}\nraw: {call_line[:200]}")
+
+    if "error" in resp:
+        raise RuntimeError(f"health_report JSON-RPC error: {resp['error']}")
+
+    result = resp.get("result") or {}
+
+    # Preferred: structuredContent (cua-driver-rs always emits it on the
+    # health_report response). Fall back to parsing the first text item
+    # as JSON for older cua-driver builds that didn't carry structuredContent.
+    sc = result.get("structuredContent")
+    if isinstance(sc, dict):
+        return sc
+
+    for item in result.get("content", []):
+        if item.get("type") == "text":
+            text = item.get("text", "")
+            try:
+                # Many health_report payloads ship JSON in the text item too.
+                parsed = json.loads(text)
+                if isinstance(parsed, dict) and "schema_version" in parsed:
+                    return parsed
+            except (ValueError, TypeError):
+                pass
+
+    raise RuntimeError(
+        "health_report response carried neither structuredContent nor a parseable "
+        f"JSON text block. Result keys: {list(result.keys())}"
+    )
+
+
+def _print_text_report(report: Dict[str, Any], color: bool) -> None:
+    """Render the report in the same style as `cua-driver call health_report`
+    would (one line per check + a summary footer)."""
+    schema = report.get("schema_version", "?")
+    platform = report.get("platform", "?")
+    driver_v = report.get("driver_version", "?")
+    overall = report.get("overall", "?")
+
+    header_glyph = _OVERALL_GLYPH.get(overall, "•")
+
+    if color and overall in _OVERALL_GLYPH:
+        # No external color library — keep ANSI inline so the doctor
+        # command stays a single self-contained module.
+        col_red = "\033[31m"
+        col_yellow = "\033[33m"
+        col_green = "\033[32m"
+        col_reset = "\033[0m"
+        col_dim = "\033[2m"
+        col_for = {"failed": col_red, "degraded": col_yellow, "ok": col_green}.get(overall, "")
+    else:
+        col_red = col_yellow = col_green = col_reset = col_dim = ""
+        col_for = ""
+
+    print(
+        f"{header_glyph} cua-driver {driver_v} on {platform} — "
+        f"{col_for}{overall}{col_reset}"
+    )
+
+    for check in report.get("checks", []):
+        name = check.get("name", "?")
+        status = check.get("status", "?")
+        glyph = _STATUS_GLYPH.get(status, "•")
+        message = check.get("message") or ""
+        if color:
+            status_col = {
+                "pass": col_green, "fail": col_red, "skip": col_dim,
+            }.get(status, "")
+            print(f"  {glyph} {status_col}{name}{col_reset}: {message}")
+        else:
+            print(f"  {glyph} {name}: {message}")
+        hint = check.get("hint")
+        if hint:
+            print(f"      → {col_dim}{hint}{col_reset}")
+        # `data` is the structured payload some checks attach (bundle id,
+        # AX permission state, version triple, etc.). Surface when present
+        # because users / support staff frequently need it.
+        data = check.get("data")
+        if isinstance(data, dict) and data:
+            for key, value in data.items():
+                rendered = value if not isinstance(value, (dict, list)) else json.dumps(value)
+                print(f"      {col_dim}{key}={rendered}{col_reset}")
+    _ = schema  # acknowledge field for forward-compat readers
+
+
+def run_doctor(
+    driver_cmd: Optional[str] = None,
+    *,
+    include: Sequence[str] = (),
+    skip: Sequence[str] = (),
+    json_output: bool = False,
+    color: Optional[bool] = None,
+) -> int:
+    """Resolve the cua-driver binary, call `health_report`, render the result.
+
+    Honors `HERMES_CUA_DRIVER_CMD` via the same `_cua_driver_cmd()` resolver
+    that `install_cua_driver` + the runtime backend use, so the doctor
+    diagnoses what your `computer_use` toolset will actually invoke.
+    """
+    # Windows ships stdout/stderr wrapped with the system ANSI codec
+    # (`cp1252` on a US locale, `cp936` on zh-CN, etc.). The check-matrix
+    # output below contains ✅ ❌ ⚠️ ⏭️ glyphs — none of them encodable
+    # in those codepages. Switch stdout to UTF-8 once, idempotently: every
+    # supported TextIOWrapper (Py3.7+) has `.reconfigure`, and a no-op
+    # re-encode is cheap if we were already UTF-8.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")  # type: ignore[union-attr]
+        except (AttributeError, OSError):
+            pass
+    if driver_cmd is None:
+        try:
+            from hermes_cli.tools_config import _cua_driver_cmd
+            driver_cmd = _cua_driver_cmd()
+        except Exception:
+            driver_cmd = os.environ.get("HERMES_CUA_DRIVER_CMD") or "cua-driver"
+
+    binary = shutil.which(driver_cmd)
+    if not binary:
+        print(f"cua-driver: not installed (looked for {driver_cmd!r}).")
+        print("  Run: hermes computer-use install")
+        return 2
+
+    try:
+        report = _drive_health_report(binary, include=include, skip=skip)
+    except RuntimeError as e:
+        print(f"cua-driver health_report failed: {e}", file=sys.stderr)
+        return 2
+
+    if json_output:
+        json.dump(report, sys.stdout, indent=2, sort_keys=True)
+        sys.stdout.write("\n")
+    else:
+        if color is None:
+            color = sys.stdout.isatty()
+        _print_text_report(report, color=bool(color))
+
+    overall = report.get("overall")
+    if overall in ("degraded", "failed"):
+        return 1
+    return 0

tools/computer_use/permissions.py

@@ -0,0 +1,200 @@
+"""
+Cross-platform Computer Use readiness + macOS permission helpers.
+
+cua-driver runs on macOS, Windows, and Linux, but "ready to drive" means
+something different on each:
+
+  * macOS — explicit TCC grants (Accessibility + Screen Recording). cua-driver
+    reports/requests them via ``permissions status`` / ``permissions grant``.
+    The grants attach to cua-driver's OWN identity (``com.trycua.driver`` /
+    the installed ``CuaDriver.app``), NOT Hermes — so no Hermes entitlement is
+    involved, and ``grant`` launches CuaDriver via LaunchServices so the macOS
+    dialog is attributed correctly.
+  * Windows — no TCC toggles; the UIAccess worker (``cua-driver-uia.exe``) may
+    trip a SmartScreen prompt on first run. Readiness == driver health.
+  * Linux — assistive control via the X11/XWayland stack. Readiness == driver
+    health.
+
+The universal signal on every platform is ``cua-driver doctor --json`` (binary
+integrity + platform support). ``computer_use_status`` folds that together with
+the macOS permission detail into one payload for the desktop card, the
+``hermes computer-use permissions`` CLI, and ``/api/tools/computer-use/status``.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+from typing import Any, Dict, List, Optional
+
+# Platforms with a cua-driver runtime backend (mirrors the toolset platform_gate).
+_RUNTIME_PLATFORMS = frozenset({"darwin", "win32", "linux"})
+_BOOLS = ("accessibility", "screen_recording", "screen_recording_capturable")
+
+
+def _driver_cmd(override: Optional[str]) -> str:
+    if override:
+        return override
+    try:
+        from hermes_cli.tools_config import _cua_driver_cmd
+
+        return _cua_driver_cmd()
+    except Exception:
+        return os.environ.get("HERMES_CUA_DRIVER_CMD", "").strip() or "cua-driver"
+
+
+def _child_env() -> Dict[str, str]:
+    """cua-driver child env: telemetry opt-in policy + secret sanitization.
+
+    cua-driver is a third-party binary — it must never inherit provider
+    API keys (#53503/#55709/#58889 lineage). Each layer degrades
+    gracefully so permission probes never break on a helper import error.
+    """
+    try:
+        from tools.computer_use.cua_backend import cua_driver_child_env
+
+        env = cua_driver_child_env()
+    except Exception:
+        env = dict(os.environ)
+    try:
+        from tools.environments.local import _sanitize_subprocess_env
+
+        return _sanitize_subprocess_env(env)
+    except Exception:
+        return env
+
+
+def _run(binary: str, *args: str, timeout: float) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        [binary, *args],
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        env=_child_env(),
+        stdin=subprocess.DEVNULL,
+    )
+
+
+def _json_out(binary: str, *args: str, timeout: float) -> Any:
+    """Run ``binary args`` and parse stdout as JSON, or ``None`` on any failure."""
+    raw = (_run(binary, *args, timeout=timeout).stdout or "").strip()
+    return json.loads(raw) if raw else None
+
+
+def _doctor(binary: str) -> Optional[Dict[str, Any]]:
+    """``cua-driver doctor --json`` → ``{ok, checks:[{label,status,message}]}``."""
+    try:
+        data = _json_out(binary, "doctor", "--json", timeout=12)
+    except Exception:
+        return None
+    if not isinstance(data, dict):
+        return None
+    checks: List[Dict[str, str]] = [
+        {
+            "label": str(p.get("label", "")),
+            "status": str(p.get("status", "")),
+            "message": str(p.get("message", "")),
+        }
+        for p in data.get("probes", [])
+        if isinstance(p, dict)
+    ]
+    return {"ok": bool(data.get("ok")), "checks": checks}
+
+
+def _mac_permissions(binary: str, out: Dict[str, Any]) -> None:
+    """Fold ``cua-driver permissions status --json`` booleans into ``out``."""
+    try:
+        data = _json_out(binary, "permissions", "status", "--json", timeout=10)
+    except subprocess.TimeoutExpired:
+        out["error"] = "cua-driver permissions status timed out"
+        return
+    except Exception as exc:  # spawn failure or malformed JSON
+        out["error"] = f"cua-driver permissions status failed: {exc}"
+        return
+    if isinstance(data, dict):
+        out.update({k: data[k] for k in _BOOLS if isinstance(data.get(k), bool)})
+        if isinstance(data.get("source"), dict):
+            out["source"] = data["source"]
+
+
+def computer_use_status(driver_cmd: Optional[str] = None) -> Dict[str, Any]:
+    """Unified, OS-aware Computer Use readiness for the desktop card.
+
+    ``ready`` is the single signal the UI keys off: on macOS it's both TCC
+    grants; elsewhere it's driver health (no TCC model). ``None`` means
+    unknown (binary missing / probe failed). ``can_grant`` is macOS-only.
+    """
+    plat = sys.platform
+    binary = shutil.which(_driver_cmd(driver_cmd))
+    out: Dict[str, Any] = {
+        "platform": plat,
+        "platform_supported": plat in _RUNTIME_PLATFORMS,
+        "installed": bool(binary),
+        "version": None,
+        "ready": None,
+        "can_grant": plat == "darwin",
+        "checks": [],
+        "source": None,
+        "error": None,
+        **{k: None for k in _BOOLS},
+    }
+    if not binary:
+        return out
+
+    try:
+        out["version"] = (_run(binary, "--version", timeout=5).stdout or "").strip() or None
+    except Exception:
+        pass
+
+    doctor = _doctor(binary)
+    if doctor is not None:
+        out["checks"] = doctor["checks"]
+
+    if plat == "darwin":
+        _mac_permissions(binary, out)
+        if out["error"] is None:
+            out["ready"] = out["accessibility"] is True and out["screen_recording"] is True
+    elif doctor is not None:
+        # No TCC model off macOS — readiness is driver health.
+        out["ready"] = doctor["ok"]
+    return out
+
+
+def request_permissions_grant(driver_cmd: Optional[str] = None) -> int:
+    """Run ``cua-driver permissions grant`` (macOS); stream its output.
+
+    Launches CuaDriver via LaunchServices so the TCC dialog is attributed to
+    ``com.trycua.driver``, then waits for the grant. Returns the driver's exit
+    code (0 ok), 2 if the binary is missing, 64 on a non-macOS platform (which
+    has no TCC permission model to grant).
+    """
+    if sys.platform != "darwin":
+        print("Computer Use permissions are a macOS concept; nothing to grant here.")
+        return 64
+
+    binary = shutil.which(_driver_cmd(driver_cmd))
+    if not binary:
+        print("cua-driver: not installed. Run: hermes computer-use install")
+        return 2
+
+    print(
+        "Requesting Accessibility + Screen Recording for CuaDriver.\n"
+        "macOS will show a dialog attributed to CuaDriver (com.trycua.driver) — "
+        "approve it, then return here."
+    )
+    try:
+        return int(
+            subprocess.run(
+                [binary, "permissions", "grant"],
+                env=_child_env(),
+                stdin=subprocess.DEVNULL,
+            ).returncode
+        )
+    except KeyboardInterrupt:  # pragma: no cover - interactive
+        return 130
+    except Exception as exc:  # pragma: no cover - defensive
+        print(f"cua-driver permissions grant failed: {exc}", file=sys.stderr)
+        return 2

tools/computer_use/schema.py

@@ -0,0 +1,222 @@
+"""Schema for the generic `computer_use` tool.
+
+Model-agnostic. Any tool-calling model can drive this. Vision-capable models
+should prefer `capture(mode='som')` then `click(element=N)` — much more
+reliable than pixel coordinates. Pixel coordinates remain supported for
+models that were trained on them (e.g. Claude's computer-use RL).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+
+# One consolidated tool with an `action` discriminator. Keeps the schema
+# compact and the per-turn token cost low.
+COMPUTER_USE_SCHEMA: Dict[str, Any] = {
+    "name": "computer_use",
+    "description": (
+        "Drive the desktop in the background via cua-driver — screenshots, "
+        "mouse, keyboard, scroll, drag — without stealing the user's cursor "
+        "or keyboard focus. Supported on macOS, Windows, and Linux. "
+        "Preferred workflow: call with "
+        "action='capture' (mode='som' gives numbered element overlays), "
+        "then click by `element` index for reliability. Pixel coordinates "
+        "are supported for models trained on them. Works on any window — "
+        "hidden, minimized, or behind another app. Requires cua-driver to "
+        "be installed."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "action": {
+                "type": "string",
+                "enum": [
+                    "capture",
+                    "click",
+                    "double_click",
+                    "right_click",
+                    "middle_click",
+                    "drag",
+                    "scroll",
+                    "type",
+                    "key",
+                    "set_value",
+                    "wait",
+                    "list_apps",
+                    "focus_app",
+                ],
+                "description": (
+                    "Which action to perform. `capture` is free (no side "
+                    "effects). All other actions require approval unless "
+                    "auto-approved. Use `set_value` for select/popup elements "
+                    "and sliders — it selects the matching option directly "
+                    "without opening the native menu (no focus steal)."
+                ),
+            },
+            # ── capture ────────────────────────────────────────────
+            "mode": {
+                "type": "string",
+                "enum": ["som", "vision", "ax"],
+                "description": (
+                    "Capture mode. `som` (default) is a screenshot with "
+                    "numbered overlays on every interactable element plus "
+                    "the AX tree — best for vision models, lets you click "
+                    "by element index. `vision` is a plain screenshot. "
+                    "`ax` is the accessibility tree only (no image; useful "
+                    "for text-only models)."
+                ),
+            },
+            "app": {
+                "type": "string",
+                "description": (
+                    "Optional. Limit capture/action to a specific app "
+                    "(by name, e.g. 'Safari', or bundle ID, "
+                    "'com.apple.Safari'). If omitted, operates on the "
+                    "frontmost app's window. Pass app='screen' (or "
+                    "'desktop') to capture the OS desktop/shell surface — "
+                    "e.g. to see the wallpaper or click the taskbar. Note: "
+                    "capture is per-window; a single image cannot span "
+                    "multiple monitors, so on a multi-screen setup capture "
+                    "one window or display at a time."
+                ),
+            },
+            "max_elements": {
+                "type": "integer",
+                "description": (
+                    "Optional cap on the AX `elements` array returned by "
+                    "`action='capture'`. Default 100, hard maximum 1000. "
+                    "Dense UIs (Electron apps such as Obsidian or VS Code, "
+                    "JetBrains IDEs) can publish 500+ AX nodes — capping "
+                    "prevents a single capture from blowing session "
+                    "context. When the cap trims the response, "
+                    "`total_elements` and `truncated_elements` are "
+                    "surfaced in the result so you can re-call with "
+                    "`app=` to narrow scope or raise `max_elements` when "
+                    "the full tree is required. Has no effect on "
+                    "`mode='som'` / `mode='vision'` when a screenshot is "
+                    "included in the response; only the rare image-"
+                    "missing fallback returns an `elements` array and is "
+                    "subject to the cap."
+                ),
+                "default": 100,
+                "minimum": 1,
+                "maximum": 1000,
+            },
+            # ── click / drag / scroll targeting ────────────────────
+            "element": {
+                "type": "integer",
+                "description": (
+                    "The 1-based SOM index returned by the last "
+                    "`capture(mode='som')` call. Strongly preferred over "
+                    "raw coordinates."
+                ),
+            },
+            "coordinate": {
+                "type": "array",
+                "items": {"type": "integer"},
+                "minItems": 2,
+                "maxItems": 2,
+                "description": (
+                    "Pixel coordinates [x, y] in logical screen space (as "
+                    "returned by capture width/height). Only use this if "
+                    "no element index is available."
+                ),
+            },
+            "button": {
+                "type": "string",
+                "enum": ["left", "right", "middle"],
+                "description": "Mouse button. Defaults to left.",
+            },
+            "modifiers": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                    "enum": [
+                        "cmd", "shift", "option", "alt", "ctrl", "fn",
+                        "win", "windows", "super", "meta",
+                    ],
+                },
+                "description": "Modifier keys held during the action.",
+            },
+            # ── drag ───────────────────────────────────────────────
+            "from_element": {"type": "integer",
+                              "description": "Source element index (drag)."},
+            "to_element": {"type": "integer",
+                            "description": "Target element index (drag)."},
+            "from_coordinate": {
+                "type": "array",
+                "items": {"type": "integer"},
+                "minItems": 2, "maxItems": 2,
+                "description": "Source [x,y] (drag; use when no element available).",
+            },
+            "to_coordinate": {
+                "type": "array",
+                "items": {"type": "integer"},
+                "minItems": 2, "maxItems": 2,
+                "description": "Target [x,y] (drag; use when no element available).",
+            },
+            # ── scroll ─────────────────────────────────────────────
+            "direction": {
+                "type": "string",
+                "enum": ["up", "down", "left", "right"],
+                "description": "Scroll direction.",
+            },
+            "amount": {
+                "type": "integer",
+                "description": "Scroll wheel ticks. Default 3.",
+            },
+            # ── set_value ──────────────────────────────────────────
+            "value": {
+                "type": "string",
+                "description": (
+                    "For action='set_value': the value to set on the element. "
+                    "For AXPopUpButton / select dropdowns, pass the option's "
+                    "display label (e.g. 'Blue'). For sliders and other "
+                    "AXValue-settable elements, pass the numeric or string value."
+                ),
+            },
+            # ── type / key / wait ──────────────────────────────────
+            "text": {
+                "type": "string",
+                "description": "Text to type (respects the current layout).",
+            },
+            "keys": {
+                "type": "string",
+                "description": (
+                    "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', "
+                    "'escape', 'tab'. Use '+' to combine."
+                ),
+            },
+            "seconds": {
+                "type": "number",
+                "description": "Seconds to wait. Max 30.",
+            },
+            # ── focus_app ──────────────────────────────────────────
+            "raise_window": {
+                "type": "boolean",
+                "description": (
+                    "Only for action='focus_app'. If true, brings the "
+                    "window to front (DISRUPTS the user). Default false "
+                    "— input is routed to the app without raising, "
+                    "matching the background co-work model."
+                ),
+            },
+            # ── return shape ───────────────────────────────────────
+            "capture_after": {
+                "type": "boolean",
+                "description": (
+                    "If true, take a follow-up capture after the action "
+                    "and include it in the response. Saves a round-trip "
+                    "when you need to verify an action's effect."
+                ),
+            },
+        },
+        "required": ["action"],
+    },
+}
+
+
+def get_computer_use_schema() -> Dict[str, Any]:
+    """Return the generic OpenAI function-calling schema."""
+    return COMPUTER_USE_SCHEMA

tools/computer_use/tool.py

@@ -0,0 +1,917 @@
+"""Entry point for the `computer_use` tool.
+
+Universal (any-model) desktop control across macOS, Windows, and Linux via
+cua-driver's background computer-use primitive. Replaces #4562's
+Anthropic-native `computer_20251124` approach — the schema here is standard
+OpenAI function-calling so every tool-capable model can drive it.
+
+Linux is the most recent runtime (X11 + Wayland, via cua-driver-rs's
+AT-SPI tree path); it is enabled here alongside macOS and Windows. When a
+host's display server or accessibility stack isn't reachable, cua-driver's
+`health_report` (surfaced by `hermes computer-use doctor`) reports the
+exact blocked check rather than the toolset silently failing.
+
+Return contract
+---------------
+For text-only results (wait, key, list_apps, focus_app, failures, etc.):
+  JSON string.
+
+For captures / actions with `capture_after=True`:
+  A dict wrapped as the OpenAI-style multi-part tool-message content:
+
+      {
+        "_multimodal": True,
+        "content": [
+            {"type": "text", "text": "<human-readable summary + SOM index>"},
+            {"type": "image_url",
+             "image_url": {"url": "data:image/png;base64,<b64>"}},
+        ],
+        "text_summary": "<text used for fallback string content>",
+      }
+
+  run_agent.py's tool-message builder inspects `_multimodal` and emits a
+  list-shaped `content` for OpenAI-compatible providers. The Anthropic
+  adapter splices the base64 image into a `tool_result` block (see
+  `agent/anthropic_adapter.py`). Every provider that supports multi-part
+  tool content gets the image; text-only providers see the summary only.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+import re
+import struct
+import sys
+import threading
+from typing import Any, Dict, List, Optional, Tuple
+
+from tools.computer_use.backend import (
+    ActionResult,
+    CaptureResult,
+    ComputerUseBackend,
+    UIElement,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Approval & safety
+# ---------------------------------------------------------------------------
+
+_approval_callback = None
+
+
+def set_approval_callback(cb) -> None:
+    """Register a callback for computer_use approval prompts (used by CLI).
+
+    Matches the terminal_tool._approval_callback pattern. The callback
+    receives (action, args, summary) and returns one of:
+      "approve_once" | "approve_session" | "always_approve" | "deny".
+    """
+    global _approval_callback
+    _approval_callback = cb
+
+
+# Actions that read, not mutate. Always allowed.
+_SAFE_ACTIONS = frozenset({"capture", "wait", "list_apps"})
+
+# Actions that mutate user-visible state. Go through approval.
+_DESTRUCTIVE_ACTIONS = frozenset({
+    "click", "double_click", "right_click", "middle_click",
+    "drag", "scroll", "type", "key", "set_value", "focus_app",
+})
+
+# Hard-blocked key combinations. Mirrored from #4562 — these are destructive
+# regardless of approval level (e.g. logout kills the session Hermes runs in).
+_BLOCKED_KEY_COMBOS = {
+    frozenset({"cmd", "shift", "backspace"}),   # empty trash
+    frozenset({"cmd", "option", "backspace"}),   # force delete
+    frozenset({"cmd", "ctrl", "q"}),             # lock screen
+    frozenset({"cmd", "shift", "q"}),            # log out
+    frozenset({"cmd", "option", "shift", "q"}),  # force log out
+    # Windows secure/session shortcuts. The Windows driver accepts Win-key
+    # combos, and Alt is canonicalized to option below, so block the
+    # destructive variants before any backend sees them.
+    frozenset({"win", "l"}),
+    frozenset({"ctrl", "option", "delete"}),
+    frozenset({"ctrl", "option", "del"}),
+    frozenset({"option", "f4"}),
+}
+
+_KEY_ALIASES = {
+    "command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option",
+    "windows": "win", "super": "win", "meta": "win",
+}
+
+
+def _canon_key_combo(keys: str) -> frozenset:
+    parts = [p.strip().lower() for p in re.split(r"\s*\+\s*", keys) if p.strip()]
+    parts = [_KEY_ALIASES.get(p, p) for p in parts]
+    return frozenset(parts)
+
+
+# Dangerous text patterns for the `type` action. Same list as #4562.
+_BLOCKED_TYPE_PATTERNS = [
+    re.compile(r"curl\s+[^|]*\|\s*bash", re.IGNORECASE),
+    re.compile(r"curl\s+[^|]*\|\s*sh", re.IGNORECASE),
+    re.compile(r"wget\s+[^|]*\|\s*bash", re.IGNORECASE),
+    re.compile(r"\bsudo\s+rm\s+-[rf]", re.IGNORECASE),
+    re.compile(r"\brm\s+-rf\s+/\s*$", re.IGNORECASE),
+    re.compile(r":\s*\(\)\s*\{\s*:\|:\s*&\s*\}", re.IGNORECASE),  # fork bomb
+]
+
+
+def _is_blocked_type(text: str) -> Optional[str]:
+    for pat in _BLOCKED_TYPE_PATTERNS:
+        if pat.search(text):
+            return pat.pattern
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Backend selection — env-swappable for tests
+# ---------------------------------------------------------------------------
+
+# Per-process cached backend; lazily instantiated on first call.
+_backend_lock = threading.Lock()
+_backend: Optional[ComputerUseBackend] = None
+# Session-scoped approval state.
+_session_auto_approve = False
+_always_allow: set = set()  # action names the user unlocked for the session
+
+
+def _get_backend() -> ComputerUseBackend:
+    global _backend
+    with _backend_lock:
+        if _backend is None:
+            backend_name = os.environ.get("HERMES_COMPUTER_USE_BACKEND", "cua").lower()
+            if backend_name in {"cua", "cua-driver", ""}:
+                from tools.computer_use.cua_backend import CuaDriverBackend
+                _backend = CuaDriverBackend()
+            elif backend_name == "noop":  # pragma: no cover
+                _backend = _NoopBackend()
+            else:
+                raise RuntimeError(f"Unknown HERMES_COMPUTER_USE_BACKEND={backend_name!r}")
+            try:
+                _backend.start()
+            except Exception:
+                # Don't cache a backend whose start() failed (e.g. a lazy
+                # dependency install was declined / failed). The next call
+                # retries cleanly instead of returning a half-initialised
+                # backend.
+                _backend = None
+                raise
+        return _backend
+
+
+def reset_backend_for_tests() -> None:  # pragma: no cover
+    """Test helper — tear down the cached backend."""
+    global _backend, _session_auto_approve, _always_allow
+    with _backend_lock:
+        if _backend is not None:
+            try:
+                _backend.stop()
+            except Exception:
+                pass
+        _backend = None
+    _session_auto_approve = False
+    _always_allow = set()
+
+
+class _NoopBackend(ComputerUseBackend):  # pragma: no cover
+    """Test/CI stub. Records calls; returns trivial results."""
+
+    def __init__(self) -> None:
+        self.calls: List[Tuple[str, Dict[str, Any]]] = []
+        self._started = False
+
+    def start(self) -> None: self._started = True
+    def stop(self) -> None: self._started = False
+    def is_available(self) -> bool: return True
+
+    def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult:
+        self.calls.append(("capture", {"mode": mode, "app": app}))
+        return CaptureResult(mode=mode, width=1024, height=768, png_b64=None,
+                             elements=[], app=app or "", window_title="")
+
+    def click(self, **kw) -> ActionResult:
+        self.calls.append(("click", kw))
+        return ActionResult(ok=True, action="click")
+
+    def drag(self, **kw) -> ActionResult:
+        self.calls.append(("drag", kw))
+        return ActionResult(ok=True, action="drag")
+
+    def scroll(self, **kw) -> ActionResult:
+        self.calls.append(("scroll", kw))
+        return ActionResult(ok=True, action="scroll")
+
+    def type_text(self, text: str) -> ActionResult:
+        self.calls.append(("type", {"text": text}))
+        return ActionResult(ok=True, action="type")
+
+    def key(self, keys: str) -> ActionResult:
+        self.calls.append(("key", {"keys": keys}))
+        return ActionResult(ok=True, action="key")
+
+    def list_apps(self) -> List[Dict[str, Any]]:
+        self.calls.append(("list_apps", {}))
+        return []
+
+    def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
+        self.calls.append(("focus_app", {"app": app, "raise": raise_window}))
+        return ActionResult(ok=True, action="focus_app")
+
+    def set_value(self, value: str, element: Optional[int] = None) -> ActionResult:
+        self.calls.append(("set_value", {"value": value, "element": element}))
+        return ActionResult(ok=True, action="set_value")
+
+
+# ---------------------------------------------------------------------------
+# Dispatch
+# ---------------------------------------------------------------------------
+
+def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any:
+    """Main entry point — dispatched by tools.registry.
+
+    Returns either a JSON string (text-only) or a dict marked `_multimodal`
+    (image + summary) which run_agent.py wraps into the tool message.
+    """
+    action = (args.get("action") or "").strip().lower()
+    if not action:
+        return json.dumps({"error": "missing `action`"})
+
+    # Safety: validate actions before approval prompt.
+    if action == "type":
+        text = args.get("text", "")
+        pat = _is_blocked_type(text)
+        if pat:
+            return json.dumps({
+                "error": f"blocked pattern in type text: {pat!r}",
+                "hint": "Dangerous shell patterns cannot be typed via computer_use.",
+            })
+
+    if action == "key":
+        keys = args.get("keys", "")
+        combo = _canon_key_combo(keys)
+        for blocked in _BLOCKED_KEY_COMBOS:
+            if blocked.issubset(combo) and len(blocked) <= len(combo):
+                return json.dumps({
+                    "error": f"blocked key combo: {sorted(blocked)}",
+                    "hint": "Destructive system shortcuts are hard-blocked.",
+                })
+
+    # Approval gate (destructive actions only).
+    if action in _DESTRUCTIVE_ACTIONS:
+        err = _request_approval(action, args)
+        if err is not None:
+            return err
+
+    # Dispatch to backend.
+    try:
+        backend = _get_backend()
+    except Exception as e:
+        return json.dumps({
+            "error": f"computer_use backend unavailable: {e}",
+            "hint": "If the cua-driver binary is missing, run `hermes computer-use install`. "
+                    "If a Python dependency is missing, the error above shows the exact install command.",
+        })
+
+    try:
+        return _dispatch(backend, action, args)
+    except Exception as e:
+        logger.exception("computer_use %s failed", action)
+        return json.dumps({"error": f"{action} failed: {e}"})
+
+
+def _request_approval(action: str, args: Dict[str, Any]) -> Optional[str]:
+    """Return None if approved, or a JSON error string if denied."""
+    global _session_auto_approve, _always_allow
+    if _session_auto_approve:
+        return None
+    if action in _always_allow:
+        return None
+    cb = _approval_callback
+    if cb is None:
+        # No CLI approval wired — default allow. Gateway approval is handled
+        # one layer out via the normal tool-approval infra.
+        return None
+    summary = _summarize_action(action, args)
+    try:
+        verdict = cb(action, args, summary)
+    except Exception as e:
+        logger.warning("approval callback failed: %s", e)
+        verdict = "deny"
+    if verdict == "approve_once":
+        return None
+    if verdict == "approve_session" or verdict == "always_approve":
+        _always_allow.add(action)
+        if verdict == "always_approve":
+            _session_auto_approve = True
+        return None
+    return json.dumps({"error": "denied by user", "action": action})
+
+
+def _summarize_action(action: str, args: Dict[str, Any]) -> str:
+    if action in {"click", "double_click", "right_click", "middle_click"}:
+        if args.get("element") is not None:
+            return f"{action} element #{args['element']}"
+        coord = args.get("coordinate")
+        if coord:
+            return f"{action} at {tuple(coord)}"
+        return action
+    if action == "drag":
+        src = args.get("from_element") or args.get("from_coordinate")
+        dst = args.get("to_element") or args.get("to_coordinate")
+        return f"drag {src} → {dst}"
+    if action == "scroll":
+        return f"scroll {args.get('direction', '?')} x{args.get('amount', 3)}"
+    if action == "type":
+        text = args.get("text", "")
+        return f"type {text[:60]!r}" + ("..." if len(text) > 60 else "")
+    if action == "key":
+        return f"key {args.get('keys', '')!r}"
+    if action == "focus_app":
+        return f"focus {args.get('app', '')!r}" + (" (raise)" if args.get("raise_window") else "")
+    return action
+
+
+def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> Any:
+    capture_after = bool(args.get("capture_after"))
+
+    if action == "capture":
+        mode = str(args.get("mode", "som"))
+        if mode not in {"som", "vision", "ax"}:
+            return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"})
+        cap = backend.capture(mode=mode, app=args.get("app"))
+        return _capture_response(cap, max_elements=_coerce_max_elements(args.get("max_elements")))
+
+    if action == "wait":
+        seconds = float(args.get("seconds", 1.0))
+        res = backend.wait(seconds)
+        return _text_response(res)
+
+    if action == "list_apps":
+        apps = backend.list_apps()
+        return json.dumps({"apps": apps, "count": len(apps)})
+
+    if action == "focus_app":
+        app = args.get("app")
+        if not app:
+            return json.dumps({"error": "focus_app requires `app`"})
+        res = backend.focus_app(app, raise_window=bool(args.get("raise_window")))
+        return _maybe_follow_capture(backend, res, capture_after)
+
+    if action in {"click", "double_click", "right_click", "middle_click"}:
+        button = args.get("button")
+        click_count = 1
+        if action == "double_click":
+            click_count = 2
+        elif action == "right_click":
+            button = "right"
+        elif action == "middle_click":
+            button = "middle"
+        else:
+            button = button or "left"
+        element = args.get("element")
+        coord = args.get("coordinate") or (None, None)
+        x, y = (coord[0], coord[1]) if coord and coord[0] is not None else (None, None)
+        res = backend.click(
+            element=element if element is not None else None,
+            x=x, y=y, button=button or "left", click_count=click_count,
+            modifiers=args.get("modifiers"),
+        )
+        return _maybe_follow_capture(backend, res, capture_after)
+
+    if action == "drag":
+        has_elements = args.get("from_element") is not None and args.get("to_element") is not None
+        has_coords = args.get("from_coordinate") and args.get("to_coordinate")
+        if not has_elements and not has_coords:
+            return json.dumps({
+                "error": "drag requires from_coordinate/to_coordinate or from_element/to_element",
+            })
+        res = backend.drag(
+            from_element=args.get("from_element"),
+            to_element=args.get("to_element"),
+            from_xy=tuple(args["from_coordinate"]) if args.get("from_coordinate") else None,
+            to_xy=tuple(args["to_coordinate"]) if args.get("to_coordinate") else None,
+            button=args.get("button", "left"),
+            modifiers=args.get("modifiers"),
+        )
+        return _maybe_follow_capture(backend, res, capture_after)
+
+    if action == "scroll":
+        coord = args.get("coordinate") or (None, None)
+        res = backend.scroll(
+            direction=args.get("direction", "down"),
+            amount=int(args.get("amount", 3)),
+            element=args.get("element"),
+            x=coord[0] if coord and coord[0] is not None else None,
+            y=coord[1] if coord and coord[1] is not None else None,
+            modifiers=args.get("modifiers"),
+        )
+        return _maybe_follow_capture(backend, res, capture_after)
+
+    if action == "type":
+        res = backend.type_text(args.get("text", ""))
+        return _maybe_follow_capture(backend, res, capture_after)
+
+    if action == "key":
+        res = backend.key(args.get("keys", ""))
+        return _maybe_follow_capture(backend, res, capture_after)
+
+    if action == "set_value":
+        value = args.get("value")
+        if value is None:
+            return json.dumps({"error": "set_value requires `value`"})
+        res = backend.set_value(value=str(value), element=args.get("element"))
+        return _maybe_follow_capture(backend, res, capture_after)
+
+    return json.dumps({"error": f"unknown action {action!r}"})
+
+
+# ---------------------------------------------------------------------------
+# Response shaping
+# ---------------------------------------------------------------------------
+
+def _text_response(res: ActionResult) -> str:
+    payload: Dict[str, Any] = {"ok": res.ok, "action": res.action}
+    if res.message:
+        payload["message"] = res.message
+    if res.meta:
+        payload["meta"] = res.meta
+    return json.dumps(payload)
+
+
+# Default cap for the AX `elements` array returned by capture. Dense UIs
+# (Electron apps, Obsidian, JetBrains IDEs) can publish 500+ AX nodes, which
+# can exhaust session context after a single capture. The model-facing
+# `max_elements` argument lets callers raise this when they need the full tree.
+_DEFAULT_MAX_ELEMENTS = 100
+# Hard upper bound on caller-supplied `max_elements`. Without this, a tool
+# call passing a very large integer would silently disable the safeguard and
+# reintroduce the original unbounded behavior.
+_MAX_ALLOWED_MAX_ELEMENTS = 1000
+_MIN_PROVIDER_IMAGE_DIMENSION = 8
+
+
+def _image_dimensions_from_b64(image_b64: str) -> Optional[Tuple[int, int]]:
+    """Return (width, height) for common inline screenshot formats.
+
+    Some providers reject images below 8x8 before the model sees the tool
+    result. Inspecting the encoded bytes here lets computer_use fall back to
+    its AX/SOM text payload instead of sending an unusable placeholder.
+    """
+    if not image_b64:
+        return None
+    try:
+        raw = base64.b64decode(image_b64, validate=False)
+    except Exception:
+        return None
+
+    # PNG: signature + IHDR width/height.
+    if raw.startswith(b"\x89PNG\r\n\x1a\n") and len(raw) >= 24:
+        try:
+            width, height = struct.unpack(">II", raw[16:24])
+            return int(width), int(height)
+        except Exception:
+            return None
+
+    # JPEG: scan for SOF markers that carry dimensions.
+    if raw.startswith(b"\xff\xd8") and len(raw) > 4:
+        i = 2
+        while i + 9 < len(raw):
+            if raw[i] != 0xFF:
+                i += 1
+                continue
+            marker = raw[i + 1]
+            i += 2
+            while marker == 0xFF and i < len(raw):
+                marker = raw[i]
+                i += 1
+            if marker in {0xD8, 0xD9}:
+                continue
+            if marker == 0xDA:
+                break
+            if i + 2 > len(raw):
+                break
+            segment_len = int.from_bytes(raw[i:i + 2], "big")
+            if segment_len < 2 or i + segment_len > len(raw):
+                break
+            if marker in {
+                0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7,
+                0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF,
+            } and segment_len >= 7:
+                height = int.from_bytes(raw[i + 3:i + 5], "big")
+                width = int.from_bytes(raw[i + 5:i + 7], "big")
+                return int(width), int(height)
+            i += segment_len
+    return None
+
+
+def _coerce_max_elements(value: Any) -> int:
+    """Validate the caller-supplied ``max_elements``.
+
+    Falls back to :data:`_DEFAULT_MAX_ELEMENTS` for missing / non-integer /
+    sub-1 inputs so the cap can never be silently disabled by a malformed
+    tool-call argument. Clamps oversized values to
+    :data:`_MAX_ALLOWED_MAX_ELEMENTS` so a caller cannot bypass the
+    safeguard by passing a very large integer.
+    """
+    if value is None:
+        return _DEFAULT_MAX_ELEMENTS
+    try:
+        n = int(value)
+    except (TypeError, ValueError):
+        return _DEFAULT_MAX_ELEMENTS
+    if n < 1:
+        return _DEFAULT_MAX_ELEMENTS
+    if n > _MAX_ALLOWED_MAX_ELEMENTS:
+        return _MAX_ALLOWED_MAX_ELEMENTS
+    return n
+
+
+def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEMENTS) -> Any:
+    total_elements = len(cap.elements)
+    visible_elements = cap.elements[:max_elements]
+    truncated_elements = max(0, total_elements - len(visible_elements))
+    image_dimensions = _image_dimensions_from_b64(cap.png_b64 or "") if cap.png_b64 else None
+    response_width = image_dimensions[0] if image_dimensions else cap.width
+    response_height = image_dimensions[1] if image_dimensions else cap.height
+    image_too_small = bool(
+        image_dimensions
+        and (
+            image_dimensions[0] < _MIN_PROVIDER_IMAGE_DIMENSION
+            or image_dimensions[1] < _MIN_PROVIDER_IMAGE_DIMENSION
+        )
+    )
+
+    # Index only what's actually surfaced in the response — otherwise the
+    # human-readable summary references element indices the model cannot
+    # find in the JSON `elements` array (e.g. max_elements=10 vs the default
+    # 40-line index window).
+    element_index = _format_elements(visible_elements)
+    summary_lines = [
+        f"capture mode={cap.mode} {response_width}x{response_height}"
+        + (f" app={cap.app}" if cap.app else "")
+        + (f" window={cap.window_title!r}" if cap.window_title else ""),
+        f"{total_elements} interactable element(s):",
+    ]
+    if element_index:
+        summary_lines.extend(element_index)
+    # Multimodal and AX paths both reference `summary`; build it once up-front
+    # so the aux-vision routing branch (which fires before either path is
+    # selected) has a valid value to hand to _route_capture_through_aux_vision.
+    # The AX path appends the "truncated to N of M" note to summary_lines
+    # below and rebuilds; the multimodal path keeps this version untouched.
+    if image_too_small:
+        summary_lines.append(
+            f"  (screenshot omitted: {image_dimensions[0]}x{image_dimensions[1]} "
+            f"is below the {_MIN_PROVIDER_IMAGE_DIMENSION}x{_MIN_PROVIDER_IMAGE_DIMENSION} "
+            "provider minimum)"
+        )
+    summary = "\n".join(summary_lines)
+
+    if cap.png_b64 and cap.mode != "ax" and not image_too_small:
+        # Decide whether to hand the screenshot to the auxiliary.vision
+        # pipeline (text-only result) or keep the multimodal envelope (main
+        # model handles vision natively). Issue #24015: previously the
+        # multimodal envelope was returned unconditionally, so non-vision
+        # main models tripped HTTP 404 / 400 at the provider boundary even
+        # when auxiliary.vision was explicitly configured to handle this.
+        if _should_route_through_aux_vision():
+            routed = _route_capture_through_aux_vision(cap, summary)
+            if routed is not None:
+                return routed
+            # Aux routing was requested but failed (vision node down, aux call
+            # raised, empty analysis, etc.). Routing being requested means the
+            # main model may not be able to consume images; falling through to
+            # the multimodal envelope can break the capture with a provider
+            # error. Degrade to the AX/SOM text payload instead so element
+            # indices remain usable while vision is unavailable.
+            summary_lines.append(
+                "  (vision unavailable: the auxiliary vision model could not "
+                "be reached; screenshot omitted. Element-index actions still "
+                "work — drive via the element list above.)"
+            )
+            if truncated_elements:
+                summary_lines.append(
+                    f"  (response truncated to {len(visible_elements)} of "
+                    f"{total_elements} elements; raise max_elements or pass "
+                    "app= to narrow)"
+                )
+            payload = {
+                "mode": cap.mode,
+                "width": response_width,
+                "height": response_height,
+                "app": cap.app,
+                "window_title": cap.window_title,
+                "elements": [_element_to_dict(e) for e in visible_elements],
+                "total_elements": total_elements,
+                "summary": "\n".join(summary_lines),
+                "vision_unavailable": True,
+            }
+            if truncated_elements:
+                payload["truncated_elements"] = truncated_elements
+            return json.dumps(payload)
+
+        # Prefer the explicit MIME type cua-driver attaches to its image
+        # parts (Surface 7 of NousResearch/hermes-agent#47072 — trycua/cua#1961
+        # made `mimeType` part of every MCP image-part response). Fall back
+        # to base64-prefix sniffing for older cua-driver builds that didn't
+        # carry the field. JPEG base64 starts with /9j/; PNG with iVBOR.
+        _mime = cap.image_mime_type
+        if not _mime:
+            _b64_prefix = cap.png_b64[:8]
+            _mime = "image/jpeg" if _b64_prefix.startswith("/9j/") else "image/png"
+        # The multimodal response carries the screenshot, not the AX
+        # elements array, so a "response truncated to N of M elements"
+        # note would be inaccurate — skip it on this branch.
+        return {
+            "_multimodal": True,
+            "content": [
+                {"type": "text", "text": summary},
+                {"type": "image_url",
+                 "image_url": {"url": f"data:{_mime};base64,{cap.png_b64}"}},
+            ],
+            "text_summary": summary,
+            "meta": {"mode": cap.mode, "width": response_width, "height": response_height,
+                     "elements": total_elements, "png_bytes": cap.png_bytes_len},
+        }
+    # AX-only (or image-missing fallback): text path actually carries the
+    # `elements` array, so the truncation note applies here.
+    if truncated_elements:
+        summary_lines.append(
+            f"  (response truncated to {len(visible_elements)} of {total_elements} elements; "
+            f"raise max_elements or pass app= to narrow)"
+        )
+    summary = "\n".join(summary_lines)
+    payload: Dict[str, Any] = {
+        "mode": cap.mode,
+        "width": response_width,
+        "height": response_height,
+        "app": cap.app,
+        "window_title": cap.window_title,
+        "elements": [_element_to_dict(e) for e in visible_elements],
+        "total_elements": total_elements,
+        "summary": summary,
+    }
+    if truncated_elements:
+        payload["truncated_elements"] = truncated_elements
+    return json.dumps(payload)
+
+
+# ---------------------------------------------------------------------------
+# auxiliary.vision routing for captured screenshots (#24015)
+# ---------------------------------------------------------------------------
+
+# Longest image side handed to the aux vision model. Full-resolution desktop
+# captures tokenize heavily and can overflow small local-model context windows;
+# ~1456px keeps SOM badges legible while cutting per-capture vision latency.
+_MAX_VISION_DIM = 1456
+
+
+def _shrink_capture_for_vision(raw: bytes, ext: str,
+                               max_dim: int = _MAX_VISION_DIM) -> bytes:
+    """Downscale encoded image bytes so the longest side is <= max_dim.
+
+    Returns the original bytes unchanged when the image already fits or when
+    Pillow is unavailable/fails — no worse than the pre-shrink behavior.
+    """
+    try:
+        from io import BytesIO
+        from PIL import Image
+        img = Image.open(BytesIO(raw))
+        if max(img.size) <= max_dim:
+            return raw
+        img.thumbnail((max_dim, max_dim))
+        out = BytesIO()
+        img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
+        return out.getvalue()
+    except Exception as exc:
+        logger.debug("computer_use: vision downscale skipped: %s", exc)
+        return raw
+
+def _should_route_through_aux_vision() -> bool:
+    """Return True when ``_capture_response`` should hand the PNG to aux vision.
+
+    Reads the active main provider/model and the loaded config and asks the
+    routing helper. Any failure (config import, runtime override missing,
+    etc.) returns False so the existing multimodal envelope continues to be
+    returned — fail open on the routing decision so a broken config can
+    never silently drop the screenshot for vision-capable main models.
+    """
+    try:
+        from agent.auxiliary_client import _read_main_model, _read_main_provider
+        from hermes_cli.config import load_config
+        from tools.computer_use.vision_routing import (
+            should_route_capture_to_aux_vision,
+        )
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug("computer_use: aux-vision routing import failed: %s", exc)
+        return False
+    try:
+        provider = _read_main_provider()
+        model = _read_main_model()
+        cfg = load_config()
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug("computer_use: aux-vision routing config read failed: %s", exc)
+        return False
+    try:
+        return bool(should_route_capture_to_aux_vision(provider, model, cfg))
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug("computer_use: aux-vision routing decision failed: %s", exc)
+        return False
+
+
+def _route_capture_through_aux_vision(
+    cap: CaptureResult,
+    summary: str,
+) -> Optional[str]:
+    """Pre-analyse the captured PNG via ``vision_analyze`` and return a text result.
+
+    The captured base64 PNG is materialised to ``$HERMES_HOME/cache/vision/``
+    and handed to ``vision_analyze_tool`` with a generic describe prompt.
+    The resulting text description is merged into the existing AX/SOM
+    summary so the main model receives a single text payload that mentions
+    every interactable element AND a description of what the screenshot
+    looked like.
+
+    Returns:
+      A JSON-encoded text response on success.
+      ``None`` on failure (caller falls back to the multimodal envelope).
+    """
+    if not cap.png_b64:
+        return None
+    try:
+        import base64 as _base64
+        import os as _os
+        import uuid as _uuid
+
+        from hermes_constants import get_hermes_dir
+        from model_tools import _run_async
+        from tools.vision_tools import vision_analyze_tool
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug("computer_use: aux-vision import failed: %s", exc)
+        return None
+
+    temp_image_path = None
+    try:
+        try:
+            raw = _base64.b64decode(cap.png_b64, validate=False)
+        except Exception as exc:
+            logger.debug("computer_use: failed to decode capture base64: %s", exc)
+            return None
+
+        # Pick an extension that matches the on-disk bytes so vision_analyze's
+        # MIME sniffing returns the right content-type.
+        # Surface 7: prefer the explicit MIME type cua-driver supplied.
+        _mime_for_ext = cap.image_mime_type or ""
+        if _mime_for_ext == "image/jpeg" or (not _mime_for_ext and cap.png_b64[:8].startswith("/9j/")):
+            ext = ".jpg"
+        else:
+            ext = ".png"
+        cache_dir = get_hermes_dir("cache/vision", "temp_vision_images")
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}"
+        raw = _shrink_capture_for_vision(raw, ext)
+        temp_image_path.write_bytes(raw)
+
+        prompt = (
+            "Describe what is visible in this desktop application screenshot in "
+            "concise but specific terms. Mention the app name and window "
+            "title if visible, the overall layout, any labelled buttons, "
+            "menus or text fields, and any prominent text content the user "
+            "would need to know about. Do not invent details that are not "
+            "actually visible.\n\n"
+            f"AX/SOM index for cross-reference:\n{summary}"
+        )
+
+        result_json = _run_async(
+            vision_analyze_tool(str(temp_image_path), prompt)
+        )
+    except Exception as exc:
+        logger.warning(
+            "computer_use: auxiliary.vision pre-analysis failed (%s); "
+            "returning to caller without aux analysis",
+            exc,
+        )
+        return None
+    finally:
+        if temp_image_path is not None:
+            try:
+                _os.unlink(str(temp_image_path))
+            except Exception:
+                pass
+
+    analysis_text = ""
+    if isinstance(result_json, str):
+        try:
+            parsed = json.loads(result_json)
+            if isinstance(parsed, dict):
+                analysis_text = str(parsed.get("analysis") or "").strip()
+        except (TypeError, json.JSONDecodeError):
+            analysis_text = result_json.strip()
+
+    if not analysis_text:
+        return None
+
+    return json.dumps({
+        "mode": cap.mode,
+        "width": cap.width,
+        "height": cap.height,
+        "app": cap.app,
+        "window_title": cap.window_title,
+        "elements": [_element_to_dict(e) for e in cap.elements],
+        "summary": summary,
+        "vision_analysis": analysis_text,
+        "vision_analysis_routed_via": "auxiliary.vision",
+    })
+
+
+def _maybe_follow_capture(
+    backend: ComputerUseBackend, res: ActionResult, do_capture: bool,
+) -> Any:
+    if not do_capture:
+        return _text_response(res)
+    # Skip the follow-up capture when the action itself failed: showing a
+    # normal-looking screenshot after a failure misleads the model into thinking
+    # the action succeeded. Return the error text instead.
+    if not res.ok:
+        return _text_response(res)
+    try:
+        # Preserve the app context established by the preceding capture/focus_app so
+        # that capture_after=True re-captures the same app rather than the frontmost
+        # window (which may have changed if the action caused a focus shift).
+        last_app = getattr(backend, "_last_app", None)
+        cap = backend.capture(mode="som", app=last_app)
+    except Exception as e:
+        logger.warning("follow-up capture failed: %s", e)
+        return _text_response(res)
+    # Combine action summary with the capture.
+    resp = _capture_response(cap)
+    if isinstance(resp, dict) and resp.get("_multimodal"):
+        prefix = f"[{res.action}] ok={res.ok}" + (f" — {res.message}" if res.message else "")
+        resp["content"][0]["text"] = prefix + "\n\n" + resp["content"][0]["text"]
+        resp["text_summary"] = prefix + "\n\n" + resp["text_summary"]
+        return resp
+    # Fallback: action + text capture merged.
+    try:
+        data = json.loads(resp)
+    except (TypeError, json.JSONDecodeError):
+        data = {"capture": resp}
+    data["action"] = res.action
+    data["ok"] = res.ok
+    if res.message:
+        data["message"] = res.message
+    return json.dumps(data)
+
+
+def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str]:
+    out: List[str] = []
+    for e in elements[:max_lines]:
+        label = e.label.replace("\n", " ")[:60]
+        out.append(f"  #{e.index} {e.role} {label!r} @ {e.bounds}"
+                   + (f" [{e.app}]" if e.app else ""))
+    if len(elements) > max_lines:
+        out.append(f"  ... +{len(elements) - max_lines} more (call capture with app= to narrow)")
+    return out
+
+
+def _element_to_dict(e: UIElement) -> Dict[str, Any]:
+    return {
+        "index": e.index,
+        "role": e.role,
+        "label": e.label,
+        "bounds": list(e.bounds),
+        "app": e.app,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Availability check (used by the tool registry check_fn)
+# ---------------------------------------------------------------------------
+
+def check_computer_use_requirements() -> bool:
+    """Return True iff computer_use can run on this host.
+
+    Conditions: macOS, Windows, or Linux + cua-driver binary installed (or
+    override via env). cua-driver runs on all three; the Linux path is
+    headed/X11 today (Wayland via XWayland), pure-Wayland progress tracked
+    upstream. Linux users see specific blocked checks via
+    `hermes computer-use doctor` if their session is incomplete (e.g. no
+    DISPLAY set).
+    """
+    if sys.platform not in ("darwin", "win32", "linux"):
+        return False
+    from tools.computer_use.cua_backend import cua_driver_binary_available
+    return cua_driver_binary_available()
+
+
+def get_computer_use_schema() -> Dict[str, Any]:
+    from tools.computer_use.schema import COMPUTER_USE_SCHEMA
+    return COMPUTER_USE_SCHEMA

tools/computer_use/vision_routing.py

@@ -0,0 +1,204 @@
+"""Vision-routing decisions for ``computer_use`` capture results.
+
+Background
+----------
+``computer_use(action='capture', mode='som'|'vision')`` returns a
+``_multimodal`` envelope containing the captured screenshot. That envelope
+is delivered back to the **active session model** as the tool result. When
+the active main model has no vision capability (e.g. text-only or
+text+code-only models), or when the active provider rejects multimodal
+content inside tool-result messages, the screenshot trips a 404 / 400 at
+the provider boundary and the agent loop reports a hard tool failure.
+
+Issue #24015 reports this regression for the ``cua-driver`` backend:
+configuring ``auxiliary.vision`` (a dedicated vision-capable model) in
+``config.yaml`` was silently ignored — the screenshot was still routed at
+the *main* model and failed with HTTP 404 ``No endpoints found that
+support image input`` even though a perfectly good vision backend was
+sitting in config waiting to be used.
+
+This module centralises the small policy decision: should a captured
+screenshot be returned as multimodal content (main model handles vision
+natively) or pre-analysed via the auxiliary vision pipeline so the main
+model only ever sees text?
+
+Behaviour (mirrors ``vision_analyze`` for consistency)
+------------------------------------------------------
+* If the user explicitly configured ``auxiliary.vision`` (any of
+  ``provider``, ``model``, or ``base_url`` non-empty / not ``"auto"``),
+  the screenshot is routed through the aux vision pipeline. Users who
+  pay for a dedicated vision model usually want it used.
+* Otherwise, if the user explicitly declared the active model vision-capable
+  via ``model.supports_vision`` / provider model config, return ``False``.
+  This is the escape hatch for custom/local OpenAI-compatible VLM routes that
+  are absent from models.dev and provider allowlists.
+* Otherwise, if the active main model+provider can carry an image inside
+  a tool-result message AND the model reports ``supports_vision=True``
+  in models.dev metadata, return ``False`` (use the multimodal path).
+* In every other case (non-vision main model, provider that does not
+  accept multimodal tool results, lookup failure), route through aux
+  vision so the main model receives a text description it can act on.
+
+The decision intentionally fails *closed* (i.e. towards aux routing) when
+metadata is missing or ambiguous: returning a screenshot to a model that
+cannot read it is a hard tool failure, while routing it through aux costs
+one extra LLM call and yields a usable description.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:
+    """True when ``auxiliary.vision`` carries a non-default user override.
+
+    Mirrors ``agent.image_routing._explicit_aux_vision_override`` so the
+    capture path and the user-attached-image path agree on what counts as
+    an explicit user request for the aux vision pipeline. ``provider:
+    "auto"``, blank values, or a missing block all count as *not*
+    explicit.
+    """
+    if not isinstance(cfg, dict):
+        return False
+    aux = cfg.get("auxiliary") or {}
+    if not isinstance(aux, dict):
+        return False
+    vision = aux.get("vision") or {}
+    if not isinstance(vision, dict):
+        return False
+
+    provider = str(vision.get("provider") or "").strip().lower()
+    model = str(vision.get("model") or "").strip()
+    base_url = str(vision.get("base_url") or "").strip()
+
+    if provider in ("", "auto") and not model and not base_url:
+        return False
+    return True
+
+
+def _lookup_user_declared_supports_vision(
+    provider: str,
+    model: str,
+    cfg: Optional[Dict[str, Any]],
+) -> Optional[bool]:
+    """Return config-declared ``supports_vision`` for the active route."""
+    try:
+        from agent.image_routing import _supports_vision_override
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug(
+            "computer_use vision_routing: config override lookup import failed: %s",
+            exc,
+        )
+        return None
+    try:
+        return _supports_vision_override(cfg, provider, model)
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug(
+            "computer_use vision_routing: config override lookup failed: %s",
+            exc,
+        )
+        return None
+
+
+def _lookup_supports_vision(
+    provider: str,
+    model: str,
+    cfg: Optional[Dict[str, Any]] = None,
+) -> Optional[bool]:
+    """Return config/models.dev ``supports_vision`` for *(provider, model)*."""
+    if not provider or not model:
+        return None
+    try:
+        from agent.image_routing import _lookup_supports_vision as _lookup_image_supports
+    except Exception:
+        _lookup_image_supports = None
+    if _lookup_image_supports is not None:
+        try:
+            return _lookup_image_supports(provider, model, cfg)
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.debug(
+                "computer_use vision_routing: image-routing caps lookup failed "
+                "for %s:%s — %s",
+                provider, model, exc,
+            )
+            return None
+    try:
+        from agent.models_dev import get_model_capabilities
+        caps = get_model_capabilities(provider, model)
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug(
+            "computer_use vision_routing: caps lookup failed for %s:%s — %s",
+            provider, model, exc,
+        )
+        return None
+    if caps is None:
+        return None
+    return bool(getattr(caps, "supports_vision", False))
+
+
+def _provider_accepts_multimodal_tool_result(provider: str, model: str) -> Optional[bool]:
+    """Return whether *provider*+*model* carries images inside tool-result messages.
+
+    Reuses ``tools.vision_tools._supports_media_in_tool_results`` so the
+    capture-routing decision stays in lockstep with the
+    ``vision_analyze`` native fast path. Returns None on import failure
+    so callers fall back to aux routing rather than guessing.
+    """
+    if not provider:
+        return None
+    try:
+        from tools.vision_tools import _supports_media_in_tool_results
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug(
+            "computer_use vision_routing: tool-result support lookup failed: %s",
+            exc,
+        )
+        return None
+    return bool(_supports_media_in_tool_results(provider, model))
+
+
+def should_route_capture_to_aux_vision(
+    provider: str,
+    model: str,
+    cfg: Optional[Dict[str, Any]],
+) -> bool:
+    """Return True iff the captured screenshot should be pre-analysed via aux vision.
+
+    Args:
+      provider: active inference provider id (e.g. ``"openrouter"``,
+        ``"anthropic"``, ``"openai-codex"``). Lower-case canonical id.
+      model:    active main model slug as it would be sent to the provider.
+      cfg:      loaded ``config.yaml`` dict (or None).
+
+    Returns:
+      ``True`` when the caller should hand the screenshot to the aux vision
+      pipeline (and surface a text-only tool result). ``False`` when the
+      caller should keep the existing multimodal envelope (main model
+      handles vision natively).
+    """
+    if _explicit_aux_vision_override(cfg):
+        return True
+
+    user_declared = _lookup_user_declared_supports_vision(provider, model, cfg)
+    if user_declared is True:
+        return False
+    if user_declared is False:
+        return True
+
+    accepts_tool_image = _provider_accepts_multimodal_tool_result(provider, model)
+    if accepts_tool_image is None or accepts_tool_image is False:
+        return True
+
+    supports_vision = _lookup_supports_vision(provider, model, cfg)
+    if supports_vision is True:
+        return False
+    return True
+
+
+__all__ = [
+    "should_route_capture_to_aux_vision",
+]