import AppKit import ApplicationServices import CoreGraphics import Darwin import Foundation import ImageIO import OrcaComputerUseMacOSCore import ScreenCaptureKit private let providerName = "orca-computer-use-macos" private let providerVersion = "1.0.0" private let providerProtocolVersion = 1 struct Request: Decodable { let id: Int let method: String let params: [String: JSONValue]? let token: String? } enum JSONValue: Decodable { case string(String) case number(Double) case bool(Bool) case object([String: JSONValue]) case array([JSONValue]) case null init(from decoder: Decoder) throws { let container = try decoder.singleValueContainer() if container.decodeNil() { self = .null } else if let value = try? container.decode(Bool.self) { self = .bool(value) } else if let value = try? container.decode(Double.self) { self = .number(value) } else if let value = try? container.decode(String.self) { self = .string(value) } else if let value = try? container.decode([JSONValue].self) { self = .array(value) } else { self = .object(try container.decode([String: JSONValue].self)) } } var string: String? { if case let .string(value) = self { return value } return nil } var number: Double? { if case let .number(value) = self { return value } return nil } var bool: Bool? { if case let .bool(value) = self { return value } return nil } } enum ProviderError: Error { case coded(String, String) var code: String { switch self { case let .coded(code, _): return code } } var message: String { switch self { case let .coded(_, message): return message } } } struct AppDescriptor { let name: String let bundleId: String? let pid: pid_t let app: NSRunningApplication var needsManualAccessibilityMode: Bool { // Chromium/Electron apps often need this private AX mode, but applying it // broadly can corrupt native Cocoa app trees into app-root-only nodes. guard let bundleId = bundleId?.lowercased() else { return false } return bundleId.hasPrefix("com.google.chrome") || bundleId.hasPrefix("com.microsoft.edgemac") || bundleId.hasPrefix("com.brave.browser") || bundleId.hasPrefix("com.operasoftware.opera") || bundleId.hasPrefix("com.vivaldi.vivaldi") || bundleId == "com.github.electron" || bundleId == "com.tinyspeck.slackmacgap" || bundleId == "com.spotify.client" || bundleId == "com.hnc.discord" || bundleId == "com.microsoft.teams2" || bundleId == "notion.id" } var isKnownBrowser: Bool { let bundle = bundleId?.lowercased() ?? "" let appName = name.lowercased() return bundle == "com.apple.safari" || bundle == "org.mozilla.firefox" || bundle == "company.thebrowser.browser" || bundle == "app.zen-browser.zen" || bundle.hasPrefix("com.google.chrome") || bundle.hasPrefix("com.microsoft.edgemac") || bundle.hasPrefix("com.brave.browser") || bundle.hasPrefix("com.operasoftware.opera") || bundle.hasPrefix("com.vivaldi.vivaldi") || appName == "safari" || appName == "firefox" || appName == "arc" || appName == "zen" || appName.contains("chrome") || appName.contains("chromium") || appName.contains("edge") || appName.contains("brave") || appName.contains("opera") || appName.contains("vivaldi") } } final class ElementRecord { let index: Int let element: AXUIElement let localFrame: CGRect? let actions: [String] let signature: String init(index: Int, element: AXUIElement, localFrame: CGRect?, actions: [String], signature: String) { self.index = index self.element = element self.localFrame = localFrame self.actions = actions self.signature = signature } } struct Snapshot { let id: String let app: AppDescriptor let windowTitle: String let windowBounds: CGRect let windowId: CGWindowID let windowLayer: Int let treeText: String let focusedElementId: Int? let screenshot: ScreenshotPayload? let screenshotStatus: ScreenshotStatus let screenshotScale: CGSize let screenshotEngine: String? let elements: [Int: ElementRecord] let truncated: Bool let maxDepthReached: Bool func withoutScreenshotPayload() -> Snapshot { Snapshot( id: id, app: app, windowTitle: windowTitle, windowBounds: windowBounds, windowId: windowId, windowLayer: windowLayer, treeText: treeText, focusedElementId: focusedElementId, screenshot: nil, screenshotStatus: .skipped, screenshotScale: CGSize(width: 1, height: 1), screenshotEngine: nil, elements: elements, truncated: truncated, maxDepthReached: maxDepthReached ) } } struct ScreenshotPayload { let data: String let width: Int let height: Int let scale: Double } struct CapturedImage { let image: CGImage let engine: String } enum ScreenshotStatus { case captured case skipped case failed(String) } private struct CachedSnapshotEntry { let snapshotId: String let keys: [String] let createdAt: Date } final class Provider { private var snapshots: [String: Snapshot] = [:] private var snapshotEntries: [CachedSnapshotEntry] = [] func handle(method: String, params: [String: JSONValue]) throws -> Any { switch method { case "handshake": return providerHandshake() case "listApps": return ["apps": listApps().map(renderListedApp)] case "listWindows": return try listWindows(params: params) case "getAppState": return try renderSnapshot(observe(params: params)) case "click": return try actionResult(params: params) { try click(params: params) } case "performSecondaryAction": return try actionResult(params: params) { try performSecondaryAction(params: params) } case "setValue": return try actionResult(params: params) { try setValue(params: params) } case "typeText": return try actionResult(params: params) { try typeText(params: params) } case "pressKey": return try actionResult(params: params) { try pressKey(params: params) } case "hotkey": return try actionResult(params: params) { try hotkey(params: params) } case "pasteText": return try actionResult(params: params) { try pasteText(params: params) } case "scroll": return try actionResult(params: params) { try scroll(params: params) } case "drag": return try actionResult(params: params) { try drag(params: params) } default: throw ProviderError.coded("invalid_argument", "unknown method '\(method)'") } } private func actionResult(params: [String: JSONValue], action runAction: () throws -> [String: Any]) throws -> [String: Any] { var action = try runAction() do { return try renderActionResult(action: action, snapshot: observe(params: params)) } catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && hasRequestedWindowSelector(params) { var fallbackParams = params fallbackParams.removeValue(forKey: "windowId") fallbackParams.removeValue(forKey: "windowIndex") if action["verification"] == nil { action["verification"] = ["state": "unverified", "reason": "window_changed"] } return try renderActionResult(action: action, snapshot: observe(params: fallbackParams)) } } private func observe(params: [String: JSONValue]) throws -> Snapshot { let query = try requiredString(params, "app") let windowId = try requestedWindowId(params) let windowIndex = try requestedWindowIndex(params) let app = try resolveApp(query) if params["restoreWindow"]?.bool == true { recoverWindow(app) } let snapshot = try buildSnapshot( app: app, includeScreenshot: params["noScreenshot"]?.bool != true, windowId: windowId, windowIndex: windowIndex, restoreWindow: params["restoreWindow"]?.bool == true ) // Why: cached snapshots only validate element identity for follow-up // actions; retaining MB-scale screenshot base64 in the long-lived agent grows memory. rememberSnapshot( query: query, app: app, snapshot: snapshot.withoutScreenshotPayload(), params: params, windowIndex: windowIndex ) return snapshot } private func rememberSnapshot( query: String, app: AppDescriptor, snapshot cachedSnapshot: Snapshot, params: [String: JSONValue], windowIndex: Int? ) { let keys = [query, app.name, app.bundleId ?? "", "pid:\(app.pid)"] .filter { !$0.isEmpty } .map { $0.lowercased() } let namespace = snapshotNamespace(params) var storedKeys: [String] = [] let canonicalWindowKey = snapshotCanonicalWindowIdKey(cachedSnapshot.windowId) if !isExplicitSnapshotNamespace(namespace) { snapshots[canonicalWindowKey.lowercased()] = cachedSnapshot storedKeys.append(canonicalWindowKey.lowercased()) } snapshots[namespacedSnapshotKey(namespace, canonicalWindowKey)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowKey)) if let windowIndex { let canonicalWindowIndexKey = snapshotCanonicalWindowIndexKey(windowIndex) if !isExplicitSnapshotNamespace(namespace) { snapshots[canonicalWindowIndexKey.lowercased()] = cachedSnapshot storedKeys.append(canonicalWindowIndexKey.lowercased()) } snapshots[namespacedSnapshotKey(namespace, canonicalWindowIndexKey)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowIndexKey)) } for key in keys { if !isExplicitSnapshotNamespace(namespace) { snapshots[key] = cachedSnapshot storedKeys.append(key) snapshots[snapshotWindowKey(key, cachedSnapshot.windowId)] = cachedSnapshot storedKeys.append(snapshotWindowKey(key, cachedSnapshot.windowId)) if let windowIndex { snapshots[snapshotWindowIndexKey(key, windowIndex)] = cachedSnapshot storedKeys.append(snapshotWindowIndexKey(key, windowIndex)) } } snapshots[namespacedSnapshotKey(namespace, key)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, key)) let namespacedWindowKey = namespacedSnapshotKey( namespace, snapshotWindowKey(key, cachedSnapshot.windowId) ) snapshots[namespacedWindowKey] = cachedSnapshot storedKeys.append(namespacedWindowKey) if let windowIndex { let namespacedWindowIndexKey = namespacedSnapshotKey( namespace, snapshotWindowIndexKey(key, windowIndex) ) snapshots[namespacedWindowIndexKey] = cachedSnapshot storedKeys.append(namespacedWindowIndexKey) } } snapshotEntries.append( CachedSnapshotEntry(snapshotId: cachedSnapshot.id, keys: storedKeys, createdAt: Date()) ) pruneSnapshotCache() } private func pruneSnapshotCache() { let now = Date() while let oldest = snapshotEntries.first, ComputerSnapshotCachePolicy.shouldPrune( entryCount: snapshotEntries.count, createdAt: oldest.createdAt, now: now ) { let expired = snapshotEntries.removeFirst() for key in expired.keys where snapshots[key]?.id == expired.snapshotId { snapshots.removeValue(forKey: key) } } } private func currentSnapshot(params: [String: JSONValue]) throws -> Snapshot { pruneSnapshotCache() let cached = try cachedSnapshot(params: params) // Why: cached AX frames can be stale after a window move or resize, and // stale geometry can turn an intended action into a misclick. let snapshot = try observe(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) try validateRequestedElements(cached: cached, current: snapshot, params: params) return snapshot } private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot { // Why: AX text replacement/select-all do not post global input, so only // synthetic fallback paths require the target window to be focused. try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) } private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? { guard let query = params["app"]?.string, !query.isEmpty else { return nil } let namespace = snapshotNamespace(params) if let targetWindowId = try requestedWindowId(params) { let canonicalKey = snapshotCanonicalWindowIdKey(targetWindowId) if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] { return cached } let windowKey = snapshotWindowKey(query.lowercased(), targetWindowId) if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] { return cached } return nil } if let targetWindowIndex = try requestedWindowIndex(params) { let canonicalKey = snapshotCanonicalWindowIndexKey(targetWindowIndex) if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] { return cached } let windowKey = snapshotWindowIndexKey(query.lowercased(), targetWindowIndex) if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] { return cached } return nil } let key = query.lowercased() return snapshots[namespacedSnapshotKey(namespace, key)] ?? (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key]) } private func validateRequestedElements(cached: Snapshot?, current: Snapshot, params: [String: JSONValue]) throws { let requestedIndexes = try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in guard params[key]?.number != nil else { return nil } return try optionalInteger(params, key) } guard !requestedIndexes.isEmpty else { return } guard let cached else { throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window") } for index in requestedIndexes { guard let expected = cached.elements[index], let actual = current.elements[index] else { throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index") } guard expected.signature == actual.signature else { throw ProviderError.coded("element_not_found", "element \(index) changed since the last snapshot; run get-app-state again and use a fresh element index") } } } private func ensureWindowStillAvailable(_ snapshot: Snapshot) throws { guard WindowCapture.candidates(pid: snapshot.app.pid).contains(where: { $0.windowId == snapshot.windowId }) else { throw ProviderError.coded("window_stale", "window \(Int(snapshot.windowId)) is no longer available; run get-app-state again to refresh the target window") } } private func listApps() -> [AppDescriptor] { var seen = Set() return NSWorkspace.shared.runningApplications .filter { !$0.isTerminated && $0.activationPolicy == .regular } .compactMap { app in guard let name = app.localizedName, !name.isEmpty else { return nil } let pid = app.processIdentifier guard pid > 0, pidIsLive(pid) else { return nil } let key = (app.bundleIdentifier ?? "pid:\(pid)").lowercased() guard seen.insert(key).inserted else { return nil } return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app) } .sorted { lhs, rhs in if lhs.app.isActive != rhs.app.isActive { return lhs.app.isActive && !rhs.app.isActive } return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending } } private func renderListedApp(_ app: AppDescriptor) -> [String: Any] { [ "name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid), "isRunning": true, "lastUsedAt": NSNull(), "useCount": NSNull(), ] } private func providerHandshake() -> [String: Any] { [ "platform": "darwin", "provider": providerName, "providerVersion": providerVersion, "protocolVersion": providerProtocolVersion, "supports": [ "apps": [ "list": true, "bundleIds": true, "pids": true, ], "windows": [ "list": true, "targetById": true, "targetByIndex": true, "focus": false, "moveResize": false, ], "observation": [ "screenshot": true, "annotatedScreenshot": false, "elementFrames": true, "ocr": false, ], "actions": [ "click": true, "typeText": true, "pressKey": true, "hotkey": true, "pasteText": true, "scroll": true, "drag": true, "setValue": true, "performAction": true, ], "surfaces": [ "menus": false, "dialogs": false, "dock": false, "menubar": false, ], ], ] } private func resolveApp(_ query: String) throws -> AppDescriptor { let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { throw ProviderError.coded("invalid_argument", "app query must not be empty") } if let pid = parsePid(trimmed) { if let app = appByPid(pid) { try rejectBlockedApp(app) return app } throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found") } if blockedBundleIds.contains(trimmed) { throw ProviderError.coded("app_blocked", "app '\(trimmed)' is blocked for safety") } if let app = listApps().first(where: { matches($0, query: trimmed) }) { try rejectBlockedApp(app) return app } throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found") } private func rejectBlockedApp(_ app: AppDescriptor) throws { if let bundle = app.bundleId, blockedBundleIds.contains(bundle) { throw ProviderError.coded("app_blocked", "app '\(bundle)' is blocked for safety") } } private func listWindows(params: [String: JSONValue]) throws -> [String: Any] { let app = try resolveApp(try requiredString(params, "app")) let windows = WindowCapture.candidates(pid: app.pid) .filter { $0.layer == 0 } .enumerated() .map { index, candidate -> [String: Any] in [ "index": index, "app": [ "name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid), ], "id": Int(candidate.windowId), "title": candidate.title ?? "", "x": Int(candidate.bounds.origin.x.rounded()), "y": Int(candidate.bounds.origin.y.rounded()), "width": Int(candidate.bounds.width.rounded()), "height": Int(candidate.bounds.height.rounded()), "isMinimized": false, "isOffscreen": !candidate.isOnScreen, "screenIndex": jsonNullable(screenIndex(for: candidate.bounds)), "isMain": NSNull(), "platform": [ "layer": candidate.layer, "alpha": candidate.alpha, ], ] } return [ "app": renderListedApp(app), "windows": windows, ] } private func appByPid(_ pid: pid_t) -> AppDescriptor? { guard let app = NSRunningApplication(processIdentifier: pid), !app.isTerminated, let name = app.localizedName else { return nil } return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app) } private func buildSnapshot( app: AppDescriptor, includeScreenshot: Bool, windowId: CGWindowID?, windowIndex: Int?, restoreWindow: Bool ) throws -> Snapshot { guard accessibilityTrusted() else { // Why: agents retry failed observations. Only the explicit setup flow // should open macOS privacy prompts/settings; runtime calls stay quiet. throw ProviderError.coded( "permission_denied", "Accessibility permission is required for Orca Computer Use. Run `orca computer permissions` or open Settings > Computer Use, grant Accessibility to Orca Computer Use, then retry." ) } let appElement = AXUIElementCreateApplication(app.pid) enableManualAccessibilityIfNeeded(appElement, app: app) let windowCandidates = WindowCapture.candidates(pid: app.pid) let focused = try focusedWindow( appElement: appElement, app: app, visibleWindowCount: windowCandidates.count, allowRecovery: restoreWindow ) let focusedTitle = stringAttribute(focused, kAXTitleAttribute as String) ?? app.name let canCaptureScreenshot = includeScreenshot && screenCaptureTrusted() guard let capture = WindowCapture.resolve( candidates: windowCandidates, titleHint: focusedTitle, windowId: windowId, windowIndex: windowIndex, captureImage: canCaptureScreenshot ) else { throw ProviderError.coded("window_not_found", "app '\(app.name)' has no on-screen window") } guard let window = matchingWindow(appElement: appElement, capture: capture, focused: focused, explicitTarget: windowId != nil || windowIndex != nil) else { throw ProviderError.coded("window_not_found", "could not match accessibility window to requested window; run get-app-state again or retry without a window selector") } let title = stringAttribute(window, kAXTitleAttribute as String) ?? capture.title ?? app.name let renderer = TreeRenderer( windowBounds: capture.bounds, focused: focusedElement(appElement: appElement), compactBrowserTabs: app.isKnownBrowser ) renderer.render(window) let screenshot = includeScreenshot ? capture.screenshotPayload() : nil let screenshotStatus: ScreenshotStatus = if screenshot != nil { .captured } else if includeScreenshot && !canCaptureScreenshot { .failed("Screen Recording permission is required for Orca Computer Use; grant permission or pass --no-screenshot to inspect accessibility state only.") } else if includeScreenshot { .failed("window screenshot capture returned no image; retry with --no-screenshot if accessibility state is sufficient.") } else { .skipped } return Snapshot( id: UUID().uuidString, app: app, windowTitle: title, windowBounds: capture.bounds, windowId: capture.windowId, windowLayer: capture.layer, treeText: renderTreeText(app: app, title: title, bounds: capture.bounds, lines: renderer.lines, focused: renderer.focusedSummary), focusedElementId: renderer.focusedElementId, screenshot: screenshot, screenshotStatus: screenshotStatus, screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds), screenshotEngine: capture.image?.engine, elements: renderer.records, truncated: renderer.truncated, maxDepthReached: renderer.maxDepthReached ) } private func renderSnapshot(_ snapshot: Snapshot) -> [String: Any] { var screenshot: Any = NSNull() if let payload = snapshot.screenshot { screenshot = [ "data": payload.data, "format": "png", "width": payload.width, "height": payload.height, "scale": payload.scale, ] } return [ "snapshot": [ "id": snapshot.id, "app": [ "name": snapshot.app.name, "bundleId": jsonNullable(snapshot.app.bundleId), "pid": Int(snapshot.app.pid), ], "window": [ "id": Int(snapshot.windowId), "title": snapshot.windowTitle, "x": Int(snapshot.windowBounds.origin.x.rounded()), "y": Int(snapshot.windowBounds.origin.y.rounded()), "width": Int(snapshot.windowBounds.width.rounded()), "height": Int(snapshot.windowBounds.height.rounded()), "isMinimized": false, "isOffscreen": false, "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)), "platform": [ "layer": snapshot.windowLayer, ], ], "coordinateSpace": "window", "treeText": snapshot.treeText, "elementCount": snapshot.elements.count, "focusedElementId": snapshot.focusedElementId as Any, "truncation": [ "truncated": snapshot.truncated, "maxNodes": TreeRenderer.maxNodes, "maxDepth": TreeRenderer.maxDepth, "maxDepthReached": snapshot.maxDepthReached, ], ], "screenshot": screenshot, "screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot), ] } private func renderActionResult(action: [String: Any], snapshot: Snapshot) -> [String: Any] { var result = renderSnapshot(snapshot) var metadata = action metadata["targetWindowId"] = Int(snapshot.windowId) result["action"] = metadata return result } private func click(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let button = params["mouseButton"]?.string ?? "left" let count = try positiveInteger(params["clickCount"]?.number, defaultValue: 1, name: "clickCount") // Why: agents expect a click into a target app to make the next // keyboard action safe, even when the click uses an AX action path. recoverWindow(snapshot.app) if let elementIndex = try optionalInteger(params, "elementIndex") { let record = try element(snapshot, elementIndex) if count <= 1, let actionName = try performClickAction(record: record, mouseButton: button) { return actionMetadata(path: "accessibility", actionName: actionName) } if let point = center(record.localFrame, in: snapshot.windowBounds) { try Input.click( pid: snapshot.app.pid, at: point, button: mouseButton(button), count: count ) return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported") } throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame") } let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot) try Input.click( pid: snapshot.app.pid, at: point, button: mouseButton(button), count: count ) return actionMetadata(path: "synthetic") } private func performClickAction(record: ElementRecord, mouseButton: String) throws -> String? { if mouseButton == "right" { return performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil } for action in ["AXPress", "AXConfirm", "AXOpen"] { if performAction(record.element, action) { return action } } return nil } private func performSecondaryAction(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let record = try element(snapshot, try requiredInteger(params, "elementIndex")) let requested = try requiredString(params, "action") let action = record.actions.first { SnapshotRenderHeuristics.prettyAction($0).caseInsensitiveCompare(requested) == .orderedSame || $0.caseInsensitiveCompare(requested) == .orderedSame } guard let action else { throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)") } guard performAction(record.element, action) else { throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed") } return actionMetadata(path: "accessibility", actionName: action) } private func setValue(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let record = try element(snapshot, try requiredInteger(params, "elementIndex")) let expected = try requiredStringAllowingEmpty(params, "value") guard isSettable(record.element, kAXValueAttribute as String) else { throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable") } let result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, expected as CFString) guard result == .success else { throw ProviderError.coded("accessibility_error", "AXUIElementSetAttributeValue failed with \(result.rawValue)") } let actual = rawStringAttribute(record.element, kAXValueAttribute as String) let verification = actual == expected ? verifiedAction(property: "value", expected: expected, actualPreview: actual) : unverifiedAction(reason: actual == nil ? "provider_unavailable" : "value_mismatch", expected: expected, actualPreview: actual) return actionMetadata(path: "accessibility", actionName: "AXSetValue", verification: verification) } private func typeText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.typeText(text, pid: snapshot.app.pid) return actionMetadata( path: "synthetic", actionName: "typeText", verification: unverifiedAction(reason: "synthetic_input") ) } private func pressKey(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.pressKey(try requiredString(params, "key"), pid: snapshot.app.pid) return actionMetadata( path: "synthetic", actionName: "pressKey", verification: unverifiedAction(reason: "synthetic_input") ) } private func hotkey(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let key = try requiredString(params, "key") if isSelectAllHotkey(key), let focused = focusedRecord(snapshot), TextInput.selectAll(focused.element) { return actionMetadata( path: "accessibility", actionName: "AXSelectAll", verification: TextInput.selectionVerification(focused.element) ) } try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.pressKey(key, pid: snapshot.app.pid) return actionMetadata( path: "synthetic", actionName: "hotkey", verification: unverifiedAction(reason: "synthetic_input") ) } private func pasteText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.pasteText(text, pid: snapshot.app.pid) return actionMetadata( path: "clipboard", actionName: "paste", verification: unverifiedAction(reason: "clipboard_paste") ) } private func scroll(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let direction = try scrollDirection(try requiredString(params, "direction")) let pages = try positiveNumber(params["pages"]?.number, defaultValue: 1, name: "pages") if let elementIndex = try optionalInteger(params, "elementIndex") { let record = try element(snapshot, elementIndex) let action = "AXScroll\(direction.capitalized)ByPage" if pages.rounded() == pages, let pageCount = boundedInteger(pages, as: Int.self), record.actions.contains(action) { for _ in 0.. [String: Any] { let snapshot = try currentSnapshot(params: params) let start: CGPoint let end: CGPoint if let fromIndex = try optionalInteger(params, "fromElementIndex"), let toIndex = try optionalInteger(params, "toElementIndex") { let from = try element(snapshot, fromIndex) let to = try element(snapshot, toIndex) guard let fromPoint = center(from.localFrame, in: snapshot.windowBounds), let toPoint = center(to.localFrame, in: snapshot.windowBounds) else { throw ProviderError.coded("element_not_found", "drag element has no frame") } start = fromPoint end = toPoint } else { start = try coordinatePoint(params: params, xKey: "fromX", yKey: "fromY", snapshot: snapshot) end = try coordinatePoint(params: params, xKey: "toX", yKey: "toY", snapshot: snapshot) } try Input.drag(pid: snapshot.app.pid, from: start, to: end) return actionMetadata(path: "synthetic") } private func element(_ snapshot: Snapshot, _ index: Int) throws -> ElementRecord { guard let record = snapshot.elements[index] else { throw ProviderError.coded("element_not_found", "element \(index) is not in the current cached snapshot for \(snapshot.app.name); run get-app-state again and use a fresh element index") } return record } private func focusedRecord(_ snapshot: Snapshot) -> ElementRecord? { guard let focusedElementId = snapshot.focusedElementId else { return nil } return snapshot.elements[focusedElementId] } } private let blockedBundleIds: Set = [ "com.1password.1password", "com.1password.safari", "com.bitwarden.desktop", "com.dashlane.dashlanephonefinal", "com.lastpass.LastPass", "com.nordsec.nordpass", "me.proton.pass.electron", "me.proton.pass.catalyst", ] private func requiredString(_ params: [String: JSONValue], _ key: String) throws -> String { guard let value = params[key]?.string, !value.isEmpty else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredStringAllowingEmpty(_ params: [String: JSONValue], _ key: String) throws -> String { guard let value = params[key]?.string else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredNumber(_ params: [String: JSONValue], _ key: String) throws -> Double { guard let value = params[key]?.number, value.isFinite else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredInteger(_ params: [String: JSONValue], _ key: String) throws -> Int { guard let value = boundedInteger(try requiredNumber(params, key), as: Int.self) else { throw ProviderError.coded("invalid_argument", "\(key) is out of range") } return value } private func optionalInteger(_ params: [String: JSONValue], _ key: String) throws -> Int? { guard let raw = params[key]?.number else { return nil } guard let value = boundedInteger(raw, as: Int.self) else { throw ProviderError.coded("invalid_argument", "\(key) is out of range") } return value } private func positiveInteger(_ value: Double?, defaultValue: Int, name: String) throws -> Int { switch ActionArgumentValidation.positiveInteger(value, defaultValue: defaultValue, name: name) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func positiveNumber(_ value: Double?, defaultValue: Double, name: String) throws -> Double { switch ActionArgumentValidation.positiveNumber(value, defaultValue: defaultValue, name: name) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func scrollDirection(_ value: String) throws -> String { switch ActionArgumentValidation.scrollDirection(value) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func parsePid(_ query: String) -> pid_t? { guard query.hasPrefix("pid:") else { return nil } guard let pid = Int32(query.dropFirst(4)), pid > 0 else { return nil } return pid } private func matches(_ app: AppDescriptor, query: String) -> Bool { app.name.caseInsensitiveCompare(query) == .orderedSame || app.bundleId?.caseInsensitiveCompare(query) == .orderedSame } private func pidIsLive(_ pid: pid_t) -> Bool { kill(pid, 0) == 0 } private func accessibilityTrusted() -> Bool { AXIsProcessTrusted() } private func screenCaptureTrusted() -> Bool { CGPreflightScreenCaptureAccess() } private func requestScreenCaptureAccess() -> Bool { CGRequestScreenCaptureAccess() } private func openAccessibilitySettings() { openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") } private func openScreenRecordingSettings() { openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") } private func openSystemSettings(_ value: String) { guard let url = URL(string: value) else { return } NSWorkspace.shared.open(url) } private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: AppDescriptor) { guard app.needsManualAccessibilityMode else { return } _ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) _ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) } private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement { let systemWide = AXUIElementCreateSystemWide() if let window = focusedSystemWindow(systemWide: systemWide, app: app) { return window } if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let windows = copyArray(appElement, kAXWindowsAttribute as String) { if let window = windows.first(where: usableWindow) { return window } } if allowRecovery { recoverWindow(app) if let window = focusedSystemWindow(systemWide: systemWide, app: app) { return window } if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let windows = copyArray(appElement, kAXWindowsAttribute as String) { if let window = windows.first(where: usableWindow) { return window } } } let permissionHint = visibleWindowCount > 0 ? " The app has visible windows, so macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings." : "" if visibleWindowCount > 0 { throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but no accessibility window.\(permissionHint)") } throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window; make sure the app has a visible window, then retry with --restore-window.") } private func focusedSystemWindow(systemWide: AXUIElement, app: AppDescriptor) -> AXUIElement? { guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String), pidAttribute(focusedApp) == app.pid else { return nil } if let window = copyElement(systemWide, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let window = copyElement(focusedApp, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let windows = copyArray(focusedApp, kAXWindowsAttribute as String) { return windows.first(where: usableWindow) } return nil } private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool { guard let focusedWindow = focusedSystemWindow(systemWide: AXUIElementCreateSystemWide(), app: snapshot.app) else { return false } if windowNumber(focusedWindow) == snapshot.windowId { return true } guard let frame = absoluteFrame(focusedWindow) else { return false } let intersection = frame.intersection(snapshot.windowBounds) return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75 } private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowRequested: Bool) throws { guard let failure = KeyboardInputSafety.syntheticInputFocusFailure( targetWindowFocused: isTargetWindowFocused(snapshot), restoreWindowRequested: restoreWindowRequested ) else { return } switch failure { case .targetNotFocused: throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements") case .targetNotFocusedAfterRestore: throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; --restore-window was requested but the target is still not focused; bring it forward manually or check Accessibility permissions") } } private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, focused: AXUIElement, explicitTarget: Bool) -> AXUIElement? { guard let windows = copyArray(appElement, kAXWindowsAttribute as String) else { return nil } if let byNumber = windows.first(where: { windowNumber($0) == capture.windowId }) { return byNumber } if let byBounds = windows.first(where: { window in guard usableWindow(window), let frame = absoluteFrame(window) else { return false } let intersection = frame.intersection(capture.bounds) return !intersection.isNull && intersection.area >= min(frame.area, capture.bounds.area) * 0.75 }) { return byBounds } if explicitTarget { return nil } guard let titleHint = capture.title, !titleHint.isEmpty else { return focused } return windows.first { usableWindow($0) && stringAttribute($0, kAXTitleAttribute as String) == titleHint } ?? focused } private func recoverWindow(_ app: AppDescriptor) { _ = app.app.unhide() _ = app.app.activate(options: [.activateAllWindows]) if let bundleId = app.bundleId { openBundle(bundleId) } let appElement = AXUIElementCreateApplication(app.pid) if let window = copyElement(appElement, kAXFocusedWindowAttribute as String) ?? copyArray(appElement, kAXWindowsAttribute as String)?.first { _ = AXUIElementSetAttributeValue(window, kAXMinimizedAttribute as CFString, kCFBooleanFalse) _ = AXUIElementPerformAction(window, kAXRaiseAction as CFString) _ = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue) _ = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue) } Thread.sleep(forTimeInterval: 0.4) } private func openBundle(_ bundleId: String) { let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/open") process.arguments = ["-b", bundleId] process.standardOutput = Pipe() process.standardError = Pipe() try? process.run() process.waitUntilExit() } private func hasRequestedWindowSelector(_ params: [String: JSONValue]) -> Bool { params["windowId"] != nil || params["windowIndex"] != nil } private func requestedWindowId(_ params: [String: JSONValue]) throws -> CGWindowID? { guard let raw = params["windowId"] else { return nil } guard let value = raw.number, value >= 0, let id = boundedInteger(value, as: UInt32.self) else { throw ProviderError.coded("invalid_argument", "windowId is out of range") } return CGWindowID(id) } private func snapshotWindowKey(_ query: String, _ windowId: CGWindowID) -> String { "\(query.lowercased())#window:\(Int(windowId))" } private func snapshotCanonicalWindowIdKey(_ windowId: CGWindowID) -> String { "window-id:\(Int(windowId))" } private func snapshotWindowIndexKey(_ query: String, _ windowIndex: Int) -> String { "\(query.lowercased())#windowIndex:\(windowIndex)" } private func snapshotCanonicalWindowIndexKey(_ windowIndex: Int) -> String { "window-index:\(windowIndex)" } private func snapshotNamespace(_ params: [String: JSONValue]) -> String { if let session = params["session"]?.string, !session.isEmpty { return "session:\(session)" } if let worktree = params["worktree"]?.string, !worktree.isEmpty { return "worktree:\(worktree)" } return "default" } private func namespacedSnapshotKey(_ namespace: String, _ key: String) -> String { "\(namespace):\(key.lowercased())" } private func isExplicitSnapshotNamespace(_ namespace: String) -> Bool { namespace != "default" } private func requestedWindowIndex(_ params: [String: JSONValue]) throws -> Int? { guard let raw = params["windowIndex"] else { return nil } guard let value = raw.number, value >= 0, let index = boundedInteger(value, as: Int.self) else { throw ProviderError.coded("invalid_argument", "windowIndex is out of range") } return index } private func usableWindow(_ element: AXUIElement) -> Bool { stringAttribute(element, kAXRoleAttribute as String) == kAXWindowRole as String && boolAttribute(element, kAXMinimizedAttribute as String) != true } private func focusedElement(appElement: AXUIElement) -> AXUIElement? { copyElement(appElement, kAXFocusedUIElementAttribute as String) } private func copyElement(_ element: AXUIElement, _ attribute: String) -> AXUIElement? { var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else { return nil } return (value as! AXUIElement) } private func copyArray(_ element: AXUIElement, _ attribute: String) -> [AXUIElement]? { var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else { return nil } return value as? [AXUIElement] } private func pidAttribute(_ element: AXUIElement) -> pid_t? { var pid: pid_t = 0 guard AXUIElementGetPid(element, &pid) == .success else { return nil } return pid } private func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? { var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else { return nil } if CFGetTypeID(value) == CFStringGetTypeID(), let string = value as? String { let trimmed = string.trimmingCharacters(in: .whitespacesAndNewlines) return trimmed.isEmpty ? nil : trimmed } if CFGetTypeID(value) == CFURLGetTypeID(), let url = value as? URL { return url.absoluteString } return nil } private func rawStringAttribute(_ element: AXUIElement, _ attribute: String) -> String? { var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value, CFGetTypeID(value) == CFStringGetTypeID() else { return nil } return value as? String } private func boolAttribute(_ element: AXUIElement, _ attribute: String) -> Bool? { var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else { return nil } return value as? Bool } private func numberAttribute(_ element: AXUIElement, _ attribute: String) -> NSNumber? { var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else { return nil } return value as? NSNumber } private func windowNumber(_ element: AXUIElement) -> CGWindowID? { guard let number = numberAttribute(element, "AXWindowNumber") else { return nil } return CGWindowID(number.uint32Value) } private func absoluteFrame(_ element: AXUIElement) -> CGRect? { var positionValue: CFTypeRef? var sizeValue: CFTypeRef? guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success, AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success, let positionValue, let sizeValue else { return nil } var point = CGPoint.zero var size = CGSize.zero guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point), AXValueGetValue(sizeValue as! AXValue, .cgSize, &size) else { return nil } return CGRect(origin: point, size: size) } private extension CGRect { var area: CGFloat { max(width, 0) * max(height, 0) } } private func actions(_ element: AXUIElement) -> [String] { var value: CFArray? guard AXUIElementCopyActionNames(element, &value) == .success, let value else { return [] } return value as? [String] ?? [] } private func performAction(_ element: AXUIElement, _ action: String) -> Bool { actions(element).contains(where: { $0.caseInsensitiveCompare(action) == .orderedSame }) && AXUIElementPerformAction(element, action as CFString) == .success } private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool { var settable = DarwinBoolean(false) return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue } private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? { var positionValue: CFTypeRef? var sizeValue: CFTypeRef? guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success, AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success, let positionValue, let sizeValue else { return nil } var point = CGPoint.zero var size = CGSize.zero guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point), AXValueGetValue(sizeValue as! AXValue, .cgSize, &size) else { return nil } return CGRect(x: point.x - windowBounds.minX, y: point.y - windowBounds.minY, width: size.width, height: size.height) } private func elementSignature(_ node: SnapshotRenderNode) -> String { // Why: cached element validation should prove identity, not reject text // controls because their value/placeholder/summary changed after focus. [ node.role, node.roleDescription ?? "", node.title ?? "", node.label ?? "", node.linkText ?? "", node.url ?? "", SnapshotRenderHeuristics.meaningfulActions(node.rawActions, role: node.role).joined(separator: ","), ].joined(separator: "\u{1f}") } private func center(_ localFrame: CGRect?, in windowBounds: CGRect) -> CGPoint? { guard let localFrame else { return nil } return CGPoint(x: windowBounds.minX + localFrame.midX, y: windowBounds.minY + localFrame.midY) } private func screenIndex(for bounds: CGRect) -> Int? { guard let index = NSScreen.screens.firstIndex(where: { $0.frame.intersects(bounds) }) else { return nil } return index } private func coordinatePoint(params: [String: JSONValue], xKey: String, yKey: String, snapshot: Snapshot) throws -> CGPoint { let x = try requiredNumber(params, xKey) let y = try requiredNumber(params, yKey) return CGPoint( x: snapshot.windowBounds.minX + x, y: snapshot.windowBounds.minY + y ) } private func screenshotScale(screenshot: ScreenshotPayload?, bounds: CGRect) -> CGSize { guard let screenshot, bounds.width > 0, bounds.height > 0 else { return CGSize(width: 1, height: 1) } return CGSize( width: CGFloat(screenshot.width) / bounds.width, height: CGFloat(screenshot.height) / bounds.height ) } private enum MouseButton { case left case right var cgButton: CGMouseButton { switch self { case .left: return .left case .right: return .right } } var downEvent: CGEventType { switch self { case .left: return .leftMouseDown case .right: return .rightMouseDown } } var upEvent: CGEventType { switch self { case .left: return .leftMouseUp case .right: return .rightMouseUp } } } private func mouseButton(_ raw: String?) throws -> MouseButton { switch raw ?? "left" { case "left": return .left case "right": return .right case "middle": throw ProviderError.coded("invalid_argument", "middle-click is not yet supported") case let value: throw ProviderError.coded("invalid_argument", "unsupported mouse button '\(value)'") } } private func renderTreeText(app: AppDescriptor, title: String, bounds: CGRect, lines: [String], focused: String?) -> String { var output = [ "App=\(app.bundleId ?? app.name.replacingOccurrences(of: " ", with: "_")) (pid \(app.pid))", "Window: \"\(sanitize(title))\", App: \(sanitize(app.name)).", "", ] output.append(contentsOf: lines) output.append("") output.append(focused.map { "The focused UI element is \($0)." } ?? "No UI element is currently focused.") return output.joined(separator: "\n") } private func renderScreenshotStatus(_ status: ScreenshotStatus, snapshot: Snapshot) -> [String: Any] { let metadata: [String: Any] = [ "engine": snapshot.screenshotEngine ?? "unknown", "windowId": Int(snapshot.windowId), ] switch status { case .captured: return ["state": "captured", "metadata": metadata] case .skipped: return ["state": "skipped", "reason": "no_screenshot_flag"] case let .failed(message): return ["state": "failed", "code": "screenshot_failed", "message": message, "metadata": metadata] } } private final class TreeRenderer { let windowBounds: CGRect let focused: AXUIElement? let compactBrowserTabs: Bool var lines: [String] = [] var records: [Int: ElementRecord] = [:] var focusedSummary: String? var focusedElementId: Int? var truncated = false var maxDepthReached = false private let reader = AXSnapshotReader() private var nextIndex = 0 static let maxNodes = 1200 static let maxDepth = 64 init(windowBounds: CGRect, focused: AXUIElement?, compactBrowserTabs: Bool) { self.windowBounds = windowBounds self.focused = focused self.compactBrowserTabs = compactBrowserTabs } func render(_ element: AXUIElement, depth: Int = 0, ancestors: [AXUIElement] = []) { guard nextIndex < Self.maxNodes else { truncated = true return } guard depth < Self.maxDepth else { truncated = true maxDepthReached = true return } guard !ancestors.contains(where: { CFEqual($0, element) }) else { return } let role = reader.stringAttribute(element, kAXRoleAttribute as String) ?? "AXUnknown" let children = reader.primaryChildren(element, role: role, windowBounds: windowBounds) let value = reader.valueString(element, role: role) let placeholder = reader.placeholderString(element) let rawActions = reader.actions(element) let rowSummary = reader.rowTextSummary(element, role: role) let roleDescription = reader.stringAttribute(element, kAXRoleDescriptionAttribute as String) let title = reader.stringAttribute(element, kAXTitleAttribute as String) let label = reader.stringAttribute(element, kAXDescriptionAttribute as String) let url = reader.stringAttribute(element, kAXURLAttribute as String) let linkText = role == "AXLink" ? reader.descendantTextSnippets(element, limit: 2, maxDepth: 3).first : nil let baseNode = SnapshotRenderNode( role: role, roleDescription: roleDescription, title: title, label: label, linkText: linkText, value: value, placeholder: placeholder, url: url, traits: [], rawActions: rawActions, childCount: children.count, rowSummary: rowSummary ) let name = SnapshotRenderHeuristics.displayName(baseNode) let meaningful = SnapshotRenderHeuristics.meaningfulActions(rawActions, role: role) let localFrame = reader.frame(element, windowBounds: windowBounds) let traits = reader.traitsFor(element, role: role) let webAreaDepth = reader.webAreaDepth(role: role, ancestors: ancestors) let summary = reader.genericTextSummary(element, role: role, name: name, actions: meaningful, traits: traits) let node = SnapshotRenderNode( role: role, roleDescription: roleDescription, title: title, label: label, linkText: linkText, value: value, placeholder: placeholder, url: url, traits: traits, rawActions: rawActions, childCount: children.count, summary: summary, rowSummary: rowSummary, webAreaDepth: webAreaDepth ) if SnapshotRenderHeuristics.shouldElide(node) { for child in children { render(child, depth: depth, ancestors: ancestors + [element]) } return } let index = nextIndex nextIndex += 1 let line = SnapshotRenderHeuristics.line(index: index, node: node) lines.append(String(repeating: "\t", count: depth) + line) records[index] = ElementRecord( index: index, element: element, localFrame: localFrame, actions: rawActions, signature: elementSignature(node) ) if let focused, CFEqual(focused, element) { focusedElementId = index focusedSummary = line } if summary != nil || SnapshotRenderHeuristics.shouldSuppressChildren(node) { return } if compactBrowserTabs, let tabStripCompaction = tabStripCompaction(parent: node, children: children) { for (childIndex, child) in children.enumerated() where tabStripCompaction.retainedIndexes.contains(childIndex) { render(child, depth: depth + 1, ancestors: ancestors + [element]) } lines.append( String(repeating: "\t", count: depth + 1) + "... \(tabStripCompaction.omittedCount) inactive browser tabs omitted" ) return } let childLineStart = lines.count for child in children { render(child, depth: depth + 1, ancestors: ancestors + [element]) } if compactBrowserTabs { compactRenderedBrowserTabs(parent: node, startLine: childLineStart, depth: depth + 1) } } private func tabStripCompaction(parent: SnapshotRenderNode, children: [AXUIElement]) -> SnapshotTabStripCompaction? { let childNodes = children.map { child in let role = reader.stringAttribute(child, kAXRoleAttribute as String) ?? "AXUnknown" return SnapshotRenderNode( role: role, roleDescription: reader.stringAttribute(child, kAXRoleDescriptionAttribute as String), title: reader.stringAttribute(child, kAXTitleAttribute as String), label: reader.stringAttribute(child, kAXDescriptionAttribute as String), value: reader.valueString(child, role: role), traits: reader.traitsFor(child, role: role) ) } // Why: browsers expose every open tab through AX; retaining only the active // tab keeps snapshots focused on the current page instead of stale tab titles. return SnapshotRenderHeuristics.tabStripCompaction(parent: parent, children: childNodes) } private func compactRenderedBrowserTabs(parent: SnapshotRenderNode, startLine: Int, depth: Int) { guard SnapshotRenderHeuristics.roleText(parent) == "scroll area" else { return } let indent = String(repeating: "\t", count: depth) let tabLineIndexes = lines.indices.dropFirst(startLine).filter { lineIndex in isDirectRenderedBrowserTabLine(lines[lineIndex], indent: indent) } guard tabLineIndexes.count >= 10 else { return } let activeLineIndexes = Set(tabLineIndexes.filter { lineIndex in isActiveRenderedBrowserTabLine(lines[lineIndex]) }) guard !activeLineIndexes.isEmpty else { return } let insertionIndex = tabLineIndexes.first! var omittedCount = 0 for lineIndex in tabLineIndexes.reversed() where !activeLineIndexes.contains(lineIndex) { if let recordIndex = renderedElementIndex(lines[lineIndex], indent: indent) { records.removeValue(forKey: recordIndex) if focusedElementId == recordIndex { focusedElementId = nil focusedSummary = nil } } lines.remove(at: lineIndex) omittedCount += 1 } guard omittedCount > 0 else { return } lines.insert("\(indent)... \(omittedCount) inactive browser tabs omitted", at: insertionIndex) } } private final class AXSnapshotReader { private enum CachedAttribute { case missing case found(CFTypeRef) } private final class ElementCache { let element: AXUIElement var loadedAttributeNames = false var advertisedAttributes: Set? var attributes: [String: CachedAttribute] = [:] var actions: [String]? var settable: [String: Bool] = [:] init(element: AXUIElement) { self.element = element } } private var elementsByHash: [CFHashCode: [ElementCache]] = [:] func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? { guard let value = copyAttribute(element, attribute) else { return nil } if CFGetTypeID(value) == CFStringGetTypeID(), let string = value as? String { let trimmed = string.trimmingCharacters(in: .whitespacesAndNewlines) return trimmed.isEmpty ? nil : trimmed } if CFGetTypeID(value) == CFURLGetTypeID(), let url = value as? URL { return url.absoluteString } return nil } func boolAttribute(_ element: AXUIElement, _ attribute: String) -> Bool? { copyAttribute(element, attribute) as? Bool } func numberAttribute(_ element: AXUIElement, _ attribute: String) -> NSNumber? { copyAttribute(element, attribute) as? NSNumber } func copyArray(_ element: AXUIElement, _ attribute: String) -> [AXUIElement]? { copyAttribute(element, attribute) as? [AXUIElement] } func actions(_ element: AXUIElement) -> [String] { let cache = cache(for: element) if let actions = cache.actions { return actions } var value: CFArray? let actions = AXUIElementCopyActionNames(element, &value) == .success ? value as? [String] ?? [] : [] cache.actions = actions return actions } func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool { let cache = cache(for: element) if let cached = cache.settable[attribute] { return cached } var settable = DarwinBoolean(false) let value = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue cache.settable[attribute] = value return value } func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? { guard let absolute = absoluteFrame(element) else { return nil } return CGRect( x: absolute.minX - windowBounds.minX, y: absolute.minY - windowBounds.minY, width: absolute.width, height: absolute.height ) } func primaryChildren(_ element: AXUIElement, role: String, windowBounds: CGRect) -> [AXUIElement] { if usesRowsAsPrimaryChildren(role: role), let rows = copyArray(element, kAXRowsAttribute as String), !rows.isEmpty { return visibleRows(rows, parent: element, windowBounds: windowBounds) } return copyArray(element, kAXChildrenAttribute as String) ?? [] } func valueString(_ element: AXUIElement, role: String) -> String? { if isSecureTextElement(element, role: role) { return "[redacted]" } if let string = stringAttribute(element, kAXValueAttribute as String) { return string } if let number = numberAttribute(element, kAXValueAttribute as String) { return number.stringValue } return nil } func placeholderString(_ element: AXUIElement) -> String? { stringAttribute(element, "AXPlaceholderValue") ?? stringAttribute(element, "AXPlaceholder") } func traitsFor(_ element: AXUIElement, role: String) -> [String] { var traits: [String] = [] if boolAttribute(element, kAXSelectedAttribute as String) == true { traits.append("selected") } if boolAttribute(element, kAXExpandedAttribute as String) == true { traits.append("expanded") } if boolAttribute(element, kAXEnabledAttribute as String) == false { traits.append("disabled") } if valueSettableRoles.contains(role), isSettable(element, kAXValueAttribute as String) { traits.append("settable") } return traits } func genericTextSummary( _ element: AXUIElement, role: String, name: String?, actions: [String], traits: [String] ) -> String? { guard (role == kAXGroupRole as String || role == kAXUnknownRole as String), name == nil, actions.isEmpty, traits.isEmpty, isPlainTextSubtree(element, maxDepth: 4) else { return nil } let texts = descendantTextSnippets(element, limit: 8, maxDepth: 4) guard texts.count >= 2 else { return nil } let summary = texts.joined(separator: " ") guard summary.count <= 220 else { return nil } return summary } func rowTextSummary(_ element: AXUIElement, role: String) -> String? { guard ["AXRow", "AXCell", "AXOutlineRow"].contains(role) else { return nil } let texts = descendantTextSnippets(element, limit: 6, maxDepth: 3) guard !texts.isEmpty else { return nil } return texts.joined(separator: " ") } func descendantTextSnippets(_ element: AXUIElement, limit: Int, maxDepth: Int) -> [String] { var values: [String] = [] var seen = Set() func collect(_ node: AXUIElement, depth: Int) { guard values.count < limit, depth <= maxDepth else { return } let role = stringAttribute(node, kAXRoleAttribute as String) ?? "" if role == kAXStaticTextRole as String || role == "AXLink" { for candidate in [ stringAttribute(node, kAXValueAttribute as String), stringAttribute(node, kAXTitleAttribute as String), stringAttribute(node, kAXDescriptionAttribute as String), ] { guard let candidate else { continue } let text = preview(candidate, maxLength: 80) guard !text.isEmpty, seen.insert(text).inserted else { continue } values.append(text) if values.count >= limit { return } } } for child in copyArray(node, kAXChildrenAttribute as String) ?? [] { collect(child, depth: depth + 1) if values.count >= limit { return } } } collect(element, depth: 0) return values } func webAreaDepth(role: String, ancestors: [AXUIElement]) -> Int? { if role == "AXWebArea" { return 0 } guard let index = ancestors.firstIndex(where: { stringAttribute($0, kAXRoleAttribute as String) == "AXWebArea" }) else { return nil } return ancestors.count - index } private func copyAttribute(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? { let cache = cache(for: element) if let cached = cache.attributes[attribute] { switch cached { case .missing: return nil case let .found(value): return value } } if let advertisedAttributes = advertisedAttributes(cache), !SnapshotRenderHeuristics.supportsAttribute(attribute, advertisedAttributes: advertisedAttributes) { cache.attributes[attribute] = .missing return nil } var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else { cache.attributes[attribute] = .missing return nil } cache.attributes[attribute] = .found(value) return value } private func advertisedAttributes(_ cache: ElementCache) -> Set? { if cache.loadedAttributeNames { return cache.advertisedAttributes } cache.loadedAttributeNames = true var value: CFArray? guard AXUIElementCopyAttributeNames(cache.element, &value) == .success, let attributes = value as? [String] else { return nil } cache.advertisedAttributes = Set(attributes) return cache.advertisedAttributes } private func absoluteFrame(_ element: AXUIElement) -> CGRect? { guard let positionValue = copyAttribute(element, kAXPositionAttribute as String), let sizeValue = copyAttribute(element, kAXSizeAttribute as String) else { return nil } var point = CGPoint.zero var size = CGSize.zero guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point), AXValueGetValue(sizeValue as! AXValue, .cgSize, &size) else { return nil } return CGRect(origin: point, size: size) } private func visibleRows(_ rows: [AXUIElement], parent: AXUIElement, windowBounds: CGRect) -> [AXUIElement] { guard let parentFrame = frame(parent, windowBounds: windowBounds) else { return Array(rows.prefix(20)) } let visible = rows.filter { row in guard let rowFrame = frame(row, windowBounds: windowBounds) else { return false } return rowFrame.intersects(parentFrame) } return Array((visible.isEmpty ? rows : visible).prefix(20)) } private func isSecureTextElement(_ element: AXUIElement, role: String) -> Bool { guard SnapshotRenderHeuristics.shouldProbeSecureTextMetadata(role: role) else { return false } let haystack = [ role, stringAttribute(element, kAXSubroleAttribute as String) ?? "", stringAttribute(element, kAXTitleAttribute as String) ?? "", stringAttribute(element, kAXDescriptionAttribute as String) ?? "", placeholderString(element) ?? "", ].joined(separator: " ").lowercased() return haystack.contains("secure") || haystack.contains("password") || haystack.contains("passcode") || haystack.contains("verification code") || haystack.contains("one-time code") } private func isPlainTextSubtree(_ element: AXUIElement, maxDepth: Int) -> Bool { var sawText = false let allowedContainerRoles: Set = [ kAXGroupRole as String, kAXUnknownRole as String, kAXStaticTextRole as String, "AXLink", "AXImage", ] func visit(_ node: AXUIElement, depth: Int) -> Bool { guard depth <= maxDepth else { return false } let role = stringAttribute(node, kAXRoleAttribute as String) ?? "AXUnknown" guard allowedContainerRoles.contains(role) else { return false } if role == kAXStaticTextRole as String || role == "AXLink" { sawText = true } guard SnapshotRenderHeuristics.meaningfulActions(actions(node), role: role).isEmpty else { return false } for child in copyArray(node, kAXChildrenAttribute as String) ?? [] { guard visit(child, depth: depth + 1) else { return false } } return true } return visit(element, depth: 0) && sawText } private func cache(for element: AXUIElement) -> ElementCache { let hash = CFHash(element) if let cache = elementsByHash[hash]?.first(where: { CFEqual($0.element, element) }) { return cache } let cache = ElementCache(element: element) elementsByHash[hash, default: []].append(cache) return cache } } private let valueSettableRoles: Set = [ kAXCheckBoxRole as String, kAXComboBoxRole as String, kAXRadioButtonRole as String, "AXSearchField", kAXSliderRole as String, kAXTextAreaRole as String, kAXTextFieldRole as String, ] private func usesRowsAsPrimaryChildren(role: String) -> Bool { [ kAXBrowserRole as String, kAXListRole as String, kAXOutlineRole as String, kAXTableRole as String, ].contains(role) } private func isDirectRenderedBrowserTabLine(_ line: String, indent: String) -> Bool { guard line.hasPrefix(indent), !line.dropFirst(indent.count).hasPrefix("\t") else { return false } let text = String(line.dropFirst(indent.count)) return text.range(of: #"^\d+ tab($| \(|,)"#, options: .regularExpression) != nil } private func isActiveRenderedBrowserTabLine(_ line: String) -> Bool { line.contains("(selected") || line.contains("Value: 1") } private func renderedElementIndex(_ line: String, indent: String) -> Int? { let text = line.dropFirst(indent.count) let digits = text.prefix { character in character >= "0" && character <= "9" } return Int(digits) } private func sanitize(_ value: String) -> String { value.replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\r", with: " ") } private func preview(_ value: String, maxLength: Int = 120) -> String { let clean = sanitize(value) if clean.count <= maxLength { return clean } return String(clean.prefix(maxLength)) + "..." } private func actionMetadata( path: String, actionName: String? = nil, fallbackReason: String? = nil, verification: [String: Any]? = nil ) -> [String: Any] { var metadata: [String: Any] = [ "path": path, "actionName": jsonNullable(actionName), "fallbackReason": jsonNullable(fallbackReason), ] if let verification { metadata["verification"] = verification } return metadata } private func verifiedAction(property: String, expected: String? = nil, actualPreview: String? = nil) -> [String: Any] { [ "state": "verified", "property": property, "expected": jsonNullable(expected), "actualPreview": jsonNullable(actualPreview), ] } private func unverifiedAction(reason: String, expected: String? = nil, actualPreview: String? = nil) -> [String: Any] { [ "state": "unverified", "reason": reason, "expected": jsonNullable(expected), "actualPreview": jsonNullable(actualPreview), ] } private func jsonNullable(_ value: T?) -> Any { value ?? NSNull() } private struct WindowCandidate { let windowId: CGWindowID let layer: Int let bounds: CGRect let title: String? let alpha: CGFloat let isOnScreen: Bool let sharingState: Int? var score: Int { var value = Int(bounds.width * bounds.height) if layer == 0 { value += 1_000_000_000 } if title != nil && title?.isEmpty == false { value += 10_000_000 } if isOnScreen { value += 1_000_000 } if alpha >= 0.99 { value += 100_000 } return value } } private struct WindowCapture { let windowId: CGWindowID let layer: Int let bounds: CGRect let title: String? let image: CapturedImage? static func resolve( pid: pid_t, titleHint: String?, windowId: CGWindowID?, windowIndex: Int?, captureImage: Bool ) -> WindowCapture? { resolve( candidates: candidates(pid: pid), titleHint: titleHint, windowId: windowId, windowIndex: windowIndex, captureImage: captureImage ) } static func resolve( candidates: [WindowCandidate], titleHint: String?, windowId: CGWindowID?, windowIndex: Int?, captureImage: Bool ) -> WindowCapture? { if let windowId { guard let candidate = candidates.first(where: { $0.windowId == windowId }) else { return nil } return WindowCapture(candidate: candidate, captureImage: captureImage) } if let windowIndex { let visibleWindows = candidates.filter { $0.layer == 0 } guard visibleWindows.indices.contains(windowIndex) else { return nil } return WindowCapture(candidate: visibleWindows[windowIndex], captureImage: captureImage) } guard let best = candidates.sorted(by: { lhs, rhs in if let titleHint, lhs.title == titleHint, rhs.title != titleHint { return true } if let titleHint, rhs.title == titleHint, lhs.title != titleHint { return false } return lhs.score > rhs.score }).first else { return nil } return WindowCapture(candidate: best, captureImage: captureImage) } private init(candidate: WindowCandidate, captureImage: Bool) { self.windowId = candidate.windowId self.layer = candidate.layer self.bounds = candidate.bounds self.title = candidate.title // Why: probing image APIs before TCC preflight can raise Screen // Recording prompts, even for --no-screenshot calls. if captureImage { self.image = Self.captureImage(windowId: candidate.windowId, bounds: candidate.bounds) } else { self.image = nil } } static func candidates(pid: pid_t) -> [WindowCandidate] { guard let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else { return [] } return infos.compactMap { info in guard let ownerPid = info[kCGWindowOwnerPID as String] as? pid_t, ownerPid == pid, let number = info[kCGWindowNumber as String] as? NSNumber, let layer = info[kCGWindowLayer as String] as? Int, let boundsDictionary = info[kCGWindowBounds as String] as? NSDictionary, let bounds = CGRect(dictionaryRepresentation: boundsDictionary), bounds.width >= 48, bounds.height >= 48 else { return nil } let alpha = info[kCGWindowAlpha as String] as? CGFloat ?? 1 guard alpha > 0.01 else { return nil } let sharing = (info[kCGWindowSharingState as String] as? NSNumber).map { $0.intValue } if sharing == 0 { return nil } let isOnScreen = (info[kCGWindowIsOnscreen as String] as? Bool) ?? true return WindowCandidate( windowId: CGWindowID(number.uint32Value), layer: layer, bounds: bounds, title: info[kCGWindowName as String] as? String, alpha: alpha, isOnScreen: isOnScreen, sharingState: sharing ) } .sorted { lhs, rhs in lhs.score > rhs.score } } func screenshotPayload() -> ScreenshotPayload? { guard let image, let bounded = boundedPngData(image.image) else { return nil } return ScreenshotPayload( data: bounded.data.base64EncodedString(), width: bounded.width, height: bounded.height, scale: Double(bounded.width) / max(Double(bounds.width), 1) ) } private static func captureImage(windowId: CGWindowID, bounds: CGRect) -> CapturedImage? { if ProcessInfo.processInfo.environment["ORCA_COMPUTER_USE_SCK_SCREENSHOTS"] == "1", let image = captureImageWithScreenCaptureKit(windowId: windowId, bounds: bounds) { return CapturedImage(image: image, engine: "screenCaptureKit") } if let image = CGWindowListCreateImage(.null, [.optionIncludingWindow], windowId, [.boundsIgnoreFraming, .bestResolution]) { return CapturedImage(image: image, engine: "cgWindowList") } return nil } private static func captureImageWithScreenCaptureKit(windowId: CGWindowID, bounds: CGRect) -> CGImage? { try? BlockingAsync.run(timeout: 3) { let content = try await SCShareableContent.current guard let window = content.windows.first(where: { $0.windowID == windowId }) else { return nil } let filter = SCContentFilter(desktopIndependentWindow: window) let configuration = SCStreamConfiguration() let scale = NSScreen.screens.first(where: { $0.frame.intersects(bounds) })?.backingScaleFactor ?? NSScreen.main?.backingScaleFactor ?? 2 configuration.width = size_t(max(1, Int((bounds.width * scale).rounded(.up)))) configuration.height = size_t(max(1, Int((bounds.height * scale).rounded(.up)))) configuration.scalesToFit = true configuration.preservesAspectRatio = true configuration.showsCursor = false configuration.ignoreShadowsSingleWindow = true configuration.ignoreGlobalClipSingleWindow = true return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration) } } } private final class AsyncBox: @unchecked Sendable { var result: Result? } private enum BlockingAsync { static func run(timeout: TimeInterval, operation: @escaping @Sendable () async throws -> T) throws -> T { let semaphore = DispatchSemaphore(value: 0) let box = AsyncBox() let task = Task.detached { do { box.result = .success(try await operation()) } catch { box.result = .failure(error) } semaphore.signal() } guard semaphore.wait(timeout: .now() + timeout) == .success else { task.cancel() throw ProviderError.coded("action_timeout", "screenshot capture timed out") } return try box.result!.get() } } private struct BoundedPNG { let data: Data let width: Int let height: Int } private func boundedPngData(_ image: CGImage) -> BoundedPNG? { let rep = NSBitmapImageRep(cgImage: image) guard let data = rep.representation(using: .png, properties: [:]) else { return nil } if data.count <= 900_000 { return BoundedPNG(data: data, width: image.width, height: image.height) } var scale = min(1, 1280 / CGFloat(max(image.width, image.height))) var best = BoundedPNG(data: data, width: image.width, height: image.height) while scale >= 0.25 { guard let resized = resizePng(image, scale: scale) else { break } best = resized if resized.data.count <= 900_000 { return resized } scale *= 0.85 } return best } private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? { let width = max(1, Int(CGFloat(image.width) * scale)) let height = max(1, Int(CGFloat(image.height) * scale)) guard let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil } context.interpolationQuality = .medium context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) guard let resized = context.makeImage() else { return nil } guard let data = NSBitmapImageRep(cgImage: resized).representation(using: .png, properties: [:]) else { return nil } return BoundedPNG(data: data, width: width, height: height) } private enum Input { static func click(pid: pid_t, at point: CGPoint, button: MouseButton, count: Int) throws { guard let source = CGEventSource(stateID: .combinedSessionState) else { throw ProviderError.coded("accessibility_error", "failed to create event source") } for _ in 0.. [String: Any]? { guard isSettable(element, kAXValueAttribute as String), let current = rawStringAttribute(element, kAXValueAttribute as String) else { return nil } let selectedRange = selectedTextRange(element) ?? CFRange(location: current.utf16.count, length: 0) let startOffset = max(0, min(selectedRange.location, current.utf16.count)) let endOffset = max(startOffset, min(startOffset + selectedRange.length, current.utf16.count)) let start = String.Index(utf16Offset: startOffset, in: current) let end = String.Index(utf16Offset: endOffset, in: current) let next = String(current[.. Bool { guard let current = rawStringAttribute(element, kAXValueAttribute as String) else { return false } return setSelectedTextRange(element, CFRange(location: 0, length: current.utf16.count)) } static func selectionVerification(_ element: AXUIElement) -> [String: Any] { guard let current = rawStringAttribute(element, kAXValueAttribute as String), let selectedRange = selectedTextRange(element), selectedRange.location == 0, selectedRange.length == current.utf16.count else { return unverifiedAction(reason: "provider_unavailable") } return verifiedAction(property: "selection", actualPreview: preview(current)) } private static func selectedTextRange(_ element: AXUIElement) -> CFRange? { var value: CFTypeRef? guard AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, &value) == .success, let value, CFGetTypeID(value) == AXValueGetTypeID() else { return nil } var range = CFRange(location: 0, length: 0) guard AXValueGetValue(value as! AXValue, .cfRange, &range) else { return nil } return range } @discardableResult private static func setSelectedTextRange(_ element: AXUIElement, _ range: CFRange) -> Bool { var mutableRange = range guard let value = AXValueCreate(.cfRange, &mutableRange) else { return false } return AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, value) == .success } } private func isSelectAllHotkey(_ key: String) -> Bool { let parts = key .lowercased() .replacingOccurrences(of: " ", with: "") .replacingOccurrences(of: "-", with: "+") .split(separator: "+") .map(String.init) guard parts.last == "a" else { return false } let modifiers = Set(parts.dropLast()) return modifiers.contains("cmd") || modifiers.contains("command") || modifiers.contains("meta") || modifiers.contains("cmdorctrl") || modifiers.contains("commandorcontrol") } private struct KeyModifier { let keyCode: CGKeyCode let flag: CGEventFlags } private struct ParsedKey { let keyCode: CGKeyCode let modifiers: [KeyModifier] } private enum KeyMap { static func parse(_ spec: String) throws -> ParsedKey { let parts = spec.split(separator: "+").map { String($0).lowercased() } var modifiers: [KeyModifier] = [] var keyName: String? for part in parts { switch part { case "cmd", "command", "meta", "super", "cmdorctrl", "commandorcontrol": modifiers.append(KeyModifier(keyCode: 55, flag: .maskCommand)) case "ctrl", "control": modifiers.append(KeyModifier(keyCode: 59, flag: .maskControl)) case "alt", "option": modifiers.append(KeyModifier(keyCode: 58, flag: .maskAlternate)) case "shift": modifiers.append(KeyModifier(keyCode: 56, flag: .maskShift)) default: keyName = part } } guard let keyName, let keyCode = codes[keyName] else { throw ProviderError.coded("invalid_argument", "unsupported key '\(spec)'") } return ParsedKey(keyCode: keyCode, modifiers: modifiers) } private static let codes: [String: CGKeyCode] = [ "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, "b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, "3": 20, "4": 21, "6": 22, "5": 23, "=": 24, "9": 25, "7": 26, "-": 27, "8": 28, "0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "return": 36, "enter": 36, "l": 37, "j": 38, "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43, "/": 44, "n": 45, "m": 46, ".": 47, "tab": 48, "space": 49, "`": 50, "backspace": 51, "delete": 51, "escape": 53, "esc": 53, "left": 123, "right": 124, "down": 125, "up": 126, "insert": 114, "home": 115, "pageup": 116, "page_up": 116, "forwarddelete": 117, "end": 119, "pagedown": 121, "page_down": 121, ] } private final class AgentRuntime: NSObject, NSApplicationDelegate { private let socketPath: String private let token: String? private var listener: SocketListener? init(socketPath: String, token: String?) { self.socketPath = socketPath self.token = token } func applicationDidFinishLaunching(_ notification: Notification) { do { let listener = try SocketListener(socketPath: socketPath, token: token) self.listener = listener listener.start() } catch { fputs("failed to start computer-use socket: \(error)\n", stderr) NSApp.terminate(nil) } } func applicationWillTerminate(_ notification: Notification) { listener?.stop() } } private final class PermissionRuntime: NSObject, NSApplicationDelegate { private let initialPermission: PermissionKind? private var windowController: PermissionWindowController? init(initialPermission: PermissionKind?) { self.initialPermission = initialPermission } func applicationDidFinishLaunching(_ notification: Notification) { windowController = PermissionWindowController( initialPermission: initialPermission, terminateWhenDragAssistantCloses: initialPermission != nil ) if let initialPermission { windowController?.openPermission(initialPermission) } else { windowController?.showWindow(nil) windowController?.window?.makeKeyAndOrderFront(nil) NSApp.activate(ignoringOtherApps: true) } } func applicationDidBecomeActive(_ notification: Notification) { windowController?.refreshPermissions() } func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { initialPermission == nil } } private final class PermissionWindowController: NSWindowController { private var dragAssistant: PermissionDragAssistantController? private var dragAssistantPermission: PermissionKind? private let initialPermission: PermissionKind? private let terminateWhenDragAssistantCloses: Bool convenience init(initialPermission: PermissionKind? = nil, terminateWhenDragAssistantCloses: Bool = false) { let window = NSWindow( contentRect: NSRect(x: 0, y: 0, width: 300, height: 315), styleMask: [.titled, .closable, .miniaturizable, .fullSizeContentView], backing: .buffered, defer: false ) window.title = "Enable Orca Computer Use" window.titleVisibility = .hidden window.titlebarAppearsTransparent = true window.backgroundColor = PermissionPalette.background window.center() window.isReleasedWhenClosed = false self.init(window: window, initialPermission: initialPermission, terminateWhenDragAssistantCloses: terminateWhenDragAssistantCloses) window.contentView = PermissionView( frame: window.contentView?.bounds ?? .zero, showDragAssistant: { [weak self] permission in self?.showDragAssistant(for: permission) }, close: { [weak self] in self?.closePermissionWindow() } ) } init(window: NSWindow?, initialPermission: PermissionKind?, terminateWhenDragAssistantCloses: Bool) { self.initialPermission = initialPermission self.terminateWhenDragAssistantCloses = terminateWhenDragAssistantCloses super.init(window: window) } required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } private func showDragAssistant(for permission: PermissionKind) { dragAssistant?.close() dragAssistantPermission = permission dragAssistant = PermissionDragAssistantController( permission: permission, fallbackVisibleFrame: window?.screen?.visibleFrame, onRefreshPermissions: { [weak self] in self?.refreshPermissions() }, onClose: { [weak self] in if self?.terminateWhenDragAssistantCloses == true { NSApp.terminate(nil) } } ) dragAssistant?.showWhenReady() } private func closeDragAssistant() { dragAssistant?.close() dragAssistant = nil dragAssistantPermission = nil } private func completeDragAssistant() { guard let dragAssistant else { dragAssistantPermission = nil if terminateWhenDragAssistantCloses { NSApp.terminate(nil) } return } self.dragAssistant = nil dragAssistantPermission = nil dragAssistant.complete() } private func closePermissionWindow() { // Why: the floating assistant is a separate retained window controller and // can keep the helper app alive after the main permission window closes. closeDragAssistant() window?.close() } func openPermission(_ permission: PermissionKind) { permission.requestAndOpenSettings() showDragAssistant(for: permission) } func refreshPermissions() { if let initialPermission, initialPermission.isGranted { // Why: targeted permission helpers should finish once the requested // grant lands, even if other Computer Use permissions remain unset. completeDragAssistant() return } if dragAssistantPermission?.isGranted == true { // Why: after one grant in full setup, the remaining missing permission // needs fresh guidance instead of the old assistant's instructions. closeDragAssistant() } if PermissionKind.allCases.allSatisfy(\.isGranted) { closeDragAssistant() } (window?.contentView as? PermissionView)?.refreshPermissions() } } private enum PermissionKind: CaseIterable { case accessibility case screenshots static func parse(_ value: String?) -> PermissionKind? { switch value { case "accessibility": .accessibility case "screenshots", "screen", "screen-recording": .screenshots default: nil } } var dragInstruction: String { switch self { case .accessibility: "Drag Orca Computer Use into the list above to allow Accessibility." case .screenshots: "Drag Orca Computer Use into the list above to allow Screenshots." } } var title: String { switch self { case .accessibility: "Accessibility" case .screenshots: "Screenshots" } } var detail: String { switch self { case .accessibility: "Read and control app interfaces" case .screenshots: "Capture windows for visual state" } } var icon: NSImage { switch self { case .accessibility: NSImage(systemSymbolName: "figure", accessibilityDescription: "Accessibility") ?? NSImage() case .screenshots: NSImage(systemSymbolName: "camera.viewfinder", accessibilityDescription: "Screen Recording") ?? NSImage() } } var isGranted: Bool { switch self { case .accessibility: accessibilityTrusted() case .screenshots: screenCaptureTrusted() } } func requestAndOpenSettings() { switch self { case .accessibility: openAccessibilitySettings() case .screenshots: _ = requestScreenCaptureAccess() openScreenRecordingSettings() } } } private final class PermissionView: NSView { private let appURL = Bundle.main.bundleURL private let showDragAssistant: (PermissionKind) -> Void private let close: () -> Void private var contentStack: NSStackView? private var contentConstraints: [NSLayoutConstraint] = [] init(frame frameRect: NSRect, showDragAssistant: @escaping (PermissionKind) -> Void, close: @escaping () -> Void) { self.showDragAssistant = showDragAssistant self.close = close super.init(frame: frameRect) wantsLayer = true layer?.backgroundColor = PermissionPalette.background.cgColor build() } required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } private func build() { NSLayoutConstraint.deactivate(contentConstraints) contentStack?.removeFromSuperview() contentConstraints = [] let stack = NSStackView() stack.orientation = .vertical stack.alignment = .leading stack.spacing = 10 stack.distribution = .gravityAreas stack.translatesAutoresizingMaskIntoConstraints = false addSubview(stack) contentStack = stack let icon = NSImageView(image: NSWorkspace.shared.icon(forFile: appURL.path)) icon.imageScaling = .scaleProportionallyUpOrDown icon.translatesAutoresizingMaskIntoConstraints = false NSLayoutConstraint.activate([ icon.widthAnchor.constraint(equalToConstant: 58), icon.heightAnchor.constraint(equalToConstant: 58) ]) let missingPermissions = PermissionKind.allCases.filter { !$0.isGranted } let ready = missingPermissions.isEmpty let title = label(ready ? "Computer Use is Ready" : "Enable Orca Computer Use", size: 22, weight: .bold) let subtitle = label( ready ? "Orca can use local apps when you ask." : "Grant permissions so Orca can use apps when you ask.", size: 12, weight: .regular ) subtitle.textColor = PermissionPalette.secondaryText subtitle.alignment = .center subtitle.maximumNumberOfLines = 3 let header = NSStackView(views: [icon, title, subtitle]) header.orientation = .vertical header.alignment = .centerX header.spacing = 6 header.translatesAutoresizingMaskIntoConstraints = false stack.addArrangedSubview(header) header.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true subtitle.widthAnchor.constraint(equalTo: stack.widthAnchor, constant: -10).isActive = true if ready { stack.addArrangedSubview(doneButton()) } else { for permission in missingPermissions { stack.addArrangedSubview(permissionRow(permission: permission) { [weak self] in permission.requestAndOpenSettings() self?.showDragAssistant(permission) }) } } contentConstraints = [ stack.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 18), stack.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -18), stack.topAnchor.constraint(equalTo: topAnchor, constant: 22), stack.bottomAnchor.constraint(lessThanOrEqualTo: bottomAnchor, constant: -20) ] NSLayoutConstraint.activate(contentConstraints) } func refreshPermissions() { // Why: TCC grants can change in System Settings while this window stays open. build() } private func permissionRow(permission: PermissionKind, action: @escaping () -> Void) -> NSView { let row = NSView() row.wantsLayer = true row.layer?.cornerRadius = 14 row.layer?.borderWidth = 1 row.layer?.borderColor = PermissionPalette.border.cgColor row.layer?.backgroundColor = PermissionPalette.card.cgColor row.translatesAutoresizingMaskIntoConstraints = false let iconView = NSImageView(image: permission.icon) iconView.symbolConfiguration = NSImage.SymbolConfiguration(pointSize: 30, weight: .regular) iconView.contentTintColor = .controlAccentColor iconView.translatesAutoresizingMaskIntoConstraints = false let titleLabel = label(permission.title, size: 13, weight: .bold) let detailLabel = label(permission.detail, size: 11, weight: .regular) detailLabel.textColor = PermissionPalette.secondaryText let textStack = NSStackView(views: [titleLabel, detailLabel]) textStack.orientation = .vertical textStack.alignment = .leading textStack.spacing = 4 textStack.translatesAutoresizingMaskIntoConstraints = false let button = NSButton(title: "Allow", target: nil, action: nil) button.bezelStyle = .rounded button.controlSize = .regular button.font = NSFont.systemFont(ofSize: 13, weight: .semibold) button.contentTintColor = .white button.bezelColor = .controlAccentColor let buttonTitleAttributes: [NSAttributedString.Key: Any] = [ .foregroundColor: NSColor.white, .font: NSFont.systemFont(ofSize: 13, weight: .semibold) ] button.attributedTitle = NSAttributedString(string: "Allow", attributes: buttonTitleAttributes) button.attributedAlternateTitle = NSAttributedString(string: "Allow", attributes: buttonTitleAttributes) let target = ButtonTarget(action) button.target = target button.action = #selector(ButtonTarget.run) objc_setAssociatedObject(button, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC) button.translatesAutoresizingMaskIntoConstraints = false row.addSubview(iconView) row.addSubview(textStack) row.addSubview(button) NSLayoutConstraint.activate([ row.heightAnchor.constraint(equalToConstant: 62), iconView.leadingAnchor.constraint(equalTo: row.leadingAnchor, constant: 12), iconView.centerYAnchor.constraint(equalTo: row.centerYAnchor), iconView.widthAnchor.constraint(equalToConstant: 34), iconView.heightAnchor.constraint(equalToConstant: 34), textStack.leadingAnchor.constraint(equalTo: iconView.trailingAnchor, constant: 10), textStack.centerYAnchor.constraint(equalTo: row.centerYAnchor), button.widthAnchor.constraint(greaterThanOrEqualToConstant: 52), button.heightAnchor.constraint(equalToConstant: 30), button.trailingAnchor.constraint(equalTo: row.trailingAnchor, constant: -12), button.centerYAnchor.constraint(equalTo: row.centerYAnchor), textStack.trailingAnchor.constraint(lessThanOrEqualTo: button.leadingAnchor, constant: -12) ]) return row } private func doneButton() -> NSView { let button = NSButton(title: "Done", target: nil, action: nil) button.bezelStyle = .rounded button.controlSize = .regular button.font = NSFont.systemFont(ofSize: 13, weight: .semibold) button.contentTintColor = .white button.bezelColor = .controlAccentColor let buttonTitleAttributes: [NSAttributedString.Key: Any] = [ .foregroundColor: NSColor.white, .font: NSFont.systemFont(ofSize: 13, weight: .semibold) ] button.attributedTitle = NSAttributedString(string: "Done", attributes: buttonTitleAttributes) button.attributedAlternateTitle = NSAttributedString(string: "Done", attributes: buttonTitleAttributes) let target = ButtonTarget(close) button.target = target button.action = #selector(ButtonTarget.run) objc_setAssociatedObject(button, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC) button.translatesAutoresizingMaskIntoConstraints = false button.widthAnchor.constraint(greaterThanOrEqualToConstant: 82).isActive = true button.heightAnchor.constraint(equalToConstant: 32).isActive = true return button } private func label(_ text: String, size: CGFloat, weight: NSFont.Weight) -> NSTextField { let label = NSTextField(labelWithString: text) label.font = NSFont.systemFont(ofSize: size, weight: weight) label.lineBreakMode = .byWordWrapping label.textColor = PermissionPalette.primaryText label.translatesAutoresizingMaskIntoConstraints = false return label } } private final class PermissionDragAssistantController: NSWindowController { private struct SettingsWindowState { let frame: NSRect let isVisible: Bool } private let fallbackVisibleFrame: NSRect? private let onRefreshPermissions: () -> Void private let onClose: () -> Void private var hasSeenSettingsWindow = false private var followTimer: Timer? private var isDismissed = false private var scheduledShowWorkItems: [DispatchWorkItem] = [] convenience init( permission: PermissionKind, fallbackVisibleFrame: NSRect?, onRefreshPermissions: @escaping () -> Void, onClose: @escaping () -> Void ) { let window = NSPanel( contentRect: NSRect(x: 0, y: 0, width: 390, height: 92), styleMask: [.borderless, .nonactivatingPanel], backing: .buffered, defer: false ) window.title = "Drag Orca Computer Use" window.backgroundColor = .clear window.isOpaque = false window.isReleasedWhenClosed = false window.level = .floating window.hidesOnDeactivate = false window.isFloatingPanel = true window.becomesKeyOnlyIfNeeded = true // Why: the assistant belongs to System Settings' position; user drags // should either start the app-file drag or do nothing, not move the panel. window.isMovable = false window.isMovableByWindowBackground = false window.hasShadow = true self.init( window: window, fallbackVisibleFrame: fallbackVisibleFrame, onRefreshPermissions: onRefreshPermissions, onClose: onClose ) window.contentView = PermissionDragAssistantView(permission: permission, appURL: Bundle.main.bundleURL) { [weak self] in self?.dismissFromCloseButton() } } init( window: NSWindow?, fallbackVisibleFrame: NSRect?, onRefreshPermissions: @escaping () -> Void, onClose: @escaping () -> Void ) { self.fallbackVisibleFrame = fallbackVisibleFrame self.onRefreshPermissions = onRefreshPermissions self.onClose = onClose super.init(window: window) } required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } func showWhenReady() { guard !isDismissed else { return } startFollowingSettingsWindow() schedulePositionAndShow() } override func close() { isDismissed = true scheduledShowWorkItems.forEach { $0.cancel() } scheduledShowWorkItems.removeAll() followTimer?.invalidate() followTimer = nil super.close() } private func dismissFromCloseButton() { // Why: closing the NSWindow directly skips this controller's timer cleanup, // letting the assistant reappear while System Settings remains visible. complete() } func complete() { close() onClose() } private func schedulePositionAndShow() { scheduledShowWorkItems.forEach { $0.cancel() } scheduledShowWorkItems.removeAll() let delays = [0.12, 0.25, 0.4, 0.65, 0.95, 1.35, 1.8, 2.5] for (index, delay) in delays.enumerated() { let workItem = DispatchWorkItem { [weak self] in guard let self, !self.isDismissed, self.window?.isVisible != true else { return } if let settingsWindow = self.systemSettingsWindowState(), settingsWindow.isVisible { self.positionNearSettingsWindow(settingsWindow.frame) guard !self.isDismissed else { return } self.showWindow(nil) self.window?.orderFrontRegardless() } else if index == delays.count - 1 && self.systemSettingsIsFrontmost() { self.positionFallback() guard !self.isDismissed else { return } self.showWindow(nil) self.window?.orderFrontRegardless() } } scheduledShowWorkItems.append(workItem) DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: workItem) } } private func startFollowingSettingsWindow() { followTimer?.invalidate() followTimer = Timer.scheduledTimer(withTimeInterval: 0.35, repeats: true) { [weak self] _ in Task { @MainActor in guard let self, !self.isDismissed else { return } self.syncVisibilityWithSettingsWindow() } } } private func syncVisibilityWithSettingsWindow() { guard !isDismissed else { followTimer?.invalidate() followTimer = nil return } guard let window else { followTimer?.invalidate() followTimer = nil return } let settingsWindow = systemSettingsWindowState() if settingsWindow != nil { hasSeenSettingsWindow = true } else if hasSeenSettingsWindow { NSApp.terminate(nil) return } // Why: System Settings can stay visible on one display while the user // works on another; follow actual occlusion instead of app focus. if let settingsWindow, settingsWindow.isVisible { onRefreshPermissions() guard !isDismissed else { return } positionNearSettingsWindow(settingsWindow.frame) guard !isDismissed else { return } if !window.isVisible { showWindow(nil) } guard !isDismissed else { return } window.orderFrontRegardless() } else if window.isVisible { window.orderOut(nil) } } private func systemSettingsIsFrontmost() -> Bool { let bundleId = NSWorkspace.shared.frontmostApplication?.bundleIdentifier return bundleId == "com.apple.systempreferences" || bundleId == "com.apple.SystemSettings" } private func positionNearSettingsWindow(_ settingsFrame: NSRect) { guard let window else { return } let visibleFrame = visibleFrameContaining(settingsFrame) let x = settingsFrame.maxX - window.frame.width - 18 let y = settingsFrame.minY + 18 window.setFrameOrigin(clampedOrigin(x: x, y: y, window: window, visibleFrame: visibleFrame)) } private func positionFallback() { guard let window else { return } let visibleFrame = fallbackVisibleFrame ?? NSScreen.main?.visibleFrame ?? NSRect(x: 0, y: 0, width: 900, height: 700) let origin = NSPoint( x: visibleFrame.midX - window.frame.width / 2, y: visibleFrame.minY + 24 ) window.setFrameOrigin(clampedOrigin(x: origin.x, y: origin.y, window: window, visibleFrame: visibleFrame)) } private func clampedOrigin(x: CGFloat, y: CGFloat, window: NSWindow, visibleFrame: NSRect) -> NSPoint { let inset: CGFloat = 10 let minX = visibleFrame.minX + inset let maxX = visibleFrame.maxX - window.frame.width - inset let minY = visibleFrame.minY + inset let maxY = visibleFrame.maxY - window.frame.height - inset return NSPoint( x: min(max(x, minX), maxX), y: min(max(y, minY), maxY) ) } private func systemSettingsWindowState() -> SettingsWindowState? { guard let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return nil } var occludingFrames: [NSRect] = [] for windowInfo in windows { guard let frame = cgWindowFrame(windowInfo), isNormalWindow(windowInfo), frame.width > 0, frame.height > 0 else { continue } if isSystemSettingsWindow(windowInfo), frame.width > 520, frame.height > 360 { return SettingsWindowState( frame: appKitFrameForCGWindowFrame(frame), isVisible: !isWindowFullyCovered(frame, by: occludingFrames) ) } occludingFrames.append(frame) } return nil } private func isSystemSettingsWindow(_ windowInfo: [String: Any]) -> Bool { guard let ownerName = windowInfo[kCGWindowOwnerName as String] as? String else { return false } return ownerName == "System Settings" || ownerName == "System Preferences" } private func isNormalWindow(_ windowInfo: [String: Any]) -> Bool { guard (windowInfo[kCGWindowLayer as String] as? Int) == 0 else { return false } if let alpha = windowInfo[kCGWindowAlpha as String] as? CGFloat, alpha <= 0 { return false } return true } private func cgWindowFrame(_ windowInfo: [String: Any]) -> NSRect? { guard let bounds = windowInfo[kCGWindowBounds as String] as? [String: CGFloat], let x = bounds["X"], let y = bounds["Y"], let width = bounds["Width"], let height = bounds["Height"] else { return nil } return NSRect(x: x, y: y, width: width, height: height) } private func isWindowFullyCovered(_ frame: NSRect, by occludingFrames: [NSRect]) -> Bool { var uncoveredFrames = [frame] for occludingFrame in occludingFrames { uncoveredFrames = uncoveredFrames.flatMap { subtract(occludingFrame, from: $0) } if uncoveredFrames.isEmpty { return true } } return false } private func subtract(_ coveringFrame: NSRect, from frame: NSRect) -> [NSRect] { let overlap = frame.intersection(coveringFrame) guard !overlap.isNull, overlap.width > 0, overlap.height > 0 else { return [frame] } var remaining: [NSRect] = [] if overlap.minY > frame.minY { remaining.append(NSRect(x: frame.minX, y: frame.minY, width: frame.width, height: overlap.minY - frame.minY)) } if overlap.maxY < frame.maxY { remaining.append(NSRect(x: frame.minX, y: overlap.maxY, width: frame.width, height: frame.maxY - overlap.maxY)) } if overlap.minX > frame.minX { remaining.append(NSRect(x: frame.minX, y: overlap.minY, width: overlap.minX - frame.minX, height: overlap.height)) } if overlap.maxX < frame.maxX { remaining.append(NSRect(x: overlap.maxX, y: overlap.minY, width: frame.maxX - overlap.maxX, height: overlap.height)) } return remaining.filter { $0.width > 0 && $0.height > 0 } } private func appKitFrameForCGWindowFrame(_ cgFrame: NSRect) -> NSRect { guard let screen = screenContainingCGFrame(cgFrame), let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber else { return cgFrame } let displayBounds = CGDisplayBounds(CGDirectDisplayID(screenNumber.uint32Value)) return NSRect( x: cgFrame.minX, y: screen.frame.maxY - (cgFrame.minY - displayBounds.minY) - cgFrame.height, width: cgFrame.width, height: cgFrame.height ) } private func screenContainingCGFrame(_ frame: NSRect) -> NSScreen? { NSScreen.screens.max { lhs, rhs in cgDisplayBounds(for: lhs).intersection(frame).width * cgDisplayBounds(for: lhs).intersection(frame).height < cgDisplayBounds(for: rhs).intersection(frame).width * cgDisplayBounds(for: rhs).intersection(frame).height } } private func cgDisplayBounds(for screen: NSScreen) -> NSRect { guard let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber else { return screen.frame } return CGDisplayBounds(CGDirectDisplayID(screenNumber.uint32Value)) } private func visibleFrameContaining(_ frame: NSRect) -> NSRect { let screen = NSScreen.screens.max { lhs, rhs in lhs.frame.intersection(frame).width * lhs.frame.intersection(frame).height < rhs.frame.intersection(frame).width * rhs.frame.intersection(frame).height } return screen?.visibleFrame ?? NSScreen.main?.visibleFrame ?? NSRect(x: 0, y: 0, width: 900, height: 700) } } private final class PermissionDragAssistantView: NSView { private let permission: PermissionKind private let appURL: URL private let close: () -> Void init(permission: PermissionKind, appURL: URL, close: @escaping () -> Void) { self.permission = permission self.appURL = appURL self.close = close super.init(frame: .zero) wantsLayer = true layer?.backgroundColor = PermissionPalette.background.cgColor layer?.cornerRadius = 14 layer?.borderWidth = 1 layer?.borderColor = PermissionPalette.border.cgColor layer?.masksToBounds = true build() } required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } private func build() { let closeButton = NSButton(title: "", target: nil, action: nil) closeButton.bezelStyle = .shadowlessSquare closeButton.isBordered = false closeButton.image = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close") closeButton.imagePosition = .imageOnly closeButton.contentTintColor = PermissionPalette.secondaryText closeButton.controlSize = .small closeButton.translatesAutoresizingMaskIntoConstraints = false let target = ButtonTarget(close) closeButton.target = target closeButton.action = #selector(ButtonTarget.run) objc_setAssociatedObject(closeButton, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC) let instruction = label(permission.dragInstruction, size: 12, weight: .semibold) instruction.textColor = PermissionPalette.primaryText instruction.maximumNumberOfLines = 2 let dragTile = DraggableAppTile(appURL: appURL) dragTile.translatesAutoresizingMaskIntoConstraints = false addSubview(closeButton) addSubview(instruction) addSubview(dragTile) NSLayoutConstraint.activate([ closeButton.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 10), closeButton.topAnchor.constraint(equalTo: topAnchor, constant: 10), closeButton.widthAnchor.constraint(equalToConstant: 18), closeButton.heightAnchor.constraint(equalToConstant: 18), instruction.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 36), instruction.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16), instruction.centerYAnchor.constraint(equalTo: closeButton.centerYAnchor), dragTile.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 16), dragTile.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16), dragTile.topAnchor.constraint(equalTo: instruction.bottomAnchor, constant: 8), dragTile.heightAnchor.constraint(equalToConstant: 42) ]) } private func label(_ text: String, size: CGFloat, weight: NSFont.Weight) -> NSTextField { let label = NSTextField(labelWithString: text) label.font = NSFont.systemFont(ofSize: size, weight: weight) label.lineBreakMode = .byWordWrapping label.textColor = PermissionPalette.primaryText label.translatesAutoresizingMaskIntoConstraints = false return label } } private final class DraggableAppTile: NSView, NSDraggingSource { private let appURL: URL override var mouseDownCanMoveWindow: Bool { false } init(appURL: URL) { self.appURL = appURL super.init(frame: .zero) wantsLayer = true layer?.cornerRadius = 12 layer?.borderWidth = 1 layer?.borderColor = PermissionPalette.border.cgColor layer?.backgroundColor = PermissionPalette.card.cgColor build() } required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } override func mouseDragged(with event: NSEvent) { let item = NSPasteboardItem() item.setString(appURL.absoluteString, forType: .fileURL) item.setString(appURL.path, forType: .string) let draggingItem = NSDraggingItem(pasteboardWriter: item) let iconSize: CGFloat = 64 let location = convert(event.locationInWindow, from: nil) let dragFrame = NSRect( x: location.x - iconSize / 2, y: location.y - iconSize / 2, width: iconSize, height: iconSize ) draggingItem.setDraggingFrame(dragFrame, contents: appIcon(size: iconSize)) beginDraggingSession(with: [draggingItem], event: event, source: self) } func draggingSession(_ session: NSDraggingSession, sourceOperationMaskFor context: NSDraggingContext) -> NSDragOperation { .copy } private func build() { let icon = NSImageView(image: appIcon(size: 34)) icon.imageScaling = .scaleProportionallyUpOrDown icon.translatesAutoresizingMaskIntoConstraints = false let title = NSTextField(labelWithString: "Orca Computer Use") title.font = NSFont.systemFont(ofSize: 15, weight: .semibold) title.textColor = PermissionPalette.primaryText title.translatesAutoresizingMaskIntoConstraints = false addSubview(icon) addSubview(title) NSLayoutConstraint.activate([ icon.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 16), icon.centerYAnchor.constraint(equalTo: centerYAnchor), icon.widthAnchor.constraint(equalToConstant: 34), icon.heightAnchor.constraint(equalToConstant: 34), title.leadingAnchor.constraint(equalTo: icon.trailingAnchor, constant: 14), title.centerYAnchor.constraint(equalTo: centerYAnchor), title.trailingAnchor.constraint(lessThanOrEqualTo: trailingAnchor, constant: -20) ]) } private func appIcon(size: CGFloat) -> NSImage { let icon = NSWorkspace.shared.icon(forFile: appURL.path) icon.size = NSSize(width: size, height: size) return icon } } private enum PermissionPalette { static let background = adaptiveColor( light: NSColor(calibratedWhite: 0.94, alpha: 0.98), dark: NSColor(calibratedWhite: 0.20, alpha: 0.98) ) static let card = adaptiveColor( light: NSColor(calibratedWhite: 0.99, alpha: 0.98), dark: NSColor(calibratedWhite: 0.25, alpha: 0.98) ) static let border = adaptiveColor( light: NSColor(calibratedWhite: 0.0, alpha: 0.14), dark: NSColor(calibratedWhite: 1.0, alpha: 0.22) ) static let primaryText = NSColor.labelColor static let secondaryText = NSColor.secondaryLabelColor private static func adaptiveColor(light: NSColor, dark: NSColor) -> NSColor { NSColor(name: nil) { appearance in let bestMatch = appearance.bestMatch(from: [ .aqua, .darkAqua, .accessibilityHighContrastAqua, .accessibilityHighContrastDarkAqua ]) return bestMatch == .darkAqua || bestMatch == .accessibilityHighContrastDarkAqua ? dark : light } } } private final class ButtonTarget: NSObject { private let actionBlock: () -> Void init(_ actionBlock: @escaping () -> Void) { self.actionBlock = actionBlock } @objc func run() { actionBlock() } } private final class SocketListener: @unchecked Sendable { private let socketPath: String private let token: String? private let provider = Provider() private let providerLock = NSLock() private var socketFd: Int32 = -1 private var isStopped = false init(socketPath: String, token: String?) throws { self.socketPath = socketPath self.token = token try bindSocket() } func start() { Thread.detachNewThread { [weak self] in self?.acceptLoop() } } func stop() { isStopped = true if socketFd >= 0 { close(socketFd) socketFd = -1 } // Why: the parent owns the private temp directory cleanup; the helper // must not unlink arbitrary caller-supplied paths on shutdown. } private func bindSocket() throws { socketFd = socket(AF_UNIX, SOCK_STREAM, 0) guard socketFd >= 0 else { throw ProviderError.coded("accessibility_error", "failed to create computer-use socket") } var address = sockaddr_un() address.sun_family = sa_family_t(AF_UNIX) let maxPathLength = MemoryLayout.size(ofValue: address.sun_path) guard socketPath.utf8.count < maxPathLength else { throw ProviderError.coded("invalid_argument", "computer-use socket path is too long") } _ = withUnsafeMutablePointer(to: &address.sun_path) { pointer in socketPath.withCString { source in strncpy(UnsafeMutableRawPointer(pointer).assumingMemoryBound(to: CChar.self), source, maxPathLength) } } let result = withUnsafePointer(to: &address) { pointer -> Int32 in pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in bind(socketFd, sockaddrPointer, socklen_t(MemoryLayout.size)) } } guard result == 0 else { let bindErrno = errno let message = String(cString: strerror(bindErrno)) close(socketFd) socketFd = -1 if UnixSocketPathSafety.shouldRejectExistingPathAfterBindFailure( bindErrno: bindErrno, existingMode: existingPathMode(socketPath) ) { throw ProviderError.coded("invalid_argument", "refusing to replace non-socket file at computer-use socket path") } throw ProviderError.coded("accessibility_error", "failed to bind computer-use socket: \(message)") } chmod(socketPath, 0o600) guard listen(socketFd, 8) == 0 else { let message = String(cString: strerror(errno)) close(socketFd) socketFd = -1 throw ProviderError.coded("accessibility_error", "failed to listen on computer-use socket: \(message)") } } private func acceptLoop() { while !isStopped { let fd = accept(socketFd, nil, nil) if fd < 0 { if !isStopped { fputs("computer-use socket accept failed: \(String(cString: strerror(errno)))\n", stderr) } continue } Thread.detachNewThread { [weak self] in self?.handleConnection(fd) } } } private func handleConnection(_ fd: Int32) { defer { close(fd) } let authorizedPeer = peerProcessId(fd).map(isAuthorizedAgentPeer) == true let decoder = JSONDecoder() while let line = readLine(from: fd) { guard let data = line.data(using: .utf8), let request = try? decoder.decode(Request.self, from: data) else { continue } let response = handleRequest( provider: provider, lock: providerLock, request: request, expectedToken: token, authorizedPeer: authorizedPeer ) writeJSON(response, to: fd) } } } private func existingPathMode(_ path: String) -> mode_t? { var statInfo = stat() guard lstat(path, &statInfo) == 0 else { return nil } return statInfo.st_mode } private func peerProcessId(_ fd: Int32) -> pid_t? { var pid = pid_t(0) var length = socklen_t(MemoryLayout.size) let result = withUnsafeMutablePointer(to: &pid) { pointer in getsockopt(fd, 0, 2, pointer, &length) } return result == 0 && pid > 0 ? pid : nil } private func isAuthorizedAgentPeer(_ pid: pid_t) -> Bool { guard let command = processCommand(pid), command.contains("/out/main/computer-sidecar.js") || command.contains("/Contents/Resources/app.asar.unpacked/out/main/computer-sidecar.js") else { return false } if isTrustedOrcaApplication(pid) { return true } guard let parentPid = parentProcessId(pid) else { return false } return isTrustedOrcaApplication(parentPid) } private func isTrustedOrcaApplication(_ pid: pid_t) -> Bool { guard let app = NSRunningApplication(processIdentifier: pid), let bundleId = app.bundleIdentifier else { return false } // Why: dev validation runs from per-worktree wrapper apps with stable // Orca-owned bundle ids; the sidecar peer check must still authorize them. return bundleId == "com.stablyai.orca" || bundleId.hasPrefix("com.stablyai.orca.dev.") || bundleId == "com.github.Electron" } private func parentProcessId(_ pid: pid_t) -> pid_t? { guard let output = processField(pid: pid, field: "ppid=") else { return nil } let trimmed = output.trimmingCharacters(in: .whitespacesAndNewlines) guard let parentPid = pid_t(trimmed), parentPid > 1 else { return nil } return parentPid } private func processCommand(_ pid: pid_t) -> String? { return processField(pid: pid, field: "command=") } private func processField(pid: pid_t, field: String) -> String? { let process = Process() let pipe = Pipe() process.executableURL = URL(fileURLWithPath: "/bin/ps") process.arguments = ["-p", "\(pid)", "-o", field] process.standardOutput = pipe process.standardError = Pipe() do { try process.run() process.waitUntilExit() guard process.terminationStatus == 0 else { return nil } let data = pipe.fileHandleForReading.readDataToEndOfFile() return String(data: data, encoding: .utf8) } catch { return nil } } @MainActor private func runAgent(socketPath: String, token: String?) { let app = NSApplication.shared let delegate = AgentRuntime(socketPath: socketPath, token: token) app.delegate = delegate // Why: SCK is reliable once this code runs as a signed app with a real TCC identity. setenv("ORCA_COMPUTER_USE_SCK_SCREENSHOTS", "1", 1) app.run() } @MainActor private func runPermissionCheck(initialPermission: PermissionKind? = nil) { let app = NSApplication.shared let delegate = PermissionRuntime(initialPermission: initialPermission) app.delegate = delegate // Why: setup must foreground reliably; the long-running agent path stays accessory-only. app.setActivationPolicy(.regular) app.run() } private func printPermissionStatus() { let accessibility = accessibilityTrusted() ? "granted" : "not-granted" let screenshots = screenCaptureTrusted() ? "granted" : "not-granted" print(#"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"#) } private func writePermissionStatus(to path: String) { let accessibility = accessibilityTrusted() ? "granted" : "not-granted" let screenshots = screenCaptureTrusted() ? "granted" : "not-granted" let text = #"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"# do { try text.write(toFile: path, atomically: true, encoding: .utf8) } catch { fputs("failed to write permission status: \(error)\n", stderr) exit(1) } } private func runStdio() { fputs("Orca Computer Use provider must be launched by Orca in app-agent mode.\n", stderr) exit(13) } private func handleRequest( provider: Provider, lock: NSLock, request: Request, expectedToken: String?, authorizedPeer: Bool ) -> Any { if let expectedToken, request.token != expectedToken { return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "invalid computer-use agent token"]] } if expectedToken != nil && !authorizedPeer { return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "computer-use agent peer is not authorized"]] } if request.method == "terminate" { DispatchQueue.main.async { NSApp.terminate(nil) } return ["id": request.id, "ok": true, "result": ["ok": true]] } do { lock.lock() defer { lock.unlock() } let result = try provider.handle(method: request.method, params: request.params ?? [:]) return ["id": request.id, "ok": true, "result": result] } catch let error as ProviderError { return ["id": request.id, "ok": false, "error": ["code": error.code, "message": error.message]] } catch { return ["id": request.id, "ok": false, "error": ["code": "accessibility_error", "message": String(describing: error)]] } } private func readLine(from fd: Int32) -> String? { var bytes: [UInt8] = [] var byte: UInt8 = 0 while true { let count = read(fd, &byte, 1) if count == 0 { return bytes.isEmpty ? nil : String(bytes: bytes, encoding: .utf8) } if count < 0 { return nil } if byte == 10 { return String(bytes: bytes, encoding: .utf8) } bytes.append(byte) } } private func writeJSON(_ object: Any, to fd: Int32?) { guard JSONSerialization.isValidJSONObject(object), let data = try? JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes]) else { return } if let fd { _ = writeAll(data, to: fd) _ = writeAll(Data([10]), to: fd) } else { guard let text = String(data: data, encoding: .utf8) else { return } print(text) fflush(stdout) } } private func writeAll(_ data: Data, to fd: Int32) -> Bool { data.withUnsafeBytes { rawBuffer in guard let baseAddress = rawBuffer.baseAddress else { return true } var offset = 0 while offset < rawBuffer.count { let written = write(fd, baseAddress.advanced(by: offset), rawBuffer.count - offset) if written < 0 { if errno == EINTR { continue } return false } if written == 0 { return false } offset += written } return true } } let arguments = Array(CommandLine.arguments.dropFirst()) if arguments.first == "--agent" { guard arguments.count >= 2 else { fputs("usage: orca-computer-use-macos --agent --token-file \n", stderr) exit(2) } let tokenFileIndex = arguments.firstIndex(of: "--token-file") let token = tokenFileIndex.flatMap { index -> String? in let valueIndex = index + 1 guard valueIndex < arguments.count else { return nil } let tokenPath = arguments[valueIndex] return try? String(contentsOfFile: tokenPath, encoding: .utf8) .trimmingCharacters(in: .whitespacesAndNewlines) } guard let token, !token.isEmpty else { fputs("orca-computer-use-macos --agent requires a non-empty --token-file\n", stderr) exit(2) } runAgent(socketPath: arguments[1], token: token) } else if arguments.first == "--permissions" { runPermissionCheck() } else if arguments.first == "--permission" { runPermissionCheck(initialPermission: PermissionKind.parse(arguments.dropFirst().first)) } else if arguments.first == "--permission-status" { printPermissionStatus() } else if arguments.first == "--permission-status-file" { guard arguments.count >= 2 else { fputs("usage: orca-computer-use-macos --permission-status-file \n", stderr) exit(2) } writePermissionStatus(to: arguments[1]) } else { runStdio() }