Files
2026-07-13 13:05:33 +08:00

3814 lines
149 KiB
Swift

import AppKit
import ApplicationServices
import CoreGraphics
import Darwin
import Foundation
import ImageIO
import OrcaComputerUseMacOSCore
import ScreenCaptureKit
private let providerName = "orca-computer-use-macos"
private let providerVersion = "1.0.0"
private let providerProtocolVersion = 1
struct Request: Decodable {
let id: Int
let method: String
let params: [String: JSONValue]?
let token: String?
}
enum JSONValue: Decodable {
case string(String)
case number(Double)
case bool(Bool)
case object([String: JSONValue])
case array([JSONValue])
case null
init(from decoder: Decoder) throws {
let container = try decoder.singleValueContainer()
if container.decodeNil() {
self = .null
} else if let value = try? container.decode(Bool.self) {
self = .bool(value)
} else if let value = try? container.decode(Double.self) {
self = .number(value)
} else if let value = try? container.decode(String.self) {
self = .string(value)
} else if let value = try? container.decode([JSONValue].self) {
self = .array(value)
} else {
self = .object(try container.decode([String: JSONValue].self))
}
}
var string: String? {
if case let .string(value) = self { return value }
return nil
}
var number: Double? {
if case let .number(value) = self { return value }
return nil
}
var bool: Bool? {
if case let .bool(value) = self { return value }
return nil
}
}
enum ProviderError: Error {
case coded(String, String)
var code: String {
switch self {
case let .coded(code, _):
return code
}
}
var message: String {
switch self {
case let .coded(_, message):
return message
}
}
}
struct AppDescriptor {
let name: String
let bundleId: String?
let pid: pid_t
let app: NSRunningApplication
var needsManualAccessibilityMode: Bool {
// Chromium/Electron apps often need this private AX mode, but applying it
// broadly can corrupt native Cocoa app trees into app-root-only nodes.
guard let bundleId = bundleId?.lowercased() else {
return false
}
return bundleId.hasPrefix("com.google.chrome") ||
bundleId.hasPrefix("com.microsoft.edgemac") ||
bundleId.hasPrefix("com.brave.browser") ||
bundleId.hasPrefix("com.operasoftware.opera") ||
bundleId.hasPrefix("com.vivaldi.vivaldi") ||
bundleId == "com.github.electron" ||
bundleId == "com.tinyspeck.slackmacgap" ||
bundleId == "com.spotify.client" ||
bundleId == "com.hnc.discord" ||
bundleId == "com.microsoft.teams2" ||
bundleId == "notion.id"
}
var isKnownBrowser: Bool {
let bundle = bundleId?.lowercased() ?? ""
let appName = name.lowercased()
return bundle == "com.apple.safari" ||
bundle == "org.mozilla.firefox" ||
bundle == "company.thebrowser.browser" ||
bundle == "app.zen-browser.zen" ||
bundle.hasPrefix("com.google.chrome") ||
bundle.hasPrefix("com.microsoft.edgemac") ||
bundle.hasPrefix("com.brave.browser") ||
bundle.hasPrefix("com.operasoftware.opera") ||
bundle.hasPrefix("com.vivaldi.vivaldi") ||
appName == "safari" ||
appName == "firefox" ||
appName == "arc" ||
appName == "zen" ||
appName.contains("chrome") ||
appName.contains("chromium") ||
appName.contains("edge") ||
appName.contains("brave") ||
appName.contains("opera") ||
appName.contains("vivaldi")
}
}
final class ElementRecord {
let index: Int
let element: AXUIElement
let localFrame: CGRect?
let actions: [String]
let signature: String
init(index: Int, element: AXUIElement, localFrame: CGRect?, actions: [String], signature: String) {
self.index = index
self.element = element
self.localFrame = localFrame
self.actions = actions
self.signature = signature
}
}
struct Snapshot {
let id: String
let app: AppDescriptor
let windowTitle: String
let windowBounds: CGRect
let windowId: CGWindowID
let windowLayer: Int
let treeText: String
let focusedElementId: Int?
let screenshot: ScreenshotPayload?
let screenshotStatus: ScreenshotStatus
let screenshotScale: CGSize
let screenshotEngine: String?
let elements: [Int: ElementRecord]
let truncated: Bool
let maxDepthReached: Bool
func withoutScreenshotPayload() -> Snapshot {
Snapshot(
id: id,
app: app,
windowTitle: windowTitle,
windowBounds: windowBounds,
windowId: windowId,
windowLayer: windowLayer,
treeText: treeText,
focusedElementId: focusedElementId,
screenshot: nil,
screenshotStatus: .skipped,
screenshotScale: CGSize(width: 1, height: 1),
screenshotEngine: nil,
elements: elements,
truncated: truncated,
maxDepthReached: maxDepthReached
)
}
}
struct ScreenshotPayload {
let data: String
let width: Int
let height: Int
let scale: Double
}
struct CapturedImage {
let image: CGImage
let engine: String
}
enum ScreenshotStatus {
case captured
case skipped
case failed(String)
}
private struct CachedSnapshotEntry {
let snapshotId: String
let keys: [String]
let createdAt: Date
}
final class Provider {
private var snapshots: [String: Snapshot] = [:]
private var snapshotEntries: [CachedSnapshotEntry] = []
func handle(method: String, params: [String: JSONValue]) throws -> Any {
switch method {
case "handshake":
return providerHandshake()
case "listApps":
return ["apps": listApps().map(renderListedApp)]
case "listWindows":
return try listWindows(params: params)
case "getAppState":
return try renderSnapshot(observe(params: params))
case "click":
return try actionResult(params: params) { try click(params: params) }
case "performSecondaryAction":
return try actionResult(params: params) { try performSecondaryAction(params: params) }
case "setValue":
return try actionResult(params: params) { try setValue(params: params) }
case "typeText":
return try actionResult(params: params) { try typeText(params: params) }
case "pressKey":
return try actionResult(params: params) { try pressKey(params: params) }
case "hotkey":
return try actionResult(params: params) { try hotkey(params: params) }
case "pasteText":
return try actionResult(params: params) { try pasteText(params: params) }
case "scroll":
return try actionResult(params: params) { try scroll(params: params) }
case "drag":
return try actionResult(params: params) { try drag(params: params) }
default:
throw ProviderError.coded("invalid_argument", "unknown method '\(method)'")
}
}
private func actionResult(params: [String: JSONValue], action runAction: () throws -> [String: Any]) throws -> [String: Any] {
var action = try runAction()
do {
return try renderActionResult(action: action, snapshot: observe(params: params))
} catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && hasRequestedWindowSelector(params) {
var fallbackParams = params
fallbackParams.removeValue(forKey: "windowId")
fallbackParams.removeValue(forKey: "windowIndex")
if action["verification"] == nil {
action["verification"] = ["state": "unverified", "reason": "window_changed"]
}
return try renderActionResult(action: action, snapshot: observe(params: fallbackParams))
}
}
private func observe(params: [String: JSONValue]) throws -> Snapshot {
let query = try requiredString(params, "app")
let windowId = try requestedWindowId(params)
let windowIndex = try requestedWindowIndex(params)
let app = try resolveApp(query)
if params["restoreWindow"]?.bool == true {
recoverWindow(app)
}
let snapshot = try buildSnapshot(
app: app,
includeScreenshot: params["noScreenshot"]?.bool != true,
windowId: windowId,
windowIndex: windowIndex,
restoreWindow: params["restoreWindow"]?.bool == true
)
// Why: cached snapshots only validate element identity for follow-up
// actions; retaining MB-scale screenshot base64 in the long-lived agent grows memory.
rememberSnapshot(
query: query,
app: app,
snapshot: snapshot.withoutScreenshotPayload(),
params: params,
windowIndex: windowIndex
)
return snapshot
}
private func rememberSnapshot(
query: String,
app: AppDescriptor,
snapshot cachedSnapshot: Snapshot,
params: [String: JSONValue],
windowIndex: Int?
) {
let keys = [query, app.name, app.bundleId ?? "", "pid:\(app.pid)"]
.filter { !$0.isEmpty }
.map { $0.lowercased() }
let namespace = snapshotNamespace(params)
var storedKeys: [String] = []
let canonicalWindowKey = snapshotCanonicalWindowIdKey(cachedSnapshot.windowId)
if !isExplicitSnapshotNamespace(namespace) {
snapshots[canonicalWindowKey.lowercased()] = cachedSnapshot
storedKeys.append(canonicalWindowKey.lowercased())
}
snapshots[namespacedSnapshotKey(namespace, canonicalWindowKey)] = cachedSnapshot
storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowKey))
if let windowIndex {
let canonicalWindowIndexKey = snapshotCanonicalWindowIndexKey(windowIndex)
if !isExplicitSnapshotNamespace(namespace) {
snapshots[canonicalWindowIndexKey.lowercased()] = cachedSnapshot
storedKeys.append(canonicalWindowIndexKey.lowercased())
}
snapshots[namespacedSnapshotKey(namespace, canonicalWindowIndexKey)] = cachedSnapshot
storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowIndexKey))
}
for key in keys {
if !isExplicitSnapshotNamespace(namespace) {
snapshots[key] = cachedSnapshot
storedKeys.append(key)
snapshots[snapshotWindowKey(key, cachedSnapshot.windowId)] = cachedSnapshot
storedKeys.append(snapshotWindowKey(key, cachedSnapshot.windowId))
if let windowIndex {
snapshots[snapshotWindowIndexKey(key, windowIndex)] = cachedSnapshot
storedKeys.append(snapshotWindowIndexKey(key, windowIndex))
}
}
snapshots[namespacedSnapshotKey(namespace, key)] = cachedSnapshot
storedKeys.append(namespacedSnapshotKey(namespace, key))
let namespacedWindowKey = namespacedSnapshotKey(
namespace,
snapshotWindowKey(key, cachedSnapshot.windowId)
)
snapshots[namespacedWindowKey] = cachedSnapshot
storedKeys.append(namespacedWindowKey)
if let windowIndex {
let namespacedWindowIndexKey = namespacedSnapshotKey(
namespace,
snapshotWindowIndexKey(key, windowIndex)
)
snapshots[namespacedWindowIndexKey] = cachedSnapshot
storedKeys.append(namespacedWindowIndexKey)
}
}
snapshotEntries.append(
CachedSnapshotEntry(snapshotId: cachedSnapshot.id, keys: storedKeys, createdAt: Date())
)
pruneSnapshotCache()
}
private func pruneSnapshotCache() {
let now = Date()
while let oldest = snapshotEntries.first,
ComputerSnapshotCachePolicy.shouldPrune(
entryCount: snapshotEntries.count,
createdAt: oldest.createdAt,
now: now
) {
let expired = snapshotEntries.removeFirst()
for key in expired.keys where snapshots[key]?.id == expired.snapshotId {
snapshots.removeValue(forKey: key)
}
}
}
private func currentSnapshot(params: [String: JSONValue]) throws -> Snapshot {
pruneSnapshotCache()
let cached = try cachedSnapshot(params: params)
// Why: cached AX frames can be stale after a window move or resize, and
// stale geometry can turn an intended action into a misclick.
let snapshot = try observe(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
try validateRequestedElements(cached: cached, current: snapshot, params: params)
return snapshot
}
private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot {
// Why: AX text replacement/select-all do not post global input, so only
// synthetic fallback paths require the target window to be focused.
try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
}
private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? {
guard let query = params["app"]?.string, !query.isEmpty else { return nil }
let namespace = snapshotNamespace(params)
if let targetWindowId = try requestedWindowId(params) {
let canonicalKey = snapshotCanonicalWindowIdKey(targetWindowId)
if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] {
return cached
}
if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] {
return cached
}
let windowKey = snapshotWindowKey(query.lowercased(), targetWindowId)
if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] {
return cached
}
if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] {
return cached
}
return nil
}
if let targetWindowIndex = try requestedWindowIndex(params) {
let canonicalKey = snapshotCanonicalWindowIndexKey(targetWindowIndex)
if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] {
return cached
}
if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] {
return cached
}
let windowKey = snapshotWindowIndexKey(query.lowercased(), targetWindowIndex)
if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] {
return cached
}
if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] {
return cached
}
return nil
}
let key = query.lowercased()
return snapshots[namespacedSnapshotKey(namespace, key)] ??
(isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key])
}
private func validateRequestedElements(cached: Snapshot?, current: Snapshot, params: [String: JSONValue]) throws {
let requestedIndexes = try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in
guard params[key]?.number != nil else { return nil }
return try optionalInteger(params, key)
}
guard !requestedIndexes.isEmpty else { return }
guard let cached else {
throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window")
}
for index in requestedIndexes {
guard let expected = cached.elements[index], let actual = current.elements[index] else {
throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index")
}
guard expected.signature == actual.signature else {
throw ProviderError.coded("element_not_found", "element \(index) changed since the last snapshot; run get-app-state again and use a fresh element index")
}
}
}
private func ensureWindowStillAvailable(_ snapshot: Snapshot) throws {
guard WindowCapture.candidates(pid: snapshot.app.pid).contains(where: { $0.windowId == snapshot.windowId }) else {
throw ProviderError.coded("window_stale", "window \(Int(snapshot.windowId)) is no longer available; run get-app-state again to refresh the target window")
}
}
private func listApps() -> [AppDescriptor] {
var seen = Set<String>()
return NSWorkspace.shared.runningApplications
.filter { !$0.isTerminated && $0.activationPolicy == .regular }
.compactMap { app in
guard let name = app.localizedName, !name.isEmpty else { return nil }
let pid = app.processIdentifier
guard pid > 0, pidIsLive(pid) else { return nil }
let key = (app.bundleIdentifier ?? "pid:\(pid)").lowercased()
guard seen.insert(key).inserted else { return nil }
return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app)
}
.sorted { lhs, rhs in
if lhs.app.isActive != rhs.app.isActive {
return lhs.app.isActive && !rhs.app.isActive
}
return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending
}
}
private func renderListedApp(_ app: AppDescriptor) -> [String: Any] {
[
"name": app.name,
"bundleId": jsonNullable(app.bundleId),
"pid": Int(app.pid),
"isRunning": true,
"lastUsedAt": NSNull(),
"useCount": NSNull(),
]
}
private func providerHandshake() -> [String: Any] {
[
"platform": "darwin",
"provider": providerName,
"providerVersion": providerVersion,
"protocolVersion": providerProtocolVersion,
"supports": [
"apps": [
"list": true,
"bundleIds": true,
"pids": true,
],
"windows": [
"list": true,
"targetById": true,
"targetByIndex": true,
"focus": false,
"moveResize": false,
],
"observation": [
"screenshot": true,
"annotatedScreenshot": false,
"elementFrames": true,
"ocr": false,
],
"actions": [
"click": true,
"typeText": true,
"pressKey": true,
"hotkey": true,
"pasteText": true,
"scroll": true,
"drag": true,
"setValue": true,
"performAction": true,
],
"surfaces": [
"menus": false,
"dialogs": false,
"dock": false,
"menubar": false,
],
],
]
}
private func resolveApp(_ query: String) throws -> AppDescriptor {
let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else {
throw ProviderError.coded("invalid_argument", "app query must not be empty")
}
if let pid = parsePid(trimmed) {
if let app = appByPid(pid) {
try rejectBlockedApp(app)
return app
}
throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found")
}
if blockedBundleIds.contains(trimmed) {
throw ProviderError.coded("app_blocked", "app '\(trimmed)' is blocked for safety")
}
if let app = listApps().first(where: { matches($0, query: trimmed) }) {
try rejectBlockedApp(app)
return app
}
throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found")
}
private func rejectBlockedApp(_ app: AppDescriptor) throws {
if let bundle = app.bundleId, blockedBundleIds.contains(bundle) {
throw ProviderError.coded("app_blocked", "app '\(bundle)' is blocked for safety")
}
}
private func listWindows(params: [String: JSONValue]) throws -> [String: Any] {
let app = try resolveApp(try requiredString(params, "app"))
let windows = WindowCapture.candidates(pid: app.pid)
.filter { $0.layer == 0 }
.enumerated()
.map { index, candidate -> [String: Any] in
[
"index": index,
"app": [
"name": app.name,
"bundleId": jsonNullable(app.bundleId),
"pid": Int(app.pid),
],
"id": Int(candidate.windowId),
"title": candidate.title ?? "",
"x": Int(candidate.bounds.origin.x.rounded()),
"y": Int(candidate.bounds.origin.y.rounded()),
"width": Int(candidate.bounds.width.rounded()),
"height": Int(candidate.bounds.height.rounded()),
"isMinimized": false,
"isOffscreen": !candidate.isOnScreen,
"screenIndex": jsonNullable(screenIndex(for: candidate.bounds)),
"isMain": NSNull(),
"platform": [
"layer": candidate.layer,
"alpha": candidate.alpha,
],
]
}
return [
"app": renderListedApp(app),
"windows": windows,
]
}
private func appByPid(_ pid: pid_t) -> AppDescriptor? {
guard let app = NSRunningApplication(processIdentifier: pid),
!app.isTerminated,
let name = app.localizedName
else {
return nil
}
return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app)
}
private func buildSnapshot(
app: AppDescriptor,
includeScreenshot: Bool,
windowId: CGWindowID?,
windowIndex: Int?,
restoreWindow: Bool
) throws -> Snapshot {
guard accessibilityTrusted() else {
// Why: agents retry failed observations. Only the explicit setup flow
// should open macOS privacy prompts/settings; runtime calls stay quiet.
throw ProviderError.coded(
"permission_denied",
"Accessibility permission is required for Orca Computer Use. Run `orca computer permissions` or open Settings > Computer Use, grant Accessibility to Orca Computer Use, then retry."
)
}
let appElement = AXUIElementCreateApplication(app.pid)
enableManualAccessibilityIfNeeded(appElement, app: app)
let windowCandidates = WindowCapture.candidates(pid: app.pid)
let focused = try focusedWindow(
appElement: appElement,
app: app,
visibleWindowCount: windowCandidates.count,
allowRecovery: restoreWindow
)
let focusedTitle = stringAttribute(focused, kAXTitleAttribute as String) ?? app.name
let canCaptureScreenshot = includeScreenshot && screenCaptureTrusted()
guard let capture = WindowCapture.resolve(
candidates: windowCandidates,
titleHint: focusedTitle,
windowId: windowId,
windowIndex: windowIndex,
captureImage: canCaptureScreenshot
) else {
throw ProviderError.coded("window_not_found", "app '\(app.name)' has no on-screen window")
}
guard let window = matchingWindow(appElement: appElement, capture: capture, focused: focused, explicitTarget: windowId != nil || windowIndex != nil) else {
throw ProviderError.coded("window_not_found", "could not match accessibility window to requested window; run get-app-state again or retry without a window selector")
}
let title = stringAttribute(window, kAXTitleAttribute as String) ?? capture.title ?? app.name
let renderer = TreeRenderer(
windowBounds: capture.bounds,
focused: focusedElement(appElement: appElement),
compactBrowserTabs: app.isKnownBrowser
)
renderer.render(window)
let screenshot = includeScreenshot ? capture.screenshotPayload() : nil
let screenshotStatus: ScreenshotStatus = if screenshot != nil {
.captured
} else if includeScreenshot && !canCaptureScreenshot {
.failed("Screen Recording permission is required for Orca Computer Use; grant permission or pass --no-screenshot to inspect accessibility state only.")
} else if includeScreenshot {
.failed("window screenshot capture returned no image; retry with --no-screenshot if accessibility state is sufficient.")
} else {
.skipped
}
return Snapshot(
id: UUID().uuidString,
app: app,
windowTitle: title,
windowBounds: capture.bounds,
windowId: capture.windowId,
windowLayer: capture.layer,
treeText: renderTreeText(app: app, title: title, bounds: capture.bounds, lines: renderer.lines, focused: renderer.focusedSummary),
focusedElementId: renderer.focusedElementId,
screenshot: screenshot,
screenshotStatus: screenshotStatus,
screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds),
screenshotEngine: capture.image?.engine,
elements: renderer.records,
truncated: renderer.truncated,
maxDepthReached: renderer.maxDepthReached
)
}
private func renderSnapshot(_ snapshot: Snapshot) -> [String: Any] {
var screenshot: Any = NSNull()
if let payload = snapshot.screenshot {
screenshot = [
"data": payload.data,
"format": "png",
"width": payload.width,
"height": payload.height,
"scale": payload.scale,
]
}
return [
"snapshot": [
"id": snapshot.id,
"app": [
"name": snapshot.app.name,
"bundleId": jsonNullable(snapshot.app.bundleId),
"pid": Int(snapshot.app.pid),
],
"window": [
"id": Int(snapshot.windowId),
"title": snapshot.windowTitle,
"x": Int(snapshot.windowBounds.origin.x.rounded()),
"y": Int(snapshot.windowBounds.origin.y.rounded()),
"width": Int(snapshot.windowBounds.width.rounded()),
"height": Int(snapshot.windowBounds.height.rounded()),
"isMinimized": false,
"isOffscreen": false,
"screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)),
"platform": [
"layer": snapshot.windowLayer,
],
],
"coordinateSpace": "window",
"treeText": snapshot.treeText,
"elementCount": snapshot.elements.count,
"focusedElementId": snapshot.focusedElementId as Any,
"truncation": [
"truncated": snapshot.truncated,
"maxNodes": TreeRenderer.maxNodes,
"maxDepth": TreeRenderer.maxDepth,
"maxDepthReached": snapshot.maxDepthReached,
],
],
"screenshot": screenshot,
"screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot),
]
}
private func renderActionResult(action: [String: Any], snapshot: Snapshot) -> [String: Any] {
var result = renderSnapshot(snapshot)
var metadata = action
metadata["targetWindowId"] = Int(snapshot.windowId)
result["action"] = metadata
return result
}
private func click(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentSnapshot(params: params)
let button = params["mouseButton"]?.string ?? "left"
let count = try positiveInteger(params["clickCount"]?.number, defaultValue: 1, name: "clickCount")
// Why: agents expect a click into a target app to make the next
// keyboard action safe, even when the click uses an AX action path.
recoverWindow(snapshot.app)
if let elementIndex = try optionalInteger(params, "elementIndex") {
let record = try element(snapshot, elementIndex)
if count <= 1, let actionName = try performClickAction(record: record, mouseButton: button) {
return actionMetadata(path: "accessibility", actionName: actionName)
}
if let point = center(record.localFrame, in: snapshot.windowBounds) {
try Input.click(
pid: snapshot.app.pid,
at: point,
button: mouseButton(button),
count: count
)
return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported")
}
throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame")
}
let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
try Input.click(
pid: snapshot.app.pid,
at: point,
button: mouseButton(button),
count: count
)
return actionMetadata(path: "synthetic")
}
private func performClickAction(record: ElementRecord, mouseButton: String) throws -> String? {
if mouseButton == "right" {
return performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil
}
for action in ["AXPress", "AXConfirm", "AXOpen"] {
if performAction(record.element, action) {
return action
}
}
return nil
}
private func performSecondaryAction(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentSnapshot(params: params)
let record = try element(snapshot, try requiredInteger(params, "elementIndex"))
let requested = try requiredString(params, "action")
let action = record.actions.first { SnapshotRenderHeuristics.prettyAction($0).caseInsensitiveCompare(requested) == .orderedSame || $0.caseInsensitiveCompare(requested) == .orderedSame }
guard let action else {
throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)")
}
guard performAction(record.element, action) else {
throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed")
}
return actionMetadata(path: "accessibility", actionName: action)
}
private func setValue(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentSnapshot(params: params)
let record = try element(snapshot, try requiredInteger(params, "elementIndex"))
let expected = try requiredStringAllowingEmpty(params, "value")
guard isSettable(record.element, kAXValueAttribute as String) else {
throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable")
}
let result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, expected as CFString)
guard result == .success else {
throw ProviderError.coded("accessibility_error", "AXUIElementSetAttributeValue failed with \(result.rawValue)")
}
let actual = rawStringAttribute(record.element, kAXValueAttribute as String)
let verification = actual == expected
? verifiedAction(property: "value", expected: expected, actualPreview: actual)
: unverifiedAction(reason: actual == nil ? "provider_unavailable" : "value_mismatch", expected: expected, actualPreview: actual)
return actionMetadata(path: "accessibility", actionName: "AXSetValue", verification: verification)
}
private func typeText(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
let text = try requiredString(params, "text")
if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
}
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
try Input.typeText(text, pid: snapshot.app.pid)
return actionMetadata(
path: "synthetic",
actionName: "typeText",
verification: unverifiedAction(reason: "synthetic_input")
)
}
private func pressKey(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
try Input.pressKey(try requiredString(params, "key"), pid: snapshot.app.pid)
return actionMetadata(
path: "synthetic",
actionName: "pressKey",
verification: unverifiedAction(reason: "synthetic_input")
)
}
private func hotkey(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
let key = try requiredString(params, "key")
if isSelectAllHotkey(key), let focused = focusedRecord(snapshot), TextInput.selectAll(focused.element) {
return actionMetadata(
path: "accessibility",
actionName: "AXSelectAll",
verification: TextInput.selectionVerification(focused.element)
)
}
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
try Input.pressKey(key, pid: snapshot.app.pid)
return actionMetadata(
path: "synthetic",
actionName: "hotkey",
verification: unverifiedAction(reason: "synthetic_input")
)
}
private func pasteText(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
let text = try requiredString(params, "text")
if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
}
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
try Input.pasteText(text, pid: snapshot.app.pid)
return actionMetadata(
path: "clipboard",
actionName: "paste",
verification: unverifiedAction(reason: "clipboard_paste")
)
}
private func scroll(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentSnapshot(params: params)
let direction = try scrollDirection(try requiredString(params, "direction"))
let pages = try positiveNumber(params["pages"]?.number, defaultValue: 1, name: "pages")
if let elementIndex = try optionalInteger(params, "elementIndex") {
let record = try element(snapshot, elementIndex)
let action = "AXScroll\(direction.capitalized)ByPage"
if pages.rounded() == pages, let pageCount = boundedInteger(pages, as: Int.self),
record.actions.contains(action) {
for _ in 0..<max(1, pageCount) {
_ = performAction(record.element, action)
}
return actionMetadata(path: "accessibility", actionName: action)
}
guard let point = center(record.localFrame, in: snapshot.windowBounds) else {
throw ProviderError.coded("element_not_found", "element \(record.index) has no scrollable frame")
}
try Input.scroll(pid: snapshot.app.pid, at: point, direction: direction, pages: pages)
return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported")
}
let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
try Input.scroll(pid: snapshot.app.pid, at: point, direction: direction, pages: pages)
return actionMetadata(path: "synthetic")
}
private func drag(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentSnapshot(params: params)
let start: CGPoint
let end: CGPoint
if let fromIndex = try optionalInteger(params, "fromElementIndex"),
let toIndex = try optionalInteger(params, "toElementIndex") {
let from = try element(snapshot, fromIndex)
let to = try element(snapshot, toIndex)
guard let fromPoint = center(from.localFrame, in: snapshot.windowBounds),
let toPoint = center(to.localFrame, in: snapshot.windowBounds)
else {
throw ProviderError.coded("element_not_found", "drag element has no frame")
}
start = fromPoint
end = toPoint
} else {
start = try coordinatePoint(params: params, xKey: "fromX", yKey: "fromY", snapshot: snapshot)
end = try coordinatePoint(params: params, xKey: "toX", yKey: "toY", snapshot: snapshot)
}
try Input.drag(pid: snapshot.app.pid, from: start, to: end)
return actionMetadata(path: "synthetic")
}
private func element(_ snapshot: Snapshot, _ index: Int) throws -> ElementRecord {
guard let record = snapshot.elements[index] else {
throw ProviderError.coded("element_not_found", "element \(index) is not in the current cached snapshot for \(snapshot.app.name); run get-app-state again and use a fresh element index")
}
return record
}
private func focusedRecord(_ snapshot: Snapshot) -> ElementRecord? {
guard let focusedElementId = snapshot.focusedElementId else {
return nil
}
return snapshot.elements[focusedElementId]
}
}
private let blockedBundleIds: Set<String> = [
"com.1password.1password",
"com.1password.safari",
"com.bitwarden.desktop",
"com.dashlane.dashlanephonefinal",
"com.lastpass.LastPass",
"com.nordsec.nordpass",
"me.proton.pass.electron",
"me.proton.pass.catalyst",
]
private func requiredString(_ params: [String: JSONValue], _ key: String) throws -> String {
guard let value = params[key]?.string, !value.isEmpty else {
throw ProviderError.coded("invalid_argument", "missing \(key)")
}
return value
}
private func requiredStringAllowingEmpty(_ params: [String: JSONValue], _ key: String) throws -> String {
guard let value = params[key]?.string else {
throw ProviderError.coded("invalid_argument", "missing \(key)")
}
return value
}
private func requiredNumber(_ params: [String: JSONValue], _ key: String) throws -> Double {
guard let value = params[key]?.number, value.isFinite else {
throw ProviderError.coded("invalid_argument", "missing \(key)")
}
return value
}
private func requiredInteger(_ params: [String: JSONValue], _ key: String) throws -> Int {
guard let value = boundedInteger(try requiredNumber(params, key), as: Int.self) else {
throw ProviderError.coded("invalid_argument", "\(key) is out of range")
}
return value
}
private func optionalInteger(_ params: [String: JSONValue], _ key: String) throws -> Int? {
guard let raw = params[key]?.number else { return nil }
guard let value = boundedInteger(raw, as: Int.self) else {
throw ProviderError.coded("invalid_argument", "\(key) is out of range")
}
return value
}
private func positiveInteger(_ value: Double?, defaultValue: Int, name: String) throws -> Int {
switch ActionArgumentValidation.positiveInteger(value, defaultValue: defaultValue, name: name) {
case let .success(value):
return value
case let .failure(error):
throw ProviderError.coded("invalid_argument", error.message)
}
}
private func positiveNumber(_ value: Double?, defaultValue: Double, name: String) throws -> Double {
switch ActionArgumentValidation.positiveNumber(value, defaultValue: defaultValue, name: name) {
case let .success(value):
return value
case let .failure(error):
throw ProviderError.coded("invalid_argument", error.message)
}
}
private func scrollDirection(_ value: String) throws -> String {
switch ActionArgumentValidation.scrollDirection(value) {
case let .success(value):
return value
case let .failure(error):
throw ProviderError.coded("invalid_argument", error.message)
}
}
private func parsePid(_ query: String) -> pid_t? {
guard query.hasPrefix("pid:") else { return nil }
guard let pid = Int32(query.dropFirst(4)), pid > 0 else { return nil }
return pid
}
private func matches(_ app: AppDescriptor, query: String) -> Bool {
app.name.caseInsensitiveCompare(query) == .orderedSame ||
app.bundleId?.caseInsensitiveCompare(query) == .orderedSame
}
private func pidIsLive(_ pid: pid_t) -> Bool {
kill(pid, 0) == 0
}
private func accessibilityTrusted() -> Bool {
AXIsProcessTrusted()
}
private func screenCaptureTrusted() -> Bool {
CGPreflightScreenCaptureAccess()
}
private func requestScreenCaptureAccess() -> Bool {
CGRequestScreenCaptureAccess()
}
private func openAccessibilitySettings() {
openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility")
}
private func openScreenRecordingSettings() {
openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture")
}
private func openSystemSettings(_ value: String) {
guard let url = URL(string: value) else { return }
NSWorkspace.shared.open(url)
}
private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: AppDescriptor) {
guard app.needsManualAccessibilityMode else {
return
}
_ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
_ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
}
private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement {
let systemWide = AXUIElementCreateSystemWide()
if let window = focusedSystemWindow(systemWide: systemWide, app: app) {
return window
}
if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) {
return window
}
if let windows = copyArray(appElement, kAXWindowsAttribute as String) {
if let window = windows.first(where: usableWindow) {
return window
}
}
if allowRecovery {
recoverWindow(app)
if let window = focusedSystemWindow(systemWide: systemWide, app: app) {
return window
}
if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) {
return window
}
if let windows = copyArray(appElement, kAXWindowsAttribute as String) {
if let window = windows.first(where: usableWindow) {
return window
}
}
}
let permissionHint = visibleWindowCount > 0
? " The app has visible windows, so macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings."
: ""
if visibleWindowCount > 0 {
throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but no accessibility window.\(permissionHint)")
}
throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window; make sure the app has a visible window, then retry with --restore-window.")
}
private func focusedSystemWindow(systemWide: AXUIElement, app: AppDescriptor) -> AXUIElement? {
guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String),
pidAttribute(focusedApp) == app.pid
else {
return nil
}
if let window = copyElement(systemWide, kAXFocusedWindowAttribute as String), usableWindow(window) {
return window
}
if let window = copyElement(focusedApp, kAXFocusedWindowAttribute as String), usableWindow(window) {
return window
}
if let windows = copyArray(focusedApp, kAXWindowsAttribute as String) {
return windows.first(where: usableWindow)
}
return nil
}
private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool {
guard let focusedWindow = focusedSystemWindow(systemWide: AXUIElementCreateSystemWide(), app: snapshot.app) else {
return false
}
if windowNumber(focusedWindow) == snapshot.windowId {
return true
}
guard let frame = absoluteFrame(focusedWindow) else {
return false
}
let intersection = frame.intersection(snapshot.windowBounds)
return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75
}
private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowRequested: Bool) throws {
guard let failure = KeyboardInputSafety.syntheticInputFocusFailure(
targetWindowFocused: isTargetWindowFocused(snapshot),
restoreWindowRequested: restoreWindowRequested
) else {
return
}
switch failure {
case .targetNotFocused:
throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements")
case .targetNotFocusedAfterRestore:
throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; --restore-window was requested but the target is still not focused; bring it forward manually or check Accessibility permissions")
}
}
private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, focused: AXUIElement, explicitTarget: Bool) -> AXUIElement? {
guard let windows = copyArray(appElement, kAXWindowsAttribute as String) else {
return nil
}
if let byNumber = windows.first(where: { windowNumber($0) == capture.windowId }) {
return byNumber
}
if let byBounds = windows.first(where: { window in
guard usableWindow(window), let frame = absoluteFrame(window) else { return false }
let intersection = frame.intersection(capture.bounds)
return !intersection.isNull && intersection.area >= min(frame.area, capture.bounds.area) * 0.75
}) {
return byBounds
}
if explicitTarget {
return nil
}
guard let titleHint = capture.title, !titleHint.isEmpty else {
return focused
}
return windows.first {
usableWindow($0) && stringAttribute($0, kAXTitleAttribute as String) == titleHint
} ?? focused
}
private func recoverWindow(_ app: AppDescriptor) {
_ = app.app.unhide()
_ = app.app.activate(options: [.activateAllWindows])
if let bundleId = app.bundleId {
openBundle(bundleId)
}
let appElement = AXUIElementCreateApplication(app.pid)
if let window = copyElement(appElement, kAXFocusedWindowAttribute as String) ?? copyArray(appElement, kAXWindowsAttribute as String)?.first {
_ = AXUIElementSetAttributeValue(window, kAXMinimizedAttribute as CFString, kCFBooleanFalse)
_ = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
_ = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
_ = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue)
}
Thread.sleep(forTimeInterval: 0.4)
}
private func openBundle(_ bundleId: String) {
let process = Process()
process.executableURL = URL(fileURLWithPath: "/usr/bin/open")
process.arguments = ["-b", bundleId]
process.standardOutput = Pipe()
process.standardError = Pipe()
try? process.run()
process.waitUntilExit()
}
private func hasRequestedWindowSelector(_ params: [String: JSONValue]) -> Bool {
params["windowId"] != nil || params["windowIndex"] != nil
}
private func requestedWindowId(_ params: [String: JSONValue]) throws -> CGWindowID? {
guard let raw = params["windowId"] else { return nil }
guard let value = raw.number, value >= 0, let id = boundedInteger(value, as: UInt32.self) else {
throw ProviderError.coded("invalid_argument", "windowId is out of range")
}
return CGWindowID(id)
}
private func snapshotWindowKey(_ query: String, _ windowId: CGWindowID) -> String {
"\(query.lowercased())#window:\(Int(windowId))"
}
private func snapshotCanonicalWindowIdKey(_ windowId: CGWindowID) -> String {
"window-id:\(Int(windowId))"
}
private func snapshotWindowIndexKey(_ query: String, _ windowIndex: Int) -> String {
"\(query.lowercased())#windowIndex:\(windowIndex)"
}
private func snapshotCanonicalWindowIndexKey(_ windowIndex: Int) -> String {
"window-index:\(windowIndex)"
}
private func snapshotNamespace(_ params: [String: JSONValue]) -> String {
if let session = params["session"]?.string, !session.isEmpty {
return "session:\(session)"
}
if let worktree = params["worktree"]?.string, !worktree.isEmpty {
return "worktree:\(worktree)"
}
return "default"
}
private func namespacedSnapshotKey(_ namespace: String, _ key: String) -> String {
"\(namespace):\(key.lowercased())"
}
private func isExplicitSnapshotNamespace(_ namespace: String) -> Bool {
namespace != "default"
}
private func requestedWindowIndex(_ params: [String: JSONValue]) throws -> Int? {
guard let raw = params["windowIndex"] else { return nil }
guard let value = raw.number, value >= 0, let index = boundedInteger(value, as: Int.self) else {
throw ProviderError.coded("invalid_argument", "windowIndex is out of range")
}
return index
}
private func usableWindow(_ element: AXUIElement) -> Bool {
stringAttribute(element, kAXRoleAttribute as String) == kAXWindowRole as String &&
boolAttribute(element, kAXMinimizedAttribute as String) != true
}
private func focusedElement(appElement: AXUIElement) -> AXUIElement? {
copyElement(appElement, kAXFocusedUIElementAttribute as String)
}
private func copyElement(_ element: AXUIElement, _ attribute: String) -> AXUIElement? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {
return nil
}
return (value as! AXUIElement)
}
private func copyArray(_ element: AXUIElement, _ attribute: String) -> [AXUIElement]? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {
return nil
}
return value as? [AXUIElement]
}
private func pidAttribute(_ element: AXUIElement) -> pid_t? {
var pid: pid_t = 0
guard AXUIElementGetPid(element, &pid) == .success else {
return nil
}
return pid
}
private func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {
return nil
}
if CFGetTypeID(value) == CFStringGetTypeID(), let string = value as? String {
let trimmed = string.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
}
if CFGetTypeID(value) == CFURLGetTypeID(), let url = value as? URL {
return url.absoluteString
}
return nil
}
private func rawStringAttribute(_ element: AXUIElement, _ attribute: String) -> String? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
let value,
CFGetTypeID(value) == CFStringGetTypeID()
else {
return nil
}
return value as? String
}
private func boolAttribute(_ element: AXUIElement, _ attribute: String) -> Bool? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {
return nil
}
return value as? Bool
}
private func numberAttribute(_ element: AXUIElement, _ attribute: String) -> NSNumber? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {
return nil
}
return value as? NSNumber
}
private func windowNumber(_ element: AXUIElement) -> CGWindowID? {
guard let number = numberAttribute(element, "AXWindowNumber") else {
return nil
}
return CGWindowID(number.uint32Value)
}
private func absoluteFrame(_ element: AXUIElement) -> CGRect? {
var positionValue: CFTypeRef?
var sizeValue: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,
AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,
let positionValue,
let sizeValue
else {
return nil
}
var point = CGPoint.zero
var size = CGSize.zero
guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point),
AXValueGetValue(sizeValue as! AXValue, .cgSize, &size)
else {
return nil
}
return CGRect(origin: point, size: size)
}
private extension CGRect {
var area: CGFloat {
max(width, 0) * max(height, 0)
}
}
private func actions(_ element: AXUIElement) -> [String] {
var value: CFArray?
guard AXUIElementCopyActionNames(element, &value) == .success, let value else {
return []
}
return value as? [String] ?? []
}
private func performAction(_ element: AXUIElement, _ action: String) -> Bool {
actions(element).contains(where: { $0.caseInsensitiveCompare(action) == .orderedSame }) &&
AXUIElementPerformAction(element, action as CFString) == .success
}
private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
var settable = DarwinBoolean(false)
return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue
}
private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? {
var positionValue: CFTypeRef?
var sizeValue: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,
AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,
let positionValue,
let sizeValue
else {
return nil
}
var point = CGPoint.zero
var size = CGSize.zero
guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point),
AXValueGetValue(sizeValue as! AXValue, .cgSize, &size)
else {
return nil
}
return CGRect(x: point.x - windowBounds.minX, y: point.y - windowBounds.minY, width: size.width, height: size.height)
}
private func elementSignature(_ node: SnapshotRenderNode) -> String {
// Why: cached element validation should prove identity, not reject text
// controls because their value/placeholder/summary changed after focus.
[
node.role,
node.roleDescription ?? "",
node.title ?? "",
node.label ?? "",
node.linkText ?? "",
node.url ?? "",
SnapshotRenderHeuristics.meaningfulActions(node.rawActions, role: node.role).joined(separator: ","),
].joined(separator: "\u{1f}")
}
private func center(_ localFrame: CGRect?, in windowBounds: CGRect) -> CGPoint? {
guard let localFrame else { return nil }
return CGPoint(x: windowBounds.minX + localFrame.midX, y: windowBounds.minY + localFrame.midY)
}
private func screenIndex(for bounds: CGRect) -> Int? {
guard let index = NSScreen.screens.firstIndex(where: { $0.frame.intersects(bounds) }) else {
return nil
}
return index
}
private func coordinatePoint(params: [String: JSONValue], xKey: String, yKey: String, snapshot: Snapshot) throws -> CGPoint {
let x = try requiredNumber(params, xKey)
let y = try requiredNumber(params, yKey)
return CGPoint(
x: snapshot.windowBounds.minX + x,
y: snapshot.windowBounds.minY + y
)
}
private func screenshotScale(screenshot: ScreenshotPayload?, bounds: CGRect) -> CGSize {
guard let screenshot, bounds.width > 0, bounds.height > 0 else {
return CGSize(width: 1, height: 1)
}
return CGSize(
width: CGFloat(screenshot.width) / bounds.width,
height: CGFloat(screenshot.height) / bounds.height
)
}
private enum MouseButton {
case left
case right
var cgButton: CGMouseButton {
switch self {
case .left:
return .left
case .right:
return .right
}
}
var downEvent: CGEventType {
switch self {
case .left:
return .leftMouseDown
case .right:
return .rightMouseDown
}
}
var upEvent: CGEventType {
switch self {
case .left:
return .leftMouseUp
case .right:
return .rightMouseUp
}
}
}
private func mouseButton(_ raw: String?) throws -> MouseButton {
switch raw ?? "left" {
case "left":
return .left
case "right":
return .right
case "middle":
throw ProviderError.coded("invalid_argument", "middle-click is not yet supported")
case let value:
throw ProviderError.coded("invalid_argument", "unsupported mouse button '\(value)'")
}
}
private func renderTreeText(app: AppDescriptor, title: String, bounds: CGRect, lines: [String], focused: String?) -> String {
var output = [
"App=\(app.bundleId ?? app.name.replacingOccurrences(of: " ", with: "_")) (pid \(app.pid))",
"Window: \"\(sanitize(title))\", App: \(sanitize(app.name)).",
"",
]
output.append(contentsOf: lines)
output.append("")
output.append(focused.map { "The focused UI element is \($0)." } ?? "No UI element is currently focused.")
return output.joined(separator: "\n")
}
private func renderScreenshotStatus(_ status: ScreenshotStatus, snapshot: Snapshot) -> [String: Any] {
let metadata: [String: Any] = [
"engine": snapshot.screenshotEngine ?? "unknown",
"windowId": Int(snapshot.windowId),
]
switch status {
case .captured:
return ["state": "captured", "metadata": metadata]
case .skipped:
return ["state": "skipped", "reason": "no_screenshot_flag"]
case let .failed(message):
return ["state": "failed", "code": "screenshot_failed", "message": message, "metadata": metadata]
}
}
private final class TreeRenderer {
let windowBounds: CGRect
let focused: AXUIElement?
let compactBrowserTabs: Bool
var lines: [String] = []
var records: [Int: ElementRecord] = [:]
var focusedSummary: String?
var focusedElementId: Int?
var truncated = false
var maxDepthReached = false
private let reader = AXSnapshotReader()
private var nextIndex = 0
static let maxNodes = 1200
static let maxDepth = 64
init(windowBounds: CGRect, focused: AXUIElement?, compactBrowserTabs: Bool) {
self.windowBounds = windowBounds
self.focused = focused
self.compactBrowserTabs = compactBrowserTabs
}
func render(_ element: AXUIElement, depth: Int = 0, ancestors: [AXUIElement] = []) {
guard nextIndex < Self.maxNodes else {
truncated = true
return
}
guard depth < Self.maxDepth else {
truncated = true
maxDepthReached = true
return
}
guard !ancestors.contains(where: { CFEqual($0, element) }) else { return }
let role = reader.stringAttribute(element, kAXRoleAttribute as String) ?? "AXUnknown"
let children = reader.primaryChildren(element, role: role, windowBounds: windowBounds)
let value = reader.valueString(element, role: role)
let placeholder = reader.placeholderString(element)
let rawActions = reader.actions(element)
let rowSummary = reader.rowTextSummary(element, role: role)
let roleDescription = reader.stringAttribute(element, kAXRoleDescriptionAttribute as String)
let title = reader.stringAttribute(element, kAXTitleAttribute as String)
let label = reader.stringAttribute(element, kAXDescriptionAttribute as String)
let url = reader.stringAttribute(element, kAXURLAttribute as String)
let linkText = role == "AXLink" ? reader.descendantTextSnippets(element, limit: 2, maxDepth: 3).first : nil
let baseNode = SnapshotRenderNode(
role: role,
roleDescription: roleDescription,
title: title,
label: label,
linkText: linkText,
value: value,
placeholder: placeholder,
url: url,
traits: [],
rawActions: rawActions,
childCount: children.count,
rowSummary: rowSummary
)
let name = SnapshotRenderHeuristics.displayName(baseNode)
let meaningful = SnapshotRenderHeuristics.meaningfulActions(rawActions, role: role)
let localFrame = reader.frame(element, windowBounds: windowBounds)
let traits = reader.traitsFor(element, role: role)
let webAreaDepth = reader.webAreaDepth(role: role, ancestors: ancestors)
let summary = reader.genericTextSummary(element, role: role, name: name, actions: meaningful, traits: traits)
let node = SnapshotRenderNode(
role: role,
roleDescription: roleDescription,
title: title,
label: label,
linkText: linkText,
value: value,
placeholder: placeholder,
url: url,
traits: traits,
rawActions: rawActions,
childCount: children.count,
summary: summary,
rowSummary: rowSummary,
webAreaDepth: webAreaDepth
)
if SnapshotRenderHeuristics.shouldElide(node) {
for child in children {
render(child, depth: depth, ancestors: ancestors + [element])
}
return
}
let index = nextIndex
nextIndex += 1
let line = SnapshotRenderHeuristics.line(index: index, node: node)
lines.append(String(repeating: "\t", count: depth) + line)
records[index] = ElementRecord(
index: index,
element: element,
localFrame: localFrame,
actions: rawActions,
signature: elementSignature(node)
)
if let focused, CFEqual(focused, element) {
focusedElementId = index
focusedSummary = line
}
if summary != nil || SnapshotRenderHeuristics.shouldSuppressChildren(node) {
return
}
if compactBrowserTabs, let tabStripCompaction = tabStripCompaction(parent: node, children: children) {
for (childIndex, child) in children.enumerated() where tabStripCompaction.retainedIndexes.contains(childIndex) {
render(child, depth: depth + 1, ancestors: ancestors + [element])
}
lines.append(
String(repeating: "\t", count: depth + 1) +
"... \(tabStripCompaction.omittedCount) inactive browser tabs omitted"
)
return
}
let childLineStart = lines.count
for child in children {
render(child, depth: depth + 1, ancestors: ancestors + [element])
}
if compactBrowserTabs {
compactRenderedBrowserTabs(parent: node, startLine: childLineStart, depth: depth + 1)
}
}
private func tabStripCompaction(parent: SnapshotRenderNode, children: [AXUIElement]) -> SnapshotTabStripCompaction? {
let childNodes = children.map { child in
let role = reader.stringAttribute(child, kAXRoleAttribute as String) ?? "AXUnknown"
return SnapshotRenderNode(
role: role,
roleDescription: reader.stringAttribute(child, kAXRoleDescriptionAttribute as String),
title: reader.stringAttribute(child, kAXTitleAttribute as String),
label: reader.stringAttribute(child, kAXDescriptionAttribute as String),
value: reader.valueString(child, role: role),
traits: reader.traitsFor(child, role: role)
)
}
// Why: browsers expose every open tab through AX; retaining only the active
// tab keeps snapshots focused on the current page instead of stale tab titles.
return SnapshotRenderHeuristics.tabStripCompaction(parent: parent, children: childNodes)
}
private func compactRenderedBrowserTabs(parent: SnapshotRenderNode, startLine: Int, depth: Int) {
guard SnapshotRenderHeuristics.roleText(parent) == "scroll area" else { return }
let indent = String(repeating: "\t", count: depth)
let tabLineIndexes = lines.indices.dropFirst(startLine).filter { lineIndex in
isDirectRenderedBrowserTabLine(lines[lineIndex], indent: indent)
}
guard tabLineIndexes.count >= 10 else { return }
let activeLineIndexes = Set(tabLineIndexes.filter { lineIndex in
isActiveRenderedBrowserTabLine(lines[lineIndex])
})
guard !activeLineIndexes.isEmpty else { return }
let insertionIndex = tabLineIndexes.first!
var omittedCount = 0
for lineIndex in tabLineIndexes.reversed() where !activeLineIndexes.contains(lineIndex) {
if let recordIndex = renderedElementIndex(lines[lineIndex], indent: indent) {
records.removeValue(forKey: recordIndex)
if focusedElementId == recordIndex {
focusedElementId = nil
focusedSummary = nil
}
}
lines.remove(at: lineIndex)
omittedCount += 1
}
guard omittedCount > 0 else { return }
lines.insert("\(indent)... \(omittedCount) inactive browser tabs omitted", at: insertionIndex)
}
}
private final class AXSnapshotReader {
private enum CachedAttribute {
case missing
case found(CFTypeRef)
}
private final class ElementCache {
let element: AXUIElement
var loadedAttributeNames = false
var advertisedAttributes: Set<String>?
var attributes: [String: CachedAttribute] = [:]
var actions: [String]?
var settable: [String: Bool] = [:]
init(element: AXUIElement) {
self.element = element
}
}
private var elementsByHash: [CFHashCode: [ElementCache]] = [:]
func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? {
guard let value = copyAttribute(element, attribute) else { return nil }
if CFGetTypeID(value) == CFStringGetTypeID(), let string = value as? String {
let trimmed = string.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
}
if CFGetTypeID(value) == CFURLGetTypeID(), let url = value as? URL {
return url.absoluteString
}
return nil
}
func boolAttribute(_ element: AXUIElement, _ attribute: String) -> Bool? {
copyAttribute(element, attribute) as? Bool
}
func numberAttribute(_ element: AXUIElement, _ attribute: String) -> NSNumber? {
copyAttribute(element, attribute) as? NSNumber
}
func copyArray(_ element: AXUIElement, _ attribute: String) -> [AXUIElement]? {
copyAttribute(element, attribute) as? [AXUIElement]
}
func actions(_ element: AXUIElement) -> [String] {
let cache = cache(for: element)
if let actions = cache.actions {
return actions
}
var value: CFArray?
let actions = AXUIElementCopyActionNames(element, &value) == .success ? value as? [String] ?? [] : []
cache.actions = actions
return actions
}
func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
let cache = cache(for: element)
if let cached = cache.settable[attribute] {
return cached
}
var settable = DarwinBoolean(false)
let value = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue
cache.settable[attribute] = value
return value
}
func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? {
guard let absolute = absoluteFrame(element) else { return nil }
return CGRect(
x: absolute.minX - windowBounds.minX,
y: absolute.minY - windowBounds.minY,
width: absolute.width,
height: absolute.height
)
}
func primaryChildren(_ element: AXUIElement, role: String, windowBounds: CGRect) -> [AXUIElement] {
if usesRowsAsPrimaryChildren(role: role), let rows = copyArray(element, kAXRowsAttribute as String), !rows.isEmpty {
return visibleRows(rows, parent: element, windowBounds: windowBounds)
}
return copyArray(element, kAXChildrenAttribute as String) ?? []
}
func valueString(_ element: AXUIElement, role: String) -> String? {
if isSecureTextElement(element, role: role) {
return "[redacted]"
}
if let string = stringAttribute(element, kAXValueAttribute as String) {
return string
}
if let number = numberAttribute(element, kAXValueAttribute as String) {
return number.stringValue
}
return nil
}
func placeholderString(_ element: AXUIElement) -> String? {
stringAttribute(element, "AXPlaceholderValue") ?? stringAttribute(element, "AXPlaceholder")
}
func traitsFor(_ element: AXUIElement, role: String) -> [String] {
var traits: [String] = []
if boolAttribute(element, kAXSelectedAttribute as String) == true { traits.append("selected") }
if boolAttribute(element, kAXExpandedAttribute as String) == true { traits.append("expanded") }
if boolAttribute(element, kAXEnabledAttribute as String) == false { traits.append("disabled") }
if valueSettableRoles.contains(role), isSettable(element, kAXValueAttribute as String) { traits.append("settable") }
return traits
}
func genericTextSummary(
_ element: AXUIElement,
role: String,
name: String?,
actions: [String],
traits: [String]
) -> String? {
guard (role == kAXGroupRole as String || role == kAXUnknownRole as String),
name == nil,
actions.isEmpty,
traits.isEmpty,
isPlainTextSubtree(element, maxDepth: 4)
else {
return nil
}
let texts = descendantTextSnippets(element, limit: 8, maxDepth: 4)
guard texts.count >= 2 else { return nil }
let summary = texts.joined(separator: " ")
guard summary.count <= 220 else { return nil }
return summary
}
func rowTextSummary(_ element: AXUIElement, role: String) -> String? {
guard ["AXRow", "AXCell", "AXOutlineRow"].contains(role) else { return nil }
let texts = descendantTextSnippets(element, limit: 6, maxDepth: 3)
guard !texts.isEmpty else { return nil }
return texts.joined(separator: " ")
}
func descendantTextSnippets(_ element: AXUIElement, limit: Int, maxDepth: Int) -> [String] {
var values: [String] = []
var seen = Set<String>()
func collect(_ node: AXUIElement, depth: Int) {
guard values.count < limit, depth <= maxDepth else { return }
let role = stringAttribute(node, kAXRoleAttribute as String) ?? ""
if role == kAXStaticTextRole as String || role == "AXLink" {
for candidate in [
stringAttribute(node, kAXValueAttribute as String),
stringAttribute(node, kAXTitleAttribute as String),
stringAttribute(node, kAXDescriptionAttribute as String),
] {
guard let candidate else { continue }
let text = preview(candidate, maxLength: 80)
guard !text.isEmpty, seen.insert(text).inserted else { continue }
values.append(text)
if values.count >= limit { return }
}
}
for child in copyArray(node, kAXChildrenAttribute as String) ?? [] {
collect(child, depth: depth + 1)
if values.count >= limit { return }
}
}
collect(element, depth: 0)
return values
}
func webAreaDepth(role: String, ancestors: [AXUIElement]) -> Int? {
if role == "AXWebArea" { return 0 }
guard let index = ancestors.firstIndex(where: { stringAttribute($0, kAXRoleAttribute as String) == "AXWebArea" }) else {
return nil
}
return ancestors.count - index
}
private func copyAttribute(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? {
let cache = cache(for: element)
if let cached = cache.attributes[attribute] {
switch cached {
case .missing:
return nil
case let .found(value):
return value
}
}
if let advertisedAttributes = advertisedAttributes(cache),
!SnapshotRenderHeuristics.supportsAttribute(attribute, advertisedAttributes: advertisedAttributes) {
cache.attributes[attribute] = .missing
return nil
}
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {
cache.attributes[attribute] = .missing
return nil
}
cache.attributes[attribute] = .found(value)
return value
}
private func advertisedAttributes(_ cache: ElementCache) -> Set<String>? {
if cache.loadedAttributeNames {
return cache.advertisedAttributes
}
cache.loadedAttributeNames = true
var value: CFArray?
guard AXUIElementCopyAttributeNames(cache.element, &value) == .success, let attributes = value as? [String] else {
return nil
}
cache.advertisedAttributes = Set(attributes)
return cache.advertisedAttributes
}
private func absoluteFrame(_ element: AXUIElement) -> CGRect? {
guard let positionValue = copyAttribute(element, kAXPositionAttribute as String),
let sizeValue = copyAttribute(element, kAXSizeAttribute as String)
else {
return nil
}
var point = CGPoint.zero
var size = CGSize.zero
guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point),
AXValueGetValue(sizeValue as! AXValue, .cgSize, &size)
else {
return nil
}
return CGRect(origin: point, size: size)
}
private func visibleRows(_ rows: [AXUIElement], parent: AXUIElement, windowBounds: CGRect) -> [AXUIElement] {
guard let parentFrame = frame(parent, windowBounds: windowBounds) else {
return Array(rows.prefix(20))
}
let visible = rows.filter { row in
guard let rowFrame = frame(row, windowBounds: windowBounds) else { return false }
return rowFrame.intersects(parentFrame)
}
return Array((visible.isEmpty ? rows : visible).prefix(20))
}
private func isSecureTextElement(_ element: AXUIElement, role: String) -> Bool {
guard SnapshotRenderHeuristics.shouldProbeSecureTextMetadata(role: role) else {
return false
}
let haystack = [
role,
stringAttribute(element, kAXSubroleAttribute as String) ?? "",
stringAttribute(element, kAXTitleAttribute as String) ?? "",
stringAttribute(element, kAXDescriptionAttribute as String) ?? "",
placeholderString(element) ?? "",
].joined(separator: " ").lowercased()
return haystack.contains("secure") ||
haystack.contains("password") ||
haystack.contains("passcode") ||
haystack.contains("verification code") ||
haystack.contains("one-time code")
}
private func isPlainTextSubtree(_ element: AXUIElement, maxDepth: Int) -> Bool {
var sawText = false
let allowedContainerRoles: Set<String> = [
kAXGroupRole as String,
kAXUnknownRole as String,
kAXStaticTextRole as String,
"AXLink",
"AXImage",
]
func visit(_ node: AXUIElement, depth: Int) -> Bool {
guard depth <= maxDepth else { return false }
let role = stringAttribute(node, kAXRoleAttribute as String) ?? "AXUnknown"
guard allowedContainerRoles.contains(role) else { return false }
if role == kAXStaticTextRole as String || role == "AXLink" {
sawText = true
}
guard SnapshotRenderHeuristics.meaningfulActions(actions(node), role: role).isEmpty else { return false }
for child in copyArray(node, kAXChildrenAttribute as String) ?? [] {
guard visit(child, depth: depth + 1) else { return false }
}
return true
}
return visit(element, depth: 0) && sawText
}
private func cache(for element: AXUIElement) -> ElementCache {
let hash = CFHash(element)
if let cache = elementsByHash[hash]?.first(where: { CFEqual($0.element, element) }) {
return cache
}
let cache = ElementCache(element: element)
elementsByHash[hash, default: []].append(cache)
return cache
}
}
private let valueSettableRoles: Set<String> = [
kAXCheckBoxRole as String,
kAXComboBoxRole as String,
kAXRadioButtonRole as String,
"AXSearchField",
kAXSliderRole as String,
kAXTextAreaRole as String,
kAXTextFieldRole as String,
]
private func usesRowsAsPrimaryChildren(role: String) -> Bool {
[
kAXBrowserRole as String,
kAXListRole as String,
kAXOutlineRole as String,
kAXTableRole as String,
].contains(role)
}
private func isDirectRenderedBrowserTabLine(_ line: String, indent: String) -> Bool {
guard line.hasPrefix(indent), !line.dropFirst(indent.count).hasPrefix("\t") else {
return false
}
let text = String(line.dropFirst(indent.count))
return text.range(of: #"^\d+ tab($| \(|,)"#, options: .regularExpression) != nil
}
private func isActiveRenderedBrowserTabLine(_ line: String) -> Bool {
line.contains("(selected") || line.contains("Value: 1")
}
private func renderedElementIndex(_ line: String, indent: String) -> Int? {
let text = line.dropFirst(indent.count)
let digits = text.prefix { character in
character >= "0" && character <= "9"
}
return Int(digits)
}
private func sanitize(_ value: String) -> String {
value.replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\r", with: " ")
}
private func preview(_ value: String, maxLength: Int = 120) -> String {
let clean = sanitize(value)
if clean.count <= maxLength {
return clean
}
return String(clean.prefix(maxLength)) + "..."
}
private func actionMetadata(
path: String,
actionName: String? = nil,
fallbackReason: String? = nil,
verification: [String: Any]? = nil
) -> [String: Any] {
var metadata: [String: Any] = [
"path": path,
"actionName": jsonNullable(actionName),
"fallbackReason": jsonNullable(fallbackReason),
]
if let verification {
metadata["verification"] = verification
}
return metadata
}
private func verifiedAction(property: String, expected: String? = nil, actualPreview: String? = nil) -> [String: Any] {
[
"state": "verified",
"property": property,
"expected": jsonNullable(expected),
"actualPreview": jsonNullable(actualPreview),
]
}
private func unverifiedAction(reason: String, expected: String? = nil, actualPreview: String? = nil) -> [String: Any] {
[
"state": "unverified",
"reason": reason,
"expected": jsonNullable(expected),
"actualPreview": jsonNullable(actualPreview),
]
}
private func jsonNullable<T>(_ value: T?) -> Any {
value ?? NSNull()
}
private struct WindowCandidate {
let windowId: CGWindowID
let layer: Int
let bounds: CGRect
let title: String?
let alpha: CGFloat
let isOnScreen: Bool
let sharingState: Int?
var score: Int {
var value = Int(bounds.width * bounds.height)
if layer == 0 { value += 1_000_000_000 }
if title != nil && title?.isEmpty == false { value += 10_000_000 }
if isOnScreen { value += 1_000_000 }
if alpha >= 0.99 { value += 100_000 }
return value
}
}
private struct WindowCapture {
let windowId: CGWindowID
let layer: Int
let bounds: CGRect
let title: String?
let image: CapturedImage?
static func resolve(
pid: pid_t,
titleHint: String?,
windowId: CGWindowID?,
windowIndex: Int?,
captureImage: Bool
) -> WindowCapture? {
resolve(
candidates: candidates(pid: pid),
titleHint: titleHint,
windowId: windowId,
windowIndex: windowIndex,
captureImage: captureImage
)
}
static func resolve(
candidates: [WindowCandidate],
titleHint: String?,
windowId: CGWindowID?,
windowIndex: Int?,
captureImage: Bool
) -> WindowCapture? {
if let windowId {
guard let candidate = candidates.first(where: { $0.windowId == windowId }) else { return nil }
return WindowCapture(candidate: candidate, captureImage: captureImage)
}
if let windowIndex {
let visibleWindows = candidates.filter { $0.layer == 0 }
guard visibleWindows.indices.contains(windowIndex) else { return nil }
return WindowCapture(candidate: visibleWindows[windowIndex], captureImage: captureImage)
}
guard let best = candidates.sorted(by: { lhs, rhs in
if let titleHint, lhs.title == titleHint, rhs.title != titleHint { return true }
if let titleHint, rhs.title == titleHint, lhs.title != titleHint { return false }
return lhs.score > rhs.score
}).first else {
return nil
}
return WindowCapture(candidate: best, captureImage: captureImage)
}
private init(candidate: WindowCandidate, captureImage: Bool) {
self.windowId = candidate.windowId
self.layer = candidate.layer
self.bounds = candidate.bounds
self.title = candidate.title
// Why: probing image APIs before TCC preflight can raise Screen
// Recording prompts, even for --no-screenshot calls.
if captureImage {
self.image = Self.captureImage(windowId: candidate.windowId, bounds: candidate.bounds)
} else {
self.image = nil
}
}
static func candidates(pid: pid_t) -> [WindowCandidate] {
guard let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else {
return []
}
return infos.compactMap { info in
guard let ownerPid = info[kCGWindowOwnerPID as String] as? pid_t, ownerPid == pid,
let number = info[kCGWindowNumber as String] as? NSNumber,
let layer = info[kCGWindowLayer as String] as? Int,
let boundsDictionary = info[kCGWindowBounds as String] as? NSDictionary,
let bounds = CGRect(dictionaryRepresentation: boundsDictionary),
bounds.width >= 48,
bounds.height >= 48
else {
return nil
}
let alpha = info[kCGWindowAlpha as String] as? CGFloat ?? 1
guard alpha > 0.01 else { return nil }
let sharing = (info[kCGWindowSharingState as String] as? NSNumber).map { $0.intValue }
if sharing == 0 { return nil }
let isOnScreen = (info[kCGWindowIsOnscreen as String] as? Bool) ?? true
return WindowCandidate(
windowId: CGWindowID(number.uint32Value),
layer: layer,
bounds: bounds,
title: info[kCGWindowName as String] as? String,
alpha: alpha,
isOnScreen: isOnScreen,
sharingState: sharing
)
}
.sorted { lhs, rhs in
lhs.score > rhs.score
}
}
func screenshotPayload() -> ScreenshotPayload? {
guard let image, let bounded = boundedPngData(image.image) else { return nil }
return ScreenshotPayload(
data: bounded.data.base64EncodedString(),
width: bounded.width,
height: bounded.height,
scale: Double(bounded.width) / max(Double(bounds.width), 1)
)
}
private static func captureImage(windowId: CGWindowID, bounds: CGRect) -> CapturedImage? {
if ProcessInfo.processInfo.environment["ORCA_COMPUTER_USE_SCK_SCREENSHOTS"] == "1",
let image = captureImageWithScreenCaptureKit(windowId: windowId, bounds: bounds) {
return CapturedImage(image: image, engine: "screenCaptureKit")
}
if let image = CGWindowListCreateImage(.null, [.optionIncludingWindow], windowId, [.boundsIgnoreFraming, .bestResolution]) {
return CapturedImage(image: image, engine: "cgWindowList")
}
return nil
}
private static func captureImageWithScreenCaptureKit(windowId: CGWindowID, bounds: CGRect) -> CGImage? {
try? BlockingAsync.run(timeout: 3) {
let content = try await SCShareableContent.current
guard let window = content.windows.first(where: { $0.windowID == windowId }) else {
return nil
}
let filter = SCContentFilter(desktopIndependentWindow: window)
let configuration = SCStreamConfiguration()
let scale = NSScreen.screens.first(where: { $0.frame.intersects(bounds) })?.backingScaleFactor ?? NSScreen.main?.backingScaleFactor ?? 2
configuration.width = size_t(max(1, Int((bounds.width * scale).rounded(.up))))
configuration.height = size_t(max(1, Int((bounds.height * scale).rounded(.up))))
configuration.scalesToFit = true
configuration.preservesAspectRatio = true
configuration.showsCursor = false
configuration.ignoreShadowsSingleWindow = true
configuration.ignoreGlobalClipSingleWindow = true
return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
}
}
}
private final class AsyncBox<T>: @unchecked Sendable {
var result: Result<T, Error>?
}
private enum BlockingAsync {
static func run<T>(timeout: TimeInterval, operation: @escaping @Sendable () async throws -> T) throws -> T {
let semaphore = DispatchSemaphore(value: 0)
let box = AsyncBox<T>()
let task = Task.detached {
do {
box.result = .success(try await operation())
} catch {
box.result = .failure(error)
}
semaphore.signal()
}
guard semaphore.wait(timeout: .now() + timeout) == .success else {
task.cancel()
throw ProviderError.coded("action_timeout", "screenshot capture timed out")
}
return try box.result!.get()
}
}
private struct BoundedPNG {
let data: Data
let width: Int
let height: Int
}
private func boundedPngData(_ image: CGImage) -> BoundedPNG? {
let rep = NSBitmapImageRep(cgImage: image)
guard let data = rep.representation(using: .png, properties: [:]) else { return nil }
if data.count <= 900_000 {
return BoundedPNG(data: data, width: image.width, height: image.height)
}
var scale = min(1, 1280 / CGFloat(max(image.width, image.height)))
var best = BoundedPNG(data: data, width: image.width, height: image.height)
while scale >= 0.25 {
guard let resized = resizePng(image, scale: scale) else {
break
}
best = resized
if resized.data.count <= 900_000 {
return resized
}
scale *= 0.85
}
return best
}
private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? {
let width = max(1, Int(CGFloat(image.width) * scale))
let height = max(1, Int(CGFloat(image.height) * scale))
guard let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else {
return nil
}
context.interpolationQuality = .medium
context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
guard let resized = context.makeImage() else { return nil }
guard let data = NSBitmapImageRep(cgImage: resized).representation(using: .png, properties: [:]) else {
return nil
}
return BoundedPNG(data: data, width: width, height: height)
}
private enum Input {
static func click(pid: pid_t, at point: CGPoint, button: MouseButton, count: Int) throws {
guard let source = CGEventSource(stateID: .combinedSessionState) else {
throw ProviderError.coded("accessibility_error", "failed to create event source")
}
for _ in 0..<max(count, 1) {
try mouse(.mouseMoved, source: source, point: point, button: button.cgButton, pid: pid)
try mouse(button.downEvent, source: source, point: point, button: button.cgButton, pid: pid)
try mouse(button.upEvent, source: source, point: point, button: button.cgButton, pid: pid)
}
}
static func scroll(pid: pid_t, at point: CGPoint, direction: String, pages: Double) throws {
guard let delta = boundedInteger(max(1, (12 * pages).rounded()), as: Int32.self) else {
throw ProviderError.coded("invalid_argument", "pages is out of range")
}
let wheel1: Int32 = direction == "up" ? delta : direction == "down" ? -delta : 0
let wheel2: Int32 = direction == "left" ? delta : direction == "right" ? -delta : 0
guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: wheel1, wheel2: wheel2, wheel3: 0) else {
throw ProviderError.coded("accessibility_error", "failed to create scroll event")
}
event.location = point
event.postToPid(pid)
}
static func drag(pid: pid_t, from start: CGPoint, to end: CGPoint) throws {
guard let source = CGEventSource(stateID: .combinedSessionState) else {
throw ProviderError.coded("accessibility_error", "failed to create event source")
}
try mouse(.mouseMoved, source: source, point: start, button: .left, pid: pid)
try mouse(.leftMouseDown, source: source, point: start, button: .left, pid: pid)
for step in 1...10 {
let progress = CGFloat(step) / 10
try mouse(
.leftMouseDragged,
source: source,
point: CGPoint(x: start.x + (end.x - start.x) * progress, y: start.y + (end.y - start.y) * progress),
button: .left,
pid: pid
)
}
try mouse(.leftMouseUp, source: source, point: end, button: .left, pid: pid)
}
static func typeText(_ text: String, pid: pid_t) throws {
for unit in text.utf16 {
var char = unit
guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
else {
throw ProviderError.coded("accessibility_error", "failed to create keyboard event")
}
down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &char)
up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &char)
down.post(tap: .cghidEventTap)
up.post(tap: .cghidEventTap)
}
}
static func pressKey(_ key: String, pid: pid_t) throws {
let parsed = try KeyMap.parse(key)
var flags = CGEventFlags()
for modifier in parsed.modifiers {
flags.insert(modifier.flag)
try keyEvent(modifier.keyCode, down: true, flags: flags, pid: pid)
}
try keyEvent(parsed.keyCode, down: true, flags: flags, pid: pid)
try keyEvent(parsed.keyCode, down: false, flags: flags, pid: pid)
for modifier in parsed.modifiers.reversed() {
try keyEvent(modifier.keyCode, down: false, flags: flags, pid: pid)
flags.remove(modifier.flag)
}
}
static func pasteText(_ text: String, pid: pid_t) throws {
let pasteboard = NSPasteboard.general
let previousItems: [NSPasteboardItem] = pasteboard.pasteboardItems?.map { item in
let copy = NSPasteboardItem()
for type in item.types {
if let data = item.data(forType: type) {
copy.setData(data, forType: type)
}
}
return copy
} ?? []
pasteboard.clearContents()
pasteboard.setString(text, forType: .string)
defer {
pasteboard.clearContents()
if !previousItems.isEmpty {
pasteboard.writeObjects(previousItems)
}
}
try pressKey("cmd+v", pid: pid)
}
private static func mouse(_ type: CGEventType, source: CGEventSource, point: CGPoint, button: CGMouseButton, pid: pid_t) throws {
guard let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else {
throw ProviderError.coded("accessibility_error", "failed to create mouse event")
}
event.postToPid(pid)
}
private static func keyEvent(_ keyCode: CGKeyCode, down: Bool, flags: CGEventFlags, pid: pid_t) throws {
guard let event = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: down) else {
throw ProviderError.coded("accessibility_error", "failed to create key event")
}
event.flags = flags
event.post(tap: .cghidEventTap)
}
}
private enum TextInput {
static func replaceSelection(_ element: AXUIElement, with text: String) -> [String: Any]? {
guard isSettable(element, kAXValueAttribute as String),
let current = rawStringAttribute(element, kAXValueAttribute as String)
else {
return nil
}
let selectedRange = selectedTextRange(element) ?? CFRange(location: current.utf16.count, length: 0)
let startOffset = max(0, min(selectedRange.location, current.utf16.count))
let endOffset = max(startOffset, min(startOffset + selectedRange.length, current.utf16.count))
let start = String.Index(utf16Offset: startOffset, in: current)
let end = String.Index(utf16Offset: endOffset, in: current)
let next = String(current[..<start]) + text + String(current[end...])
guard AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, next as CFString) == .success else {
return nil
}
setSelectedTextRange(element, CFRange(location: startOffset + text.utf16.count, length: 0))
guard rawStringAttribute(element, kAXValueAttribute as String) == next else {
return nil
}
return verifiedAction(
property: "focusedText",
expected: text,
actualPreview: preview(next)
)
}
static func selectAll(_ element: AXUIElement) -> Bool {
guard let current = rawStringAttribute(element, kAXValueAttribute as String) else {
return false
}
return setSelectedTextRange(element, CFRange(location: 0, length: current.utf16.count))
}
static func selectionVerification(_ element: AXUIElement) -> [String: Any] {
guard let current = rawStringAttribute(element, kAXValueAttribute as String),
let selectedRange = selectedTextRange(element),
selectedRange.location == 0,
selectedRange.length == current.utf16.count
else {
return unverifiedAction(reason: "provider_unavailable")
}
return verifiedAction(property: "selection", actualPreview: preview(current))
}
private static func selectedTextRange(_ element: AXUIElement) -> CFRange? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, &value) == .success,
let value,
CFGetTypeID(value) == AXValueGetTypeID()
else {
return nil
}
var range = CFRange(location: 0, length: 0)
guard AXValueGetValue(value as! AXValue, .cfRange, &range) else {
return nil
}
return range
}
@discardableResult
private static func setSelectedTextRange(_ element: AXUIElement, _ range: CFRange) -> Bool {
var mutableRange = range
guard let value = AXValueCreate(.cfRange, &mutableRange) else {
return false
}
return AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, value) == .success
}
}
private func isSelectAllHotkey(_ key: String) -> Bool {
let parts = key
.lowercased()
.replacingOccurrences(of: " ", with: "")
.replacingOccurrences(of: "-", with: "+")
.split(separator: "+")
.map(String.init)
guard parts.last == "a" else {
return false
}
let modifiers = Set(parts.dropLast())
return modifiers.contains("cmd") ||
modifiers.contains("command") ||
modifiers.contains("meta") ||
modifiers.contains("cmdorctrl") ||
modifiers.contains("commandorcontrol")
}
private struct KeyModifier {
let keyCode: CGKeyCode
let flag: CGEventFlags
}
private struct ParsedKey {
let keyCode: CGKeyCode
let modifiers: [KeyModifier]
}
private enum KeyMap {
static func parse(_ spec: String) throws -> ParsedKey {
let parts = spec.split(separator: "+").map { String($0).lowercased() }
var modifiers: [KeyModifier] = []
var keyName: String?
for part in parts {
switch part {
case "cmd", "command", "meta", "super", "cmdorctrl", "commandorcontrol":
modifiers.append(KeyModifier(keyCode: 55, flag: .maskCommand))
case "ctrl", "control":
modifiers.append(KeyModifier(keyCode: 59, flag: .maskControl))
case "alt", "option":
modifiers.append(KeyModifier(keyCode: 58, flag: .maskAlternate))
case "shift":
modifiers.append(KeyModifier(keyCode: 56, flag: .maskShift))
default:
keyName = part
}
}
guard let keyName, let keyCode = codes[keyName] else {
throw ProviderError.coded("invalid_argument", "unsupported key '\(spec)'")
}
return ParsedKey(keyCode: keyCode, modifiers: modifiers)
}
private static let codes: [String: CGKeyCode] = [
"a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9,
"b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19,
"3": 20, "4": 21, "6": 22, "5": 23, "=": 24, "9": 25, "7": 26, "-": 27, "8": 28,
"0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "return": 36,
"enter": 36, "l": 37, "j": 38, "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43,
"/": 44, "n": 45, "m": 46, ".": 47, "tab": 48, "space": 49, "`": 50,
"backspace": 51, "delete": 51, "escape": 53, "esc": 53, "left": 123, "right": 124,
"down": 125, "up": 126, "insert": 114, "home": 115, "pageup": 116, "page_up": 116,
"forwarddelete": 117, "end": 119, "pagedown": 121, "page_down": 121,
]
}
private final class AgentRuntime: NSObject, NSApplicationDelegate {
private let socketPath: String
private let token: String?
private var listener: SocketListener?
init(socketPath: String, token: String?) {
self.socketPath = socketPath
self.token = token
}
func applicationDidFinishLaunching(_ notification: Notification) {
do {
let listener = try SocketListener(socketPath: socketPath, token: token)
self.listener = listener
listener.start()
} catch {
fputs("failed to start computer-use socket: \(error)\n", stderr)
NSApp.terminate(nil)
}
}
func applicationWillTerminate(_ notification: Notification) {
listener?.stop()
}
}
private final class PermissionRuntime: NSObject, NSApplicationDelegate {
private let initialPermission: PermissionKind?
private var windowController: PermissionWindowController?
init(initialPermission: PermissionKind?) {
self.initialPermission = initialPermission
}
func applicationDidFinishLaunching(_ notification: Notification) {
windowController = PermissionWindowController(
initialPermission: initialPermission,
terminateWhenDragAssistantCloses: initialPermission != nil
)
if let initialPermission {
windowController?.openPermission(initialPermission)
} else {
windowController?.showWindow(nil)
windowController?.window?.makeKeyAndOrderFront(nil)
NSApp.activate(ignoringOtherApps: true)
}
}
func applicationDidBecomeActive(_ notification: Notification) {
windowController?.refreshPermissions()
}
func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
initialPermission == nil
}
}
private final class PermissionWindowController: NSWindowController {
private var dragAssistant: PermissionDragAssistantController?
private var dragAssistantPermission: PermissionKind?
private let initialPermission: PermissionKind?
private let terminateWhenDragAssistantCloses: Bool
convenience init(initialPermission: PermissionKind? = nil, terminateWhenDragAssistantCloses: Bool = false) {
let window = NSWindow(
contentRect: NSRect(x: 0, y: 0, width: 300, height: 315),
styleMask: [.titled, .closable, .miniaturizable, .fullSizeContentView],
backing: .buffered,
defer: false
)
window.title = "Enable Orca Computer Use"
window.titleVisibility = .hidden
window.titlebarAppearsTransparent = true
window.backgroundColor = PermissionPalette.background
window.center()
window.isReleasedWhenClosed = false
self.init(window: window, initialPermission: initialPermission, terminateWhenDragAssistantCloses: terminateWhenDragAssistantCloses)
window.contentView = PermissionView(
frame: window.contentView?.bounds ?? .zero,
showDragAssistant: { [weak self] permission in
self?.showDragAssistant(for: permission)
},
close: { [weak self] in
self?.closePermissionWindow()
}
)
}
init(window: NSWindow?, initialPermission: PermissionKind?, terminateWhenDragAssistantCloses: Bool) {
self.initialPermission = initialPermission
self.terminateWhenDragAssistantCloses = terminateWhenDragAssistantCloses
super.init(window: window)
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
private func showDragAssistant(for permission: PermissionKind) {
dragAssistant?.close()
dragAssistantPermission = permission
dragAssistant = PermissionDragAssistantController(
permission: permission,
fallbackVisibleFrame: window?.screen?.visibleFrame,
onRefreshPermissions: { [weak self] in
self?.refreshPermissions()
},
onClose: { [weak self] in
if self?.terminateWhenDragAssistantCloses == true {
NSApp.terminate(nil)
}
}
)
dragAssistant?.showWhenReady()
}
private func closeDragAssistant() {
dragAssistant?.close()
dragAssistant = nil
dragAssistantPermission = nil
}
private func completeDragAssistant() {
guard let dragAssistant else {
dragAssistantPermission = nil
if terminateWhenDragAssistantCloses {
NSApp.terminate(nil)
}
return
}
self.dragAssistant = nil
dragAssistantPermission = nil
dragAssistant.complete()
}
private func closePermissionWindow() {
// Why: the floating assistant is a separate retained window controller and
// can keep the helper app alive after the main permission window closes.
closeDragAssistant()
window?.close()
}
func openPermission(_ permission: PermissionKind) {
permission.requestAndOpenSettings()
showDragAssistant(for: permission)
}
func refreshPermissions() {
if let initialPermission, initialPermission.isGranted {
// Why: targeted permission helpers should finish once the requested
// grant lands, even if other Computer Use permissions remain unset.
completeDragAssistant()
return
}
if dragAssistantPermission?.isGranted == true {
// Why: after one grant in full setup, the remaining missing permission
// needs fresh guidance instead of the old assistant's instructions.
closeDragAssistant()
}
if PermissionKind.allCases.allSatisfy(\.isGranted) {
closeDragAssistant()
}
(window?.contentView as? PermissionView)?.refreshPermissions()
}
}
private enum PermissionKind: CaseIterable {
case accessibility
case screenshots
static func parse(_ value: String?) -> PermissionKind? {
switch value {
case "accessibility":
.accessibility
case "screenshots", "screen", "screen-recording":
.screenshots
default:
nil
}
}
var dragInstruction: String {
switch self {
case .accessibility:
"Drag Orca Computer Use into the list above to allow Accessibility."
case .screenshots:
"Drag Orca Computer Use into the list above to allow Screenshots."
}
}
var title: String {
switch self {
case .accessibility:
"Accessibility"
case .screenshots:
"Screenshots"
}
}
var detail: String {
switch self {
case .accessibility:
"Read and control app interfaces"
case .screenshots:
"Capture windows for visual state"
}
}
var icon: NSImage {
switch self {
case .accessibility:
NSImage(systemSymbolName: "figure", accessibilityDescription: "Accessibility") ?? NSImage()
case .screenshots:
NSImage(systemSymbolName: "camera.viewfinder", accessibilityDescription: "Screen Recording") ?? NSImage()
}
}
var isGranted: Bool {
switch self {
case .accessibility:
accessibilityTrusted()
case .screenshots:
screenCaptureTrusted()
}
}
func requestAndOpenSettings() {
switch self {
case .accessibility:
openAccessibilitySettings()
case .screenshots:
_ = requestScreenCaptureAccess()
openScreenRecordingSettings()
}
}
}
private final class PermissionView: NSView {
private let appURL = Bundle.main.bundleURL
private let showDragAssistant: (PermissionKind) -> Void
private let close: () -> Void
private var contentStack: NSStackView?
private var contentConstraints: [NSLayoutConstraint] = []
init(frame frameRect: NSRect, showDragAssistant: @escaping (PermissionKind) -> Void, close: @escaping () -> Void) {
self.showDragAssistant = showDragAssistant
self.close = close
super.init(frame: frameRect)
wantsLayer = true
layer?.backgroundColor = PermissionPalette.background.cgColor
build()
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
private func build() {
NSLayoutConstraint.deactivate(contentConstraints)
contentStack?.removeFromSuperview()
contentConstraints = []
let stack = NSStackView()
stack.orientation = .vertical
stack.alignment = .leading
stack.spacing = 10
stack.distribution = .gravityAreas
stack.translatesAutoresizingMaskIntoConstraints = false
addSubview(stack)
contentStack = stack
let icon = NSImageView(image: NSWorkspace.shared.icon(forFile: appURL.path))
icon.imageScaling = .scaleProportionallyUpOrDown
icon.translatesAutoresizingMaskIntoConstraints = false
NSLayoutConstraint.activate([
icon.widthAnchor.constraint(equalToConstant: 58),
icon.heightAnchor.constraint(equalToConstant: 58)
])
let missingPermissions = PermissionKind.allCases.filter { !$0.isGranted }
let ready = missingPermissions.isEmpty
let title = label(ready ? "Computer Use is Ready" : "Enable Orca Computer Use", size: 22, weight: .bold)
let subtitle = label(
ready ? "Orca can use local apps when you ask." : "Grant permissions so Orca can use apps when you ask.",
size: 12,
weight: .regular
)
subtitle.textColor = PermissionPalette.secondaryText
subtitle.alignment = .center
subtitle.maximumNumberOfLines = 3
let header = NSStackView(views: [icon, title, subtitle])
header.orientation = .vertical
header.alignment = .centerX
header.spacing = 6
header.translatesAutoresizingMaskIntoConstraints = false
stack.addArrangedSubview(header)
header.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true
subtitle.widthAnchor.constraint(equalTo: stack.widthAnchor, constant: -10).isActive = true
if ready {
stack.addArrangedSubview(doneButton())
} else {
for permission in missingPermissions {
stack.addArrangedSubview(permissionRow(permission: permission) { [weak self] in
permission.requestAndOpenSettings()
self?.showDragAssistant(permission)
})
}
}
contentConstraints = [
stack.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 18),
stack.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -18),
stack.topAnchor.constraint(equalTo: topAnchor, constant: 22),
stack.bottomAnchor.constraint(lessThanOrEqualTo: bottomAnchor, constant: -20)
]
NSLayoutConstraint.activate(contentConstraints)
}
func refreshPermissions() {
// Why: TCC grants can change in System Settings while this window stays open.
build()
}
private func permissionRow(permission: PermissionKind, action: @escaping () -> Void) -> NSView {
let row = NSView()
row.wantsLayer = true
row.layer?.cornerRadius = 14
row.layer?.borderWidth = 1
row.layer?.borderColor = PermissionPalette.border.cgColor
row.layer?.backgroundColor = PermissionPalette.card.cgColor
row.translatesAutoresizingMaskIntoConstraints = false
let iconView = NSImageView(image: permission.icon)
iconView.symbolConfiguration = NSImage.SymbolConfiguration(pointSize: 30, weight: .regular)
iconView.contentTintColor = .controlAccentColor
iconView.translatesAutoresizingMaskIntoConstraints = false
let titleLabel = label(permission.title, size: 13, weight: .bold)
let detailLabel = label(permission.detail, size: 11, weight: .regular)
detailLabel.textColor = PermissionPalette.secondaryText
let textStack = NSStackView(views: [titleLabel, detailLabel])
textStack.orientation = .vertical
textStack.alignment = .leading
textStack.spacing = 4
textStack.translatesAutoresizingMaskIntoConstraints = false
let button = NSButton(title: "Allow", target: nil, action: nil)
button.bezelStyle = .rounded
button.controlSize = .regular
button.font = NSFont.systemFont(ofSize: 13, weight: .semibold)
button.contentTintColor = .white
button.bezelColor = .controlAccentColor
let buttonTitleAttributes: [NSAttributedString.Key: Any] = [
.foregroundColor: NSColor.white,
.font: NSFont.systemFont(ofSize: 13, weight: .semibold)
]
button.attributedTitle = NSAttributedString(string: "Allow", attributes: buttonTitleAttributes)
button.attributedAlternateTitle = NSAttributedString(string: "Allow", attributes: buttonTitleAttributes)
let target = ButtonTarget(action)
button.target = target
button.action = #selector(ButtonTarget.run)
objc_setAssociatedObject(button, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC)
button.translatesAutoresizingMaskIntoConstraints = false
row.addSubview(iconView)
row.addSubview(textStack)
row.addSubview(button)
NSLayoutConstraint.activate([
row.heightAnchor.constraint(equalToConstant: 62),
iconView.leadingAnchor.constraint(equalTo: row.leadingAnchor, constant: 12),
iconView.centerYAnchor.constraint(equalTo: row.centerYAnchor),
iconView.widthAnchor.constraint(equalToConstant: 34),
iconView.heightAnchor.constraint(equalToConstant: 34),
textStack.leadingAnchor.constraint(equalTo: iconView.trailingAnchor, constant: 10),
textStack.centerYAnchor.constraint(equalTo: row.centerYAnchor),
button.widthAnchor.constraint(greaterThanOrEqualToConstant: 52),
button.heightAnchor.constraint(equalToConstant: 30),
button.trailingAnchor.constraint(equalTo: row.trailingAnchor, constant: -12),
button.centerYAnchor.constraint(equalTo: row.centerYAnchor),
textStack.trailingAnchor.constraint(lessThanOrEqualTo: button.leadingAnchor, constant: -12)
])
return row
}
private func doneButton() -> NSView {
let button = NSButton(title: "Done", target: nil, action: nil)
button.bezelStyle = .rounded
button.controlSize = .regular
button.font = NSFont.systemFont(ofSize: 13, weight: .semibold)
button.contentTintColor = .white
button.bezelColor = .controlAccentColor
let buttonTitleAttributes: [NSAttributedString.Key: Any] = [
.foregroundColor: NSColor.white,
.font: NSFont.systemFont(ofSize: 13, weight: .semibold)
]
button.attributedTitle = NSAttributedString(string: "Done", attributes: buttonTitleAttributes)
button.attributedAlternateTitle = NSAttributedString(string: "Done", attributes: buttonTitleAttributes)
let target = ButtonTarget(close)
button.target = target
button.action = #selector(ButtonTarget.run)
objc_setAssociatedObject(button, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC)
button.translatesAutoresizingMaskIntoConstraints = false
button.widthAnchor.constraint(greaterThanOrEqualToConstant: 82).isActive = true
button.heightAnchor.constraint(equalToConstant: 32).isActive = true
return button
}
private func label(_ text: String, size: CGFloat, weight: NSFont.Weight) -> NSTextField {
let label = NSTextField(labelWithString: text)
label.font = NSFont.systemFont(ofSize: size, weight: weight)
label.lineBreakMode = .byWordWrapping
label.textColor = PermissionPalette.primaryText
label.translatesAutoresizingMaskIntoConstraints = false
return label
}
}
private final class PermissionDragAssistantController: NSWindowController {
private struct SettingsWindowState {
let frame: NSRect
let isVisible: Bool
}
private let fallbackVisibleFrame: NSRect?
private let onRefreshPermissions: () -> Void
private let onClose: () -> Void
private var hasSeenSettingsWindow = false
private var followTimer: Timer?
private var isDismissed = false
private var scheduledShowWorkItems: [DispatchWorkItem] = []
convenience init(
permission: PermissionKind,
fallbackVisibleFrame: NSRect?,
onRefreshPermissions: @escaping () -> Void,
onClose: @escaping () -> Void
) {
let window = NSPanel(
contentRect: NSRect(x: 0, y: 0, width: 390, height: 92),
styleMask: [.borderless, .nonactivatingPanel],
backing: .buffered,
defer: false
)
window.title = "Drag Orca Computer Use"
window.backgroundColor = .clear
window.isOpaque = false
window.isReleasedWhenClosed = false
window.level = .floating
window.hidesOnDeactivate = false
window.isFloatingPanel = true
window.becomesKeyOnlyIfNeeded = true
// Why: the assistant belongs to System Settings' position; user drags
// should either start the app-file drag or do nothing, not move the panel.
window.isMovable = false
window.isMovableByWindowBackground = false
window.hasShadow = true
self.init(
window: window,
fallbackVisibleFrame: fallbackVisibleFrame,
onRefreshPermissions: onRefreshPermissions,
onClose: onClose
)
window.contentView = PermissionDragAssistantView(permission: permission, appURL: Bundle.main.bundleURL) { [weak self] in
self?.dismissFromCloseButton()
}
}
init(
window: NSWindow?,
fallbackVisibleFrame: NSRect?,
onRefreshPermissions: @escaping () -> Void,
onClose: @escaping () -> Void
) {
self.fallbackVisibleFrame = fallbackVisibleFrame
self.onRefreshPermissions = onRefreshPermissions
self.onClose = onClose
super.init(window: window)
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
func showWhenReady() {
guard !isDismissed else { return }
startFollowingSettingsWindow()
schedulePositionAndShow()
}
override func close() {
isDismissed = true
scheduledShowWorkItems.forEach { $0.cancel() }
scheduledShowWorkItems.removeAll()
followTimer?.invalidate()
followTimer = nil
super.close()
}
private func dismissFromCloseButton() {
// Why: closing the NSWindow directly skips this controller's timer cleanup,
// letting the assistant reappear while System Settings remains visible.
complete()
}
func complete() {
close()
onClose()
}
private func schedulePositionAndShow() {
scheduledShowWorkItems.forEach { $0.cancel() }
scheduledShowWorkItems.removeAll()
let delays = [0.12, 0.25, 0.4, 0.65, 0.95, 1.35, 1.8, 2.5]
for (index, delay) in delays.enumerated() {
let workItem = DispatchWorkItem { [weak self] in
guard let self, !self.isDismissed, self.window?.isVisible != true else { return }
if let settingsWindow = self.systemSettingsWindowState(), settingsWindow.isVisible {
self.positionNearSettingsWindow(settingsWindow.frame)
guard !self.isDismissed else { return }
self.showWindow(nil)
self.window?.orderFrontRegardless()
} else if index == delays.count - 1 && self.systemSettingsIsFrontmost() {
self.positionFallback()
guard !self.isDismissed else { return }
self.showWindow(nil)
self.window?.orderFrontRegardless()
}
}
scheduledShowWorkItems.append(workItem)
DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: workItem)
}
}
private func startFollowingSettingsWindow() {
followTimer?.invalidate()
followTimer = Timer.scheduledTimer(withTimeInterval: 0.35, repeats: true) { [weak self] _ in
Task { @MainActor in
guard let self, !self.isDismissed else { return }
self.syncVisibilityWithSettingsWindow()
}
}
}
private func syncVisibilityWithSettingsWindow() {
guard !isDismissed else {
followTimer?.invalidate()
followTimer = nil
return
}
guard let window else {
followTimer?.invalidate()
followTimer = nil
return
}
let settingsWindow = systemSettingsWindowState()
if settingsWindow != nil {
hasSeenSettingsWindow = true
} else if hasSeenSettingsWindow {
NSApp.terminate(nil)
return
}
// Why: System Settings can stay visible on one display while the user
// works on another; follow actual occlusion instead of app focus.
if let settingsWindow, settingsWindow.isVisible {
onRefreshPermissions()
guard !isDismissed else { return }
positionNearSettingsWindow(settingsWindow.frame)
guard !isDismissed else { return }
if !window.isVisible {
showWindow(nil)
}
guard !isDismissed else { return }
window.orderFrontRegardless()
} else if window.isVisible {
window.orderOut(nil)
}
}
private func systemSettingsIsFrontmost() -> Bool {
let bundleId = NSWorkspace.shared.frontmostApplication?.bundleIdentifier
return bundleId == "com.apple.systempreferences" || bundleId == "com.apple.SystemSettings"
}
private func positionNearSettingsWindow(_ settingsFrame: NSRect) {
guard let window else { return }
let visibleFrame = visibleFrameContaining(settingsFrame)
let x = settingsFrame.maxX - window.frame.width - 18
let y = settingsFrame.minY + 18
window.setFrameOrigin(clampedOrigin(x: x, y: y, window: window, visibleFrame: visibleFrame))
}
private func positionFallback() {
guard let window else { return }
let visibleFrame = fallbackVisibleFrame ?? NSScreen.main?.visibleFrame ?? NSRect(x: 0, y: 0, width: 900, height: 700)
let origin = NSPoint(
x: visibleFrame.midX - window.frame.width / 2,
y: visibleFrame.minY + 24
)
window.setFrameOrigin(clampedOrigin(x: origin.x, y: origin.y, window: window, visibleFrame: visibleFrame))
}
private func clampedOrigin(x: CGFloat, y: CGFloat, window: NSWindow, visibleFrame: NSRect) -> NSPoint {
let inset: CGFloat = 10
let minX = visibleFrame.minX + inset
let maxX = visibleFrame.maxX - window.frame.width - inset
let minY = visibleFrame.minY + inset
let maxY = visibleFrame.maxY - window.frame.height - inset
return NSPoint(
x: min(max(x, minX), maxX),
y: min(max(y, minY), maxY)
)
}
private func systemSettingsWindowState() -> SettingsWindowState? {
guard let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
return nil
}
var occludingFrames: [NSRect] = []
for windowInfo in windows {
guard let frame = cgWindowFrame(windowInfo), isNormalWindow(windowInfo), frame.width > 0, frame.height > 0 else {
continue
}
if isSystemSettingsWindow(windowInfo), frame.width > 520, frame.height > 360 {
return SettingsWindowState(
frame: appKitFrameForCGWindowFrame(frame),
isVisible: !isWindowFullyCovered(frame, by: occludingFrames)
)
}
occludingFrames.append(frame)
}
return nil
}
private func isSystemSettingsWindow(_ windowInfo: [String: Any]) -> Bool {
guard let ownerName = windowInfo[kCGWindowOwnerName as String] as? String else {
return false
}
return ownerName == "System Settings" || ownerName == "System Preferences"
}
private func isNormalWindow(_ windowInfo: [String: Any]) -> Bool {
guard (windowInfo[kCGWindowLayer as String] as? Int) == 0 else {
return false
}
if let alpha = windowInfo[kCGWindowAlpha as String] as? CGFloat, alpha <= 0 {
return false
}
return true
}
private func cgWindowFrame(_ windowInfo: [String: Any]) -> NSRect? {
guard
let bounds = windowInfo[kCGWindowBounds as String] as? [String: CGFloat],
let x = bounds["X"],
let y = bounds["Y"],
let width = bounds["Width"],
let height = bounds["Height"]
else {
return nil
}
return NSRect(x: x, y: y, width: width, height: height)
}
private func isWindowFullyCovered(_ frame: NSRect, by occludingFrames: [NSRect]) -> Bool {
var uncoveredFrames = [frame]
for occludingFrame in occludingFrames {
uncoveredFrames = uncoveredFrames.flatMap { subtract(occludingFrame, from: $0) }
if uncoveredFrames.isEmpty {
return true
}
}
return false
}
private func subtract(_ coveringFrame: NSRect, from frame: NSRect) -> [NSRect] {
let overlap = frame.intersection(coveringFrame)
guard !overlap.isNull, overlap.width > 0, overlap.height > 0 else {
return [frame]
}
var remaining: [NSRect] = []
if overlap.minY > frame.minY {
remaining.append(NSRect(x: frame.minX, y: frame.minY, width: frame.width, height: overlap.minY - frame.minY))
}
if overlap.maxY < frame.maxY {
remaining.append(NSRect(x: frame.minX, y: overlap.maxY, width: frame.width, height: frame.maxY - overlap.maxY))
}
if overlap.minX > frame.minX {
remaining.append(NSRect(x: frame.minX, y: overlap.minY, width: overlap.minX - frame.minX, height: overlap.height))
}
if overlap.maxX < frame.maxX {
remaining.append(NSRect(x: overlap.maxX, y: overlap.minY, width: frame.maxX - overlap.maxX, height: overlap.height))
}
return remaining.filter { $0.width > 0 && $0.height > 0 }
}
private func appKitFrameForCGWindowFrame(_ cgFrame: NSRect) -> NSRect {
guard let screen = screenContainingCGFrame(cgFrame),
let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber
else {
return cgFrame
}
let displayBounds = CGDisplayBounds(CGDirectDisplayID(screenNumber.uint32Value))
return NSRect(
x: cgFrame.minX,
y: screen.frame.maxY - (cgFrame.minY - displayBounds.minY) - cgFrame.height,
width: cgFrame.width,
height: cgFrame.height
)
}
private func screenContainingCGFrame(_ frame: NSRect) -> NSScreen? {
NSScreen.screens.max { lhs, rhs in
cgDisplayBounds(for: lhs).intersection(frame).width * cgDisplayBounds(for: lhs).intersection(frame).height <
cgDisplayBounds(for: rhs).intersection(frame).width * cgDisplayBounds(for: rhs).intersection(frame).height
}
}
private func cgDisplayBounds(for screen: NSScreen) -> NSRect {
guard let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber else {
return screen.frame
}
return CGDisplayBounds(CGDirectDisplayID(screenNumber.uint32Value))
}
private func visibleFrameContaining(_ frame: NSRect) -> NSRect {
let screen = NSScreen.screens.max { lhs, rhs in
lhs.frame.intersection(frame).width * lhs.frame.intersection(frame).height <
rhs.frame.intersection(frame).width * rhs.frame.intersection(frame).height
}
return screen?.visibleFrame ?? NSScreen.main?.visibleFrame ?? NSRect(x: 0, y: 0, width: 900, height: 700)
}
}
private final class PermissionDragAssistantView: NSView {
private let permission: PermissionKind
private let appURL: URL
private let close: () -> Void
init(permission: PermissionKind, appURL: URL, close: @escaping () -> Void) {
self.permission = permission
self.appURL = appURL
self.close = close
super.init(frame: .zero)
wantsLayer = true
layer?.backgroundColor = PermissionPalette.background.cgColor
layer?.cornerRadius = 14
layer?.borderWidth = 1
layer?.borderColor = PermissionPalette.border.cgColor
layer?.masksToBounds = true
build()
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
private func build() {
let closeButton = NSButton(title: "", target: nil, action: nil)
closeButton.bezelStyle = .shadowlessSquare
closeButton.isBordered = false
closeButton.image = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")
closeButton.imagePosition = .imageOnly
closeButton.contentTintColor = PermissionPalette.secondaryText
closeButton.controlSize = .small
closeButton.translatesAutoresizingMaskIntoConstraints = false
let target = ButtonTarget(close)
closeButton.target = target
closeButton.action = #selector(ButtonTarget.run)
objc_setAssociatedObject(closeButton, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC)
let instruction = label(permission.dragInstruction, size: 12, weight: .semibold)
instruction.textColor = PermissionPalette.primaryText
instruction.maximumNumberOfLines = 2
let dragTile = DraggableAppTile(appURL: appURL)
dragTile.translatesAutoresizingMaskIntoConstraints = false
addSubview(closeButton)
addSubview(instruction)
addSubview(dragTile)
NSLayoutConstraint.activate([
closeButton.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 10),
closeButton.topAnchor.constraint(equalTo: topAnchor, constant: 10),
closeButton.widthAnchor.constraint(equalToConstant: 18),
closeButton.heightAnchor.constraint(equalToConstant: 18),
instruction.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 36),
instruction.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16),
instruction.centerYAnchor.constraint(equalTo: closeButton.centerYAnchor),
dragTile.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 16),
dragTile.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16),
dragTile.topAnchor.constraint(equalTo: instruction.bottomAnchor, constant: 8),
dragTile.heightAnchor.constraint(equalToConstant: 42)
])
}
private func label(_ text: String, size: CGFloat, weight: NSFont.Weight) -> NSTextField {
let label = NSTextField(labelWithString: text)
label.font = NSFont.systemFont(ofSize: size, weight: weight)
label.lineBreakMode = .byWordWrapping
label.textColor = PermissionPalette.primaryText
label.translatesAutoresizingMaskIntoConstraints = false
return label
}
}
private final class DraggableAppTile: NSView, NSDraggingSource {
private let appURL: URL
override var mouseDownCanMoveWindow: Bool { false }
init(appURL: URL) {
self.appURL = appURL
super.init(frame: .zero)
wantsLayer = true
layer?.cornerRadius = 12
layer?.borderWidth = 1
layer?.borderColor = PermissionPalette.border.cgColor
layer?.backgroundColor = PermissionPalette.card.cgColor
build()
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
override func mouseDragged(with event: NSEvent) {
let item = NSPasteboardItem()
item.setString(appURL.absoluteString, forType: .fileURL)
item.setString(appURL.path, forType: .string)
let draggingItem = NSDraggingItem(pasteboardWriter: item)
let iconSize: CGFloat = 64
let location = convert(event.locationInWindow, from: nil)
let dragFrame = NSRect(
x: location.x - iconSize / 2,
y: location.y - iconSize / 2,
width: iconSize,
height: iconSize
)
draggingItem.setDraggingFrame(dragFrame, contents: appIcon(size: iconSize))
beginDraggingSession(with: [draggingItem], event: event, source: self)
}
func draggingSession(_ session: NSDraggingSession, sourceOperationMaskFor context: NSDraggingContext) -> NSDragOperation {
.copy
}
private func build() {
let icon = NSImageView(image: appIcon(size: 34))
icon.imageScaling = .scaleProportionallyUpOrDown
icon.translatesAutoresizingMaskIntoConstraints = false
let title = NSTextField(labelWithString: "Orca Computer Use")
title.font = NSFont.systemFont(ofSize: 15, weight: .semibold)
title.textColor = PermissionPalette.primaryText
title.translatesAutoresizingMaskIntoConstraints = false
addSubview(icon)
addSubview(title)
NSLayoutConstraint.activate([
icon.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 16),
icon.centerYAnchor.constraint(equalTo: centerYAnchor),
icon.widthAnchor.constraint(equalToConstant: 34),
icon.heightAnchor.constraint(equalToConstant: 34),
title.leadingAnchor.constraint(equalTo: icon.trailingAnchor, constant: 14),
title.centerYAnchor.constraint(equalTo: centerYAnchor),
title.trailingAnchor.constraint(lessThanOrEqualTo: trailingAnchor, constant: -20)
])
}
private func appIcon(size: CGFloat) -> NSImage {
let icon = NSWorkspace.shared.icon(forFile: appURL.path)
icon.size = NSSize(width: size, height: size)
return icon
}
}
private enum PermissionPalette {
static let background = adaptiveColor(
light: NSColor(calibratedWhite: 0.94, alpha: 0.98),
dark: NSColor(calibratedWhite: 0.20, alpha: 0.98)
)
static let card = adaptiveColor(
light: NSColor(calibratedWhite: 0.99, alpha: 0.98),
dark: NSColor(calibratedWhite: 0.25, alpha: 0.98)
)
static let border = adaptiveColor(
light: NSColor(calibratedWhite: 0.0, alpha: 0.14),
dark: NSColor(calibratedWhite: 1.0, alpha: 0.22)
)
static let primaryText = NSColor.labelColor
static let secondaryText = NSColor.secondaryLabelColor
private static func adaptiveColor(light: NSColor, dark: NSColor) -> NSColor {
NSColor(name: nil) { appearance in
let bestMatch = appearance.bestMatch(from: [
.aqua,
.darkAqua,
.accessibilityHighContrastAqua,
.accessibilityHighContrastDarkAqua
])
return bestMatch == .darkAqua || bestMatch == .accessibilityHighContrastDarkAqua ? dark : light
}
}
}
private final class ButtonTarget: NSObject {
private let actionBlock: () -> Void
init(_ actionBlock: @escaping () -> Void) {
self.actionBlock = actionBlock
}
@objc func run() {
actionBlock()
}
}
private final class SocketListener: @unchecked Sendable {
private let socketPath: String
private let token: String?
private let provider = Provider()
private let providerLock = NSLock()
private var socketFd: Int32 = -1
private var isStopped = false
init(socketPath: String, token: String?) throws {
self.socketPath = socketPath
self.token = token
try bindSocket()
}
func start() {
Thread.detachNewThread { [weak self] in
self?.acceptLoop()
}
}
func stop() {
isStopped = true
if socketFd >= 0 {
close(socketFd)
socketFd = -1
}
// Why: the parent owns the private temp directory cleanup; the helper
// must not unlink arbitrary caller-supplied paths on shutdown.
}
private func bindSocket() throws {
socketFd = socket(AF_UNIX, SOCK_STREAM, 0)
guard socketFd >= 0 else {
throw ProviderError.coded("accessibility_error", "failed to create computer-use socket")
}
var address = sockaddr_un()
address.sun_family = sa_family_t(AF_UNIX)
let maxPathLength = MemoryLayout.size(ofValue: address.sun_path)
guard socketPath.utf8.count < maxPathLength else {
throw ProviderError.coded("invalid_argument", "computer-use socket path is too long")
}
_ = withUnsafeMutablePointer(to: &address.sun_path) { pointer in
socketPath.withCString { source in
strncpy(UnsafeMutableRawPointer(pointer).assumingMemoryBound(to: CChar.self), source, maxPathLength)
}
}
let result = withUnsafePointer(to: &address) { pointer -> Int32 in
pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in
bind(socketFd, sockaddrPointer, socklen_t(MemoryLayout<sockaddr_un>.size))
}
}
guard result == 0 else {
let bindErrno = errno
let message = String(cString: strerror(bindErrno))
close(socketFd)
socketFd = -1
if UnixSocketPathSafety.shouldRejectExistingPathAfterBindFailure(
bindErrno: bindErrno,
existingMode: existingPathMode(socketPath)
) {
throw ProviderError.coded("invalid_argument", "refusing to replace non-socket file at computer-use socket path")
}
throw ProviderError.coded("accessibility_error", "failed to bind computer-use socket: \(message)")
}
chmod(socketPath, 0o600)
guard listen(socketFd, 8) == 0 else {
let message = String(cString: strerror(errno))
close(socketFd)
socketFd = -1
throw ProviderError.coded("accessibility_error", "failed to listen on computer-use socket: \(message)")
}
}
private func acceptLoop() {
while !isStopped {
let fd = accept(socketFd, nil, nil)
if fd < 0 {
if !isStopped {
fputs("computer-use socket accept failed: \(String(cString: strerror(errno)))\n", stderr)
}
continue
}
Thread.detachNewThread { [weak self] in
self?.handleConnection(fd)
}
}
}
private func handleConnection(_ fd: Int32) {
defer { close(fd) }
let authorizedPeer = peerProcessId(fd).map(isAuthorizedAgentPeer) == true
let decoder = JSONDecoder()
while let line = readLine(from: fd) {
guard let data = line.data(using: .utf8),
let request = try? decoder.decode(Request.self, from: data)
else {
continue
}
let response = handleRequest(
provider: provider,
lock: providerLock,
request: request,
expectedToken: token,
authorizedPeer: authorizedPeer
)
writeJSON(response, to: fd)
}
}
}
private func existingPathMode(_ path: String) -> mode_t? {
var statInfo = stat()
guard lstat(path, &statInfo) == 0 else {
return nil
}
return statInfo.st_mode
}
private func peerProcessId(_ fd: Int32) -> pid_t? {
var pid = pid_t(0)
var length = socklen_t(MemoryLayout<pid_t>.size)
let result = withUnsafeMutablePointer(to: &pid) { pointer in
getsockopt(fd, 0, 2, pointer, &length)
}
return result == 0 && pid > 0 ? pid : nil
}
private func isAuthorizedAgentPeer(_ pid: pid_t) -> Bool {
guard let command = processCommand(pid),
command.contains("/out/main/computer-sidecar.js")
|| command.contains("/Contents/Resources/app.asar.unpacked/out/main/computer-sidecar.js")
else {
return false
}
if isTrustedOrcaApplication(pid) {
return true
}
guard let parentPid = parentProcessId(pid) else { return false }
return isTrustedOrcaApplication(parentPid)
}
private func isTrustedOrcaApplication(_ pid: pid_t) -> Bool {
guard let app = NSRunningApplication(processIdentifier: pid),
let bundleId = app.bundleIdentifier
else {
return false
}
// Why: dev validation runs from per-worktree wrapper apps with stable
// Orca-owned bundle ids; the sidecar peer check must still authorize them.
return bundleId == "com.stablyai.orca" ||
bundleId.hasPrefix("com.stablyai.orca.dev.") ||
bundleId == "com.github.Electron"
}
private func parentProcessId(_ pid: pid_t) -> pid_t? {
guard let output = processField(pid: pid, field: "ppid=") else {
return nil
}
let trimmed = output.trimmingCharacters(in: .whitespacesAndNewlines)
guard let parentPid = pid_t(trimmed), parentPid > 1 else {
return nil
}
return parentPid
}
private func processCommand(_ pid: pid_t) -> String? {
return processField(pid: pid, field: "command=")
}
private func processField(pid: pid_t, field: String) -> String? {
let process = Process()
let pipe = Pipe()
process.executableURL = URL(fileURLWithPath: "/bin/ps")
process.arguments = ["-p", "\(pid)", "-o", field]
process.standardOutput = pipe
process.standardError = Pipe()
do {
try process.run()
process.waitUntilExit()
guard process.terminationStatus == 0 else { return nil }
let data = pipe.fileHandleForReading.readDataToEndOfFile()
return String(data: data, encoding: .utf8)
} catch {
return nil
}
}
@MainActor
private func runAgent(socketPath: String, token: String?) {
let app = NSApplication.shared
let delegate = AgentRuntime(socketPath: socketPath, token: token)
app.delegate = delegate
// Why: SCK is reliable once this code runs as a signed app with a real TCC identity.
setenv("ORCA_COMPUTER_USE_SCK_SCREENSHOTS", "1", 1)
app.run()
}
@MainActor
private func runPermissionCheck(initialPermission: PermissionKind? = nil) {
let app = NSApplication.shared
let delegate = PermissionRuntime(initialPermission: initialPermission)
app.delegate = delegate
// Why: setup must foreground reliably; the long-running agent path stays accessory-only.
app.setActivationPolicy(.regular)
app.run()
}
private func printPermissionStatus() {
let accessibility = accessibilityTrusted() ? "granted" : "not-granted"
let screenshots = screenCaptureTrusted() ? "granted" : "not-granted"
print(#"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"#)
}
private func writePermissionStatus(to path: String) {
let accessibility = accessibilityTrusted() ? "granted" : "not-granted"
let screenshots = screenCaptureTrusted() ? "granted" : "not-granted"
let text = #"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"#
do {
try text.write(toFile: path, atomically: true, encoding: .utf8)
} catch {
fputs("failed to write permission status: \(error)\n", stderr)
exit(1)
}
}
private func runStdio() {
fputs("Orca Computer Use provider must be launched by Orca in app-agent mode.\n", stderr)
exit(13)
}
private func handleRequest(
provider: Provider,
lock: NSLock,
request: Request,
expectedToken: String?,
authorizedPeer: Bool
) -> Any {
if let expectedToken, request.token != expectedToken {
return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "invalid computer-use agent token"]]
}
if expectedToken != nil && !authorizedPeer {
return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "computer-use agent peer is not authorized"]]
}
if request.method == "terminate" {
DispatchQueue.main.async {
NSApp.terminate(nil)
}
return ["id": request.id, "ok": true, "result": ["ok": true]]
}
do {
lock.lock()
defer { lock.unlock() }
let result = try provider.handle(method: request.method, params: request.params ?? [:])
return ["id": request.id, "ok": true, "result": result]
} catch let error as ProviderError {
return ["id": request.id, "ok": false, "error": ["code": error.code, "message": error.message]]
} catch {
return ["id": request.id, "ok": false, "error": ["code": "accessibility_error", "message": String(describing: error)]]
}
}
private func readLine(from fd: Int32) -> String? {
var bytes: [UInt8] = []
var byte: UInt8 = 0
while true {
let count = read(fd, &byte, 1)
if count == 0 {
return bytes.isEmpty ? nil : String(bytes: bytes, encoding: .utf8)
}
if count < 0 {
return nil
}
if byte == 10 {
return String(bytes: bytes, encoding: .utf8)
}
bytes.append(byte)
}
}
private func writeJSON(_ object: Any, to fd: Int32?) {
guard JSONSerialization.isValidJSONObject(object),
let data = try? JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes])
else {
return
}
if let fd {
_ = writeAll(data, to: fd)
_ = writeAll(Data([10]), to: fd)
} else {
guard let text = String(data: data, encoding: .utf8) else {
return
}
print(text)
fflush(stdout)
}
}
private func writeAll(_ data: Data, to fd: Int32) -> Bool {
data.withUnsafeBytes { rawBuffer in
guard let baseAddress = rawBuffer.baseAddress else {
return true
}
var offset = 0
while offset < rawBuffer.count {
let written = write(fd, baseAddress.advanced(by: offset), rawBuffer.count - offset)
if written < 0 {
if errno == EINTR {
continue
}
return false
}
if written == 0 {
return false
}
offset += written
}
return true
}
}
let arguments = Array(CommandLine.arguments.dropFirst())
if arguments.first == "--agent" {
guard arguments.count >= 2 else {
fputs("usage: orca-computer-use-macos --agent <socket-path> --token-file <token-path>\n", stderr)
exit(2)
}
let tokenFileIndex = arguments.firstIndex(of: "--token-file")
let token = tokenFileIndex.flatMap { index -> String? in
let valueIndex = index + 1
guard valueIndex < arguments.count else { return nil }
let tokenPath = arguments[valueIndex]
return try? String(contentsOfFile: tokenPath, encoding: .utf8)
.trimmingCharacters(in: .whitespacesAndNewlines)
}
guard let token, !token.isEmpty else {
fputs("orca-computer-use-macos --agent requires a non-empty --token-file\n", stderr)
exit(2)
}
runAgent(socketPath: arguments[1], token: token)
} else if arguments.first == "--permissions" {
runPermissionCheck()
} else if arguments.first == "--permission" {
runPermissionCheck(initialPermission: PermissionKind.parse(arguments.dropFirst().first))
} else if arguments.first == "--permission-status" {
printPermissionStatus()
} else if arguments.first == "--permission-status-file" {
guard arguments.count >= 2 else {
fputs("usage: orca-computer-use-macos --permission-status-file <path>\n", stderr)
exit(2)
}
writePermissionStatus(to: arguments[1])
} else {
runStdio()
}