From 7b2b946b6ce760a1b43692108ccbad6a93b96d2c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 16 Nov 2026 18:01:06 -0800 Subject: [PATCH 1/2] fix(computer-use): treat an attribute-unsupported refusal of an advertised action as unsupported --- apps/desktop/native/computer-use/ComputerUse.swift | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 3529be2c61a..0ab2aa29e06 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -56,9 +56,10 @@ func dispatchAX(_ element: AXUIElement, press: Bool = false, _ operation: (AXUIE let result = operation(element) if ![.illegalArgument, .attributeUnsupported, .actionUnsupported, .notImplemented, .apiDisabled].contains(result) { inputDispatched = true } if result == .success || result == .cannotComplete && press { return } - // Some elements advertise an action they then refuse (Finder and file-panel cells with AXOpen); nothing reached the app. - if result == .actionUnsupported { - throw ComputerError("unsupported_action", "The element advertises this action but refused it. Use another of its actions or click it instead (clickCount 2 opens most files and folders).") + // An element can refuse a request it advertises; Finder and file-panel cells answer AXOpen with + // attributeUnsupported (-25205). Nothing reached the app, so the snapshot stays valid. + if result == .actionUnsupported || result == .attributeUnsupported { + throw ComputerError("unsupported_action", "The element refused this request although it advertises it. Use another of its actions or click it instead (clickCount 2 opens most files and folders).") } try axCheck(result) } From 5113ca842d1ffd593ec5af86c6f447d43c2bdfe4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 16 Nov 2026 18:11:16 -0800 Subject: [PATCH 2/2] scope the refusal mapping to performed actions; value and selection writes keep the generic error --- .../native/computer-use/ComputerUse.swift | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 0ab2aa29e06..85f858f6c36 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -56,13 +56,22 @@ func dispatchAX(_ element: AXUIElement, press: Bool = false, _ operation: (AXUIE let result = operation(element) if ![.illegalArgument, .attributeUnsupported, .actionUnsupported, .notImplemented, .apiDisabled].contains(result) { inputDispatched = true } if result == .success || result == .cannotComplete && press { return } - // An element can refuse a request it advertises; Finder and file-panel cells answer AXOpen with - // attributeUnsupported (-25205). Nothing reached the app, so the snapshot stays valid. - if result == .actionUnsupported || result == .attributeUnsupported { - throw ComputerError("unsupported_action", "The element refused this request although it advertises it. Use another of its actions or click it instead (clickCount 2 opens most files and folders).") - } try axCheck(result) } +/// Performs an advertised Accessibility action. An element can still refuse it (Finder and file-panel +/// cells answer AXOpen with attributeUnsupported, -25205); nothing reached the app, so the snapshot stays valid. +func performAXAction(_ element: AXUIElement, _ action: String, press: Bool = false) throws { + var refused = false + do { + try dispatchAX(element, press: press) { element in + let result = AXUIElementPerformAction(element, action as CFString) + refused = result == .actionUnsupported || result == .attributeUnsupported + return result + } + } catch where refused { + throw ComputerError("unsupported_action", "The element refused this action although it advertises it. Use another of its actions or click it instead (clickCount 2 opens most files and folders).") + } +} /// Some apps (Calendar on macOS 26) answer AXPress on a menu item by opening its menu with the item /// highlighted instead of running it. Return runs the highlighted item, as it would for a keyboard user. func completeMenuPress(_ item: AXUIElement, pid: pid_t) async throws { @@ -877,7 +886,7 @@ final class Driver { guard (names as? [String] ?? []).contains(action) else { throw ComputerError("unsupported_action", "Action was not advertised by the selected element.") } // Actions that open UI (a sheet, a menu) hold the reply like a press; others fail as uncertain on timeout. let opensUI = [kAXPressAction, kAXPickAction, kAXShowMenuAction].contains(action) - try dispatchAX(element, press: opensUI) { AXUIElementPerformAction($0, action as CFString) } + try performAXAction(element, action, press: opensUI) if action == kAXPressAction || action == kAXPickAction { try await completeMenuPress(element, pid: pid) } case "set_value": let element = try snapshot.element(p.elementId); let value = try required(p.value, "value") @@ -889,7 +898,7 @@ final class Driver { let button = p.button ?? "left"; let count = p.clickCount ?? 1 guard ["left", "right"].contains(button), (1...3).contains(count) else { throw ComputerError("invalid_arguments", "Invalid button or clickCount.") } if p.elementId != nil && button == "left" && count == 1, let element = try? snapshot.element(p.elementId), axActions(element).contains(kAXPressAction) { - try dispatchAX(element, press: true) { AXUIElementPerformAction($0, kAXPressAction as CFString) } + try performAXAction(element, kAXPressAction, press: true) try await completeMenuPress(element, pid: pid) } else { try await coordinateAction { dispatch in