From 3244a4cda48fb4824fa9351c57c8e24cd03a323e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 16 Nov 2026 18:26:53 -0800 Subject: [PATCH 1/2] improvement(computer-use): wait for an app's first observable window after launching it --- .../native/computer-use/ComputerUse.swift | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 85f858f6c36..345a7dae8b6 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -220,12 +220,12 @@ func appFor(_ bundleId: String?, launch: Bool = false) async throws -> NSRunning let configuration = NSWorkspace.OpenConfiguration(); configuration.activates = false let app = try await NSWorkspace.shared.openApplication(at: url, configuration: configuration) guard app.bundleIdentifier == bundle, !app.isTerminated else { throw ComputerError("app_identity_mismatch", "Launched app identity does not match the requested bundle ID.") } - let root = AXUIElementCreateApplication(app.processIdentifier) - AXUIElementSetMessagingTimeout(root, 0.1) + // Apps built on Electron or Catalyst take several seconds to show their first window; wait for + // one an observation can target rather than failing the read the agent will only repeat. var windowReady = false - for _ in 0..<20 { + for _ in 0..<80 { try checkCancellation() - if let windows = attribute(root, kAXWindowsAttribute) as? [AXUIElement], !windows.isEmpty { windowReady = true; break } + if hasObservableWindow(app.processIdentifier) { windowReady = true; break } guard !app.isTerminated else { throw ComputerError("app_unavailable", "App exited while opening its window.") } try await Task.sleep(for: .milliseconds(100)) } @@ -245,6 +245,20 @@ func windowsFor(_ pid: pid_t) -> [[String: Any]] { guard let entries = CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return [] } return entries.filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid && ($0[kCGWindowLayer as String] as? Int) == 0 } } +/// Whether the app shows a window an observation can target: an Accessibility window whose frame +/// matches one of its on-screen windows, the same match `get_app_state` uses to list windows. +func hasObservableWindow(_ pid: pid_t) -> Bool { + let root = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(root, 0.1) + guard let windows = attribute(root, kAXWindowsAttribute) as? [AXUIElement] else { return false } + let frames = windows.compactMap(elementRect) + return windowsFor(pid).contains { entry in + guard entry[kCGWindowIsOnscreen as String] as? Bool == true, + let bounds = entry[kCGWindowBounds as String] as? [String: Any], + let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary), rect.width > 0, rect.height > 0 else { return false } + return frames.contains(rect) + } +} func windowRect(_ id: String?, pid: pid_t) throws -> CGRect { let id = try required(id, "windowId") guard let number = UInt32(id), let entry = windowsFor(pid).first(where: { ($0[kCGWindowNumber as String] as? NSNumber)?.uint32Value == number }), From de76f2aba6761c28e6a761ce2c21116cb687d92f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 16 Nov 2026 18:49:43 -0800 Subject: [PATCH 2/2] count a minimized window as ready and time out each window read --- apps/desktop/native/computer-use/ComputerUse.swift | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 345a7dae8b6..17cc5afdedb 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -245,16 +245,19 @@ func windowsFor(_ pid: pid_t) -> [[String: Any]] { guard let entries = CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return [] } return entries.filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid && ($0[kCGWindowLayer as String] as? Int) == 0 } } -/// Whether the app shows a window an observation can target: an Accessibility window whose frame -/// matches one of its on-screen windows, the same match `get_app_state` uses to list windows. +/// Whether the app has a window an observation can list: an Accessibility window whose frame matches +/// one of its window-list entries, the match `get_app_state` uses. A minimized window counts, since it +/// is listed and `activate_app` can bring it forward. func hasObservableWindow(_ pid: pid_t) -> Bool { let root = AXUIElementCreateApplication(pid) AXUIElementSetMessagingTimeout(root, 0.1) guard let windows = attribute(root, kAXWindowsAttribute) as? [AXUIElement] else { return false } - let frames = windows.compactMap(elementRect) + let frames = windows.compactMap { window -> CGRect? in + AXUIElementSetMessagingTimeout(window, 0.1) + return elementRect(window) + } return windowsFor(pid).contains { entry in - guard entry[kCGWindowIsOnscreen as String] as? Bool == true, - let bounds = entry[kCGWindowBounds as String] as? [String: Any], + guard let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary), rect.width > 0, rect.height > 0 else { return false } return frames.contains(rect) }