Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 131 additions & 20 deletions apps/desktop/native/computer-use/ComputerUse.swift
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,26 @@ struct Parameters: Decodable {
var clickCount: Int?
var accessibilityAction: String?
var command: AppCommand?
var expected: ReminderExpectation?
}
/// The reminder as the user saw it when approving an update; the write is refused if a field it changes has moved since.
struct ReminderExpectation: Decodable {
var title: String
var due: String?
var priority: Int
var completed: Bool
var notes: String?
}
/// One allowlisted app command. Commands go through EventKit and Contacts, never Apple Events or
/// scripts, so they cannot reach a shell; each is a typed create or a list read, with no update,
/// delete or search.
/// scripts, so they cannot reach a shell; each is a typed create, a bounded read, or a reminder
/// update by id. Nothing is ever deleted.
struct AppCommand: Decodable {
var name: String
var id: String?
var titleContains: String?
var includeCompleted: Bool?
var completed: Bool?
var clearDue: Bool?
var title: String?
var notes: String?
var due: String?
Expand All @@ -67,6 +81,16 @@ struct AppCommand: Decodable {
var phones: [String]?
var groupId: String?
}
/// At most `max` UTF-16 units, the unit the reply's schema counts, cut on a character boundary:
/// Swift's `prefix` counts characters, and one emoji can be several units.
func bounded(_ text: String, _ max: Int) -> String {
guard text.utf16.count > max else { return text }
var units = 0
return String(text.prefix { character in
units += character.utf16.count
return units <= max
})
}
func invalidCommand(_ message: String) -> ComputerError { ComputerError("invalid_arguments", message, dispatchState: "not_started") }
/// A required, trimmed text field of bounded length.
func commandText(_ value: String?, _ field: String, max: Int = 1024) throws -> String {
Expand Down Expand Up @@ -171,12 +195,96 @@ func eventDate(_ text: String?, _ field: String, allDay: Bool) throws -> Date {
guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be a real ISO 8601 date-time.") }
return date
}
func runAppCommand(_ command: AppCommand) throws -> [String: Any] {
/// Reminders matching a predicate; EventKit only offers the asynchronous fetch.
func fetchReminders(_ store: EKEventStore, _ predicate: NSPredicate) async -> [EKReminder] {
await withCheckedContinuation { continuation in
store.fetchReminders(matching: predicate) { continuation.resume(returning: $0 ?? []) }
}
}
/// A reminder as reads and confirmations report it.
func reminderItem(_ reminder: EKReminder) -> [String: Any] {
var item: [String: Any] = [
"id": reminder.calendarItemIdentifier, "title": bounded((reminder.title ?? ""), 1024),
"list": bounded(reminder.calendar.title, 1024), "priority": reminder.priority, "completed": reminder.isCompleted,
]
if let notes = reminder.notes, !notes.isEmpty { item["notes"] = bounded(notes, 8000) }
if let due = reminder.dueDateComponents { item["due"] = dueText(due) }
return item
}
/// Due components written back the way commands take them: `YYYY-MM-DD`, or this Mac's local `YYYY-MM-DDTHH:MM:SS`.
func dueText(_ due: DateComponents) -> String {
let day = String(format: "%04d-%02d-%02d", due.year ?? 0, due.month ?? 0, due.day ?? 0)
guard due.hour != nil else { return day }
var source = Calendar(identifier: .gregorian); source.timeZone = due.timeZone ?? .current
guard let date = source.date(from: due) else { return day }
let local = localGregorian().dateComponents([.year, .month, .day, .hour, .minute, .second], from: date)
return String(format: "%04d-%02d-%02dT%02d:%02d:%02d", local.year ?? 0, local.month ?? 0, local.day ?? 0, local.hour ?? 0, local.minute ?? 0, local.second ?? 0)
}
/// Up to 50 reminders, incomplete unless asked otherwise, optionally in one list and matching part of a
/// title; or the one reminder with `id`, which an update reads to show its current values.
func findReminders(_ command: AppCommand) async throws -> [String: Any] {
let contains = try optionalText(command.titleContains, "titleContains", max: 256)
let store = try remindersStore()
if let id = command.id {
let found = (store.calendarItem(withIdentifier: id) as? EKReminder).map { [reminderItem($0)] } ?? []
return ["kind": "command", "command": command.name, "reminders": found, "truncated": false]
}
var lists: [EKCalendar]?
if let listId = command.listId {
guard let found = store.calendar(withIdentifier: listId), found.allowedEntityTypes.contains(.reminder) else {
throw ComputerError("list_unavailable", "No reminder list has this id; call list_reminder_lists.", dispatchState: "not_started")
}
lists = [found]
}
let predicate = command.includeCompleted == true
? store.predicateForReminders(in: lists)
: store.predicateForIncompleteReminders(withDueDateStarting: nil, ending: nil, calendars: lists)
var matches = await fetchReminders(store, predicate)
if let contains { matches = matches.filter { ($0.title ?? "").localizedCaseInsensitiveContains(contains) } }
return ["kind": "command", "command": command.name, "reminders": matches.prefix(50).map(reminderItem), "truncated": matches.count > 50]
}
func runAppCommand(_ command: AppCommand, expected: ReminderExpectation? = nil) throws -> [String: Any] {
switch command.name {
case "update_reminder":
let id = try commandText(command.id, "id", max: 256)
let title = try command.title.map { try commandText($0, "Title") }
// Blank notes clear them, as deleting them in the app would; others are trimmed, as a create trims them.
if let notes = command.notes, notes.count > 8000 { throw invalidCommand("Notes are limited to 8000 characters.") }
if let priority = command.priority, ![0, 1, 5, 9].contains(priority) { throw invalidCommand("Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).") }
if command.due != nil && command.clearDue == true { throw invalidCommand("Pass due or clearDue, not both.") }
let due = try command.due.map { try dueComponents($0) }
guard title != nil || command.notes != nil || command.priority != nil || due != nil || command.clearDue == true || command.completed != nil else {
throw invalidCommand("An update needs at least one field to change.")
}
let store = try remindersStore()
guard let reminder = store.calendarItem(withIdentifier: id) as? EKReminder else {
throw ComputerError("not_found", "No reminder has this id; call find_reminders.", dispatchState: "not_started")
}
guard reminder.calendar.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") }
guard let expected else { throw invalidCommand("An update needs the values the user approved.") }
let now = reminderItem(reminder)
let moved = (title != nil && now["title"] as? String != expected.title)
|| ((due != nil || command.clearDue == true) && now["due"] as? String != expected.due)
|| (command.priority != nil && now["priority"] as? Int != expected.priority)
|| (command.completed != nil && now["completed"] as? Bool != expected.completed)
|| (command.notes != nil && now["notes"] as? String != expected.notes)
guard !moved else {
throw ComputerError("reminder_changed", "This reminder changed while the user was confirming; read it again with find_reminders before retrying.", dispatchState: "not_started")
}
if let title { reminder.title = title }
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
if let notes = command.notes { reminder.notes = try optionalText(notes, "Notes", max: 8000) }
if let priority = command.priority { reminder.priority = priority }
if let due { reminder.dueDateComponents = due }
if command.clearDue == true { reminder.dueDateComponents = nil }
if let completed = command.completed { reminder.isCompleted = completed }
try checkCancellation()
inputDispatched = true
try store.save(reminder, commit: true)
return ["kind": "command", "command": command.name, "updated": ["id": reminder.calendarItemIdentifier, "title": bounded((reminder.title ?? ""), 1024), "list": bounded(reminder.calendar.title, 1024)]]
case "list_reminder_lists":
let allLists = try remindersStore().calendars(for: .reminder)
let lists = allLists.prefix(200).map { list -> [String: Any] in
["id": list.calendarIdentifier, "title": String(list.title.prefix(1024)), "allowsModifications": list.allowsContentModifications]
["id": list.calendarIdentifier, "title": bounded(list.title, 1024), "allowsModifications": list.allowsContentModifications]
}
return ["kind": "command", "command": command.name, "items": lists, "truncated": allLists.count > 200]
case "create_reminder":
Expand Down Expand Up @@ -206,7 +314,7 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] {
case "list_calendars":
let allCalendars = try calendarStore().calendars(for: .event)
let calendars = allCalendars.prefix(200).map { calendar -> [String: Any] in
["id": calendar.calendarIdentifier, "title": String(calendar.title.prefix(1024)), "allowsModifications": calendar.allowsContentModifications]
["id": calendar.calendarIdentifier, "title": bounded(calendar.title, 1024), "allowsModifications": calendar.allowsContentModifications]
}
return ["kind": "command", "command": command.name, "items": calendars, "truncated": allCalendars.count > 200]
case "create_event":
Expand Down Expand Up @@ -245,7 +353,7 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] {
case "list_contact_groups":
let allGroups = try contactStore().groups(matching: nil)
let groups = allGroups.prefix(200).map { group -> [String: Any] in
["id": group.identifier, "title": String(group.name.prefix(1024)), "allowsModifications": true]
["id": group.identifier, "title": bounded(group.name, 1024), "allowsModifications": true]
}
return ["kind": "command", "command": command.name, "items": groups, "truncated": allGroups.count > 200]
case "create_contact_group":
Expand Down Expand Up @@ -290,8 +398,8 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] {
inputDispatched = true
try store.execute(request)
let display = [given, family].compactMap { $0 }.joined(separator: " ")
let title = String((display.isEmpty ? organization ?? "" : display).prefix(1024))
return ["kind": "command", "command": command.name, "created": ["id": contact.identifier, "title": title, "list": String((group?.name ?? "Contacts").prefix(1024))]]
let title = bounded((display.isEmpty ? organization ?? "" : display), 1024)
return ["kind": "command", "command": command.name, "created": ["id": contact.identifier, "title": title, "list": bounded((group?.name ?? "Contacts"), 1024)]]
default:
throw ComputerError("unknown_command", "Unknown app command.", dispatchState: "not_started")
}
Expand Down Expand Up @@ -688,9 +796,9 @@ final class Snapshot {
var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty }
if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } }
if role == kAXMenuItemRole, let shortcut = menuShortcut(element) { labels.append(shortcut) }
if !isSecure, !labels.isEmpty { node["label"] = String(labels.joined(separator: " ").prefix(1024)) }
if !isSecure, !labels.isEmpty { node["label"] = bounded(labels.joined(separator: " "), 1024) }
if !isSecure, let raw = values[kAXValueAttribute] {
if let value = raw as? String { node["value"] = String(value.prefix(2048)) }
if let value = raw as? String { node["value"] = bounded(value, 2048) }
else if let value = raw as? NSNumber { node["value"] = value.stringValue }
}
if !isSecure {
Expand All @@ -699,12 +807,12 @@ final class Snapshot {
let writableValue = [kAXTextAreaRole, kAXTextFieldRole].contains(role)
&& AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue
node["editable"] = writableValue
if let placeholder = values[kAXPlaceholderValueAttribute] as? String { node["placeholder"] = String(placeholder.prefix(1024)) }
if let placeholder = values[kAXPlaceholderValueAttribute] as? String { node["placeholder"] = bounded(placeholder, 1024) }
}
if let enabled = values[kAXEnabledAttribute] as? Bool { node["enabled"] = enabled }
if let rect = axRect(position: values[kAXPositionAttribute], size: values[kAXSizeAttribute]) { node.merge(rectJSON(rect)) { _, new in new } }
var actions: CFArray?
if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { String($0.prefix(128)) } }
if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { bounded($0, 128) } }
nodes.append(node)
if role == kAXWindowRole, let preferredWindow, !CFEqual(element, preferredWindow) { continue }
if !isSecure, entry.depth < 64 {
Expand Down Expand Up @@ -997,13 +1105,13 @@ func discoverApps() throws -> [[String: Any]] {
enumerator.skipDescendants()
guard let bundle = Bundle(url: url), let id = bundle.bundleIdentifier, id.utf8.count <= 255 else { continue }
let name = bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String ?? bundle.object(forInfoDictionaryKey: "CFBundleName") as? String ?? url.deletingPathExtension().lastPathComponent
entries[id] = ["bundleId": id, "name": String(name.prefix(1024)), "isActive": false]
entries[id] = ["bundleId": id, "name": bounded(name, 1024), "isActive": false]
} else if enumerator.level >= 3 { enumerator.skipDescendants() }
}
}
for app in NSWorkspace.shared.runningApplications where app.activationPolicy == .regular {
guard let id = app.bundleIdentifier, id.utf8.count <= 255 else { continue }
entries[id] = ["bundleId": id, "name": String((app.localizedName ?? id).prefix(1024)), "pid": app.processIdentifier, "isActive": app.isActive]
entries[id] = ["bundleId": id, "name": bounded((app.localizedName ?? id), 1024), "pid": app.processIdentifier, "isActive": app.isActive]
}
return Array(entries.values.sorted { ($0["bundleId"] as? String ?? "") < ($1["bundleId"] as? String ?? "") }.prefix(1000))
}
Expand Down Expand Up @@ -1064,7 +1172,10 @@ final class Driver {
case "list_apps": return ["kind": "apps", "apps": try discoverApps()]
case "app_command":
inputDispatched = false
return try runAppCommand(required(p.command, "command"))
let command = try required(p.command, "command")
// EventKit only fetches reminders asynchronously.
if command.name == "find_reminders" { return try await findReminders(command) }
return try runAppCommand(command, expected: p.expected)
case "activate_app":
try requireAccessibility()
let app = try await appFor(p.bundleId, launch: true)
Expand Down Expand Up @@ -1095,7 +1206,7 @@ final class Driver {
let matches = axWindows.filter { elementRect($0) == rect }
if matches.count == 1, onScreen(entry) { snapshot.windowElements[id.stringValue] = matches[0] }
var result: [String: Any] = ["windowId": id.stringValue, "title": entry[kCGWindowName as String] as? String ?? ""]
if let title = result["title"] as? String { result["title"] = String(title.prefix(4096)) }
if let title = result["title"] as? String { result["title"] = bounded(title, 4096) }
result.merge(rectJSON(rect)) { _, new in new }; return result
}
if snapshots.count >= 16 { snapshots.removeAll() }
Expand All @@ -1117,8 +1228,8 @@ final class Driver {
do {
guard #available(macOS 14.0, *) else { throw ComputerError("unsupported_os", "Screenshots require macOS 14 or later.") }
result["screenshot"] = try await screenshot(pid: app.processIdentifier, windowID: selectedWindowId, observedFrame: try required(snapshot.windowFrames[selectedWindowId], "observed window frame"))
} catch let error as ComputerError { result["screenshotError"] = String((error.code + ": " + error.message).prefix(2000)) }
catch { result["screenshotError"] = String(("capture_failed: " + String(describing: error)).prefix(2000)) }
} catch let error as ComputerError { result["screenshotError"] = bounded((error.code + ": " + error.message), 2000) }
catch { result["screenshotError"] = bounded(("capture_failed: " + String(describing: error)), 2000) }
}
result["isActive"] = app.isActive
return result
Expand Down Expand Up @@ -1153,7 +1264,7 @@ final class Driver {
var completed = 0
func stopped(_ message: String) -> [String: Any] {
["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false,
"sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": String((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch.").prefix(2000))]]
"sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": bounded((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch."), 2000)]]
}
for (index, step) in steps.enumerated() {
// Stop can arrive during the previous step's render wait; no later step may start after it.
Expand Down Expand Up @@ -1481,7 +1592,7 @@ struct Main {
if let dispatchState = error.dispatchState { details["dispatchState"] = dispatchState }
response = ["id": id, "error": details]
}
catch { response = ["id": id, "error": ["code": "request_failed", "message": String(String(describing: error).prefix(2000))]] }
catch { response = ["id": id, "error": ["code": "request_failed", "message": bounded(String(describing: error), 2000)]] }
let data: Data?
do { data = try encodeReply(response) } catch let error as ComputerError {
data = try? encodeReply(["id": id, "error": ["code": error.code, "message": error.message]])
Expand Down
Loading
Loading