From 0e073d2c8cb18f5d5f94e7de646a0fd55fc1d703 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 19:16:32 -0800 Subject: [PATCH 1/2] feat(computer-use): find and update reminders through app commands --- .../native/computer-use/ComputerUse.swift | 88 ++++++++++++++++++- .../computer-use/tests/NativeTests.swift | 7 ++ .../src/main/computer-use/service.test.ts | 74 +++++++++++++++- apps/desktop/src/main/computer-use/service.ts | 59 ++++++++++++- apps/desktop/src/main/index.ts | 9 +- .../src/computer-use.generated.ts | 80 ++++++++++++++--- 6 files changed, 294 insertions(+), 23 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index a55a3fde106..a14fbb65df1 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -44,10 +44,15 @@ struct Parameters: Decodable { var command: AppCommand? } /// One allowlisted app command. Commands go through EventKit and Contacts, never Apple Events or -/// scripts, so they cannot reach a shell; each is a typed create or a list read, with no update, -/// delete or search. +/// scripts, so they cannot reach a shell; each is a typed create, a bounded read, or a reminder +/// update by id. Nothing is ever deleted. struct AppCommand: Decodable { var name: String + var id: String? + var titleContains: String? + var includeCompleted: Bool? + var completed: Bool? + var clearDue: Bool? var title: String? var notes: String? var due: String? @@ -171,8 +176,82 @@ func eventDate(_ text: String?, _ field: String, allDay: Bool) throws -> Date { guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be a real ISO 8601 date-time.") } return date } +/// Reminders matching a predicate; EventKit only offers the asynchronous fetch. +func fetchReminders(_ store: EKEventStore, _ predicate: NSPredicate) async -> [EKReminder] { + await withCheckedContinuation { continuation in + store.fetchReminders(matching: predicate) { continuation.resume(returning: $0 ?? []) } + } +} +/// A reminder as reads and confirmations report it. +func reminderItem(_ reminder: EKReminder) -> [String: Any] { + var item: [String: Any] = [ + "id": reminder.calendarItemIdentifier, "title": String((reminder.title ?? "").prefix(1024)), + "list": String(reminder.calendar.title.prefix(1024)), "priority": reminder.priority, "completed": reminder.isCompleted, + ] + if let notes = reminder.notes, !notes.isEmpty { item["notes"] = String(notes.prefix(8000)) } + if let due = reminder.dueDateComponents { item["due"] = dueText(due) } + return item +} +/// Due components written back the way commands take them: `YYYY-MM-DD`, or this Mac's local `YYYY-MM-DDTHH:MM:SS`. +func dueText(_ due: DateComponents) -> String { + let day = String(format: "%04d-%02d-%02d", due.year ?? 0, due.month ?? 0, due.day ?? 0) + guard due.hour != nil else { return day } + var source = Calendar(identifier: .gregorian); source.timeZone = due.timeZone ?? .current + guard let date = source.date(from: due) else { return day } + let local = localGregorian().dateComponents([.year, .month, .day, .hour, .minute, .second], from: date) + return String(format: "%04d-%02d-%02dT%02d:%02d:%02d", local.year ?? 0, local.month ?? 0, local.day ?? 0, local.hour ?? 0, local.minute ?? 0, local.second ?? 0) +} +/// Up to 50 reminders, incomplete unless asked otherwise, optionally in one list and matching part of a +/// title; or the one reminder with `id`, which an update reads to show its current values. +func findReminders(_ command: AppCommand) async throws -> [String: Any] { + let contains = try optionalText(command.titleContains, "titleContains", max: 256) + let store = try remindersStore() + if let id = command.id { + let found = (store.calendarItem(withIdentifier: id) as? EKReminder).map { [reminderItem($0)] } ?? [] + return ["kind": "command", "command": command.name, "reminders": found, "truncated": false] + } + var lists: [EKCalendar]? + if let listId = command.listId { + guard let found = store.calendar(withIdentifier: listId), found.allowedEntityTypes.contains(.reminder) else { + throw ComputerError("list_unavailable", "No reminder list has this id; call list_reminder_lists.", dispatchState: "not_started") + } + lists = [found] + } + let predicate = command.includeCompleted == true + ? store.predicateForReminders(in: lists) + : store.predicateForIncompleteReminders(withDueDateStarting: nil, ending: nil, calendars: lists) + var matches = await fetchReminders(store, predicate) + if let contains { matches = matches.filter { ($0.title ?? "").localizedCaseInsensitiveContains(contains) } } + return ["kind": "command", "command": command.name, "reminders": matches.prefix(50).map(reminderItem), "truncated": matches.count > 50] +} func runAppCommand(_ command: AppCommand) throws -> [String: Any] { switch command.name { + case "update_reminder": + let id = try commandText(command.id, "id", max: 256) + let title = try command.title.map { try commandText($0, "Title") } + // An empty string clears the notes, as deleting them in the app would. + if let notes = command.notes, notes.count > 8000 { throw invalidCommand("Notes are limited to 8000 characters.") } + if let priority = command.priority, ![0, 1, 5, 9].contains(priority) { throw invalidCommand("Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).") } + if command.due != nil && command.clearDue == true { throw invalidCommand("Pass due or clearDue, not both.") } + let due = try command.due.map { try dueComponents($0) } + guard title != nil || command.notes != nil || command.priority != nil || due != nil || command.clearDue == true || command.completed != nil else { + throw invalidCommand("An update needs at least one field to change.") + } + let store = try remindersStore() + guard let reminder = store.calendarItem(withIdentifier: id) as? EKReminder else { + throw ComputerError("not_found", "No reminder has this id; call find_reminders.", dispatchState: "not_started") + } + guard reminder.calendar.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") } + if let title { reminder.title = title } + if let notes = command.notes { reminder.notes = notes.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : notes } + if let priority = command.priority { reminder.priority = priority } + if let due { reminder.dueDateComponents = due } + if command.clearDue == true { reminder.dueDateComponents = nil } + if let completed = command.completed { reminder.isCompleted = completed } + try checkCancellation() + inputDispatched = true + try store.save(reminder, commit: true) + return ["kind": "command", "command": command.name, "updated": ["id": reminder.calendarItemIdentifier, "title": String((reminder.title ?? "").prefix(1024)), "list": String(reminder.calendar.title.prefix(1024))]] case "list_reminder_lists": let allLists = try remindersStore().calendars(for: .reminder) let lists = allLists.prefix(200).map { list -> [String: Any] in @@ -1064,7 +1143,10 @@ final class Driver { case "list_apps": return ["kind": "apps", "apps": try discoverApps()] case "app_command": inputDispatched = false - return try runAppCommand(required(p.command, "command")) + let command = try required(p.command, "command") + // EventKit only fetches reminders asynchronously. + if command.name == "find_reminders" { return try await findReminders(command) } + return try runAppCommand(command) case "activate_app": try requireAccessibility() let app = try await appFor(p.bundleId, launch: true) diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index bd29405fe49..2e854211c1e 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -51,6 +51,13 @@ import AppKit expectError("unknown_command") { _ = try runAppCommand(AppCommand(name: "run_script")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: " ")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: "Pay rent", priority: 3)) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1", title: " ")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1", priority: 3)) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1", clearDue: true, due: "2027-05-04")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", completed: true)) } + precondition(dueText(DateComponents(year: 2027, month: 5, day: 4)) == "2027-05-04") + let localDueText = dueText(try dueComponents("2027-05-04T09:30")); precondition(localDueText == "2027-05-04T09:30:00") var local = Calendar(identifier: .gregorian); local.timeZone = .current let floating = try eventDate("2027-06-09T07:15:00", "Start", allDay: false) precondition(local.dateComponents([.hour, .minute], from: floating) == DateComponents(hour: 7, minute: 15)) diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index 4684e5c7f20..184d520667c 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -56,7 +56,9 @@ function setup(enabled = true, askForApps = true) { ) const native = { request, stop: vi.fn() } const approveApp = vi.fn(async (): Promise<'always' | 'deny' | 'once'> => 'always') - const confirmAppCommand = vi.fn(async (_request: { appName: string; summary: string }) => true) + const confirmAppCommand = vi.fn( + async (_request: { appName: string; summary: string; change: 'add' | 'update' }) => true + ) const onActivity = vi.fn() const openPermissionSettings = vi.fn(async () => {}) const setStopShortcutActive = vi.fn((active: boolean) => active) @@ -231,6 +233,76 @@ describe('native computer use authority and lifecycle', () => { expect(confirmAppCommand.mock.lastCall?.[0].summary).toContain('List: deleted-list') }) + it('confirms a reminder update against its current values, and refuses an unknown id first', async () => { + const { service, native, confirmAppCommand } = setup() + native.request.mockImplementation(async (method, params): Promise => { + if (method !== 'app_command') return { kind: 'apps', apps: [] } + const command = params?.command as { name: string; id: string } + if (command.name === 'find_reminders') + return { + kind: 'command', + command: 'find_reminders', + reminders: + command.id === 'r1' + ? [ + { + id: 'r1', + title: 'Pay rent', + list: 'Home', + due: '2027-05-04', + priority: 0, + completed: false, + }, + ] + : [], + truncated: false, + } + return { + kind: 'command', + command: 'update_reminder', + updated: { id: 'r1', title: 'Pay rent', list: 'Home' }, + } + }) + const sent = () => + native.request.mock.calls.filter( + ([method, params]) => + method === 'app_command' && + (params?.command as { name: string }).name === 'update_reminder' + ).length + + await expect( + service.execute('missing', 'chat', { + action: 'app_command', + command: { name: 'update_reminder', id: 'gone', completed: true }, + }) + ).rejects.toMatchObject({ details: { code: 'not_found', dispatchState: 'not_started' } }) + expect(confirmAppCommand).not.toHaveBeenCalled() + + confirmAppCommand.mockResolvedValueOnce(false) + const update = { + action: 'app_command', + command: { + name: 'update_reminder', + id: 'r1', + due: '2027-05-11', + priority: 1, + completed: true, + }, + } + await expect(service.execute('declined', 'chat', update)).rejects.toMatchObject({ + details: { code: 'user_declined' }, + }) + expect(sent()).toBe(0) + expect(confirmAppCommand.mock.calls[0]?.[0]).toMatchObject({ + change: 'update', + summary: + 'Reminder: Pay rent\nList: Home\nDue: 2027-05-04 → 2027-05-11\nPriority: None → High\nStatus: Open → Completed', + }) + + await service.execute('accepted', 'chat', update) + expect(sent()).toBe(1) + }) + it('refuses an app command when the user denies the app, before asking to confirm', async () => { const { service, native, approveApp, confirmAppCommand } = setup() approveApp.mockResolvedValue('deny') diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index d142a91572c..89d94715526 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -27,7 +27,7 @@ interface ComputerUseServiceDeps { ) => Promise<'once' | 'always' | 'deny'> /** Confirms one app-command write. It shows the exact fields, since nothing happens on screen. */ confirmAppCommand: ( - request: { appName: string; summary: string }, + request: { appName: string; summary: string; change: 'add' | 'update' }, signal: AbortSignal ) => Promise onActivity: (activity: ComputerUseActivity | null) => void @@ -53,6 +53,8 @@ interface SnapshotOwnership { const APP_COMMAND_APPS: Record = { list_reminder_lists: 'com.apple.reminders', create_reminder: 'com.apple.reminders', + find_reminders: 'com.apple.reminders', + update_reminder: 'com.apple.reminders', list_calendars: 'com.apple.iCal', create_event: 'com.apple.iCal', list_contact_groups: 'com.apple.AddressBook', @@ -81,6 +83,39 @@ function chosenDestination( return null } +type ReminderUpdate = Extract +type ReminderItem = NonNullable< + Extract['reminders'] +>[number] + +/** + * What an update changes, read against the reminder as it is now: the user sees which reminder + * and each field's current and new value, since nothing happens on screen. + */ +function updateSummary(command: ReminderUpdate, current: ReminderItem): string { + const priority = (value: number) => PRIORITY_NAMES[value] ?? 'None' + const changes: Array<[string, string | undefined, string]> = [] + const title = command.title?.trim() + if (title !== undefined && title !== current.title) changes.push(['Title', current.title, title]) + if (command.due !== undefined) changes.push(['Due', current.due, command.due]) + if (command.clearDue) changes.push(['Due', current.due, 'none']) + if (command.priority !== undefined) + changes.push(['Priority', priority(current.priority), priority(command.priority)]) + if (command.completed !== undefined) + changes.push([ + 'Status', + current.completed ? 'Completed' : 'Open', + command.completed ? 'Completed' : 'Open', + ]) + if (command.notes !== undefined) + changes.push(['Notes', current.notes, command.notes.trim() || 'none']) + return [ + `Reminder: ${current.title}`, + `List: ${current.list}`, + ...changes.map(([label, before, after]) => `${label}: ${before || 'none'} → ${after}`), + ].join('\n') +} + /** * The exact fields a write will save, for the user to confirm; null for a read, which needs none. * `destination` is the chosen list, calendar or group by name, so the user can tell where it goes. @@ -453,17 +488,20 @@ export class ComputerUseService { this.activity = { ...this.activity, appName } this.deps.onActivity(this.activity) } - const summary = writeSummary(command, await this.destinationName(command, work)) + const summary = + command.name === 'update_reminder' + ? updateSummary(command, await this.currentReminder(command.id, work)) + : writeSummary(command, await this.destinationName(command, work)) if (summary !== null) { const confirmed = await this.deps.confirmAppCommand( - { appName, summary }, + { appName, summary, change: command.name === 'update_reminder' ? 'update' : 'add' }, work.controller.signal ) this.check(work) if (!confirmed) throw new ComputerUseError({ code: 'user_declined', - message: 'The user declined to add this item. Do not retry it.', + message: 'The user declined this change. Do not retry it.', dispatchState: 'not_started', }) } @@ -481,6 +519,19 @@ export class ComputerUseService { return result } + /** The reminder an update names, read when it is confirmed; an unknown id is refused before the user is asked. */ + private async currentReminder(id: string, work: ToolWork): Promise { + const found = await this.appCommand({ name: 'find_reminders', id }) + this.check(work) + const reminder = found.reminders?.find((item) => item.id === id) + if (reminder) return reminder + throw new ComputerUseError({ + code: 'not_found', + message: 'No reminder has this id; call find_reminders.', + dispatchState: 'not_started', + }) + } + /** * The name of the list, calendar or group a write chose by id, read when it is confirmed so the * name is current. An id missing from a complete read is refused before the user is asked; past diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 0fc6fdc6e0a..cd074cbe5ee 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -237,13 +237,16 @@ function main(): void { }) return result.response === 0 ? 'once' : result.response === 1 ? 'always' : 'deny' }, - confirmAppCommand: async ({ appName, summary }, signal) => { + confirmAppCommand: async ({ appName, summary, change }, signal) => { const result = await dialog.showMessageBox({ type: 'question', title: 'Computer Use', - message: `Let Mothership add this to ${appName}?`, + message: + change === 'update' + ? `Let Mothership change this in ${appName}?` + : `Let Mothership add this to ${appName}?`, detail: summary, - buttons: ['Add', 'Cancel'], + buttons: [change === 'update' ? 'Change' : 'Add', 'Cancel'], defaultId: 1, cancelId: 1, noLink: true, diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index 69fe3e86753..0b049f86cb0 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -255,7 +255,7 @@ const EventBoundary = DateOrDateTime.describe( "ISO 8601 date-time, or YYYY-MM-DD when allDay is true. Without an offset it is this Mac's local time, as typing it into Calendar would be.", ); -/** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */ +/** Typed app commands through the OS's own data frameworks, never a script: allowlisted creates, bounded reads and reminder updates by id. Nothing is deleted. */ export const AppCommandSchema = z.discriminatedUnion("name", [ z.strictObject({ name: z.literal("list_reminder_lists") }), z.strictObject({ @@ -276,6 +276,44 @@ export const AppCommandSchema = z.discriminatedUnion("name", [ .optional() .describe("0 none, 1 high, 5 medium, 9 low."), }), + z.strictObject({ + name: z.literal("find_reminders"), + id: z.string().min(1).max(256).optional().describe("Read just this reminder."), + listId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_reminder_lists; omit for every list."), + titleContains: z.string().min(1).max(256).optional().describe("Case-insensitive part of the title."), + includeCompleted: z.boolean().optional().describe("Completed reminders are left out unless true."), + }), + z + .strictObject({ + name: z.literal("update_reminder"), + id: z.string().min(1).max(256).describe("An id from find_reminders or create_reminder."), + title: CommandTitle.optional(), + notes: CommandNotes.describe("Replaces the notes; an empty string clears them."), + due: DateOrDateTime.optional().describe( + "YYYY-MM-DD for a date-only reminder, or an ISO 8601 date-time (this Mac's local time without an offset).", + ), + clearDue: z.literal(true).optional().describe("Removes the due date."), + priority: z + .union([z.literal(0), z.literal(1), z.literal(5), z.literal(9)]) + .optional() + .describe("0 none, 1 high, 5 medium, 9 low."), + completed: z.boolean().optional().describe("true marks it done; false reopens it."), + }) + .refine((update) => !(update.due !== undefined && update.clearDue), { + message: "Pass due or clearDue, not both", + }) + .refine( + (update) => + [update.title, update.notes, update.due, update.clearDue, update.priority, update.completed].some( + (value) => value !== undefined, + ), + { message: "An update needs at least one field to change" }, + ), z.strictObject({ name: z.literal("list_calendars") }), z.strictObject({ name: z.literal("create_event"), @@ -414,6 +452,22 @@ const CommandItem = z.strictObject({ allowsModifications: z.boolean(), }); +const ReminderItem = z.strictObject({ + id: z.string().min(1).max(256), + title: z.string().max(1024), + list: z.string().max(1024), + due: z.string().max(64).optional(), + priority: z.number().int().min(0).max(9), + completed: z.boolean(), + notes: z.string().max(8000).optional(), +}); + +const SavedItem = z.strictObject({ + id: z.string().min(1).max(256), + title: z.string().max(1024), + list: z.string().max(1024), +}); + /** A create must return the id it committed, so a lost reply is never retried as a second create. */ export const ComputerUseCommandResultSchema = z .strictObject({ @@ -421,30 +475,32 @@ export const ComputerUseCommandResultSchema = z command: z.enum([ "list_reminder_lists", "create_reminder", + "find_reminders", + "update_reminder", "list_calendars", "create_event", "list_contact_groups", "create_contact_group", "create_contact", ]), - created: z - .strictObject({ - id: z.string().min(1).max(256), - title: z.string().max(1024), - list: z.string().max(1024), - }) - .optional(), + created: SavedItem.optional(), + updated: SavedItem.optional(), items: z.array(CommandItem).max(200).optional(), + reminders: z.array(ReminderItem).max(50).optional(), truncated: z .boolean() .optional() - .describe("True when more than 200 items exist and only the first 200 are listed."), + .describe("True when more items exist than the read returns (200 for lists, 50 for reminders)."), }) .refine( - (result) => - result.command.startsWith("create_") ? result.created !== undefined : result.items !== undefined, + (result) => { + if (result.command.startsWith("create_")) return result.created !== undefined; + if (result.command.startsWith("update_")) return result.updated !== undefined; + if (result.command.startsWith("find_")) return result.reminders !== undefined; + return result.items !== undefined; + }, { - message: "A create returns created; a list returns items", + message: "A create returns created, an update updated, a find reminders, and a list items", }, ); From f177ec7ffe3921f1b0f29d8dd38f1eae7614a1e1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 20:00:31 -0800 Subject: [PATCH 2/2] fix(computer-use): refuse stale reminder updates and bound replies in UTF-16 units --- .../native/computer-use/ComputerUse.swift | 77 +++++++++++++------ .../computer-use/tests/NativeTests.swift | 2 + .../src/main/computer-use/service.test.ts | 3 + apps/desktop/src/main/computer-use/service.ts | 20 +++-- 4 files changed, 72 insertions(+), 30 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index a14fbb65df1..cc0509d092d 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -42,6 +42,15 @@ struct Parameters: Decodable { var clickCount: Int? var accessibilityAction: String? var command: AppCommand? + var expected: ReminderExpectation? +} +/// The reminder as the user saw it when approving an update; the write is refused if a field it changes has moved since. +struct ReminderExpectation: Decodable { + var title: String + var due: String? + var priority: Int + var completed: Bool + var notes: String? } /// One allowlisted app command. Commands go through EventKit and Contacts, never Apple Events or /// scripts, so they cannot reach a shell; each is a typed create, a bounded read, or a reminder @@ -72,6 +81,16 @@ struct AppCommand: Decodable { var phones: [String]? var groupId: String? } +/// At most `max` UTF-16 units, the unit the reply's schema counts, cut on a character boundary: +/// Swift's `prefix` counts characters, and one emoji can be several units. +func bounded(_ text: String, _ max: Int) -> String { + guard text.utf16.count > max else { return text } + var units = 0 + return String(text.prefix { character in + units += character.utf16.count + return units <= max + }) +} func invalidCommand(_ message: String) -> ComputerError { ComputerError("invalid_arguments", message, dispatchState: "not_started") } /// A required, trimmed text field of bounded length. func commandText(_ value: String?, _ field: String, max: Int = 1024) throws -> String { @@ -185,10 +204,10 @@ func fetchReminders(_ store: EKEventStore, _ predicate: NSPredicate) async -> [E /// A reminder as reads and confirmations report it. func reminderItem(_ reminder: EKReminder) -> [String: Any] { var item: [String: Any] = [ - "id": reminder.calendarItemIdentifier, "title": String((reminder.title ?? "").prefix(1024)), - "list": String(reminder.calendar.title.prefix(1024)), "priority": reminder.priority, "completed": reminder.isCompleted, + "id": reminder.calendarItemIdentifier, "title": bounded((reminder.title ?? ""), 1024), + "list": bounded(reminder.calendar.title, 1024), "priority": reminder.priority, "completed": reminder.isCompleted, ] - if let notes = reminder.notes, !notes.isEmpty { item["notes"] = String(notes.prefix(8000)) } + if let notes = reminder.notes, !notes.isEmpty { item["notes"] = bounded(notes, 8000) } if let due = reminder.dueDateComponents { item["due"] = dueText(due) } return item } @@ -224,12 +243,12 @@ func findReminders(_ command: AppCommand) async throws -> [String: Any] { if let contains { matches = matches.filter { ($0.title ?? "").localizedCaseInsensitiveContains(contains) } } return ["kind": "command", "command": command.name, "reminders": matches.prefix(50).map(reminderItem), "truncated": matches.count > 50] } -func runAppCommand(_ command: AppCommand) throws -> [String: Any] { +func runAppCommand(_ command: AppCommand, expected: ReminderExpectation? = nil) throws -> [String: Any] { switch command.name { case "update_reminder": let id = try commandText(command.id, "id", max: 256) let title = try command.title.map { try commandText($0, "Title") } - // An empty string clears the notes, as deleting them in the app would. + // Blank notes clear them, as deleting them in the app would; others are trimmed, as a create trims them. if let notes = command.notes, notes.count > 8000 { throw invalidCommand("Notes are limited to 8000 characters.") } if let priority = command.priority, ![0, 1, 5, 9].contains(priority) { throw invalidCommand("Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).") } if command.due != nil && command.clearDue == true { throw invalidCommand("Pass due or clearDue, not both.") } @@ -242,8 +261,18 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] { throw ComputerError("not_found", "No reminder has this id; call find_reminders.", dispatchState: "not_started") } guard reminder.calendar.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") } + guard let expected else { throw invalidCommand("An update needs the values the user approved.") } + let now = reminderItem(reminder) + let moved = (title != nil && now["title"] as? String != expected.title) + || ((due != nil || command.clearDue == true) && now["due"] as? String != expected.due) + || (command.priority != nil && now["priority"] as? Int != expected.priority) + || (command.completed != nil && now["completed"] as? Bool != expected.completed) + || (command.notes != nil && now["notes"] as? String != expected.notes) + guard !moved else { + throw ComputerError("reminder_changed", "This reminder changed while the user was confirming; read it again with find_reminders before retrying.", dispatchState: "not_started") + } if let title { reminder.title = title } - if let notes = command.notes { reminder.notes = notes.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : notes } + if let notes = command.notes { reminder.notes = try optionalText(notes, "Notes", max: 8000) } if let priority = command.priority { reminder.priority = priority } if let due { reminder.dueDateComponents = due } if command.clearDue == true { reminder.dueDateComponents = nil } @@ -251,11 +280,11 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] { try checkCancellation() inputDispatched = true try store.save(reminder, commit: true) - return ["kind": "command", "command": command.name, "updated": ["id": reminder.calendarItemIdentifier, "title": String((reminder.title ?? "").prefix(1024)), "list": String(reminder.calendar.title.prefix(1024))]] + return ["kind": "command", "command": command.name, "updated": ["id": reminder.calendarItemIdentifier, "title": bounded((reminder.title ?? ""), 1024), "list": bounded(reminder.calendar.title, 1024)]] case "list_reminder_lists": let allLists = try remindersStore().calendars(for: .reminder) let lists = allLists.prefix(200).map { list -> [String: Any] in - ["id": list.calendarIdentifier, "title": String(list.title.prefix(1024)), "allowsModifications": list.allowsContentModifications] + ["id": list.calendarIdentifier, "title": bounded(list.title, 1024), "allowsModifications": list.allowsContentModifications] } return ["kind": "command", "command": command.name, "items": lists, "truncated": allLists.count > 200] case "create_reminder": @@ -285,7 +314,7 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] { case "list_calendars": let allCalendars = try calendarStore().calendars(for: .event) let calendars = allCalendars.prefix(200).map { calendar -> [String: Any] in - ["id": calendar.calendarIdentifier, "title": String(calendar.title.prefix(1024)), "allowsModifications": calendar.allowsContentModifications] + ["id": calendar.calendarIdentifier, "title": bounded(calendar.title, 1024), "allowsModifications": calendar.allowsContentModifications] } return ["kind": "command", "command": command.name, "items": calendars, "truncated": allCalendars.count > 200] case "create_event": @@ -324,7 +353,7 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] { case "list_contact_groups": let allGroups = try contactStore().groups(matching: nil) let groups = allGroups.prefix(200).map { group -> [String: Any] in - ["id": group.identifier, "title": String(group.name.prefix(1024)), "allowsModifications": true] + ["id": group.identifier, "title": bounded(group.name, 1024), "allowsModifications": true] } return ["kind": "command", "command": command.name, "items": groups, "truncated": allGroups.count > 200] case "create_contact_group": @@ -369,8 +398,8 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] { inputDispatched = true try store.execute(request) let display = [given, family].compactMap { $0 }.joined(separator: " ") - let title = String((display.isEmpty ? organization ?? "" : display).prefix(1024)) - return ["kind": "command", "command": command.name, "created": ["id": contact.identifier, "title": title, "list": String((group?.name ?? "Contacts").prefix(1024))]] + let title = bounded((display.isEmpty ? organization ?? "" : display), 1024) + return ["kind": "command", "command": command.name, "created": ["id": contact.identifier, "title": title, "list": bounded((group?.name ?? "Contacts"), 1024)]] default: throw ComputerError("unknown_command", "Unknown app command.", dispatchState: "not_started") } @@ -767,9 +796,9 @@ final class Snapshot { var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } } if role == kAXMenuItemRole, let shortcut = menuShortcut(element) { labels.append(shortcut) } - if !isSecure, !labels.isEmpty { node["label"] = String(labels.joined(separator: " ").prefix(1024)) } + if !isSecure, !labels.isEmpty { node["label"] = bounded(labels.joined(separator: " "), 1024) } if !isSecure, let raw = values[kAXValueAttribute] { - if let value = raw as? String { node["value"] = String(value.prefix(2048)) } + if let value = raw as? String { node["value"] = bounded(value, 2048) } else if let value = raw as? NSNumber { node["value"] = value.stringValue } } if !isSecure { @@ -778,12 +807,12 @@ final class Snapshot { let writableValue = [kAXTextAreaRole, kAXTextFieldRole].contains(role) && AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue node["editable"] = writableValue - if let placeholder = values[kAXPlaceholderValueAttribute] as? String { node["placeholder"] = String(placeholder.prefix(1024)) } + if let placeholder = values[kAXPlaceholderValueAttribute] as? String { node["placeholder"] = bounded(placeholder, 1024) } } if let enabled = values[kAXEnabledAttribute] as? Bool { node["enabled"] = enabled } if let rect = axRect(position: values[kAXPositionAttribute], size: values[kAXSizeAttribute]) { node.merge(rectJSON(rect)) { _, new in new } } var actions: CFArray? - if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { String($0.prefix(128)) } } + if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { bounded($0, 128) } } nodes.append(node) if role == kAXWindowRole, let preferredWindow, !CFEqual(element, preferredWindow) { continue } if !isSecure, entry.depth < 64 { @@ -1076,13 +1105,13 @@ func discoverApps() throws -> [[String: Any]] { enumerator.skipDescendants() guard let bundle = Bundle(url: url), let id = bundle.bundleIdentifier, id.utf8.count <= 255 else { continue } let name = bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String ?? bundle.object(forInfoDictionaryKey: "CFBundleName") as? String ?? url.deletingPathExtension().lastPathComponent - entries[id] = ["bundleId": id, "name": String(name.prefix(1024)), "isActive": false] + entries[id] = ["bundleId": id, "name": bounded(name, 1024), "isActive": false] } else if enumerator.level >= 3 { enumerator.skipDescendants() } } } for app in NSWorkspace.shared.runningApplications where app.activationPolicy == .regular { guard let id = app.bundleIdentifier, id.utf8.count <= 255 else { continue } - entries[id] = ["bundleId": id, "name": String((app.localizedName ?? id).prefix(1024)), "pid": app.processIdentifier, "isActive": app.isActive] + entries[id] = ["bundleId": id, "name": bounded((app.localizedName ?? id), 1024), "pid": app.processIdentifier, "isActive": app.isActive] } return Array(entries.values.sorted { ($0["bundleId"] as? String ?? "") < ($1["bundleId"] as? String ?? "") }.prefix(1000)) } @@ -1146,7 +1175,7 @@ final class Driver { let command = try required(p.command, "command") // EventKit only fetches reminders asynchronously. if command.name == "find_reminders" { return try await findReminders(command) } - return try runAppCommand(command) + return try runAppCommand(command, expected: p.expected) case "activate_app": try requireAccessibility() let app = try await appFor(p.bundleId, launch: true) @@ -1177,7 +1206,7 @@ final class Driver { let matches = axWindows.filter { elementRect($0) == rect } if matches.count == 1, onScreen(entry) { snapshot.windowElements[id.stringValue] = matches[0] } var result: [String: Any] = ["windowId": id.stringValue, "title": entry[kCGWindowName as String] as? String ?? ""] - if let title = result["title"] as? String { result["title"] = String(title.prefix(4096)) } + if let title = result["title"] as? String { result["title"] = bounded(title, 4096) } result.merge(rectJSON(rect)) { _, new in new }; return result } if snapshots.count >= 16 { snapshots.removeAll() } @@ -1199,8 +1228,8 @@ final class Driver { do { guard #available(macOS 14.0, *) else { throw ComputerError("unsupported_os", "Screenshots require macOS 14 or later.") } result["screenshot"] = try await screenshot(pid: app.processIdentifier, windowID: selectedWindowId, observedFrame: try required(snapshot.windowFrames[selectedWindowId], "observed window frame")) - } catch let error as ComputerError { result["screenshotError"] = String((error.code + ": " + error.message).prefix(2000)) } - catch { result["screenshotError"] = String(("capture_failed: " + String(describing: error)).prefix(2000)) } + } catch let error as ComputerError { result["screenshotError"] = bounded((error.code + ": " + error.message), 2000) } + catch { result["screenshotError"] = bounded(("capture_failed: " + String(describing: error)), 2000) } } result["isActive"] = app.isActive return result @@ -1235,7 +1264,7 @@ final class Driver { var completed = 0 func stopped(_ message: String) -> [String: Any] { ["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false, - "sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": String((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch.").prefix(2000))]] + "sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": bounded((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch."), 2000)]] } for (index, step) in steps.enumerated() { // Stop can arrive during the previous step's render wait; no later step may start after it. @@ -1563,7 +1592,7 @@ struct Main { if let dispatchState = error.dispatchState { details["dispatchState"] = dispatchState } response = ["id": id, "error": details] } - catch { response = ["id": id, "error": ["code": "request_failed", "message": String(String(describing: error).prefix(2000))]] } + catch { response = ["id": id, "error": ["code": "request_failed", "message": bounded(String(describing: error), 2000)]] } let data: Data? do { data = try encodeReply(response) } catch let error as ComputerError { data = try? encodeReply(["id": id, "error": ["code": error.code, "message": error.message]]) diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 2e854211c1e..9f6e80a2dde 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -57,6 +57,8 @@ import AppKit expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1", clearDue: true, due: "2027-05-04")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", completed: true)) } precondition(dueText(DateComponents(year: 2027, month: 5, day: 4)) == "2027-05-04") + let padded = String(repeating: "a", count: 7999) + precondition(bounded(padded + "😀", 8000) == padded && bounded("👍🏽ok", 4) == "👍🏽") let localDueText = dueText(try dueComponents("2027-05-04T09:30")); precondition(localDueText == "2027-05-04T09:30:00") var local = Calendar(identifier: .gregorian); local.timeZone = .current let floating = try eventDate("2027-06-09T07:15:00", "Start", allDay: false) diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index 184d520667c..46eb525f7a9 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -301,6 +301,9 @@ describe('native computer use authority and lifecycle', () => { await service.execute('accepted', 'chat', update) expect(sent()).toBe(1) + expect(native.request.mock.lastCall?.[1]).toMatchObject({ + expected: { title: 'Pay rent', due: '2027-05-04', priority: 0, completed: false }, + }) }) it('refuses an app command when the user denies the app, before asking to confirm', async () => { diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 89d94715526..9ec4365ec0a 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -93,7 +93,8 @@ type ReminderItem = NonNullable< * and each field's current and new value, since nothing happens on screen. */ function updateSummary(command: ReminderUpdate, current: ReminderItem): string { - const priority = (value: number) => PRIORITY_NAMES[value] ?? 'None' + const priority = (value: number) => + value === 0 ? 'None' : (PRIORITY_NAMES[value] ?? String(value)) const changes: Array<[string, string | undefined, string]> = [] const title = command.title?.trim() if (title !== undefined && title !== current.title) changes.push(['Title', current.title, title]) @@ -488,9 +489,11 @@ export class ComputerUseService { this.activity = { ...this.activity, appName } this.deps.onActivity(this.activity) } + const current = + command.name === 'update_reminder' ? await this.currentReminder(command.id, work) : undefined const summary = - command.name === 'update_reminder' - ? updateSummary(command, await this.currentReminder(command.id, work)) + command.name === 'update_reminder' && current + ? updateSummary(command, current) : writeSummary(command, await this.destinationName(command, work)) if (summary !== null) { const confirmed = await this.deps.confirmAppCommand( @@ -506,14 +509,19 @@ export class ComputerUseService { }) } this.check(work) - return await this.appCommand(command) + // The helper refuses the update if a field it changes no longer has the value the user approved. + return await this.appCommand(command, current && omit(current, ['id', 'list'])) } /** Runs one app command in the helper and checks the reply is for that command. */ private async appCommand( - command: AppCommand + command: AppCommand, + expected?: Omit ): Promise> { - const result = await this.deps.native.request('app_command', { command }) + const result = await this.deps.native.request( + 'app_command', + expected ? { command, expected } : { command } + ) if (result.kind !== 'command' || result.command !== command.name) throw new Error('Computer Use returned a result for a different command.') return result