From 728cbd3e82d7b06f24aea8dd97072abfd86b005a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 20:19:12 -0800 Subject: [PATCH 1/6] feat(computer-use): find and update calendar events through app commands --- .../native/computer-use/ComputerUse.swift | 117 +++++++++++++-- .../computer-use/tests/NativeTests.swift | 9 ++ .../src/main/computer-use/service.test.ts | 50 +++++++ apps/desktop/src/main/computer-use/service.ts | 135 +++++++++++++----- .../src/computer-use.generated.ts | 67 ++++++++- 5 files changed, 325 insertions(+), 53 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index cc0509d092d..86eec874666 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -42,23 +42,28 @@ struct Parameters: Decodable { var clickCount: Int? var accessibilityAction: String? var command: AppCommand? - var expected: ReminderExpectation? + var expected: Expectation? } -/// The reminder as the user saw it when approving an update; the write is refused if a field it changes has moved since. -struct ReminderExpectation: Decodable { +/// The reminder or event as the user saw it when approving an update; the write is refused if a field it changes has moved since. +struct Expectation: Decodable { var title: String var due: String? - var priority: Int - var completed: Bool + var priority: Int? + var completed: Bool? + var start: String? + var end: String? + var location: String? var notes: String? } /// One allowlisted app command. Commands go through EventKit and Contacts, never Apple Events or -/// scripts, so they cannot reach a shell; each is a typed create, a bounded read, or a reminder -/// update by id. Nothing is ever deleted. +/// scripts, so they cannot reach a shell; each is a typed create, a bounded read, or a reminder or +/// event update by id. Nothing is ever deleted. struct AppCommand: Decodable { var name: String var id: String? var titleContains: String? + var from: String? + var to: String? var includeCompleted: Bool? var completed: Bool? var clearDue: Bool? @@ -243,7 +248,32 @@ func findReminders(_ command: AppCommand) async throws -> [String: Any] { if let contains { matches = matches.filter { ($0.title ?? "").localizedCaseInsensitiveContains(contains) } } return ["kind": "command", "command": command.name, "reminders": matches.prefix(50).map(reminderItem), "truncated": matches.count > 50] } -func runAppCommand(_ command: AppCommand, expected: ReminderExpectation? = nil) throws -> [String: Any] { +/// A date as commands take it: `YYYY-MM-DD` for all-day, else this Mac's local `YYYY-MM-DDTHH:MM:SS`. +func localText(_ date: Date, allDay: Bool) -> String { + let parts = localGregorian().dateComponents([.year, .month, .day, .hour, .minute, .second], from: date) + let day = String(format: "%04d-%02d-%02d", parts.year ?? 0, parts.month ?? 0, parts.day ?? 0) + return allDay ? day : day + String(format: "T%02d:%02d:%02d", parts.hour ?? 0, parts.minute ?? 0, parts.second ?? 0) +} +/// An event as reads and confirmations report it. +func eventItem(_ event: EKEvent) -> [String: Any] { + var item: [String: Any] = [ + "id": event.eventIdentifier ?? event.calendarItemIdentifier, "title": bounded(event.title ?? "", 1024), + "list": bounded(event.calendar.title, 1024), "allDay": event.isAllDay, "recurring": event.hasRecurrenceRules, + "start": localText(event.startDate, allDay: event.isAllDay), "end": localText(event.endDate, allDay: event.isAllDay), + ] + if let location = event.location, !location.isEmpty { item["location"] = bounded(location, 1024) } + if let notes = event.notes, !notes.isEmpty { item["notes"] = bounded(notes, 8000) } + return item +} +/// A non-recurring event's start and end within the same 31-day limit a create has. +func checkEventSpan(_ start: Date, _ end: Date, allDay: Bool) throws { + guard allDay ? end >= start : end > start else { throw invalidCommand("End must be after start.") } + // Calendar days, so a daylight-saving change inside the span does not shorten the limit. + guard let limit = localGregorian().date(byAdding: .day, value: 31, to: start), end <= limit else { + throw invalidCommand("An event can last at most 31 days.") + } +} +func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws -> [String: Any] { switch command.name { case "update_reminder": let id = try commandText(command.id, "id", max: 256) @@ -281,6 +311,71 @@ func runAppCommand(_ command: AppCommand, expected: ReminderExpectation? = nil) inputDispatched = true try store.save(reminder, commit: true) return ["kind": "command", "command": command.name, "updated": ["id": reminder.calendarItemIdentifier, "title": bounded((reminder.title ?? ""), 1024), "list": bounded(reminder.calendar.title, 1024)]] + case "find_events": + let contains = try optionalText(command.titleContains, "titleContains", max: 256) + if let id = command.id { + let store = try calendarStore() + return ["kind": "command", "command": command.name, "events": store.event(withIdentifier: id).map { [eventItem($0)] } ?? [], "truncated": false] + } + guard let fromText = command.from, let toText = command.to else { throw invalidCommand("Pass id, or from and to.") } + let from = try eventDate(fromText, "from", allDay: !fromText.contains("T")) + // A date-only `to` includes that whole day. + let toDay = !toText.contains("T") + let toStart = try eventDate(toText, "to", allDay: toDay) + let to = toDay ? localGregorian().date(byAdding: .day, value: 1, to: toStart) ?? toStart : toStart + try checkEventSpan(from, to, allDay: false) + let store = try calendarStore() + var calendars: [EKCalendar]? + if let calendarId = command.calendarId { + guard let found = store.calendar(withIdentifier: calendarId), found.allowedEntityTypes.contains(.event) else { + throw ComputerError("list_unavailable", "No calendar has this id; call list_calendars.", dispatchState: "not_started") + } + calendars = [found] + } + var matches = store.events(matching: store.predicateForEvents(withStart: from, end: to, calendars: calendars)) + if let contains { matches = matches.filter { ($0.title ?? "").localizedCaseInsensitiveContains(contains) } } + matches.sort { $0.startDate < $1.startDate } + return ["kind": "command", "command": command.name, "events": matches.prefix(50).map(eventItem), "truncated": matches.count > 50] + case "update_event": + let id = try commandText(command.id, "id", max: 256) + let title = try command.title.map { try commandText($0, "Title") } + if let location = command.location, location.count > 1024 { throw invalidCommand("Location is limited to 1024 characters.") } + if let notes = command.notes, notes.count > 8000 { throw invalidCommand("Notes are limited to 8000 characters.") } + guard title != nil || command.start != nil || command.end != nil || command.location != nil || command.notes != nil else { + throw invalidCommand("An update needs at least one field to change.") + } + let store = try calendarStore() + guard let event = store.event(withIdentifier: id) else { + throw ComputerError("not_found", "No event has this id; call find_events.", dispatchState: "not_started") + } + guard !event.hasRecurrenceRules else { + throw ComputerError("recurring_unsupported", "This event repeats; change it in Calendar so the user picks which occurrences.", dispatchState: "not_started") + } + guard event.calendar.allowsContentModifications else { throw ComputerError("list_read_only", "This calendar is read-only.", dispatchState: "not_started") } + guard let currentStart = event.startDate, let currentEnd = event.endDate else { + throw ComputerError("not_found", "This event has no dates; call find_events.", dispatchState: "not_started") + } + let start = try command.start.map { try eventDate($0, "Start", allDay: event.isAllDay) } ?? currentStart + let end = try command.end.map { try eventDate($0, "End", allDay: event.isAllDay) } ?? currentEnd + try checkEventSpan(start, end, allDay: event.isAllDay) + guard let expected else { throw invalidCommand("An update needs the values the user approved.") } + let now = eventItem(event) + let moved = (title != nil && now["title"] as? String != expected.title) + || (command.start != nil && now["start"] as? String != expected.start) + || (command.end != nil && now["end"] as? String != expected.end) + || (command.location != nil && now["location"] as? String != expected.location) + || (command.notes != nil && now["notes"] as? String != expected.notes) + guard !moved else { + throw ComputerError("event_changed", "This event changed while the user was confirming; read it again with find_events before retrying.", dispatchState: "not_started") + } + if let title { event.title = title } + event.startDate = start; event.endDate = end + if let location = command.location { event.location = try optionalText(location, "Location") } + if let notes = command.notes { event.notes = try optionalText(notes, "Notes", max: 8000) } + try checkCancellation() + inputDispatched = true + try store.save(event, span: .thisEvent, commit: true) + return ["kind": "command", "command": command.name, "updated": ["id": event.eventIdentifier ?? id, "title": bounded(event.title ?? "", 1024), "list": bounded(event.calendar.title, 1024)]] case "list_reminder_lists": let allLists = try remindersStore().calendars(for: .reminder) let lists = allLists.prefix(200).map { list -> [String: Any] in @@ -322,11 +417,7 @@ func runAppCommand(_ command: AppCommand, expected: ReminderExpectation? = nil) let allDay = command.allDay ?? false let start = try eventDate(command.start, "Start", allDay: allDay) let end = try eventDate(command.end, "End", allDay: allDay) - guard allDay ? end >= start : end > start else { throw invalidCommand("End must be after start.") } - // Calendar days, so a daylight-saving change inside the event does not shorten the limit. - guard let limit = localGregorian().date(byAdding: .day, value: 31, to: start), end <= limit else { - throw invalidCommand("An event can last at most 31 days.") - } + try checkEventSpan(start, end, allDay: allDay) let location = try optionalText(command.location, "Location") let notes = try optionalText(command.notes, "Notes", max: 8000) if let minutes = command.alertMinutesBefore, !(0...40_320).contains(minutes) { throw invalidCommand("alertMinutesBefore must be 0 to 40320.") } diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 9f6e80a2dde..9dc3088a170 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -56,6 +56,15 @@ import AppKit expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1", priority: 3)) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1", clearDue: true, due: "2027-05-04")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", completed: true)) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_event", id: "e1")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_event", id: "e1", title: " ")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "find_events", from: "2027-06-01")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "find_events", from: "2027-06-01", to: "2027-08-01")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "find_events", from: "2027-06-09T10:00", to: "2027-06-09T09:00")) } + expectError("invalid_arguments") { _ = try checkEventSpan(Date(timeIntervalSince1970: 100), Date(timeIntervalSince1970: 100), allDay: false) } + try checkEventSpan(Date(timeIntervalSince1970: 100), Date(timeIntervalSince1970: 100), allDay: true) + let timedStart = try eventDate("2027-06-09T07:15", "Start", allDay: false), dayStart = try eventDate("2027-06-09", "Start", allDay: true) + precondition(localText(timedStart, allDay: false) == "2027-06-09T07:15:00" && localText(dayStart, allDay: true) == "2027-06-09") precondition(dueText(DateComponents(year: 2027, month: 5, day: 4)) == "2027-05-04") let padded = String(repeating: "a", count: 7999) precondition(bounded(padded + "😀", 8000) == padded && bounded("👍🏽ok", 4) == "👍🏽") diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index 46eb525f7a9..ed7bfab7135 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -306,6 +306,56 @@ describe('native computer use authority and lifecycle', () => { }) }) + it('confirms an event update against its current times, and refuses a repeating event first', async () => { + const { service, native, confirmAppCommand } = setup() + native.request.mockImplementation(async (method, params): Promise => { + if (method !== 'app_command') return { kind: 'apps', apps: [] } + const command = params?.command as { name: string; id: string } + if (command.name === 'find_events') + return { + kind: 'command', + command: 'find_events', + events: [ + { + id: command.id, + title: 'Flight', + list: 'Travel', + start: '2027-06-09T07:15:00', + end: '2027-06-09T08:10:00', + allDay: false, + recurring: command.id === 'weekly', + }, + ], + truncated: false, + } + return { + kind: 'command', + command: 'update_event', + updated: { id: command.id, title: 'Flight', list: 'Travel' }, + } + }) + const update = (id: string) => ({ + action: 'app_command', + command: { name: 'update_event', id, start: '2027-06-09T09:15', end: '2027-06-09T10:10' }, + }) + + await expect(service.execute('weekly', 'chat', update('weekly'))).rejects.toMatchObject({ + details: { code: 'recurring_unsupported', dispatchState: 'not_started' }, + }) + expect(confirmAppCommand).not.toHaveBeenCalled() + + await service.execute('moved', 'chat', update('e1')) + expect(confirmAppCommand.mock.calls[0]?.[0]).toMatchObject({ + change: 'update', + summary: + 'Event: Flight\nCalendar: Travel\nStarts: 2027-06-09T07:15:00 → 2027-06-09T09:15\nEnds: 2027-06-09T08:10:00 → 2027-06-09T10:10', + }) + expect(native.request.mock.lastCall?.[1]).toEqual({ + command: update('e1').command, + expected: { title: 'Flight', start: '2027-06-09T07:15:00', end: '2027-06-09T08:10:00' }, + }) + }) + it('refuses an app command when the user denies the app, before asking to confirm', async () => { const { service, native, approveApp, confirmAppCommand } = setup() approveApp.mockResolvedValue('deny') diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 9ec4365ec0a..027f8142372 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -57,6 +57,8 @@ const APP_COMMAND_APPS: Record = { update_reminder: 'com.apple.reminders', list_calendars: 'com.apple.iCal', create_event: 'com.apple.iCal', + find_events: 'com.apple.iCal', + update_event: 'com.apple.iCal', list_contact_groups: 'com.apple.AddressBook', create_contact_group: 'com.apple.AddressBook', create_contact: 'com.apple.AddressBook', @@ -83,19 +85,32 @@ function chosenDestination( return null } -type ReminderUpdate = Extract -type ReminderItem = NonNullable< - Extract['reminders'] ->[number] +type CommandResult = Extract +type ReminderItem = NonNullable[number] +type EventItem = NonNullable[number] +type Change = [label: string, before: string | undefined, after: string] -/** - * What an update changes, read against the reminder as it is now: the user sees which reminder - * and each field's current and new value, since nothing happens on screen. - */ -function updateSummary(command: ReminderUpdate, current: ReminderItem): string { +/** An update as the user approves it, and the values the helper must still find before it writes. */ +interface UpdateApproval { + summary: string + expected: Record +} + +/** Which item, then each changed field's current and new value, since nothing happens on screen. */ +function changeSummary(header: string[], changes: Change[]): string { + return [ + ...header, + ...changes.map(([label, before, after]) => `${label}: ${before || 'none'} → ${after}`), + ].join('\n') +} + +function reminderChanges( + command: Extract, + current: ReminderItem +): Change[] { const priority = (value: number) => value === 0 ? 'None' : (PRIORITY_NAMES[value] ?? String(value)) - const changes: Array<[string, string | undefined, string]> = [] + const changes: Change[] = [] const title = command.title?.trim() if (title !== undefined && title !== current.title) changes.push(['Title', current.title, title]) if (command.due !== undefined) changes.push(['Due', current.due, command.due]) @@ -110,11 +125,23 @@ function updateSummary(command: ReminderUpdate, current: ReminderItem): string { ]) if (command.notes !== undefined) changes.push(['Notes', current.notes, command.notes.trim() || 'none']) - return [ - `Reminder: ${current.title}`, - `List: ${current.list}`, - ...changes.map(([label, before, after]) => `${label}: ${before || 'none'} → ${after}`), - ].join('\n') + return changes +} + +function eventChanges( + command: Extract, + current: EventItem +): Change[] { + const changes: Change[] = [] + const title = command.title?.trim() + if (title !== undefined && title !== current.title) changes.push(['Title', current.title, title]) + if (command.start !== undefined) changes.push(['Starts', current.start, command.start]) + if (command.end !== undefined) changes.push(['Ends', current.end, command.end]) + if (command.location !== undefined) + changes.push(['Location', current.location, command.location.trim() || 'none']) + if (command.notes !== undefined) + changes.push(['Notes', current.notes, command.notes.trim() || 'none']) + return changes } /** @@ -170,6 +197,14 @@ function writeSummary(command: AppCommand, destination: string | undefined): str } } +function notFound(kind: string, read: string): ComputerUseError { + return new ComputerUseError({ + code: 'not_found', + message: `No ${kind} has this id; call ${read}.`, + dispatchState: 'not_started', + }) +} + /** Native snapshots cannot transfer authority between chats or restarted helpers. */ export class ComputerUseService { private admissions = new Map() @@ -489,15 +524,13 @@ export class ComputerUseService { this.activity = { ...this.activity, appName } this.deps.onActivity(this.activity) } - const current = - command.name === 'update_reminder' ? await this.currentReminder(command.id, work) : undefined - const summary = - command.name === 'update_reminder' && current - ? updateSummary(command, current) - : writeSummary(command, await this.destinationName(command, work)) + const update = await this.updateApproval(command, work) + const summary = update + ? update.summary + : writeSummary(command, await this.destinationName(command, work)) if (summary !== null) { const confirmed = await this.deps.confirmAppCommand( - { appName, summary, change: command.name === 'update_reminder' ? 'update' : 'add' }, + { appName, summary, change: update ? 'update' : 'add' }, work.controller.signal ) this.check(work) @@ -510,14 +543,14 @@ export class ComputerUseService { } this.check(work) // The helper refuses the update if a field it changes no longer has the value the user approved. - return await this.appCommand(command, current && omit(current, ['id', 'list'])) + return await this.appCommand(command, update?.expected) } /** Runs one app command in the helper and checks the reply is for that command. */ private async appCommand( command: AppCommand, - expected?: Omit - ): Promise> { + expected?: Record + ): Promise { const result = await this.deps.native.request( 'app_command', expected ? { command, expected } : { command } @@ -527,17 +560,47 @@ export class ComputerUseService { return result } - /** The reminder an update names, read when it is confirmed; an unknown id is refused before the user is asked. */ - private async currentReminder(id: string, work: ToolWork): Promise { - const found = await this.appCommand({ name: 'find_reminders', id }) - this.check(work) - const reminder = found.reminders?.find((item) => item.id === id) - if (reminder) return reminder - throw new ComputerUseError({ - code: 'not_found', - message: 'No reminder has this id; call find_reminders.', - dispatchState: 'not_started', - }) + /** + * For an update, reads the item it names when it is confirmed. An unknown id, or a repeating + * event (whose occurrences the user must choose in Calendar), is refused before the user is asked. + */ + private async updateApproval( + command: AppCommand, + work: ToolWork + ): Promise { + if (command.name === 'update_reminder') { + const found = await this.appCommand({ name: 'find_reminders', id: command.id }) + this.check(work) + const current = found.reminders?.find((item) => item.id === command.id) + if (!current) throw notFound('reminder', 'find_reminders') + return { + summary: changeSummary( + [`Reminder: ${current.title}`, `List: ${current.list}`], + reminderChanges(command, current) + ), + expected: omit(current, ['id', 'list']), + } + } + if (command.name === 'update_event') { + const found = await this.appCommand({ name: 'find_events', id: command.id }) + this.check(work) + const current = found.events?.find((item) => item.id === command.id) + if (!current) throw notFound('event', 'find_events') + if (current.recurring) + throw new ComputerUseError({ + code: 'recurring_unsupported', + message: 'This event repeats; change it in Calendar so the user picks which occurrences.', + dispatchState: 'not_started', + }) + return { + summary: changeSummary( + [`Event: ${current.title}`, `Calendar: ${current.list}`], + eventChanges(command, current) + ), + expected: omit(current, ['id', 'list', 'allDay', 'recurring']), + } + } + return undefined } /** diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index 0b049f86cb0..56643c5dbc6 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -255,7 +255,7 @@ const EventBoundary = DateOrDateTime.describe( "ISO 8601 date-time, or YYYY-MM-DD when allDay is true. Without an offset it is this Mac's local time, as typing it into Calendar would be.", ); -/** Typed app commands through the OS's own data frameworks, never a script: allowlisted creates, bounded reads and reminder updates by id. Nothing is deleted. */ +/** Typed app commands through the OS's own data frameworks, never a script: allowlisted creates, bounded reads and reminder or event updates by id. Nothing is deleted. */ export const AppCommandSchema = z.discriminatedUnion("name", [ z.strictObject({ name: z.literal("list_reminder_lists") }), z.strictObject({ @@ -331,6 +331,46 @@ export const AppCommandSchema = z.discriminatedUnion("name", [ .describe("An id from list_calendars; omit for the default calendar."), alertMinutesBefore: z.number().int().min(0).max(40_320).optional(), }), + z + .strictObject({ + name: z.literal("find_events"), + id: z.string().min(1).max(256).optional().describe("Read just this event."), + calendarId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_calendars; omit for every calendar."), + from: DateOrDateTime.optional().describe("Range start (this Mac's local time without an offset)."), + to: DateOrDateTime.optional().describe( + "Range end, at most 31 days after from; a YYYY-MM-DD end includes that whole day.", + ), + titleContains: z.string().min(1).max(256).optional().describe("Case-insensitive part of the title."), + }) + .refine((find) => find.id !== undefined || (find.from !== undefined && find.to !== undefined), { + message: "Pass id, or from and to", + }), + z + .strictObject({ + name: z.literal("update_event"), + id: z + .string() + .min(1) + .max(256) + .describe("An id from find_events or create_event; repeating events are refused."), + title: CommandTitle.optional(), + start: EventBoundary.optional(), + end: EventBoundary.optional(), + location: z.string().max(1024).optional().describe("Replaces the location; an empty string clears it."), + notes: CommandNotes.describe("Replaces the notes; an empty string clears them."), + }) + .refine( + (update) => + [update.title, update.start, update.end, update.location, update.notes].some( + (value) => value !== undefined, + ), + { message: "An update needs at least one field to change" }, + ), z.strictObject({ name: z.literal("list_contact_groups") }), z.strictObject({ name: z.literal("create_contact_group"), @@ -462,6 +502,18 @@ const ReminderItem = z.strictObject({ notes: z.string().max(8000).optional(), }); +const EventItem = z.strictObject({ + id: z.string().min(1).max(256), + title: z.string().max(1024), + list: z.string().max(1024), + start: z.string().max(64), + end: z.string().max(64), + allDay: z.boolean(), + recurring: z.boolean(), + location: z.string().max(1024).optional(), + notes: z.string().max(8000).optional(), +}); + const SavedItem = z.strictObject({ id: z.string().min(1).max(256), title: z.string().max(1024), @@ -479,6 +531,8 @@ export const ComputerUseCommandResultSchema = z "update_reminder", "list_calendars", "create_event", + "find_events", + "update_event", "list_contact_groups", "create_contact_group", "create_contact", @@ -487,20 +541,25 @@ export const ComputerUseCommandResultSchema = z updated: SavedItem.optional(), items: z.array(CommandItem).max(200).optional(), reminders: z.array(ReminderItem).max(50).optional(), + events: z.array(EventItem).max(50).optional(), truncated: z .boolean() .optional() - .describe("True when more items exist than the read returns (200 for lists, 50 for reminders)."), + .describe( + "True when more items exist than the read returns (200 for lists, 50 for reminders or events).", + ), }) .refine( (result) => { if (result.command.startsWith("create_")) return result.created !== undefined; if (result.command.startsWith("update_")) return result.updated !== undefined; - if (result.command.startsWith("find_")) return result.reminders !== undefined; + if (result.command === "find_reminders") return result.reminders !== undefined; + if (result.command === "find_events") return result.events !== undefined; return result.items !== undefined; }, { - message: "A create returns created, an update updated, a find reminders, and a list items", + message: + "A create returns created, an update updated, a find its reminders or events, and a list items", }, ); From 24f0accd7fd37234cc41c2c84d71b22474fcf4cd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 20:55:33 -0800 Subject: [PATCH 2/6] fix(computer-use): refuse unverifiable long fields, keep long events editable, one line per confirmed value --- .../native/computer-use/ComputerUse.swift | 11 ++++++- .../computer-use/tests/NativeTests.swift | 1 + .../src/main/computer-use/service.test.ts | 10 +++++-- apps/desktop/src/main/computer-use/service.ts | 29 +++++++++++++++---- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 86eec874666..fd8f40367a9 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -265,6 +265,8 @@ func eventItem(_ event: EKEvent) -> [String: Any] { if let notes = event.notes, !notes.isEmpty { item["notes"] = bounded(notes, 8000) } return item } +/// Whether a read had to shorten this value; an update cannot check a value it never saw in full. +func shortened(_ value: String?, _ max: Int) -> Bool { (value?.utf16.count ?? 0) > max } /// A non-recurring event's start and end within the same 31-day limit a create has. func checkEventSpan(_ start: Date, _ end: Date, allDay: Bool) throws { guard allDay ? end >= start : end > start else { throw invalidCommand("End must be after start.") } @@ -292,6 +294,9 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - } guard reminder.calendar.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") } guard let expected else { throw invalidCommand("An update needs the values the user approved.") } + if (title != nil && shortened(reminder.title, 1024)) || (command.notes != nil && shortened(reminder.notes, 8000)) { + throw ComputerError("too_long_to_compare", "This reminder's title or notes are too long to show in full; change them in Reminders.", dispatchState: "not_started") + } let now = reminderItem(reminder) let moved = (title != nil && now["title"] as? String != expected.title) || ((due != nil || command.clearDue == true) && now["due"] as? String != expected.due) @@ -357,8 +362,12 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - } let start = try command.start.map { try eventDate($0, "Start", allDay: event.isAllDay) } ?? currentStart let end = try command.end.map { try eventDate($0, "End", allDay: event.isAllDay) } ?? currentEnd - try checkEventSpan(start, end, allDay: event.isAllDay) + // An existing event may already be longer than a new one can be; only a new time is checked. + if command.start != nil || command.end != nil { try checkEventSpan(start, end, allDay: event.isAllDay) } guard let expected else { throw invalidCommand("An update needs the values the user approved.") } + if (title != nil && shortened(event.title, 1024)) || (command.location != nil && shortened(event.location, 1024)) || (command.notes != nil && shortened(event.notes, 8000)) { + throw ComputerError("too_long_to_compare", "This event's title, location or notes are too long to show in full; change them in Calendar.", dispatchState: "not_started") + } let now = eventItem(event) let moved = (title != nil && now["title"] as? String != expected.title) || (command.start != nil && now["start"] as? String != expected.start) diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 9dc3088a170..19542a95969 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -67,6 +67,7 @@ import AppKit precondition(localText(timedStart, allDay: false) == "2027-06-09T07:15:00" && localText(dayStart, allDay: true) == "2027-06-09") precondition(dueText(DateComponents(year: 2027, month: 5, day: 4)) == "2027-05-04") let padded = String(repeating: "a", count: 7999) + precondition(shortened(padded + "😀", 8000) && !shortened(padded, 8000) && !shortened(nil, 8000)) precondition(bounded(padded + "😀", 8000) == padded && bounded("👍🏽ok", 4) == "👍🏽") let localDueText = dueText(try dueComponents("2027-05-04T09:30")); precondition(localDueText == "2027-05-04T09:30:00") var local = Calendar(identifier: .gregorian); local.timeZone = .current diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index ed7bfab7135..451d3910b05 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -318,7 +318,7 @@ describe('native computer use authority and lifecycle', () => { events: [ { id: command.id, - title: 'Flight', + title: 'Flight\nCalendar: Work', list: 'Travel', start: '2027-06-09T07:15:00', end: '2027-06-09T08:10:00', @@ -348,11 +348,15 @@ describe('native computer use authority and lifecycle', () => { expect(confirmAppCommand.mock.calls[0]?.[0]).toMatchObject({ change: 'update', summary: - 'Event: Flight\nCalendar: Travel\nStarts: 2027-06-09T07:15:00 → 2027-06-09T09:15\nEnds: 2027-06-09T08:10:00 → 2027-06-09T10:10', + 'Event: Flight Calendar: Work\nCalendar: Travel\nWhen: 2027-06-09T07:15:00 – 2027-06-09T08:10:00\nStarts: 2027-06-09T07:15:00 → 2027-06-09T09:15\nEnds: 2027-06-09T08:10:00 → 2027-06-09T10:10', }) expect(native.request.mock.lastCall?.[1]).toEqual({ command: update('e1').command, - expected: { title: 'Flight', start: '2027-06-09T07:15:00', end: '2027-06-09T08:10:00' }, + expected: { + title: 'Flight\nCalendar: Work', + start: '2027-06-09T07:15:00', + end: '2027-06-09T08:10:00', + }, }) }) diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 027f8142372..4baf6976037 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -96,11 +96,21 @@ interface UpdateApproval { expected: Record } +/** + * One line per value: a title or note from a shared calendar could otherwise add lines that read + * like extra fields in the confirmation. + */ +function oneLine(value: string): string { + return value.replace(/[\u0000-\u001f\u007f\u2028\u2029]+/g, ' ').trim() +} + /** Which item, then each changed field's current and new value, since nothing happens on screen. */ -function changeSummary(header: string[], changes: Change[]): string { +function changeSummary(header: Array<[label: string, value: string]>, changes: Change[]): string { return [ - ...header, - ...changes.map(([label, before, after]) => `${label}: ${before || 'none'} → ${after}`), + ...header.map(([label, value]) => `${label}: ${oneLine(value)}`), + ...changes.map( + ([label, before, after]) => `${label}: ${oneLine(before || 'none')} → ${oneLine(after)}` + ), ].join('\n') } @@ -154,7 +164,7 @@ function writeSummary(command: AppCommand, destination: string | undefined): str entries .map(([label, value]) => [label, typeof value === 'string' ? value.trim() : value] as const) .filter(([, value]) => value !== undefined && value !== '') - .map(([label, value]) => `${label}: ${value}`) + .map(([label, value]) => `${label}: ${typeof value === 'string' ? oneLine(value) : value}`) .join('\n') switch (command.name) { case 'create_reminder': @@ -575,7 +585,10 @@ export class ComputerUseService { if (!current) throw notFound('reminder', 'find_reminders') return { summary: changeSummary( - [`Reminder: ${current.title}`, `List: ${current.list}`], + [ + ['Reminder', current.title], + ['List', current.list], + ], reminderChanges(command, current) ), expected: omit(current, ['id', 'list']), @@ -594,7 +607,11 @@ export class ComputerUseService { }) return { summary: changeSummary( - [`Event: ${current.title}`, `Calendar: ${current.list}`], + [ + ['Event', current.title], + ['Calendar', current.list], + ['When', `${current.start} – ${current.end}`], + ], eventChanges(command, current) ), expected: omit(current, ['id', 'list', 'allDay', 'recurring']), From 9c8def78341ec9739765d9cf6b0ab6d0d81910d2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 21:07:46 -0800 Subject: [PATCH 3/6] feat(computer-use): find contacts and add details to them through app commands --- .../native/computer-use/ComputerUse.swift | 73 +++++++++++++++++-- .../computer-use/tests/NativeTests.swift | 3 + .../src/main/computer-use/service.test.ts | 45 ++++++++++++ apps/desktop/src/main/computer-use/service.ts | 42 +++++++++-- .../src/computer-use.generated.ts | 51 +++++++++++-- 5 files changed, 198 insertions(+), 16 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index fd8f40367a9..933f68da58f 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -56,12 +56,13 @@ struct Expectation: Decodable { var notes: String? } /// One allowlisted app command. Commands go through EventKit and Contacts, never Apple Events or -/// scripts, so they cannot reach a shell; each is a typed create, a bounded read, or a reminder or -/// event update by id. Nothing is ever deleted. +/// scripts, so they cannot reach a shell; each is a typed create, a bounded read, a reminder or +/// event update by id, or additions to a contact. Nothing is ever deleted or overwritten on a contact. struct AppCommand: Decodable { var name: String var id: String? var titleContains: String? + var nameContains: String? var from: String? var to: String? var includeCompleted: Bool? @@ -275,6 +276,26 @@ func checkEventSpan(_ start: Date, _ end: Date, allDay: Bool) throws { throw invalidCommand("An event can last at most 31 days.") } } +/// The keys a contact read or confirmation needs: its name, organization, emails and phones. +let contactReadKeys: [CNKeyDescriptor] = [ + CNContactFormatter.descriptorForRequiredKeys(for: .fullName), CNContactOrganizationNameKey as CNKeyDescriptor, + CNContactEmailAddressesKey as CNKeyDescriptor, CNContactPhoneNumbersKey as CNKeyDescriptor, +] +/// A contact as reads report it: its name only, never its emails, phones or notes. +func contactItem(_ contact: CNContact) -> [String: Any] { + let name = CNContactFormatter.string(from: contact, style: .fullName) ?? "" + var item: [String: Any] = ["id": contact.identifier, "title": bounded(name.isEmpty ? contact.organizationName : name, 1024)] + if !contact.organizationName.isEmpty { item["organization"] = bounded(contact.organizationName, 1024) } + return item +} +/// Emails and phones a command adds, checked as a create checks them. +func contactValues(_ command: AppCommand) throws -> (emails: [String], phones: [String]) { + let emails = try (command.emails ?? []).map { try commandText($0, "Email", max: 320) } + let phones = try (command.phones ?? []).map { try commandText($0, "Phone", max: 64) } + guard emails.count <= 5, phones.count <= 5 else { throw invalidCommand("A contact takes at most 5 emails and 5 phones.") } + guard emails.allSatisfy({ $0.contains("@") }) else { throw invalidCommand("Each email needs an @.") } + return (emails, phones) +} func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws -> [String: Any] { switch command.name { case "update_reminder": @@ -385,6 +406,49 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - inputDispatched = true try store.save(event, span: .thisEvent, commit: true) return ["kind": "command", "command": command.name, "updated": ["id": event.eventIdentifier ?? id, "title": bounded(event.title ?? "", 1024), "list": bounded(event.calendar.title, 1024)]] + case "find_contacts": + let store = try contactStore() + if let id = command.id { + let found = try? store.unifiedContact(withIdentifier: id, keysToFetch: contactReadKeys) + return ["kind": "command", "command": command.name, "contacts": found.map { [contactItem($0)] } ?? [], "truncated": false] + } + let name = try commandText(command.nameContains, "nameContains", max: 256) + guard name.count >= 2 else { throw invalidCommand("nameContains needs at least 2 characters.") } + let matches = try store.unifiedContacts(matching: CNContact.predicateForContacts(matchingName: name), keysToFetch: contactReadKeys) + return ["kind": "command", "command": command.name, "contacts": matches.prefix(50).map(contactItem), "truncated": matches.count > 50] + case "update_contact": + let id = try commandText(command.id, "id", max: 256) + let (emails, phones) = try contactValues(command) + guard !emails.isEmpty || !phones.isEmpty || command.groupId != nil else { throw invalidCommand("An update needs an email, phone or group to add.") } + let store = try contactStore() + guard let contact = try? store.unifiedContact(withIdentifier: id, keysToFetch: contactReadKeys), let changed = contact.mutableCopy() as? CNMutableContact else { + throw ComputerError("not_found", "No contact has this id; call find_contacts.", dispatchState: "not_started") + } + var group: CNGroup? + if let groupId = command.groupId { + guard let found = try store.groups(matching: CNGroup.predicateForGroups(withIdentifiers: [groupId])).first else { + throw ComputerError("list_unavailable", "No contact group has this id; call list_contact_groups.", dispatchState: "not_started") + } + // A group's members must live in the group's own account. + let groupAccount = try store.containers(matching: CNContainer.predicateForContainerOfGroup(withIdentifier: found.identifier)).first?.identifier + let contactAccount = try store.containers(matching: CNContainer.predicateForContainerOfContact(withIdentifier: contact.identifier)).first?.identifier + guard groupAccount != nil, groupAccount == contactAccount else { + throw ComputerError("group_other_account", "This group is in a different account from the contact; add it in Contacts.", dispatchState: "not_started") + } + group = found + } + // Only additions: existing values stay, and a value the contact already has is not added twice. + let knownEmails = Set(contact.emailAddresses.map { ($0.value as String).lowercased() }) + let knownPhones = Set(contact.phoneNumbers.map { $0.value.stringValue.filter(\.isNumber) }) + changed.emailAddresses += emails.filter { !knownEmails.contains($0.lowercased()) }.map { CNLabeledValue(label: CNLabelWork, value: $0 as NSString) } + changed.phoneNumbers += phones.filter { !knownPhones.contains($0.filter(\.isNumber)) }.map { CNLabeledValue(label: CNLabelPhoneNumberMain, value: CNPhoneNumber(stringValue: $0)) } + let request = CNSaveRequest(); request.update(changed) + if let group { request.addMember(contact, to: group) } + try checkCancellation() + inputDispatched = true + try store.execute(request) + let item = contactItem(contact) + return ["kind": "command", "command": command.name, "updated": ["id": contact.identifier, "title": item["title"] ?? "", "list": bounded(group?.name ?? "Contacts", 1024)]] case "list_reminder_lists": let allLists = try remindersStore().calendars(for: .reminder) let lists = allLists.prefix(200).map { list -> [String: Any] in @@ -473,10 +537,7 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - let given = try optionalText(command.givenName, "Given name"), family = try optionalText(command.familyName, "Family name") let organization = try optionalText(command.organization, "Organization"), jobTitle = try optionalText(command.jobTitle, "Job title") guard given != nil || family != nil || organization != nil else { throw invalidCommand("A contact needs a name or an organization.") } - let emails = try (command.emails ?? []).map { try commandText($0, "Email", max: 320) } - let phones = try (command.phones ?? []).map { try commandText($0, "Phone", max: 64) } - guard emails.count <= 5, phones.count <= 5 else { throw invalidCommand("A contact takes at most 5 emails and 5 phones.") } - guard emails.allSatisfy({ $0.contains("@") }) else { throw invalidCommand("Each email needs an @.") } + let (emails, phones) = try contactValues(command) let store = try contactStore() var group: CNGroup? if let groupId = command.groupId { diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 19542a95969..279f7dfddd8 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -57,6 +57,9 @@ import AppKit expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", id: "r1", clearDue: true, due: "2027-05-04")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_reminder", completed: true)) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_event", id: "e1")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_contact", id: "c1")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_contact", id: "c1", emails: ["not-an-email"])) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_contact", id: "c1", phones: ["1", "2", "3", "4", "5", "6"])) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "update_event", id: "e1", title: " ")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "find_events", from: "2027-06-01")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "find_events", from: "2027-06-01", to: "2027-08-01")) } diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index 451d3910b05..a7fe0777e72 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -360,6 +360,51 @@ describe('native computer use authority and lifecycle', () => { }) }) + it('confirms what a contact update adds, by name and group, and refuses an unknown id first', async () => { + const { service, native, confirmAppCommand } = setup() + native.request.mockImplementation(async (method, params): Promise => { + if (method !== 'app_command') return { kind: 'apps', apps: [] } + const command = params?.command as { name: string; id: string } + if (command.name === 'find_contacts') + return { + kind: 'command', + command: 'find_contacts', + contacts: + command.id === 'c1' ? [{ id: 'c1', title: 'Ana Ruiz', organization: 'Acme' }] : [], + truncated: false, + } + if (command.name === 'list_contact_groups') + return { + kind: 'command', + command: 'list_contact_groups', + items: [{ id: 'g1', title: 'Hires', allowsModifications: true }], + truncated: false, + } + return { + kind: 'command', + command: 'update_contact', + updated: { id: 'c1', title: 'Ana Ruiz', list: 'Hires' }, + } + }) + const update = (id: string) => ({ + action: 'app_command', + command: { name: 'update_contact', id, phones: [' +1 555 0100 '], groupId: 'g1' }, + }) + + await expect(service.execute('unknown', 'chat', update('gone'))).rejects.toMatchObject({ + details: { code: 'not_found', dispatchState: 'not_started' }, + }) + expect(confirmAppCommand).not.toHaveBeenCalled() + + await service.execute('added', 'chat', update('c1')) + expect(confirmAppCommand.mock.calls[0]?.[0]).toMatchObject({ + change: 'update', + summary: + 'Contact: Ana Ruiz\nOrganization: Acme\nAdd phones: +1 555 0100\nAdd to group: Hires', + }) + expect(native.request.mock.lastCall?.[1]).toEqual({ command: update('c1').command }) + }) + it('refuses an app command when the user denies the app, before asking to confirm', async () => { const { service, native, approveApp, confirmAppCommand } = setup() approveApp.mockResolvedValue('deny') diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 4baf6976037..ab9ebf81ab1 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -62,6 +62,8 @@ const APP_COMMAND_APPS: Record = { list_contact_groups: 'com.apple.AddressBook', create_contact_group: 'com.apple.AddressBook', create_contact: 'com.apple.AddressBook', + find_contacts: 'com.apple.AddressBook', + update_contact: 'com.apple.AddressBook', } const PRIORITY_NAMES: Record = { 1: 'High', 5: 'Medium', 9: 'Low' } @@ -80,7 +82,7 @@ function chosenDestination( return { id: command.listId, list: 'list_reminder_lists' } if (command.name === 'create_event' && command.calendarId) return { id: command.calendarId, list: 'list_calendars' } - if (command.name === 'create_contact' && command.groupId) + if ((command.name === 'create_contact' || command.name === 'update_contact') && command.groupId) return { id: command.groupId, list: 'list_contact_groups' } return null } @@ -90,10 +92,13 @@ type ReminderItem = NonNullable[number] type EventItem = NonNullable[number] type Change = [label: string, before: string | undefined, after: string] -/** An update as the user approves it, and the values the helper must still find before it writes. */ +/** + * An update as the user approves it, and the values the helper must still find before it writes; + * a contact update only adds, so it has none to check. + */ interface UpdateApproval { summary: string - expected: Record + expected?: Record } /** @@ -104,13 +109,21 @@ function oneLine(value: string): string { return value.replace(/[\u0000-\u001f\u007f\u2028\u2029]+/g, ' ').trim() } -/** Which item, then each changed field's current and new value, since nothing happens on screen. */ -function changeSummary(header: Array<[label: string, value: string]>, changes: Change[]): string { +/** + * Which item, then each changed field's current and new value, then anything only added, since + * nothing happens on screen. + */ +function changeSummary( + header: Array<[label: string, value: string]>, + changes: Change[], + additions: Array<[label: string, value: string | undefined]> = [] +): string { return [ ...header.map(([label, value]) => `${label}: ${oneLine(value)}`), ...changes.map( ([label, before, after]) => `${label}: ${oneLine(before || 'none')} → ${oneLine(after)}` ), + ...additions.flatMap(([label, value]) => (value ? [`${label}: ${oneLine(value)}`] : [])), ].join('\n') } @@ -617,6 +630,25 @@ export class ComputerUseService { expected: omit(current, ['id', 'list', 'allDay', 'recurring']), } } + if (command.name === 'update_contact') { + const found = await this.appCommand({ name: 'find_contacts', id: command.id }) + this.check(work) + const current = found.contacts?.find((item) => item.id === command.id) + if (!current) throw notFound('contact', 'find_contacts') + const header: Array<[string, string]> = [['Contact', current.title]] + if (current.organization) header.push(['Organization', current.organization]) + return { + summary: changeSummary( + header, + [], + [ + ['Add emails', command.emails?.map((email) => email.trim()).join(', ')], + ['Add phones', command.phones?.map((phone) => phone.trim()).join(', ')], + ['Add to group', await this.destinationName(command, work)], + ] + ), + } + } return undefined } diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index 56643c5dbc6..29908ea9ba6 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -255,7 +255,7 @@ const EventBoundary = DateOrDateTime.describe( "ISO 8601 date-time, or YYYY-MM-DD when allDay is true. Without an offset it is this Mac's local time, as typing it into Calendar would be.", ); -/** Typed app commands through the OS's own data frameworks, never a script: allowlisted creates, bounded reads and reminder or event updates by id. Nothing is deleted. */ +/** Typed app commands through the OS's own data frameworks, never a script: allowlisted creates, bounded reads, reminder or event updates by id, and additions to a contact. Nothing is deleted. */ export const AppCommandSchema = z.discriminatedUnion("name", [ z.strictObject({ name: z.literal("list_reminder_lists") }), z.strictObject({ @@ -401,6 +401,39 @@ export const AppCommandSchema = z.discriminatedUnion("name", [ message: "A contact needs a name or an organization", }, ), + z + .strictObject({ + name: z.literal("find_contacts"), + id: z.string().min(1).max(256).optional().describe("Read just this contact."), + nameContains: z.string().min(2).max(256).optional().describe("Part of a person's name."), + }) + .refine((find) => find.id !== undefined || find.nameContains !== undefined, { + message: "Pass id or nameContains", + }), + z + .strictObject({ + name: z.literal("update_contact"), + id: z.string().min(1).max(256).describe("An id from find_contacts or create_contact."), + emails: z + .array(z.string().min(3).max(320)) + .max(5) + .optional() + .describe("Emails to add; existing ones stay."), + phones: z + .array(z.string().min(1).max(64)) + .max(5) + .optional() + .describe("Phones to add; existing ones stay."), + groupId: z + .string() + .min(1) + .max(256) + .optional() + .describe("A group from list_contact_groups or create_contact_group to add the contact to."), + }) + .refine((update) => Boolean(update.emails?.length || update.phones?.length || update.groupId), { + message: "An update needs an email, phone or group to add", + }), ]); export type AppCommand = z.infer; @@ -514,6 +547,12 @@ const EventItem = z.strictObject({ notes: z.string().max(8000).optional(), }); +const ContactItem = z.strictObject({ + id: z.string().min(1).max(256), + title: z.string().max(1024), + organization: z.string().max(1024).optional(), +}); + const SavedItem = z.strictObject({ id: z.string().min(1).max(256), title: z.string().max(1024), @@ -536,18 +575,19 @@ export const ComputerUseCommandResultSchema = z "list_contact_groups", "create_contact_group", "create_contact", + "find_contacts", + "update_contact", ]), created: SavedItem.optional(), updated: SavedItem.optional(), items: z.array(CommandItem).max(200).optional(), reminders: z.array(ReminderItem).max(50).optional(), events: z.array(EventItem).max(50).optional(), + contacts: z.array(ContactItem).max(50).optional(), truncated: z .boolean() .optional() - .describe( - "True when more items exist than the read returns (200 for lists, 50 for reminders or events).", - ), + .describe("True when more items exist than the read returns (200 for lists, 50 for finds)."), }) .refine( (result) => { @@ -555,11 +595,12 @@ export const ComputerUseCommandResultSchema = z if (result.command.startsWith("update_")) return result.updated !== undefined; if (result.command === "find_reminders") return result.reminders !== undefined; if (result.command === "find_events") return result.events !== undefined; + if (result.command === "find_contacts") return result.contacts !== undefined; return result.items !== undefined; }, { message: - "A create returns created, an update updated, a find its reminders or events, and a list items", + "A create returns created, an update updated, a find its reminders, events or contacts, and a list items", }, ); From d5f6e5da0104a2edde090de71bd516852aa6921e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 21:19:43 -0800 Subject: [PATCH 4/6] fix(computer-use): keep dialing controls when de-duplicating phones, and de-duplicate within a request --- .../desktop/native/computer-use/ComputerUse.swift | 15 ++++++++++----- .../native/computer-use/tests/NativeTests.swift | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 933f68da58f..a2b462a8588 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -288,6 +288,11 @@ func contactItem(_ contact: CNContact) -> [String: Any] { if !contact.organizationName.isEmpty { item["organization"] = bounded(contact.organizationName, 1024) } return item } +/// An email compared without case. +func emailKey(_ email: String) -> String { email.lowercased() } +/// A phone compared without its formatting, but with its dialing controls: `5550100,123` pauses +/// before the extension and `5550100;123` waits, so they are different numbers. +func phoneKey(_ phone: String) -> String { phone.filter { $0.isASCII && ($0.isNumber || "+*#,;".contains($0)) } } /// Emails and phones a command adds, checked as a create checks them. func contactValues(_ command: AppCommand) throws -> (emails: [String], phones: [String]) { let emails = try (command.emails ?? []).map { try commandText($0, "Email", max: 320) } @@ -437,11 +442,11 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - } group = found } - // Only additions: existing values stay, and a value the contact already has is not added twice. - let knownEmails = Set(contact.emailAddresses.map { ($0.value as String).lowercased() }) - let knownPhones = Set(contact.phoneNumbers.map { $0.value.stringValue.filter(\.isNumber) }) - changed.emailAddresses += emails.filter { !knownEmails.contains($0.lowercased()) }.map { CNLabeledValue(label: CNLabelWork, value: $0 as NSString) } - changed.phoneNumbers += phones.filter { !knownPhones.contains($0.filter(\.isNumber)) }.map { CNLabeledValue(label: CNLabelPhoneNumberMain, value: CNPhoneNumber(stringValue: $0)) } + // Only additions: existing values stay, and no value is added twice, from the contact or the request. + var knownEmails = Set(contact.emailAddresses.map { emailKey($0.value as String) }) + var knownPhones = Set(contact.phoneNumbers.map { phoneKey($0.value.stringValue) }) + changed.emailAddresses += emails.filter { knownEmails.insert(emailKey($0)).inserted }.map { CNLabeledValue(label: CNLabelWork, value: $0 as NSString) } + changed.phoneNumbers += phones.filter { knownPhones.insert(phoneKey($0)).inserted }.map { CNLabeledValue(label: CNLabelPhoneNumberMain, value: CNPhoneNumber(stringValue: $0)) } let request = CNSaveRequest(); request.update(changed) if let group { request.addMember(contact, to: group) } try checkCancellation() diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 279f7dfddd8..34d190085f6 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -70,6 +70,7 @@ import AppKit precondition(localText(timedStart, allDay: false) == "2027-06-09T07:15:00" && localText(dayStart, allDay: true) == "2027-06-09") precondition(dueText(DateComponents(year: 2027, month: 5, day: 4)) == "2027-05-04") let padded = String(repeating: "a", count: 7999) + precondition(phoneKey("+1 (555) 010-0100") == phoneKey("+15550100100") && phoneKey("5550100,123") != phoneKey("5550100;123") && emailKey("Ana@Example.com") == emailKey("ana@example.com")) precondition(shortened(padded + "😀", 8000) && !shortened(padded, 8000) && !shortened(nil, 8000)) precondition(bounded(padded + "😀", 8000) == padded && bounded("👍🏽ok", 4) == "👍🏽") let localDueText = dueText(try dueComponents("2027-05-04T09:30")); precondition(localDueText == "2027-05-04T09:30:00") From d27f17dbb80870915ef3a0e82db2de7ff37872ec Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 21:31:15 -0800 Subject: [PATCH 5/6] fix(computer-use): read phone digits from any script, and report Contacts failures as failures --- .../native/computer-use/ComputerUse.swift | 17 ++++++++++++++--- .../native/computer-use/tests/NativeTests.swift | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index a2b462a8588..7f73664232e 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -292,7 +292,18 @@ func contactItem(_ contact: CNContact) -> [String: Any] { func emailKey(_ email: String) -> String { email.lowercased() } /// A phone compared without its formatting, but with its dialing controls: `5550100,123` pauses /// before the extension and `5550100;123` waits, so they are different numbers. -func phoneKey(_ phone: String) -> String { phone.filter { $0.isASCII && ($0.isNumber || "+*#,;".contains($0)) } } +/// Digits from any script read as the same number. +func phoneKey(_ phone: String) -> String { + String(phone.compactMap { character -> Character? in + if let digit = character.wholeNumberValue, character.isNumber, (0...9).contains(digit) { return Character(String(digit)) } + return "+*#,;".contains(character) ? character : nil + }) +} +/// The contact with this id, or nil when there is none; any other Contacts failure is thrown, not read as missing. +func fetchContact(_ store: CNContactStore, _ id: String) throws -> CNContact? { + do { return try store.unifiedContact(withIdentifier: id, keysToFetch: contactReadKeys) } + catch let error as CNError where error.code == .recordDoesNotExist { return nil } +} /// Emails and phones a command adds, checked as a create checks them. func contactValues(_ command: AppCommand) throws -> (emails: [String], phones: [String]) { let emails = try (command.emails ?? []).map { try commandText($0, "Email", max: 320) } @@ -414,7 +425,7 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - case "find_contacts": let store = try contactStore() if let id = command.id { - let found = try? store.unifiedContact(withIdentifier: id, keysToFetch: contactReadKeys) + let found = try fetchContact(store, id) return ["kind": "command", "command": command.name, "contacts": found.map { [contactItem($0)] } ?? [], "truncated": false] } let name = try commandText(command.nameContains, "nameContains", max: 256) @@ -426,7 +437,7 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - let (emails, phones) = try contactValues(command) guard !emails.isEmpty || !phones.isEmpty || command.groupId != nil else { throw invalidCommand("An update needs an email, phone or group to add.") } let store = try contactStore() - guard let contact = try? store.unifiedContact(withIdentifier: id, keysToFetch: contactReadKeys), let changed = contact.mutableCopy() as? CNMutableContact else { + guard let contact = try fetchContact(store, id), let changed = contact.mutableCopy() as? CNMutableContact else { throw ComputerError("not_found", "No contact has this id; call find_contacts.", dispatchState: "not_started") } var group: CNGroup? diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 34d190085f6..d526a287e14 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -70,7 +70,7 @@ import AppKit precondition(localText(timedStart, allDay: false) == "2027-06-09T07:15:00" && localText(dayStart, allDay: true) == "2027-06-09") precondition(dueText(DateComponents(year: 2027, month: 5, day: 4)) == "2027-05-04") let padded = String(repeating: "a", count: 7999) - precondition(phoneKey("+1 (555) 010-0100") == phoneKey("+15550100100") && phoneKey("5550100,123") != phoneKey("5550100;123") && emailKey("Ana@Example.com") == emailKey("ana@example.com")) + precondition(phoneKey("+1 (555) 010-0100") == phoneKey("+15550100100") && phoneKey("5550100,123") != phoneKey("5550100;123") && phoneKey("٥٥٥٠١٠٠") == "5550100" && phoneKey("٥٥٥٠١٠٠") != phoneKey("٥٥٥٠٢٠٠") && emailKey("Ana@Example.com") == emailKey("ana@example.com")) precondition(shortened(padded + "😀", 8000) && !shortened(padded, 8000) && !shortened(nil, 8000)) precondition(bounded(padded + "😀", 8000) == padded && bounded("👍🏽ok", 4) == "👍🏽") let localDueText = dueText(try dueComponents("2027-05-04T09:30")); precondition(localDueText == "2027-05-04T09:30:00") From 2c408a49f78d4fae5647056e94c75ff7687061e8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 21:42:48 -0800 Subject: [PATCH 6/6] fix(computer-use): find a unified contact's record in the group's account --- .../native/computer-use/ComputerUse.swift | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 7f73664232e..9f00ae0a1e1 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -299,6 +299,18 @@ func phoneKey(_ phone: String) -> String { return "+*#,;".contains(character) ? character : nil }) } +/// Whether the contact has a record in this account. A unified contact links records from several +/// accounts and its own id names no container, so the account's contacts are searched for it too. +func contactHasRecord(_ store: CNContactStore, _ id: String, in account: String) throws -> Bool { + if try store.containers(matching: CNContainer.predicateForContainerOfContact(withIdentifier: id)).first?.identifier == account { return true } + let request = CNContactFetchRequest(keysToFetch: [CNContactIdentifierKey as CNKeyDescriptor]) + request.predicate = CNContact.predicateForContactsInContainer(withIdentifier: account) + var found = false + try store.enumerateContacts(with: request) { candidate, stop in + if candidate.identifier == id { found = true; stop.pointee = true } + } + return found +} /// The contact with this id, or nil when there is none; any other Contacts failure is thrown, not read as missing. func fetchContact(_ store: CNContactStore, _ id: String) throws -> CNContact? { do { return try store.unifiedContact(withIdentifier: id, keysToFetch: contactReadKeys) } @@ -446,9 +458,8 @@ func runAppCommand(_ command: AppCommand, expected: Expectation? = nil) throws - throw ComputerError("list_unavailable", "No contact group has this id; call list_contact_groups.", dispatchState: "not_started") } // A group's members must live in the group's own account. - let groupAccount = try store.containers(matching: CNContainer.predicateForContainerOfGroup(withIdentifier: found.identifier)).first?.identifier - let contactAccount = try store.containers(matching: CNContainer.predicateForContainerOfContact(withIdentifier: contact.identifier)).first?.identifier - guard groupAccount != nil, groupAccount == contactAccount else { + guard let groupAccount = try store.containers(matching: CNContainer.predicateForContainerOfGroup(withIdentifier: found.identifier)).first?.identifier, + try contactHasRecord(store, contact.identifier, in: groupAccount) else { throw ComputerError("group_other_account", "This group is in a different account from the contact; add it in Contacts.", dispatchState: "not_started") } group = found