diff --git a/apps/desktop/build/entitlements.mac.plist b/apps/desktop/build/entitlements.mac.plist index edf7bcaf4b9..d7593a8f68b 100644 --- a/apps/desktop/build/entitlements.mac.plist +++ b/apps/desktop/build/entitlements.mac.plist @@ -14,5 +14,12 @@ Hardened Runtime denies the camera no matter what the user allowed. --> com.apple.security.device.camera + + com.apple.security.personal-information.calendars + + com.apple.security.personal-information.addressbook + diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index 5b303ac360b..bad2a1939a0 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -70,6 +70,9 @@ mac: # unless the bundle declares why it wants the device. extendInfo: NSScreenCaptureUsageDescription: Mothership captures approved app windows to understand and verify computer-use actions. + NSRemindersFullAccessUsageDescription: Mothership adds reminders you confirm and lists your reminder lists so it can file them correctly. + NSCalendarsFullAccessUsageDescription: Mothership adds calendar events you confirm and lists your calendars so it can file them correctly. + NSContactsUsageDescription: Mothership adds contacts and contact groups you confirm. NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat and for meetings you join in the built-in browser. NSCameraUsageDescription: Sim uses your camera for meetings you join in the built-in browser, such as Google Meet. # macOS 15 blocks connections to private addresses (self-hosted servers on a diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 907084daef8..2b4a7acb02b 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -1,5 +1,7 @@ import AppKit import ApplicationServices +import Contacts +import EventKit import ScreenCaptureKit nonisolated(unsafe) var cancellationRequested: sig_atomic_t = 0 @@ -39,6 +41,249 @@ struct Parameters: Decodable { var button: String? var clickCount: Int? var accessibilityAction: String? + var command: AppCommand? +} +/// One allowlisted app command. Commands go through EventKit and Contacts, never Apple Events or +/// scripts, so they cannot reach a shell; each is a typed create or a list read, with no update, +/// delete or search. +struct AppCommand: Decodable { + var name: String + var title: String? + var notes: String? + var due: String? + var listId: String? + var priority: Int? + var start: String? + var end: String? + var allDay: Bool? + var location: String? + var calendarId: String? + var alertMinutesBefore: Int? + var givenName: String? + var familyName: String? + var organization: String? + var jobTitle: String? + var emails: [String]? + var phones: [String]? + var groupId: String? +} +func invalidCommand(_ message: String) -> ComputerError { ComputerError("invalid_arguments", message, dispatchState: "not_started") } +/// A required, trimmed text field of bounded length. +func commandText(_ value: String?, _ field: String, max: Int = 1024) throws -> String { + let text = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty, text.count <= max else { throw invalidCommand("\(field) is required and limited to \(max) characters.") } + return text +} +func optionalText(_ value: String?, _ field: String, max: Int = 1024) throws -> String? { + guard let value, !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } + return try commandText(value, field, max: max) +} +/// The event store for reminders, available only once the user has granted full Reminders access. +/// Checking never prompts; the grant happens from settings, outside the tool call. +func remindersStore() throws -> EKEventStore { + guard EKEventStore.authorizationStatus(for: .reminder) == .fullAccess else { + throw ComputerError("reminders_permission_required", "Allow Reminders access for Mothership in System Settings, then retry.", dispatchState: "not_started") + } + return EKEventStore() +} +/// A due date: `YYYY-MM-DD` is date-only; a date-time is ISO 8601, and one without an offset is this +/// Mac's local time, as typing it into the app would be. +func dueComponents(_ text: String) throws -> DateComponents { + var calendar = Calendar(identifier: .gregorian) + if text.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil { + let parts = text.split(separator: "-").compactMap { Int($0) } + let components = DateComponents(calendar: calendar, year: parts[0], month: parts[1], day: parts[2]) + guard components.isValidDate else { throw ComputerError("invalid_arguments", "Due date is not a real date.", dispatchState: "not_started") } + return DateComponents(year: parts[0], month: parts[1], day: parts[2]) + } + guard let date = isoDateTime(text) else { + throw ComputerError("invalid_arguments", "Due must be YYYY-MM-DD or an ISO 8601 date-time.", dispatchState: "not_started") + } + calendar.timeZone = .current + return calendar.dateComponents([.year, .month, .day, .hour, .minute, .second, .timeZone], from: date) +} +/// Reminders or Calendar access as the status reports it, read without prompting. +func eventKitAuthorization(_ type: EKEntityType) -> String { + switch EKEventStore.authorizationStatus(for: type) { + case .fullAccess: return "granted" + case .notDetermined: return "not_determined" + default: return "denied" + } +} +func contactsAuthorization() -> String { + switch CNContactStore.authorizationStatus(for: .contacts) { + case .authorized: return "granted" + case .notDetermined: return "not_determined" + default: return "denied" + } +} +func calendarStore() throws -> EKEventStore { + guard EKEventStore.authorizationStatus(for: .event) == .fullAccess else { + throw ComputerError("calendars_permission_required", "Allow Calendar access for Mothership in System Settings, then retry.", dispatchState: "not_started") + } + return EKEventStore() +} +func contactStore() throws -> CNContactStore { + guard CNContactStore.authorizationStatus(for: .contacts) == .authorized else { + throw ComputerError("contacts_permission_required", "Allow Contacts access for Mothership in System Settings, then retry.", dispatchState: "not_started") + } + return CNContactStore() +} +/// The Gregorian calendar in this Mac's time zone, so `YYYY-MM-DD` means the same day whatever calendar the user prefers. +func localGregorian() -> Calendar { + var calendar = Calendar(identifier: .gregorian); calendar.timeZone = .current + return calendar +} +/// An ISO 8601 date-time; without an offset it is this Mac's local time. +func isoDateTime(_ text: String) -> Date? { + for options: ISO8601DateFormatter.Options in [[.withInternetDateTime], [.withInternetDateTime, .withFractionalSeconds]] { + let zoned = ISO8601DateFormatter(); zoned.formatOptions = options + if let date = zoned.date(from: text) { return date } + } + // Fractions of a second do not matter to a calendar item, so a local time drops them. + guard text.range(of: #"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?$"#, options: .regularExpression) != nil else { return nil } + let whole = String(text.prefix(19)) + let local = DateFormatter(); local.locale = Locale(identifier: "en_US_POSIX"); local.timeZone = .current + local.dateFormat = whole.count == 16 ? "yyyy-MM-dd'T'HH:mm" : "yyyy-MM-dd'T'HH:mm:ss" + local.isLenient = false + return local.date(from: whole) +} +/// An event boundary: an ISO 8601 date-time (local without an offset), or `YYYY-MM-DD` for all-day events. +func eventDate(_ text: String?, _ field: String, allDay: Bool) throws -> Date { + let text = try commandText(text, field, max: 64) + if allDay { + let parts = try dueComponents(text) + guard parts.hour == nil, let date = localGregorian().date(from: parts) else { throw invalidCommand("All-day \(field) must be YYYY-MM-DD.") } + return date + } + guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be an ISO 8601 date-time.") } + return date +} +func runAppCommand(_ command: AppCommand) throws -> [String: Any] { + switch command.name { + case "list_reminder_lists": + let allLists = try remindersStore().calendars(for: .reminder) + let lists = allLists.prefix(200).map { list -> [String: Any] in + ["id": list.calendarIdentifier, "title": String(list.title.prefix(1024)), "allowsModifications": list.allowsContentModifications] + } + return ["kind": "command", "command": command.name, "items": lists, "truncated": allLists.count > 200] + case "create_reminder": + let title = (command.title ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !title.isEmpty, title.count <= 1024 else { throw ComputerError("invalid_arguments", "A reminder needs a title of at most 1024 characters.", dispatchState: "not_started") } + guard (command.notes?.count ?? 0) <= 8000 else { throw ComputerError("invalid_arguments", "Notes are limited to 8000 characters.", dispatchState: "not_started") } + guard [0, 1, 5, 9].contains(command.priority ?? 0) else { throw ComputerError("invalid_arguments", "Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).", dispatchState: "not_started") } + let due = try command.due.map(dueComponents) + let store = try remindersStore() + let list: EKCalendar + if let listId = command.listId { + guard let found = store.calendar(withIdentifier: listId), found.allowedEntityTypes.contains(.reminder) else { + throw ComputerError("list_unavailable", "No reminder list has this id; call list_reminder_lists.", dispatchState: "not_started") + } + list = found + } else { + guard let fallback = store.defaultCalendarForNewReminders() else { throw ComputerError("list_unavailable", "Reminders has no default list; pass listId.", dispatchState: "not_started") } + list = fallback + } + guard list.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") } + let reminder = EKReminder(eventStore: store) + reminder.calendar = list; reminder.title = title; reminder.notes = command.notes + reminder.priority = command.priority ?? 0; reminder.dueDateComponents = due + try checkCancellation() + inputDispatched = true + try store.save(reminder, commit: true) + return ["kind": "command", "command": command.name, "created": ["id": reminder.calendarItemIdentifier, "title": title, "list": list.title]] + case "list_calendars": + let allCalendars = try calendarStore().calendars(for: .event) + let calendars = allCalendars.prefix(200).map { calendar -> [String: Any] in + ["id": calendar.calendarIdentifier, "title": String(calendar.title.prefix(1024)), "allowsModifications": calendar.allowsContentModifications] + } + return ["kind": "command", "command": command.name, "items": calendars, "truncated": allCalendars.count > 200] + case "create_event": + let title = try commandText(command.title, "Title") + let allDay = command.allDay ?? false + let start = try eventDate(command.start, "Start", allDay: allDay) + let end = try eventDate(command.end, "End", allDay: allDay) + guard allDay ? end >= start : end > start else { throw invalidCommand("End must be after start.") } + // Calendar days, so a daylight-saving change inside the event does not shorten the limit. + guard let limit = localGregorian().date(byAdding: .day, value: 31, to: start), end <= limit else { + throw invalidCommand("An event can last at most 31 days.") + } + let location = try optionalText(command.location, "Location") + let notes = try optionalText(command.notes, "Notes", max: 8000) + if let minutes = command.alertMinutesBefore, !(0...40_320).contains(minutes) { throw invalidCommand("alertMinutesBefore must be 0 to 40320.") } + let store = try calendarStore() + let calendar: EKCalendar + if let calendarId = command.calendarId { + guard let found = store.calendar(withIdentifier: calendarId), found.allowedEntityTypes.contains(.event) else { + throw ComputerError("list_unavailable", "No calendar has this id; call list_calendars.", dispatchState: "not_started") + } + calendar = found + } else { + guard let fallback = store.defaultCalendarForNewEvents else { throw ComputerError("list_unavailable", "Calendar has no default calendar; pass calendarId.", dispatchState: "not_started") } + calendar = fallback + } + guard calendar.allowsContentModifications else { throw ComputerError("list_read_only", "This calendar is read-only.", dispatchState: "not_started") } + let event = EKEvent(eventStore: store) + event.calendar = calendar; event.title = title; event.isAllDay = allDay + event.startDate = start; event.endDate = end; event.location = location; event.notes = notes + if let minutes = command.alertMinutesBefore { event.addAlarm(EKAlarm(relativeOffset: TimeInterval(-minutes * 60))) } + try checkCancellation() + inputDispatched = true + try store.save(event, span: .thisEvent, commit: true) + return ["kind": "command", "command": command.name, "created": ["id": event.eventIdentifier ?? event.calendarItemIdentifier, "title": title, "list": calendar.title]] + case "list_contact_groups": + let allGroups = try contactStore().groups(matching: nil) + let groups = allGroups.prefix(200).map { group -> [String: Any] in + ["id": group.identifier, "title": String(group.name.prefix(1024)), "allowsModifications": true] + } + return ["kind": "command", "command": command.name, "items": groups, "truncated": allGroups.count > 200] + case "create_contact_group": + let name = try commandText(command.title, "Group name") + let store = try contactStore() + // A group with this name already exists after a lost reply; return it rather than add a twin. + if let existing = try store.groups(matching: nil).first(where: { $0.name == name }) { + return ["kind": "command", "command": command.name, "created": ["id": existing.identifier, "title": name, "list": "Contacts"]] + } + let group = CNMutableGroup(); group.name = name + let request = CNSaveRequest(); request.add(group, toContainerWithIdentifier: nil) + try checkCancellation() + inputDispatched = true + try store.execute(request) + return ["kind": "command", "command": command.name, "created": ["id": group.identifier, "title": name, "list": "Contacts"]] + case "create_contact": + let given = try optionalText(command.givenName, "Given name"), family = try optionalText(command.familyName, "Family name") + let organization = try optionalText(command.organization, "Organization"), jobTitle = try optionalText(command.jobTitle, "Job title") + guard given != nil || family != nil || organization != nil else { throw invalidCommand("A contact needs a name or an organization.") } + let emails = try (command.emails ?? []).map { try commandText($0, "Email", max: 320) } + let phones = try (command.phones ?? []).map { try commandText($0, "Phone", max: 64) } + guard emails.count <= 5, phones.count <= 5 else { throw invalidCommand("A contact takes at most 5 emails and 5 phones.") } + guard emails.allSatisfy({ $0.contains("@") }) else { throw invalidCommand("Each email needs an @.") } + let store = try contactStore() + var group: CNGroup? + if let groupId = command.groupId { + guard let found = try store.groups(matching: CNGroup.predicateForGroups(withIdentifiers: [groupId])).first else { + throw ComputerError("list_unavailable", "No contact group has this id; call list_contact_groups.", dispatchState: "not_started") + } + group = found + } + let contact = CNMutableContact() + contact.givenName = given ?? ""; contact.familyName = family ?? "" + contact.organizationName = organization ?? ""; contact.jobTitle = jobTitle ?? "" + contact.emailAddresses = emails.map { CNLabeledValue(label: CNLabelWork, value: $0 as NSString) } + contact.phoneNumbers = phones.map { CNLabeledValue(label: CNLabelPhoneNumberMain, value: CNPhoneNumber(stringValue: $0)) } + // A group's members must live in the group's own account, so the contact is saved there. + let container = try group.flatMap { try store.containers(matching: CNContainer.predicateForContainerOfGroup(withIdentifier: $0.identifier)).first?.identifier } + let request = CNSaveRequest(); request.add(contact, toContainerWithIdentifier: container) + if let group { request.addMember(contact, to: group) } + try checkCancellation() + inputDispatched = true + try store.execute(request) + let display = [given, family].compactMap { $0 }.joined(separator: " ") + let title = String((display.isEmpty ? organization ?? "" : display).prefix(1024)) + return ["kind": "command", "command": command.name, "created": ["id": contact.identifier, "title": title, "list": String((group?.name ?? "Contacts").prefix(1024))]] + default: + throw ComputerError("unknown_command", "Unknown app command.", dispatchState: "not_started") + } } struct Request: Decodable { let id: String; let method: String; let params: Parameters } func required(_ value: T?, _ name: String) throws -> T { @@ -798,11 +1043,17 @@ final class Driver { let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary _ = AXIsProcessTrustedWithOptions(options) case "screenCapture": _ = CGRequestScreenCaptureAccess() + case "reminders": _ = try? await EKEventStore().requestFullAccessToReminders() + case "calendars": _ = try? await EKEventStore().requestFullAccessToEvents() + case "contacts": _ = try? await CNContactStore().requestAccess(for: .contacts) default: throw ComputerError("invalid_arguments", "Unknown permission.") } - return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] - case "status": return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] + return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess(), "reminders": eventKitAuthorization(.reminder), "calendars": eventKitAuthorization(.event), "contacts": contactsAuthorization()] + case "status": return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess(), "reminders": eventKitAuthorization(.reminder), "calendars": eventKitAuthorization(.event), "contacts": contactsAuthorization()] case "list_apps": return ["kind": "apps", "apps": try discoverApps()] + case "app_command": + inputDispatched = false + return try runAppCommand(required(p.command, "command")) case "activate_app": try requireAccessibility() let app = try await appFor(p.bundleId, launch: true) diff --git a/apps/desktop/native/computer-use/Info.plist b/apps/desktop/native/computer-use/Info.plist index 2ba6eb3d71d..1db88f73464 100644 --- a/apps/desktop/native/computer-use/Info.plist +++ b/apps/desktop/native/computer-use/Info.plist @@ -11,5 +11,8 @@ LSMinimumSystemVersion14.0 LSUIElement NSHighResolutionCapable +NSCalendarsFullAccessUsageDescriptionMothership adds calendar events you confirm and lists your calendars so it can file them correctly. +NSContactsUsageDescriptionMothership adds contacts and contact groups you confirm. +NSRemindersFullAccessUsageDescriptionMothership adds reminders you confirm and lists your reminder lists so it can file them correctly. NSScreenCaptureUsageDescriptionMothership captures the app window you select to understand and verify computer-use actions. diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index 1603675f8e1..a0721da9222 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -6,7 +6,9 @@ Compile each architecture with `swiftc -parse-as-library -O -target arm64-apple- ## Protocol -Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"}`. Status/preflight never triggers prompts. Errors go through the protocol, and stdout contains no logs. +Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"|"reminders"|"calendars"|"contacts"}`. Status/preflight never triggers prompts; it reports `reminders`, `calendars` and `contacts` as `granted`, `denied` or `not_determined`. + +`app_command` takes `{command:{name,...}}` and runs one allowlisted command through EventKit or Contacts, never Apple Events or scripts. Reads: `list_reminder_lists`, `list_calendars` and `list_contact_groups` return `{kind:"command",items:[{id,title,allowsModifications}]}`. Creates return `{kind:"command",created:{id,title,list}}`: `create_reminder` (`title`, `notes`, `due` as `YYYY-MM-DD` or an ISO 8601 date-time, `listId`, `priority` 0/1/5/9), `create_event` (`title`, `start`/`end` as ISO 8601 date-times or `YYYY-MM-DD` with `allDay`, at most 31 days, `location`, `notes`, `calendarId`, `alertMinutesBefore`), `create_contact_group` (`title`; an existing group with that name is returned instead of a duplicate) and `create_contact` (`givenName`, `familyName`, `organization`, `jobTitle`, up to 5 `emails` and `phones`, `groupId`). A date-time without an offset is this Mac's local time, as typing it into the app would be. Without the matching access a command fails with `reminders_permission_required`, `calendars_permission_required` or `contacts_permission_required` before touching anything. There is no update, delete or search. The desktop service confirms each create with the user before sending it. The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise. diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index dccb74ed7ef..269fc6bda39 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -36,6 +36,34 @@ import AppKit precondition(isProtectedApp(bundle), bundle) } for bundle in ["com.apple.TextEdit", "com.apple.finder", "com.apple.Safari"] { precondition(!isProtectedApp(bundle), bundle) } + let dateOnly = try dueComponents("2027-05-04") + precondition(dateOnly.year == 2027 && dateOnly.month == 5 && dateOnly.day == 4 && dateOnly.hour == nil) + let timed = try dueComponents("2027-05-04T09:30:45Z") + precondition(timed.hour != nil && timed.minute != nil && timed.second == 45) + let localDue = try dueComponents("2027-05-04T09:30") + precondition(localDue.hour == 9 && localDue.minute == 30) + for due in ["2027-02-30", "2027-05-04T25:30:00", "May 4", "2027-5-4"] { + expectError("invalid_arguments") { _ = try dueComponents(due) } + } + expectError("unknown_command") { _ = try runAppCommand(AppCommand(name: "run_script")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: " ")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: "Pay rent", priority: 3)) } + var local = Calendar(identifier: .gregorian); local.timeZone = .current + let floating = try eventDate("2027-06-09T07:15:00", "Start", allDay: false) + precondition(local.dateComponents([.hour, .minute], from: floating) == DateComponents(hour: 7, minute: 15)) + expectError("invalid_arguments") { _ = try eventDate("2027-06-09 07:15", "Start", allDay: false) } + let fractional = try eventDate("2027-06-09T06:15:00.000Z", "Start", allDay: false) + precondition(fractional == Date(timeIntervalSince1970: 1_812_521_700)) + let localFraction = try eventDate("2027-06-09T07:15:00.250", "Start", allDay: false) + precondition(local.dateComponents([.hour, .minute], from: localFraction) == DateComponents(hour: 7, minute: 15)) + expectError("invalid_arguments") { _ = try eventDate("2027-06-09T07:15:00+01:00", "Start", allDay: true) } + let zoned = try eventDate("2027-06-09T07:15:00+01:00", "Start", allDay: false) + precondition(zoned == Date(timeIntervalSince1970: 1_812_521_700)) + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_event", title: "Flight", start: "2027-06-09T08:10:00Z", end: "2027-06-09T07:15:00Z")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_event", title: "Trip", start: "2027-06-01T00:00:00Z", end: "2027-08-01T00:00:00Z")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_contact", jobTitle: "Engineer")) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_contact", givenName: "Ana", emails: ["not-an-email"])) } + expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_contact_group", title: " ")) } var queue = AXTraversalQueue(limit: 1000) queue.append(AXTraversalEntry(element: "menu", parent: nil, depth: 0, menu: true, priority: false)) queue.append(AXTraversalEntry(element: "content", parent: nil, depth: 0, menu: false, priority: false)) diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index c5d463b75d1..3fd3b00e3b8 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -28,6 +28,12 @@ function setup(enabled = true, askForApps = true) { kind: 'apps', apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }], } + if (method === 'app_command') + return { + kind: 'command', + command: 'create_reminder', + created: { id: 'r1', title: 'Pay rent', list: 'Reminders' }, + } if (method === 'get_app_state') return { kind: 'state', @@ -48,6 +54,7 @@ function setup(enabled = true, askForApps = true) { }) const native = { request, stop: vi.fn() } const approveApp = vi.fn(async (): Promise<'always' | 'deny' | 'once'> => 'always') + const confirmAppCommand = vi.fn(async (_request: { appName: string; summary: string }) => true) const onActivity = vi.fn() const openPermissionSettings = vi.fn(async () => {}) const setStopShortcutActive = vi.fn((active: boolean) => active) @@ -56,6 +63,7 @@ function setup(enabled = true, askForApps = true) { native, supported: true, approveApp, + confirmAppCommand, onActivity, setStopShortcutActive, openPermissionSettings, @@ -77,6 +85,7 @@ function setup(enabled = true, askForApps = true) { config, native, approveApp, + confirmAppCommand, onActivity, setStopShortcutActive, openPermissionSettings, @@ -116,6 +125,91 @@ describe('native computer use authority and lifecycle', () => { expect(native.request.mock.calls.map(([method]) => method)).toEqual(['list_apps']) }) + it('confirms each app-command write before the helper runs it, and never observes after', async () => { + const { service, native, confirmAppCommand } = setup() + const create = { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent', due: '2027-05-04', priority: 1 }, + } + confirmAppCommand.mockResolvedValueOnce(false) + await expect(service.execute('declined', 'chat', create)).rejects.toMatchObject({ + details: { code: 'user_declined', dispatchState: 'not_started' }, + }) + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + expect(confirmAppCommand.mock.calls[0]?.[0].summary).toContain('Priority: High') + + await expect(service.execute('confirmed', 'chat', create)).resolves.toMatchObject({ + kind: 'command', + created: { id: 'r1' }, + }) + const methods = native.request.mock.calls.map(([method]) => method) + expect(methods.filter((method) => method === 'app_command')).toHaveLength(1) + expect(methods).not.toContain('get_app_state') + }) + + it.each([ + { + name: 'create_event', + title: 'TP 1947: LIS to OPO', + start: '2027-06-09T07:15:00+01:00', + end: '2027-06-09T08:10:00+01:00', + }, + { name: 'create_contact_group', title: 'Hires Q1 2027' }, + { name: 'create_contact', givenName: 'Ana', emails: ['ana@example.com'] }, + ])('asks the user before every $name write', async (command) => { + const { service, native, confirmAppCommand } = setup() + confirmAppCommand.mockResolvedValueOnce(false) + await expect( + service.execute(`declined-${command.name}`, 'chat', { action: 'app_command', command }) + ).rejects.toMatchObject({ details: { code: 'user_declined' } }) + expect(confirmAppCommand).toHaveBeenCalledOnce() + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + }) + + it('shows the trimmed values the helper will save', async () => { + const { service, confirmAppCommand } = setup() + await service.execute('padded', 'chat', { + action: 'app_command', + command: { name: 'create_reminder', title: ' Pay rent ', notes: '\tBy Friday ' }, + }) + expect(confirmAppCommand.mock.calls[0]?.[0].summary).toBe( + 'Reminder: Pay rent\nList: default\nNotes: By Friday' + ) + }) + + it('refuses an app command when the user denies the app, before asking to confirm', async () => { + const { service, native, approveApp, confirmAppCommand } = setup() + approveApp.mockResolvedValue('deny') + await expect( + service.execute('denied', 'chat', { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent' }, + }) + ).rejects.toThrow('denied') + expect(confirmAppCommand).not.toHaveBeenCalled() + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + }) + + it('lists reminder lists without a write confirmation', async () => { + const { service, native, confirmAppCommand } = setup() + native.request.mockImplementation(async (method: string) => + method === 'app_command' + ? { + kind: 'command', + command: 'list_reminder_lists', + items: [{ id: 'l1', title: 'Sprint 14', allowsModifications: true }], + } + : { kind: 'apps', apps: [] } + ) + await expect( + service.execute('lists', 'chat', { + action: 'app_command', + command: { name: 'list_reminder_lists' }, + }) + ).resolves.toMatchObject({ items: [{ title: 'Sprint 14' }] }) + expect(confirmAppCommand).not.toHaveBeenCalled() + }) + it('binds observed elements to their originating chat', async () => { const { state, click, native } = setup() await state('owner') @@ -492,6 +586,23 @@ describe('native computer use authority and lifecycle', () => { await expect(click('s1')).rejects.toThrow('stale') }) + it('revoking an app stops its app command while the write waits for confirmation', async () => { + const { service, native, confirmAppCommand } = setup() + let confirm: (answer: boolean) => void = () => {} + confirmAppCommand.mockImplementation( + () => new Promise((resolve) => (confirm = resolve)) + ) + const pending = service.execute('revoked', 'chat', { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent' }, + }) + await vi.waitFor(() => expect(confirmAppCommand).toHaveBeenCalledOnce()) + service.revokeApp('com.apple.reminders') + confirm(true) + await expect(pending).rejects.toThrow('stopped') + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('app_command') + }) + it('blocks self-automation before showing an approval', async () => { const { service, approveApp, native } = setup() await expect( diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 397f5041589..08b4139de13 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -2,13 +2,15 @@ import { type ComputerUseActivity, type ComputerUseAppPermission, ComputerUseError, + type ComputerUsePermission, type ComputerUseStatus, } from '@sim/desktop-bridge' import { - type ComputerUseInput, + type AppCommand, type ComputerUseResult, - ComputerUseSchema, type ComputerUseSnapshot, + type ComputerUseWithAppCommandsInput, + ComputerUseWithAppCommandsSchema, } from '@sim/desktop-bridge/computer-use' import { getErrorMessage } from '@sim/utils/errors' import { omit } from '@sim/utils/object' @@ -23,15 +25,20 @@ interface ComputerUseServiceDeps { app: ComputerUseAppPermission, signal: AbortSignal ) => Promise<'once' | 'always' | 'deny'> + /** Confirms one app-command write. It shows the exact fields, since nothing happens on screen. */ + confirmAppCommand: ( + request: { appName: string; summary: string }, + signal: AbortSignal + ) => Promise onActivity: (activity: ComputerUseActivity | null) => void setStopShortcutActive?: (active: boolean) => boolean - openPermissionSettings?: (permission: 'accessibility' | 'screenCapture') => Promise + openPermissionSettings?: (permission: ComputerUsePermission) => Promise } interface ToolWork { toolCallId: string scopeId: string - input: ComputerUseInput + input: ComputerUseWithAppCommandsInput controller: AbortController } @@ -42,6 +49,75 @@ interface SnapshotOwnership { elementWindows: Map } +/** The app each command writes to; its Computer Use app permission governs the command. */ +const APP_COMMAND_APPS: Record = { + list_reminder_lists: 'com.apple.reminders', + create_reminder: 'com.apple.reminders', + list_calendars: 'com.apple.iCal', + create_event: 'com.apple.iCal', + list_contact_groups: 'com.apple.AddressBook', + create_contact_group: 'com.apple.AddressBook', + create_contact: 'com.apple.AddressBook', +} + +const PRIORITY_NAMES: Record = { 1: 'High', 5: 'Medium', 9: 'Low' } + +/** The app a call operates; an app command operates the app its data belongs to. */ +function workBundleId(input: ComputerUseWithAppCommandsInput): string | undefined { + if (input.action === 'app_command') return APP_COMMAND_APPS[input.command.name] + return 'bundleId' in input ? input.bundleId : undefined +} + +/** The exact fields a write will save, for the user to confirm; null for a read, which needs none. */ +function writeSummary(command: AppCommand): string | null { + // Trimmed exactly as the helper trims before saving, so the user confirms the saved values. + const lines = (entries: Array<[string, string | number | undefined]>) => + entries + .map(([label, value]) => [label, typeof value === 'string' ? value.trim() : value] as const) + .filter(([, value]) => value !== undefined && value !== '') + .map(([label, value]) => `${label}: ${value}`) + .join('\n') + switch (command.name) { + case 'create_reminder': + return lines([ + ['Reminder', command.title], + ['Due', command.due], + ['Priority', command.priority ? PRIORITY_NAMES[command.priority] : undefined], + ['List', command.listId ?? 'default'], + ['Notes', command.notes], + ]) + case 'create_event': + return lines([ + ['Event', command.title], + ['Starts', command.start], + ['Ends', command.end], + ['All day', command.allDay ? 'yes' : undefined], + ['Location', command.location], + [ + 'Alert', + command.alertMinutesBefore !== undefined + ? `${command.alertMinutesBefore} minutes before` + : undefined, + ], + ['Calendar', command.calendarId ?? 'default'], + ['Notes', command.notes], + ]) + case 'create_contact_group': + return lines([['Contact group', command.title]]) + case 'create_contact': + return lines([ + ['Contact', [command.givenName, command.familyName].filter(Boolean).join(' ') || undefined], + ['Organization', command.organization], + ['Job title', command.jobTitle], + ['Emails', command.emails?.map((email) => email.trim()).join(', ')], + ['Phones', command.phones?.map((phone) => phone.trim()).join(', ')], + ['Group', command.groupId], + ]) + default: + return null + } +} + /** Native snapshots cannot transfer authority between chats or restarted helpers. */ export class ComputerUseService { private admissions = new Map() @@ -82,6 +158,9 @@ export class ComputerUseService { status.permissions = { accessibility: native.accessibility, screenCapture: native.screenRecording, + reminders: native.reminders === 'granted', + calendars: native.calendars === 'granted', + contacts: native.contacts === 'granted', } status.enabled = this.isEnabled() status.activeAction = this.activity @@ -111,9 +190,7 @@ export class ComputerUseService { return this.getStatus() } - async requestPermission( - permission: 'accessibility' | 'screenCapture' - ): Promise { + async requestPermission(permission: ComputerUsePermission): Promise { if (!this.deps.supported) return this.getStatus() await this.deps.native.request('request_permission', { permission }) await this.deps.openPermissionSettings?.(permission) @@ -131,7 +208,7 @@ export class ComputerUseService { ) const persisted = this.deps.config.flush() for (const work of this.queued.values()) { - if ('bundleId' in work.input && work.input.bundleId === bundleId) this.cancel(work.toolCallId) + if (workBundleId(work.input) === bundleId) this.cancel(work.toolCallId) } for (const apps of this.taskGrants.values()) apps.delete(bundleId) this.invalidateSnapshots() @@ -200,7 +277,7 @@ export class ComputerUseService { if (this.queued.has(toolCallId)) return Promise.reject(new Error('This Computer Use action is already running.')) // A newer server can send arguments this build predates; nothing has been queued yet. - const parsed = ComputerUseSchema.safeParse(raw) + const parsed = ComputerUseWithAppCommandsSchema.safeParse(raw) if (!parsed.success) return Promise.reject( new ComputerUseError({ @@ -240,6 +317,7 @@ export class ComputerUseService { stopShortcutAvailable: this.deps.setStopShortcutActive?.(true) ?? false, } this.deps.onActivity(this.activity) + if (input.action === 'app_command') return await this.runAppCommand(input.command, work) if ('bundleId' in input) { if (/^ai\.sim\.desktop(?:\.|$)/.test(input.bundleId)) { throw new Error('Computer Use cannot operate Sim or its own permission controls.') @@ -349,6 +427,37 @@ export class ComputerUseService { } } + /** + * A typed write through the OS's data frameworks leaves nothing on screen to watch, so each one + * needs the app's usual Computer Use permission and its own confirmation of the exact fields. + */ + private async runAppCommand(command: AppCommand, work: ToolWork): Promise { + const appName = await this.authorizeApp(APP_COMMAND_APPS[command.name], work) + if (this.activity) { + this.activity = { ...this.activity, appName } + this.deps.onActivity(this.activity) + } + const summary = writeSummary(command) + if (summary !== null) { + const confirmed = await this.deps.confirmAppCommand( + { appName, summary }, + work.controller.signal + ) + this.check(work) + if (!confirmed) + throw new ComputerUseError({ + code: 'user_declined', + message: 'The user declined to add this item. Do not retry it.', + dispatchState: 'not_started', + }) + } + this.check(work) + const result = await this.deps.native.request('app_command', { command }) + if (result.kind !== 'command' || result.command !== command.name) + throw new Error('Computer Use returned a result for a different command.') + return result + } + private rememberSnapshot(work: ToolWork, snapshot: ComputerUseSnapshot, windowId?: string): void { if (!('bundleId' in work.input) || snapshot.bundleId !== work.input.bundleId) throw new Error('Computer Use returned state for a different app.') diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index c6781c8666d..0fc6fdc6e0a 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -1,5 +1,6 @@ import { release } from 'node:os' import { join } from 'node:path' +import type { ComputerUsePermission } from '@sim/desktop-bridge' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { OpenDialogOptions, Session, WebContents } from 'electron' @@ -136,6 +137,15 @@ function reportHandoffFailure(error: unknown): void { /** Bounds what a hand-edited settings file can make an update relaunch open. */ const MAX_RELAUNCH_WINDOWS = 16 +/** The System Settings privacy pane for each Computer Use permission. */ +const PRIVACY_PANES: Record = { + accessibility: 'Privacy_Accessibility', + screenCapture: 'Privacy_ScreenCapture', + reminders: 'Privacy_Reminders', + calendars: 'Privacy_Calendars', + contacts: 'Privacy_Contacts', +} + const DOCK_ICON_FOR_CHANNEL = { prod: 'dock-icon.png', staging: 'dock-icon-staging.png', @@ -200,9 +210,7 @@ function main(): void { native: computerNative, openPermissionSettings: (permission) => shell.openExternal( - permission === 'accessibility' - ? 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility' - : 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture' + `x-apple.systempreferences:com.apple.preference.security?${PRIVACY_PANES[permission]}` ), setStopShortcutActive: (active) => { if (active && !computerStopShortcutRegistered) { @@ -229,6 +237,20 @@ function main(): void { }) return result.response === 0 ? 'once' : result.response === 1 ? 'always' : 'deny' }, + confirmAppCommand: async ({ appName, summary }, signal) => { + const result = await dialog.showMessageBox({ + type: 'question', + title: 'Computer Use', + message: `Let Mothership add this to ${appName}?`, + detail: summary, + buttons: ['Add', 'Cancel'], + defaultId: 1, + cancelId: 1, + noLink: true, + signal, + }) + return result.response === 0 + }, onActivity: (activity) => { for (const win of getWindows()) { if (isAppOrigin(win.webContents.getURL(), appOrigin())) { diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index c4fab9ad534..e1844913b03 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -607,6 +607,7 @@ describe('registerIpcHandlers', () => { native, supported: true, approveApp, + confirmAppCommand: vi.fn(async () => true), onActivity: vi.fn(), }) deps.computerUse = service diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 0d60c338715..f40e1aa23d6 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -19,6 +19,7 @@ import { type DesktopUpdateState, type DesktopWindowState, type DesktopZoomPercent, + isComputerUsePermission, isDesktopAppearanceTheme, isDesktopScopeId, isDesktopZoomPercent, @@ -747,7 +748,7 @@ export function registerIpcHandlers(deps: IpcDeps): void { needsUserActivation: true, denied: null, handler: (permission) => { - if (permission !== 'accessibility' && permission !== 'screenCapture') + if (!isComputerUsePermission(permission)) throw new Error('Unknown Computer Use permission.') return deps.computerUse?.requestPermission(permission) }, diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 873b880fba2..93cbc618c52 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -29,6 +29,7 @@ import type { BrowserSiteInfo, BrowserToolbarCommand, ComputerUseActivity, + ComputerUsePermission, DesktopAppearanceTheme, DesktopCommand, DesktopExecutorDevice, @@ -51,7 +52,7 @@ import type { TerminalShortcutCommand, TerminalThemeProfile, } from '@sim/desktop-bridge' -import type { ComputerUseInput } from '@sim/desktop-bridge/computer-use' +import type { ComputerUseWithAppCommandsInput } from '@sim/desktop-bridge/computer-use' import { type ScopedTerminalCommandEvent, type ScopedTerminalTabsState, @@ -128,15 +129,16 @@ const api: SimDesktopApi = { ? { computerUse: { stableElementIds: true, + appCommands: true, getActivity: () => ipcRenderer.invoke('computer-use:get-activity'), getStatus: () => ipcRenderer.invoke('computer-use:status'), setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled), - requestPermission: (permission: 'accessibility' | 'screenCapture') => + requestPermission: (permission: ComputerUsePermission) => ipcRenderer.invoke('computer-use:request-permission', permission), setAskForApps: (ask: boolean) => ipcRenderer.invoke('computer-use:set-ask-for-apps', ask), listAppPermissions: () => ipcRenderer.invoke('computer-use:list-permissions'), revokeApp: (bundleId: string) => ipcRenderer.invoke('computer-use:revoke-app', bundleId), - executeTool: (toolCallId: string, params: ComputerUseInput) => + executeTool: (toolCallId: string, params: ComputerUseWithAppCommandsInput) => ipcRenderer.invoke('computer-use:execute-tool', toolCallId, params), cancel: (toolCallId?: string) => ipcRenderer.invoke('computer-use:cancel', toolCallId), onActivity: (callback: (activity: ComputerUseActivity | null) => void) => { diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx index 146524f724b..31fea8e7c32 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx @@ -1,7 +1,11 @@ 'use client' import { useCallback, useEffect, useRef, useState } from 'react' -import type { ComputerUseAppPermission } from '@sim/desktop-bridge' +import { + COMPUTER_USE_PERMISSIONS, + type ComputerUseAppPermission, + type ComputerUsePermission, +} from '@sim/desktop-bridge' import { Chip, ChipSwitch, Label, toast } from '@sim/emcn' import { useParams } from 'next/navigation' import { ComputerUseActivity } from '@/components/computer-use/activity' @@ -16,6 +20,15 @@ const ON_OFF_OPTIONS = [ ] /** The activity stays visible so an in-flight action can be stopped; the device controls need the permission group's consent. */ +/** Shells without app commands report no app-data permissions, so their rows stay hidden. */ +const PERMISSION_LABELS: Record = { + accessibility: 'Accessibility', + screenCapture: 'Screen Recording', + reminders: 'Reminders', + calendars: 'Calendar', + contacts: 'Contacts', +} + export function ComputerUseSettings() { const { workspaceId, organizationId } = useParams<{ workspaceId?: string @@ -102,8 +115,10 @@ function ComputerUseSettingsControls() { )} {status?.enabled && ( <> - {(['accessibility', 'screenCapture'] as const).map((permission) => { - const label = permission === 'accessibility' ? 'Accessibility' : 'Screen Recording' + {COMPUTER_USE_PERMISSIONS.filter( + (permission) => status.permissions[permission] !== undefined + ).map((permission) => { + const label = PERMISSION_LABELS[permission] return (
diff --git a/apps/sim/components/computer-use/activity.tsx b/apps/sim/components/computer-use/activity.tsx index f365b9bddee..ab527d35be1 100644 --- a/apps/sim/components/computer-use/activity.tsx +++ b/apps/sim/components/computer-use/activity.tsx @@ -18,6 +18,7 @@ const ACTION_LABELS: Record = { set_value: 'Editing', perform_action: 'Interacting', batch: 'Running steps', + app_command: 'Working with app data', } /** Remains visible while a native action is active, even if the permission group or device access revokes it. */ diff --git a/apps/sim/lib/api/contracts/computer-use.ts b/apps/sim/lib/api/contracts/computer-use.ts index 778701bd379..042fd3f4cb9 100644 --- a/apps/sim/lib/api/contracts/computer-use.ts +++ b/apps/sim/lib/api/contracts/computer-use.ts @@ -1,4 +1,4 @@ -import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { ComputerUseWithAppCommandsSchema } from '@sim/desktop-bridge/computer-use' import { z } from 'zod' import { desktopToolCallIdSchema } from '@/lib/api/contracts/desktop-tool-authorization' import { organizationIdSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' @@ -8,7 +8,7 @@ const authorizeComputerUseBodySchema = z.object({ toolCallId: desktopToolCallIdS const authorizeComputerUseResponseSchema = z .object({ toolName: z.literal('computer'), - args: ComputerUseSchema, + args: ComputerUseWithAppCommandsSchema, chatId: z.string().min(1), }) .strict() diff --git a/apps/sim/lib/computer-use/application/authorize.test.ts b/apps/sim/lib/computer-use/application/authorize.test.ts index 51afc06f4a5..9bac0e44ca1 100644 --- a/apps/sim/lib/computer-use/application/authorize.test.ts +++ b/apps/sim/lib/computer-use/application/authorize.test.ts @@ -75,6 +75,13 @@ describe('native computer authorization', () => { chatId: 'chat-1', }) }) + it('returns a claimed app command to the desktop', async () => { + const command = { name: 'create_reminder', title: 'Pay rent' } + mocks.claim.mockResolvedValue({ args: { action: 'app_command', command } }) + await expect(authorizeComputerUse.execute({ principal, input })).resolves.toMatchObject({ + args: { action: 'app_command', command }, + }) + }) it('rejects non-session principals before protected loading', async () => { await expect( authorizeComputerUse.execute({ diff --git a/apps/sim/lib/computer-use/application/authorize.ts b/apps/sim/lib/computer-use/application/authorize.ts index 20f0e0f1f0a..fcb55b793a2 100644 --- a/apps/sim/lib/computer-use/application/authorize.ts +++ b/apps/sim/lib/computer-use/application/authorize.ts @@ -1,5 +1,5 @@ import type { SessionPrincipal } from '@sim/auth/principal' -import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { ComputerUseWithAppCommandsSchema } from '@sim/desktop-bridge/computer-use' import { claimComputerUseTool } from '@/lib/computer-use/repository' import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' @@ -72,7 +72,7 @@ export const authorizeComputerUse = defineAuthorizedChatUseCase({ ) return { toolName: 'computer' as const, - args: ComputerUseSchema.parse(claimed.args), + args: ComputerUseWithAppCommandsSchema.parse(claimed.args), chatId: context.chatId, } }, diff --git a/apps/sim/lib/computer-use/repository.integration.ts b/apps/sim/lib/computer-use/repository.integration.ts index c8811287e69..f88430488fc 100644 --- a/apps/sim/lib/computer-use/repository.integration.ts +++ b/apps/sim/lib/computer-use/repository.integration.ts @@ -114,6 +114,12 @@ describe.skipIf(!connection)('computer use one-shot admission in PostgreSQL', () await requireConnection()`UPDATE copilot_chats SET user_id = 'user-2'` expect(await claimComputerUseTool(input)).toBeNull() }) + it('admits a typed app command with its canonical arguments', async () => { + await requireConnection()`UPDATE copilot_async_tool_calls SET args = '{"action":"app_command","command":{"name":"create_reminder","title":"Pay rent"},"activity":{"title":"Adding a reminder"}}'` + expect(await claimComputerUseTool(input)).toEqual({ + args: { action: 'app_command', command: { name: 'create_reminder', title: 'Pay rent' } }, + }) + }) it('rejects malformed canonical targets before consuming a claim', async () => { await requireConnection()`UPDATE copilot_async_tool_calls SET args = '{"action":"click","bundleId":"com.apple.Notes"}'` expect(await claimComputerUseTool(input)).toBeNull() diff --git a/apps/sim/lib/computer-use/repository.ts b/apps/sim/lib/computer-use/repository.ts index 162e538b2c8..3ec5f00a19f 100644 --- a/apps/sim/lib/computer-use/repository.ts +++ b/apps/sim/lib/computer-use/repository.ts @@ -1,6 +1,6 @@ import { db } from '@sim/db' import { copilotAsyncToolCalls, copilotChats, copilotRuns } from '@sim/db/schema' -import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { ComputerUseWithAppCommandsSchema } from '@sim/desktop-bridge/computer-use' import { omit, toRecord } from '@sim/utils/object' import { and, eq, isNull, notInArray, sql } from 'drizzle-orm' import { executableToolPermission } from '@/lib/mothership/async-runs/executable-tool-permission' @@ -41,7 +41,8 @@ export async function claimComputerUseTool(input: ComputerUseClaim) { .limit(1) if ( !pending || - !ComputerUseSchema.safeParse(omit(toRecord(pending.args), ['activity'])).success + !ComputerUseWithAppCommandsSchema.safeParse(omit(toRecord(pending.args), ['activity'])) + .success ) return null const now = new Date() @@ -62,7 +63,7 @@ export async function claimComputerUseTool(input: ComputerUseClaim) { ) .returning({ args: copilotAsyncToolCalls.args }) return claimed - ? { args: ComputerUseSchema.parse(omit(toRecord(claimed.args), ['activity'])) } + ? { args: ComputerUseWithAppCommandsSchema.parse(omit(toRecord(claimed.args), ['activity'])) } : null }) } diff --git a/apps/sim/lib/computer-use/transport.ts b/apps/sim/lib/computer-use/transport.ts index 1e9346e1ede..f08dd278389 100644 --- a/apps/sim/lib/computer-use/transport.ts +++ b/apps/sim/lib/computer-use/transport.ts @@ -1,16 +1,16 @@ import { ComputerUseError } from '@sim/desktop-bridge' import { - type ComputerUseInput, ComputerUseNativeErrorSchema, type ComputerUseResult, ComputerUseResultSchema, + type ComputerUseWithAppCommandsInput, } from '@sim/desktop-bridge/computer-use' import { getDesktopBridge } from '@/lib/desktop' /** Desktop main independently authorizes the tool ID and executes only the server's canonical args. */ export async function executeComputerUseTool( toolCallId: string, - input: ComputerUseInput, + input: ComputerUseWithAppCommandsInput, signal?: AbortSignal ): Promise { signal?.throwIfAborted() diff --git a/apps/sim/lib/desktop/index.ts b/apps/sim/lib/desktop/index.ts index 307c72b9a75..113ef0ce6ee 100644 --- a/apps/sim/lib/desktop/index.ts +++ b/apps/sim/lib/desktop/index.ts @@ -160,6 +160,7 @@ export interface DesktopChatCapabilities { browser?: true terminal?: true computerUse?: true + computerUseAppCommands?: true browserSessions?: BrowserKnownSession[] terminals?: DesktopTerminalHint[] /** The desktop's background executor, offered to run this turn's desktop tools. */ @@ -238,6 +239,9 @@ export async function getDesktopChatCapabilities( ...(browser ? { browser: true as const } : {}), ...(terminal ? { terminal: true as const } : {}), ...(computerUse ? { computerUse: true as const } : {}), + ...(computerUse && bridge?.computerUse?.appCommands === true + ? { computerUseAppCommands: true as const } + : {}), ...(terminals.length > 0 ? { terminals } : {}), ...(browserSessions.length > 0 ? { browserSessions } : {}), ...(executorDevice diff --git a/apps/sim/lib/mothership/chat/payload.test.ts b/apps/sim/lib/mothership/chat/payload.test.ts index b300f46a5d3..34a8a9c5a03 100644 --- a/apps/sim/lib/mothership/chat/payload.test.ts +++ b/apps/sim/lib/mothership/chat/payload.test.ts @@ -888,6 +888,26 @@ describe('desktop request capabilities', () => { } ) + it.each([false, true])( + 'offers app commands only where computer use itself is allowed (allowed=%s)', + async (allowed) => { + mockComputerUseAvailable.mockResolvedValueOnce(allowed) + const payload = await buildCopilotRequestPayload({ + message: 'Add my sprint reminders', + workspaceId: 'workspace', + userId: 'user', + principal: { kind: 'session', userId: 'user', sessionId: 'session' }, + userMessageId: 'message', + mode: 'agent', + model: 'gpt-6-astra', + browser: true, + computerUse: true, + computerUseAppCommands: true, + }) + expect(payload.desktop?.computerUseAppCommands ?? false).toBe(allowed) + } + ) + it('preserves desktop capabilities and current session hints on the worker wire', async () => { const payload = await buildCopilotRequestPayload({ message: 'Inspect my local page', @@ -905,6 +925,7 @@ describe('desktop request capabilities', () => { }) expect(payload.desktop).toEqual({ computerUse: false, + computerUseAppCommands: false, browser: true, terminal: true, terminals: [{ id: 'terminal-1', cwd: '/work/app', active: true }], @@ -926,6 +947,7 @@ describe('desktop request capabilities', () => { }) expect(payload.desktop).toEqual({ computerUse: false, + computerUseAppCommands: false, localFiles: true, browser: false, terminal: false, diff --git a/apps/sim/lib/mothership/chat/payload.ts b/apps/sim/lib/mothership/chat/payload.ts index a6cace00771..de83de7fa29 100644 --- a/apps/sim/lib/mothership/chat/payload.ts +++ b/apps/sim/lib/mothership/chat/payload.ts @@ -99,6 +99,7 @@ interface BuildPayloadParams { browser?: boolean terminalCapable?: boolean computerUse?: boolean + computerUseAppCommands?: boolean terminals?: Array<{ id: string cwd?: string @@ -556,6 +557,7 @@ export async function buildCopilotRequestPayload(params: BuildPayloadParams): Pr browser: params.browser === true, terminal: params.terminalCapable === true, computerUse, + computerUseAppCommands: computerUse && params.computerUseAppCommands === true, terminals: params.terminalCapable ? (params.terminals ?? []).slice(0, 20) : [], browserSessions: params.browser ? (params.browserSessions ?? []).slice(0, 20) : [], }, diff --git a/apps/sim/lib/mothership/chat/post.ts b/apps/sim/lib/mothership/chat/post.ts index dc5fbb0e553..3d6018bf194 100644 --- a/apps/sim/lib/mothership/chat/post.ts +++ b/apps/sim/lib/mothership/chat/post.ts @@ -163,6 +163,7 @@ const ChatMessageSchema = z browser: z.boolean().optional(), terminal: z.boolean().optional(), computerUse: z.boolean().optional(), + computerUseAppCommands: z.boolean().optional(), terminals: z .array( z.object({ @@ -287,6 +288,7 @@ type UnifiedChatBranch = browser?: boolean terminalCapable?: boolean computerUse?: boolean + computerUseAppCommands?: boolean terminals?: Terminals browserSessions?: BrowserSessions }) => Promise @@ -329,6 +331,7 @@ type UnifiedChatBranch = browser?: boolean terminalCapable?: boolean computerUse?: boolean + computerUseAppCommands?: boolean terminals?: Terminals browserSessions?: BrowserSessions }) => Promise @@ -627,6 +630,7 @@ async function resolveBranch(params: { browser: payloadParams.browser, terminalCapable: payloadParams.terminalCapable, computerUse: payloadParams.computerUse, + computerUseAppCommands: payloadParams.computerUseAppCommands, terminals: payloadParams.terminals, browserSessions: payloadParams.browserSessions, }), @@ -691,6 +695,7 @@ async function resolveBranch(params: { browser: payloadParams.browser, terminalCapable: payloadParams.terminalCapable, computerUse: payloadParams.computerUse, + computerUseAppCommands: payloadParams.computerUseAppCommands, terminals: payloadParams.terminals, browserSessions: payloadParams.browserSessions, }), @@ -1206,6 +1211,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { browser: body.desktopCapabilities?.browser === true, terminalCapable: body.desktopCapabilities?.terminal === true, computerUse: body.desktopCapabilities?.computerUse === true, + computerUseAppCommands: body.desktopCapabilities?.computerUseAppCommands === true, terminals: body.desktopCapabilities?.terminals, browserSessions: body.desktopCapabilities?.browserSessions, }) @@ -1234,6 +1240,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { browser: body.desktopCapabilities?.browser === true, terminalCapable: body.desktopCapabilities?.terminal === true, computerUse: body.desktopCapabilities?.computerUse === true, + computerUseAppCommands: body.desktopCapabilities?.computerUseAppCommands === true, terminals: body.desktopCapabilities?.terminals, browserSessions: body.desktopCapabilities?.browserSessions, }) diff --git a/apps/sim/lib/mothership/generated/protocol.ts b/apps/sim/lib/mothership/generated/protocol.ts index f42d7216caa..3c84a60c102 100644 --- a/apps/sim/lib/mothership/generated/protocol.ts +++ b/apps/sim/lib/mothership/generated/protocol.ts @@ -108,6 +108,7 @@ export type ModelSelection = z.infer; export const DesktopContextSchema = z.object({ localFiles: z.boolean().optional(), computerUse: z.boolean().default(false), + computerUseAppCommands: z.boolean().default(false), browser: z.boolean().default(false), terminal: z.boolean().default(false), terminals: z diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts index 4539b3e6f8d..69c674a9990 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts @@ -1,5 +1,5 @@ import { COMPUTER_USE_TOOL_TIMEOUT_MS, ComputerUseError } from '@sim/desktop-bridge' -import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { ComputerUseWithAppCommandsSchema } from '@sim/desktop-bridge/computer-use' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { omit } from '@sim/utils/object' @@ -148,7 +148,7 @@ export async function executeComputerToolOnClient( 'This computer action is stale. Inspect the app again before deciding what to do.', { doNotRetry: true, outcomeUnknown: true } ) - const parsed = ComputerUseSchema.safeParse(omit(params, ['activity'])) + const parsed = ComputerUseWithAppCommandsSchema.safeParse(omit(params, ['activity'])) if (!parsed.success) return reject('Computer action arguments are invalid. Inspect the tool schema and try again.') if (claim !== 'claimed') @@ -245,7 +245,9 @@ export async function executeComputerToolOnClient( ? `Input was dispatched, but observation failed: ${result.observationError} Inspect the app without repeating the input.` : result.kind === 'action' && !result.verified ? 'Input was dispatched. Inspect the returned observation or the app to verify its effect.' - : 'Computer observation completed', + : result.kind === 'command' && result.created + ? `Created "${result.created.title}" in ${result.created.list}. It exists now: never repeat this create.` + : 'Computer observation completed', data: computerToolResultForModel(result, { baseSnapshotId: 'snapshotId' in parsed.data ? parsed.data.snapshotId : undefined, stableElementIds: getDesktopBridge()?.computerUse?.stableElementIds === true, diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index feef762b713..2c0dddfebe4 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -229,11 +229,111 @@ export const ComputerUseSchema = z.union([ ]); export type ComputerUseInput = z.infer; +/** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */ +const CommandTitle = z + .string() + .max(1024) + .refine((title) => title.trim().length > 0, "A title must not be blank"); +/** `YYYY-MM-DD`, or a date-time with optional seconds, fraction and offset; the helper checks it is a real date. */ +const ISO_DATE_OR_TIME = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:\d{2})?)?$/; +const CommandNotes = z.string().max(8000).optional(); +const EventBoundary = z + .string() + .regex(ISO_DATE_OR_TIME, "Use YYYY-MM-DD or an ISO 8601 date-time") + .describe( + "ISO 8601 date-time, or YYYY-MM-DD when allDay is true. Without an offset it is this Mac's local time, as typing it into Calendar would be.", + ); + +/** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */ +export const AppCommandSchema = z.discriminatedUnion("name", [ + z.strictObject({ name: z.literal("list_reminder_lists") }), + z.strictObject({ + name: z.literal("create_reminder"), + title: CommandTitle, + notes: CommandNotes, + due: z + .string() + .regex(ISO_DATE_OR_TIME, "Use YYYY-MM-DD or an ISO 8601 date-time") + .optional() + .describe( + "YYYY-MM-DD for a date-only reminder, or an ISO 8601 date-time (this Mac's local time without an offset).", + ), + listId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_reminder_lists; omit for the default list."), + priority: z + .union([z.literal(0), z.literal(1), z.literal(5), z.literal(9)]) + .optional() + .describe("0 none, 1 high, 5 medium, 9 low."), + }), + z.strictObject({ name: z.literal("list_calendars") }), + z.strictObject({ + name: z.literal("create_event"), + title: CommandTitle, + start: EventBoundary, + end: EventBoundary, + allDay: z.boolean().optional(), + location: z.string().max(1024).optional(), + notes: CommandNotes, + calendarId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_calendars; omit for the default calendar."), + alertMinutesBefore: z.number().int().min(0).max(40_320).optional(), + }), + z.strictObject({ name: z.literal("list_contact_groups") }), + z.strictObject({ + name: z.literal("create_contact_group"), + title: CommandTitle.describe( + "Group name; an existing group with this name is returned instead of a duplicate.", + ), + }), + z + .strictObject({ + name: z.literal("create_contact"), + givenName: z.string().max(1024).optional(), + familyName: z.string().max(1024).optional(), + organization: z.string().max(1024).optional(), + jobTitle: z.string().max(1024).optional(), + emails: z.array(z.string().min(3).max(320)).max(5).optional(), + phones: z.array(z.string().min(1).max(64)).max(5).optional(), + groupId: z + .string() + .min(1) + .max(256) + .optional() + .describe("An id from list_contact_groups or create_contact_group."), + }) + .refine( + (contact) => + Boolean(contact.givenName?.trim() || contact.familyName?.trim() || contact.organization?.trim()), + { + message: "A contact needs a name or an organization", + }, + ), +]); +export type AppCommand = z.infer; + +/** The computer tool for desktops that support app commands. */ +export const ComputerUseWithAppCommandsSchema = z.union([ + ...ComputerUseSchema.options, + z.strictObject({ action: z.literal("app_command"), command: AppCommandSchema }), +]); +export type ComputerUseWithAppCommandsInput = z.infer; + export const ComputerUseStatusSchema = z.strictObject({ kind: z.literal("status"), platform: z.literal("darwin"), accessibility: z.boolean(), screenRecording: z.boolean(), + reminders: z.enum(["granted", "denied", "not_determined"]).optional(), + calendars: z.enum(["granted", "denied", "not_determined"]).optional(), + contacts: z.enum(["granted", "denied", "not_determined"]).optional(), }); export const ComputerUseAppSchema = z.strictObject({ @@ -301,8 +401,49 @@ export const ComputerUseSnapshotSchema = z.strictObject({ }); export type ComputerUseSnapshot = z.infer; +const CommandItem = z.strictObject({ + id: z.string().min(1).max(256), + title: z.string().max(1024), + allowsModifications: z.boolean(), +}); + +/** A create must return the id it committed, so a lost reply is never retried as a second create. */ +export const ComputerUseCommandResultSchema = z + .strictObject({ + kind: z.literal("command"), + command: z.enum([ + "list_reminder_lists", + "create_reminder", + "list_calendars", + "create_event", + "list_contact_groups", + "create_contact_group", + "create_contact", + ]), + created: z + .strictObject({ + id: z.string().min(1).max(256), + title: z.string().max(1024), + list: z.string().max(1024), + }) + .optional(), + items: z.array(CommandItem).max(200).optional(), + truncated: z + .boolean() + .optional() + .describe("True when more than 200 items exist and only the first 200 are listed."), + }) + .refine( + (result) => + result.command.startsWith("create_") ? result.created !== undefined : result.items !== undefined, + { + message: "A create returns created; a list returns items", + }, + ); + export const ComputerUseResultSchema = z.discriminatedUnion("kind", [ ComputerUseStatusSchema, + ComputerUseCommandResultSchema, z.strictObject({ kind: z.literal("apps"), apps: z.array(ComputerUseAppSchema).max(1000) }), ComputerUseSnapshotSchema, z diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts index 4ec4b1bf27c..c6ef6ee150a 100644 --- a/packages/desktop-bridge/src/index.ts +++ b/packages/desktop-bridge/src/index.ts @@ -1,7 +1,7 @@ import type { - ComputerUseInput, ComputerUseNativeError, ComputerUseResult, + ComputerUseWithAppCommandsInput, } from './computer-use.generated' import type { DesktopLocalFileRequest, DesktopLocalFileResponse } from './local-files' @@ -58,7 +58,14 @@ export interface ComputerUseActivity { export interface ComputerUseStatus { supported: boolean enabled: boolean - permissions: { accessibility: boolean; screenCapture: boolean } + /** The app-data permissions are absent in shells without app commands. */ + permissions: { + accessibility: boolean + screenCapture: boolean + reminders?: boolean + calendars?: boolean + contacts?: boolean + } activeAction: ComputerUseActivity | null /** Whether a first use of each app needs approval; absent in older shells, which always ask. */ askForApps?: boolean @@ -83,6 +90,20 @@ export class ComputerUseError extends Error { } } +/** The macOS privacy permissions Computer Use can ask for. */ +export const COMPUTER_USE_PERMISSIONS = [ + 'accessibility', + 'screenCapture', + 'reminders', + 'calendars', + 'contacts', +] as const +export type ComputerUsePermission = (typeof COMPUTER_USE_PERMISSIONS)[number] + +export function isComputerUsePermission(value: unknown): value is ComputerUsePermission { + return COMPUTER_USE_PERMISSIONS.some((permission) => permission === value) +} + /** Optional so a new web deployment remains compatible with older desktop shells. */ interface SimDesktopComputerUseApi { /** @@ -90,18 +111,20 @@ interface SimDesktopComputerUseApi { * can list only what changed. Absent in older shells, whose ids are positional. */ stableElementIds?: true + /** The computer tool accepts typed app commands (reminders); absent in older shells. */ + appCommands?: true /** Current activity without starting the helper; absent in older shells. */ getActivity?(): Promise getStatus(): Promise setEnabled(enabled: boolean): Promise - requestPermission(permission: 'accessibility' | 'screenCapture'): Promise + requestPermission(permission: ComputerUsePermission): Promise /** Absent in older shells, which always ask before a first use of each app. */ setAskForApps?(ask: boolean): Promise listAppPermissions(): Promise revokeApp(bundleId: string): Promise executeTool( toolCallId: string, - params: ComputerUseInput + params: ComputerUseWithAppCommandsInput ): Promise cancel(toolCallId?: string): Promise onActivity(callback: (activity: ComputerUseActivity | null) => void): () => void