diff --git a/.changeset/better-cases-cheer.md b/.changeset/better-cases-cheer.md new file mode 100644 index 00000000000..ea76e4155ea --- /dev/null +++ b/.changeset/better-cases-cheer.md @@ -0,0 +1,12 @@ +--- +"@fluentui-react-native/storybook-desktop-runtime": patch +"@fluentui-react-native/storybook-desktop": patch +"@fluentui-react-native/desktop-driver": patch +"@fluentui-react-native/components": patch +"@fluentui-react-native/focus-zone": patch +--- + +Add the native macOS and Windows implementations, explicit native build +verification, and prebuilt-only Storybook PR pipeline integration for the +desktop-driver package. Keep the FocusZone Windows WinMD compatible with the +consuming Storybook application's target SDK. diff --git a/.changeset/quiet-drivers-build.md b/.changeset/quiet-drivers-build.md new file mode 100644 index 00000000000..c71bbddd1b5 --- /dev/null +++ b/.changeset/quiet-drivers-build.md @@ -0,0 +1,16 @@ +--- +"@fluentui-react-native/desktop-driver": minor +"@fluentui-react-native/components": patch +"@fluentui-react-native/storybook-desktop": minor +"@fluentui-react-native/storybook-desktop-runtime": patch +--- + +Build the source-shipped native desktop helper explicitly, reuse verified +content-addressed artifacts, and add the Windows/Win32 C++ and macOS Swift +providers. Storybook can build the helper independently, ensures it during +prep, and attaches authored smoke tests to the exact app process it launched. +macOS cache resolution pins stable signatures to the leaf certificate and +designated requirement, makes source builds reproducible, verifies Hardened +Runtime and secure timestamps, reports TCC/AX diagnostics, and normalizes Fabric +accessibility roles, window identity, input, and Retina ScreenCaptureKit +evidence. diff --git a/.github/actions/setup-desktop-driver/action.yml b/.github/actions/setup-desktop-driver/action.yml new file mode 100644 index 00000000000..3ef48b768b8 --- /dev/null +++ b/.github/actions/setup-desktop-driver/action.yml @@ -0,0 +1,195 @@ +name: Set up Desktop Driver +description: Build and verify the native Desktop Driver helper for a desktop endpoint. + +inputs: + platform: + description: Desktop endpoint to provision. + required: true + working-directory: + description: Workspace that exposes the desktop-driver script. + required: false + default: apps/storybook + run-native-contract: + description: Run the target-OS native package contract before provisioning. + required: false + default: 'false' + macos-signing-identity: + description: Pre-provisioned macOS code-signing identity. Leave empty for an ad hoc CI signature. + required: false + default: '' + disabled-input-features: + description: Comma-separated input features unavailable on the runner. + required: false + default: '' + +outputs: + cache-root: + description: Job-local native helper cache. + value: ${{ steps.configure-macos.outputs.cache-root || steps.configure-windows.outputs.cache-root }} + doctor-path: + description: Native helper diagnostic report. + value: ${{ steps.build-macos.outputs.doctor-path || steps.build-windows.outputs.doctor-path }} + +runs: + using: composite + steps: + - name: Validate Desktop Driver platform + shell: bash + env: + DRIVER_PLATFORM: ${{ inputs.platform }} + run: | + set -euo pipefail + + case "$DRIVER_PLATFORM:$RUNNER_OS" in + macos:macOS|windows:Windows|win32:Windows) ;; + macos:*|windows:*|win32:*) + echo "Desktop Driver endpoint '$DRIVER_PLATFORM' cannot run on '$RUNNER_OS'." >&2 + exit 1 + ;; + *) + echo "Unsupported Desktop Driver endpoint '$DRIVER_PLATFORM'." >&2 + exit 1 + ;; + esac + + - name: Configure macOS Desktop Driver + id: configure-macos + if: inputs.platform == 'macos' + shell: bash + env: + DISABLED_INPUT_FEATURES: ${{ inputs.disabled-input-features }} + MACOS_SIGNING_IDENTITY: ${{ inputs.macos-signing-identity }} + run: | + set -euo pipefail + + cache_root="$RUNNER_TEMP/furn-desktop-driver-native" + mkdir -p "$cache_root" + echo "FURN_DESKTOP_DRIVER_CACHE_ROOT=$cache_root" >> "$GITHUB_ENV" + echo "cache-root=$cache_root" >> "$GITHUB_OUTPUT" + + echo "FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY=$MACOS_SIGNING_IDENTITY" >> "$GITHUB_ENV" + if [[ -n "$DISABLED_INPUT_FEATURES" ]]; then + echo "FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES=$DISABLED_INPUT_FEATURES" >> "$GITHUB_ENV" + fi + + - name: Configure Windows Desktop Driver + id: configure-windows + if: inputs.platform == 'windows' || inputs.platform == 'win32' + shell: pwsh + env: + DISABLED_INPUT_FEATURES: ${{ inputs.disabled-input-features }} + run: | + $ErrorActionPreference = 'Stop' + + $cacheRoot = Join-Path $env:RUNNER_TEMP 'furn-desktop-driver-native' + New-Item -ItemType Directory -Force -Path $cacheRoot | Out-Null + "FURN_DESKTOP_DRIVER_CACHE_ROOT=$cacheRoot" | + Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "cache-root=$cacheRoot" | + Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + if ($env:DISABLED_INPUT_FEATURES) { + "FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES=$env:DISABLED_INPUT_FEATURES" | + Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + } + + - name: Verify macOS Desktop Driver native contract + if: inputs.run-native-contract == 'true' && inputs.platform == 'macos' + shell: bash + working-directory: ${{ github.workspace }} + env: + FURN_NATIVE_DRIVER_TEST: '1' + run: yarn workspace @fluentui-react-native/desktop-driver test --runInBand + + - name: Verify Windows Desktop Driver native contract + if: inputs.run-native-contract == 'true' && (inputs.platform == 'windows' || inputs.platform == 'win32') + shell: pwsh + working-directory: ${{ github.workspace }} + env: + FURN_NATIVE_DRIVER_TEST: '1' + run: | + yarn workspace @fluentui-react-native/desktop-driver test --runInBand + if ($LASTEXITCODE -ne 0) { + throw "Desktop Driver native contract exited with code $LASTEXITCODE." + } + + - name: Build and diagnose macOS Desktop Driver + id: build-macos + if: inputs.platform == 'macos' + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + DOCTOR_PATH: ${{ runner.temp }}/desktop-driver-macos-doctor.json + MACOS_SIGNING_IDENTITY: ${{ inputs.macos-signing-identity }} + run: | + set -euo pipefail + + doctor_path="$RUNNER_TEMP/desktop-driver-macos-doctor.json" + echo "doctor-path=$doctor_path" >> "$GITHUB_OUTPUT" + yarn desktop-driver build-driver --platform macos + # Hosted CI reports actual TCC authority; it must not prompt or mutate the permission database. + yarn desktop-driver doctor --platform macos --permissions | tee "$doctor_path" + + node <<'NODE' + const fs = require('node:fs'); + + const report = JSON.parse(fs.readFileSync(process.env.DOCTOR_PATH, 'utf8')); + const expectedIdentity = process.env.MACOS_SIGNING_IDENTITY.trim(); + const expectedMode = expectedIdentity ? 'signed' : 'adhoc'; + if (!report.ready || report.result?.signing?.mode !== expectedMode) { + throw new Error(`The macOS Desktop Driver helper is not ready with the expected ${expectedMode} signature.`); + } + if (expectedIdentity) { + const expectedHash = /^[0-9a-f]{40}$/i.test(expectedIdentity) ? expectedIdentity.toUpperCase() : undefined; + if ( + (expectedHash && report.result.signing.certificateHash !== expectedHash) || + (!expectedHash && report.result.signing.identity !== expectedIdentity) + ) { + throw new Error('The verified helper certificate does not match the configured macOS signing identity.'); + } + } + if (report.permissions?.schemaVersion !== 1 || report.permissions?.type !== 'permissions') { + throw new Error('The macOS Desktop Driver permission diagnostic is missing or unsupported.'); + } + NODE + echo "FURN_DESKTOP_DRIVER_BUILD_POLICY=never" >> "$GITHUB_ENV" + echo "FURN_DESKTOP_DRIVER_BUILD_POLICY=never" >> "$GITHUB_ENV" + + - name: Build and diagnose Windows Desktop Driver + id: build-windows + if: inputs.platform == 'windows' || inputs.platform == 'win32' + shell: pwsh + working-directory: ${{ inputs.working-directory }} + env: + DRIVER_PLATFORM: ${{ inputs.platform }} + run: | + $ErrorActionPreference = 'Stop' + + yarn desktop-driver build-driver --platform $env:DRIVER_PLATFORM + if ($LASTEXITCODE -ne 0) { + throw "Desktop Driver build exited with code $LASTEXITCODE." + } + + $doctorPath = Join-Path $env:RUNNER_TEMP "desktop-driver-$env:DRIVER_PLATFORM-doctor.json" + "doctor-path=$doctorPath" | + Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + $doctorOutput = yarn desktop-driver doctor --platform $env:DRIVER_PLATFORM + $doctorExitCode = $LASTEXITCODE + $doctorOutput | Set-Content -Path $doctorPath -Encoding utf8 + if ($doctorExitCode -ne 0) { + throw "Desktop Driver doctor exited with code $doctorExitCode." + } + + $report = ($doctorOutput -join [Environment]::NewLine) | ConvertFrom-Json + if ( + -not $report.ready -or + $report.result.provider -ne 'windows' -or + $report.result.architecture -ne 'x64' -or + $report.result.wireProtocol.major -ne 1 -or + -not ($report.result.endpoints -contains $env:DRIVER_PLATFORM) + ) { + throw "The verified helper does not satisfy the $env:DRIVER_PLATFORM x64 provider contract." + } + + "FURN_DESKTOP_DRIVER_BUILD_POLICY=never" | + Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8cc9ece1c0f..831b08bbd59 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -205,6 +205,13 @@ jobs: - name: Build packages run: yarn build + - name: Set up macOS Desktop Driver + id: desktop-driver + uses: ./.github/actions/setup-desktop-driver + with: + platform: macos + disabled-input-features: physicalClick + - name: Bundle macOS run: | set -eox pipefail @@ -221,6 +228,15 @@ jobs: env: CCACHE_DISABLE: 1 + - name: Upload macOS Storybook artifacts + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: Storybook_macos_Dump + path: | + apps/storybook/artifacts/macos + ${{ steps.desktop-driver.outputs.doctor-path }} + ios: name: iOS PR runs-on: macos-26 @@ -336,7 +352,7 @@ jobs: windows-storybook: name: Windows Storybook PR runs-on: windows-latest - timeout-minutes: 40 + timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -355,6 +371,14 @@ jobs: - name: Build packages run: yarn build + - name: Set up Windows Desktop Driver + id: desktop-driver + uses: ./.github/actions/setup-desktop-driver + with: + platform: windows + run-native-contract: 'true' + disabled-input-features: keyboard,physicalClick,wheel + - name: Prep Windows Storybook run: yarn storybook prep --windows working-directory: apps/storybook @@ -383,7 +407,12 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: Storybook_windows_Dump - path: apps/storybook/artifacts/windows + path: | + apps/storybook/artifacts/windows + ${{ steps.desktop-driver.outputs.doctor-path }} + ${{ steps.desktop-driver.outputs.cache-root }}/v1/artifacts/**/build.log + ${{ steps.desktop-driver.outputs.cache-root }}/v1/diagnostics/** + if-no-files-found: warn win32: name: Win32 PR @@ -453,6 +482,13 @@ jobs: - name: Build packages run: yarn build + - name: Set up Win32 Desktop Driver + id: desktop-driver + uses: ./.github/actions/setup-desktop-driver + with: + platform: win32 + disabled-input-features: keyboard,physicalClick,wheel + - name: Smoke test Win32 Storybook run: yarn storybook smoke --win32 --mode stories-and-tests working-directory: apps/storybook @@ -462,7 +498,12 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: Storybook_win32_Dump - path: apps/storybook/artifacts/win32 + path: | + apps/storybook/artifacts/win32 + ${{ steps.desktop-driver.outputs.doctor-path }} + ${{ steps.desktop-driver.outputs.cache-root }}/v1/artifacts/**/build.log + ${{ steps.desktop-driver.outputs.cache-root }}/v1/diagnostics/** + if-no-files-found: warn check-changesets: name: Check for Changesets diff --git a/apps/storybook/AGENTS.md b/apps/storybook/AGENTS.md index a2428bda7c8..366136cdab3 100644 --- a/apps/storybook/AGENTS.md +++ b/apps/storybook/AGENTS.md @@ -37,12 +37,14 @@ Read [`agent-map.yaml`](agent-map.yaml) first for the compact architecture, look - Use `yarn storybook manifest --` to validate static story-plan extraction. +- Use `yarn storybook build-driver --` for an isolated native helper + build. `prep` ensures the same verified helper before app preparation. - Use `yarn storybook driver --` to start Metro, the Storybook channel/MCP listener, and the WebDriver listener under one owned supervisor. - Use the app's `yarn desktop-driver` script for JSON story-run and agent commands against that listener. -- The Stage 1 provider is deliberately fake. Do not add Windows or macOS native - automation code until the corresponding Stage 2 plan begins. +- macOS, Windows, and Win32 use their source-built native helpers. Keep the + deterministic fake provider limited to package contract tests. - Authored tests belong in component story `parameters.desktopDriver`, not in this app. The app owns identity, package discovery, platform exclusions, and generated manifests. @@ -56,6 +58,13 @@ Read [`agent-map.yaml`](agent-map.yaml) first for the compact architecture, look ## macOS native workflow +- In CI, use `.github/actions/setup-desktop-driver` to create the job-local + cache, build and diagnose the helper, and pin later resolution to the + verified artifact. GitHub-hosted runners use an ad hoc signature because + changing certificate trust requires interactive authorization, and disable + physical clicks because the hosted desktop is not the input qualification + environment. Managed self-hosted runners pass a pre-provisioned stable + signing identity and leave physical input enabled. - Run `yarn storybook prep --macos` for project generation and Pod installation. Do not run CocoaPods from the repository root because subprocess dependency resolution must start in this workspace. - Run `yarn storybook bundle --macos` for the JavaScript bundle, `yarn storybook build --macos` for a non-launching @@ -73,6 +82,13 @@ Read [`agent-map.yaml`](agent-map.yaml) first for the compact architecture, look ## Windows native workflow +- In CI, use `.github/actions/setup-desktop-driver` to run the shared Windows + native contract, build and diagnose the helper in a job-local cache, and pin + later resolution to the verified artifact before `prep` or `smoke`. +- GitHub-hosted Windows is not the physical-input qualification environment. + Disable keyboard, physical-click, and wheel through + `FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES`; leave them enabled on the + interactive self-hosted runner. - Use `yarn storybook prep --windows`, `bundle --windows`, `build --windows`, and `run --windows` for individual stages. Use `yarn storybook smoke --windows --mode stories` for the package-owned generation, channel server, native build and registration, Metro launch, full indexed-story traversal, and ownership-safe cleanup. Use @@ -90,6 +106,10 @@ Read [`agent-map.yaml`](agent-map.yaml) first for the compact architecture, look ## Win32 native workflow +- In CI, use `.github/actions/setup-desktop-driver` to build and diagnose the + `win32` endpoint in a job-local cache and pin later resolution before smoke. + The separate Windows Storybook job owns the shared provider's opt-in native + contract. - Win32 is the `@office-iss/react-native-win32` Paper endpoint hosted by `@office-iss/rex-win32`; do not treat it as the React Native Windows Fabric endpoint or generate a `react-native-test-app` project for it. diff --git a/apps/storybook/README.md b/apps/storybook/README.md index b9b9a100fea..929e5af860b 100644 --- a/apps/storybook/README.md +++ b/apps/storybook/README.md @@ -78,6 +78,27 @@ uses that suffix in the native bundle identifier, and selects dedicated Storyboo Parallel smoke tests from different enlistments therefore launch, drive, and stop only their own app and services, even when the default ports are already occupied. +Use `yarn storybook build-driver --macos` to build the signed Swift helper in +isolation. `yarn storybook smoke --macos --mode stories-and-tests` traverses the +catalog, writes a nonce-bound lease for the exact launched bundle, and runs the +same component-authored plans through the native macOS provider. + +For repeatable local TCC identity, configure an Apple Development or reusable +local development certificate: + +```sh +FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY="FURN Desktop Driver Development" \ + yarn storybook smoke --macos --mode stories-and-tests +``` + +The helper is a direct child of the Storybook/Node supervisor, so macOS privacy +authorization can be attributed to the terminal or IDE that owns that process +tree. Use `yarn desktop-driver doctor --platform macos --permissions` to inspect +the exact helper, signer, designated requirement, parent PID, Accessibility, +PostEvent, and Screen Recording state. Standalone +`yarn storybook run --macos` applies the same enlistment-specific xcconfig used +by smoke so its bundle identity does not fall back to the generated app default. + > `react-native-safe-area-context` note: Storybook's UI imports it, but its native module is > iOS-only (UIKit) and uses a Yoga API that doesn't compile for react-native-macos 0.81. It is > therefore not installed; the shared Metro helper aliases the import to a JS-only stub, so no @@ -99,6 +120,7 @@ yarn storybook smoke --windows --mode stories yarn storybook smoke --windows --mode stories-and-tests # Individual development stages +yarn storybook build-driver --windows yarn storybook prep --windows yarn storybook build --windows yarn storybook run --windows @@ -108,7 +130,10 @@ yarn storybook run --windows solution, `ExperimentalFeatures.props`, registrations, and build outputs are git-ignored. The shared smoke command bundles the Windows catalog, generates the solution, starts the platform-scoped channel server, builds and registers the Debug app, starts Metro, launches the exact app window, renders every indexed story, optionally runs the component-authored plans, and stops only the processes it recorded. -During Stage 1 the authored plans use the manifest-derived fake target; the full story traversal remains native. Logs are written +The authored plans use the source-built native Windows helper and attach to the +exact app process launched by the smoke lifecycle. After traversal, the Windows +Fabric lifecycle restarts only that owned app and rewrites its lease before +running authored tests against the warm Metro bundle. Logs are written beneath `artifacts/windows/smoke-logs`. The Accordion and Callout stories remain excluded from the Windows catalog because the current RNW 0.81 Fabric host still fail-fasts while traversing @@ -130,6 +155,9 @@ Windows Fabric endpoint above and does not use a generated # Produce dist/index.win32.bundle from the same story catalog as the other endpoints yarn storybook bundle --win32 +# Build the Windows helper shared by Windows Fabric and Win32 Paper +yarn storybook build-driver --win32 + # Launch the prebuilt Paper host yarn storybook run --win32 @@ -176,14 +204,14 @@ Nine ListItem stories and seven Accordion stories are omitted from the Win32-generated catalog because those components terminate the current REX 0.81.1 host with fail-fast code `0xC0000409`; macOS continues to include them, while Windows also omits Accordion and Callout. The three standalone Callout stories run through the same Paper -`RCTCallout` implementation as the portal chrome. All 130 included stories +`RCTCallout` implementation as the portal chrome. All 134 included stories render through the Win32 control-plane smoke sweep. Run `yarn storybook-server --win32` with this endpoint so the server exposes the -same 130-story index as the app; the ordinary `storybook-server` command keeps the full macOS and Windows catalog. +same 134-story index as the app; the ordinary `storybook-server` command keeps the full macOS and Windows catalog. `yarn storybook smoke --win32` defaults to `--mode stories` and verifies the package-owned desktop regions, resize -handles, addon surface, the complete 130-story sweep, host liveness, and ownership-safe cleanup. The -`stories-and-tests` mode then runs the component-authored plans through the Stage 1 manifest-derived fake target. Native -plan execution begins with the Stage 2 providers. Logs are written +handles, addon surface, the complete 134-story sweep, host liveness, and ownership-safe cleanup. The +`stories-and-tests` mode then runs the component-authored plans through the +same source-built Windows helper used by the Fabric endpoint. Logs are written beneath `artifacts/win32/smoke-logs`. A native `build --win32` operation is intentionally unsupported because this endpoint uses the prebuilt REX host. @@ -212,9 +240,11 @@ yarn storybook-server --win32 # explicit Win32 catalog # WebSocket: ws://127.0.0.1:7007/ MCP: http://127.0.0.1:7007/mcp ``` -For the Stage 1 desktop-driver control plane, use the combined supervisor: +For the native desktop-driver control plane, use the combined supervisor: ```sh +yarn storybook build-driver --macos +yarn storybook build-driver --windows yarn storybook driver --windows yarn storybook manifest --windows yarn storybook instance --windows @@ -224,8 +254,9 @@ yarn storybook instance --windows listener in the same Node process. `instance` reports the enlistment-specific ports and target identity. The generated manifest contains the exact platform catalog, relocatable source paths, serializable story-test plans, and platform -and portable-plan digests. The current provider is a deterministic fake host; -native Windows, Win32, and macOS providers are a later implementation stage. +and portable-plan digests plus the verified helper and trusted application +descriptor. macOS uses the native Swift provider, Windows and Win32 use the native C++ +provider, and the fake host remains limited to package tests. The app exposes the shared JSON CLI as `yarn desktop-driver`. After the supervisor and app are running, list or run the component-authored plans: diff --git a/apps/storybook/agent-map.yaml b/apps/storybook/agent-map.yaml index edc85a0d7f9..f9c54bc42a0 100644 --- a/apps/storybook/agent-map.yaml +++ b/apps/storybook/agent-map.yaml @@ -47,6 +47,12 @@ interaction_order: - Use screenshots for visual evidence, not element lookup. native: + driver: + build: yarn storybook build-driver -- + doctor: yarn desktop-driver doctor --platform + cache_env: FURN_DESKTOP_DRIVER_CACHE_ROOT + disabled_input_env: FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES + ci_policy: explicit-build-then-prebuilt-only macos: process: ReactTestApp app_bundle: macos/DerivedData/Build/Products/Debug/ReactTestApp.app @@ -57,6 +63,12 @@ native: windows: project_policy: generated generation_command: yarn storybook prep --windows + helper_provider: windows-x64 + win32: + host: prebuilt-rex + build_policy: unsupported + bundle_command: yarn storybook bundle --win32 + helper_provider: windows-x64 validation: javascript: diff --git a/apps/storybook/storybook.config.mts b/apps/storybook/storybook.config.mts index 926dcc6f44b..a2c85410840 100644 --- a/apps/storybook/storybook.config.mts +++ b/apps/storybook/storybook.config.mts @@ -46,6 +46,11 @@ export default makeDesktopStorybookConfig({ }), }, win32: { + nativeDriver: { + application: { + windowTitle: win32Host.windowTitle, + }, + }, run: createWin32RunCommand(win32Host), smoke: { command: createWin32SmokeCommand({ diff --git a/packages/agentic/components/src/components/button/button.stories.tsx b/packages/agentic/components/src/components/button/button.stories.tsx index c9977294adc..ea49a1937d9 100644 --- a/packages/agentic/components/src/components/button/button.stories.tsx +++ b/packages/agentic/components/src/components/button/button.stories.tsx @@ -84,8 +84,9 @@ export const Default: Story = { tests: [ { id: 'pointer-focus', + platforms: ['windows', 'win32'], title: 'Responds to activation and receives focus', - requires: ['element-screenshot', 'focus'], + requires: ['element-screenshot', 'focus', 'physical-click'], steps: [ { action: 'wait', target: { testId: 'agentic-storybook-button' } }, { expect: { state: 'role', target: { testId: 'agentic-storybook-button' }, value: 'button' } }, @@ -95,6 +96,19 @@ export const Default: Story = { { action: 'screenshot', name: 'button-focused', target: { testId: 'agentic-storybook-button' } }, ], }, + { + id: 'pointer-activation', + platforms: ['macos'], + title: 'Accepts native pointer activation', + requires: ['element-screenshot', 'physical-click'], + steps: [ + { action: 'wait', target: { testId: 'agentic-storybook-button' } }, + { expect: { state: 'role', target: { testId: 'agentic-storybook-button' }, value: 'button' } }, + { expect: { state: 'enabled', target: { testId: 'agentic-storybook-button' }, value: true } }, + { action: 'click', target: { testId: 'agentic-storybook-button' } }, + { action: 'screenshot', name: 'button-after-click', target: { testId: 'agentic-storybook-button' } }, + ], + }, ], } satisfies DesktopStoryTests, }, diff --git a/packages/agentic/components/src/components/checkbox/checkbox.stories.tsx b/packages/agentic/components/src/components/checkbox/checkbox.stories.tsx index 7c03a22d74d..d05f1bcaf83 100644 --- a/packages/agentic/components/src/components/checkbox/checkbox.stories.tsx +++ b/packages/agentic/components/src/components/checkbox/checkbox.stories.tsx @@ -73,6 +73,7 @@ export const Default: Story = { { id: 'toggles-checked-state', title: 'Toggles through native activation', + requires: ['physical-click'], steps: [ { action: 'wait', target: { testId: 'agentic-storybook-checkbox' } }, { expect: { state: 'role', target: { testId: 'agentic-storybook-checkbox' }, value: 'checkbox' } }, diff --git a/packages/agentic/desktop-driver/.gitignore b/packages/agentic/desktop-driver/.gitignore new file mode 100644 index 00000000000..ffc421cd41b --- /dev/null +++ b/packages/agentic/desktop-driver/.gitignore @@ -0,0 +1 @@ +native/macos/.build \ No newline at end of file diff --git a/packages/agentic/desktop-driver/AGENTS.md b/packages/agentic/desktop-driver/AGENTS.md index 97b843a0bce..54f80ea875d 100644 --- a/packages/agentic/desktop-driver/AGENTS.md +++ b/packages/agentic/desktop-driver/AGENTS.md @@ -1,68 +1,86 @@ -# Desktop driver development +# Desktop Driver agent instructions -These instructions apply to `packages/agentic/desktop-driver`. +This is the instruction root for +`packages/agentic/desktop-driver` and every descendant. -## Module ownership +## Start every task -- `protocol/` owns W3C parsing, errors, capabilities, actions, and deadlines. -- `server/` owns target/session/window/element state and HTTP routing. -- `host/` defines the platform-neutral native contract. -- `hosts/fake/` is the deterministic Stage 1 provider. -- `authoring/` owns serializable plan and result contracts. -- `runner/` executes plans and classifies outcomes. -- `wdio/` is the sanctioned high-level automation integration. -- `agent/` exposes bounded JSON-safe operations. -- `artifacts/` confines and atomically persists evidence. -- `cli/` parses commands and adapts structured APIs to JSON and exit codes. -- `testing/` owns reusable fake harnesses, not production fallbacks. +1. Read [README.md](README.md) and the narrowest relevant reference: + [architecture](references/architecture.md), + [service integration](references/service.md), + [native helpers](references/native-helpers.md), + [test integration](references/test-integration.md), or + [security](references/security.md). +2. For macOS native work, also read + [native/macos/README.md](native/macos/README.md). For Windows or Win32 native + work, read [native/windows/README.md](native/windows/README.md). +3. Read [PLAN.md](PLAN.md) only when the task changes qualification, release + readiness, or deferred scope. It contains remaining work, not the current + architecture. +4. Inspect the owning source and tests before editing. Use package scripts from + this directory or the corresponding root workspace command. -## Invariants +## Non-negotiable invariants -- Do not import Storybook, React, or React Native from this package. -- Keep protocol, client, authoring, runner, evidence, and fake-host code - platform-neutral. -- Put future operating-system integrations behind `DesktopHost`; do not branch - on `process.platform` outside host-provider selection. -- Keep the W3C server client-neutral. WebdriverIO belongs only in `wdio/`, - agent/CLI composition, and contract tests. -- Register targets on the server. Never accept arbitrary commands, environment - variables, or output paths from WebDriver capabilities. -- Permit one active session per target and reserve the target before any async - launch/attach work. -- Serialize commands per session and serialize all physical input across - sessions. Never dispatch reset, release, deletion, or another action around - an in-flight command. -- Preserve attached applications and clean up only resources recorded as owned. -- Apply deadlines to probe, launch, host command, cleanup, and dispose paths. -- Honor every `DesktopHost` `AbortSignal`: stop side effects and settle promptly - before the command queue advances. A provider may never complete input after - timeout cleanup. -- Release depressed input on failure, cancellation, and session deletion. -- Reject browser-origin requests and non-loopback serving by default. +- Do not import Storybook, React, or React Native into this package. +- Keep protocol, client, authoring, runner, artifacts, and fake-host code + platform-neutral. Operating-system behavior belongs behind `DesktopHost`. +- Keep the W3C server client-neutral. WebdriverIO belongs in `/wdio`, composed + agent/CLI operations, and contract tests, never in routing or host state. +- Register controlled targets on startup. Never accept executable paths, + environment variables, helper selection, or artifact roots from WebDriver + capabilities. +- Bind only to loopback, reject browser-origin requests, and fail closed when a + helper, signature, target identity, permission, or capability cannot be + verified. +- Permit one active session per target. Serialize each session's commands and + all physical input, including across independent helper processes. +- Every host operation must honor its `AbortSignal`, stop side effects, settle + before the queue advances, and release owned input on cancellation, failure, + or deletion. +- Preserve attached applications. Clean up only helpers, applications, ports, + and files whose exact ownership was recorded. +- Never compile during install, download a helper, or write native output under + the package, `node_modules`, or a pnpm store. Publish immutable, verified + artifacts only to the configured native store. +- Keep plans, manifests, results, CLI output, and agent output serializable. + Reject unknown or dynamic plan fields instead of silently dropping them. +- Never translate an unsupported native state into `false`. Capability-check it + and return an explicit skip or unsupported result. +- Confine artifacts beneath the configured root, preserve the original failure + when evidence capture also fails, and never expose native handles or + unrestricted commands through agent APIs. -## Authored plans +## Detailed development guidance -- Plans are versioned static JSON under `parameters.desktopDriver`. -- Keep selectors, steps, capability requirements, results, manifests, CLI - output, and agent output serializable. -- Reject unknown fields and dynamic values rather than silently dropping them. -- Never translate an unsupported native property into `false`. -- Use stable `testID` selectors for deterministic actions; role/name selectors - validate accessibility semantics. -- Keep platform differences declarative through `platforms`, `requires`, and - explicit skip results. +Use [Contributor reference](references/contributing.md) for module ownership, +change placement, generated files, test expectations, and validation order. +Protocol behavior is authoritative in +[references/protocol.md](references/protocol.md); private native transport is +authoritative in [native/PROTOCOL.md](native/PROTOCOL.md). -## Evidence and agents - -- Confine every artifact beneath the configured run root. -- Preserve the original test failure when evidence capture also fails. -- Bound agent tree depth and node count. -- Do not expose native handles or unrestricted command execution. -- The Storybook MCP remains separate; do not add a schema-only MCP claim. +When changing user-visible behavior, update the relevant reference and keep +[README.md](README.md) as a concise entry point. When work is completed, move +its durable behavior into documentation and remove it from [PLAN.md](PLAN.md); +the plan must describe only unfinished or explicitly deferred work. ## Validation -Run the package's declared format, lint, build, and test scripts. Contract tests -must cover raw W3C, the typed client, WebdriverIO custom commands, JSON CLI, -agent operations, artifacts, target concurrency, shutdown races, and -representative Storybook plans before repository-level validation. +Run the package's declared checks: + +```sh +yarn format +yarn lint +yarn build +yarn test +``` + +Set `FURN_NATIVE_DRIVER_TEST=1` only on the matching target operating system to +exercise the real native build, handshake, self-test, cache, and resolution +contract. Stable-signed macOS coverage additionally needs +`FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY`. + +Use [CI integration](references/ci-integration.md) and +[Test integration](references/test-integration.md) for native and consuming-app +gates. Run root validation when public types, exports, manifests, dependencies, +project references, or package contents change. diff --git a/packages/agentic/desktop-driver/PLAN.md b/packages/agentic/desktop-driver/PLAN.md index f17191a1438..b8c149eeb07 100644 --- a/packages/agentic/desktop-driver/PLAN.md +++ b/packages/agentic/desktop-driver/PLAN.md @@ -1,1394 +1,179 @@ -# Desktop Driver Plan +# Desktop Driver remaining work ## Status -Active architecture and implementation plan. This document starts from the -current checked-out tree and public platform/protocol documentation. It does -not depend on work from other branches. - -The effort starts with the platform-neutral protocol, fake host, Storybook -orchestration, WebdriverIO authoring, and agent contracts. Windows and macOS -native code is an explicit later stage, so native transport, signing, and -distribution choices do not block the initial implementation. - -## Implementation status - -Updated 2026-08-28. - -### Stage 1 Phase 1: Complete - -- Added the public `@fluentui-react-native/desktop-driver` package and repository - project references. -- Implemented W3C response/error routing, capability negotiation, - server-registered targets, one-session-per-target reservation, sessions, - timeouts, windows, elements, actions, screenshots, source, and unsupported - browser-command handling. -- Implemented stable WebDriver element references, native liveness checks, - preview-scoped staleness, configurable click modes, input-state tracking, - action validation, element-origin resolution, per-session command queues, - a global input mutex, abortable host deadlines, drained runner cancellation, - and ownership-safe shutdown. -- Added the deterministic fake host, typed low-level client, raw HTTP contract - coverage, and a WebdriverIO remote-session contract with no Appium service. -- Portable fake-host coverage exercises element lookup, click, text entry, - actions, waits, screenshots, stale references, concurrent session rejection, - and shutdown during session creation. - -### Stage 1 Phase 2: Complete - -- Added exact-platform Story Manifest generation with statically extracted, - validated `parameters.desktopDriver` plans, relocatable source paths, - platform digests, and portable-plan digests. -- Added a per-enlistment driver port and generated driver manifest containing - target identity, test-ID prefix, nonce, catalog digests, and service ports. -- Added authenticated, bridge-only runtime hello/readiness/error events with - request/run correlation, explicit hello challenges, same-story reset, and - preview generation. -- Added stable native app and story-root markers, native marker verification, - preview-only element invalidation, and a keyed per-run remount/error boundary. -- Added Storybook selection, reset, manifest, current-story, and args extension - commands to the WebDriver session. -- Added `storybook-desktop manifest`, `instance`, and `driver` flows. The - `driver` supervisor runs Metro plus separate Storybook and WebDriver listeners - while keeping both server protocols in one Node process. -- Added an embedded-server integration test and verified the live Windows - Stage 1 supervisor exposes equivalent 136-story channel and driver manifests. -- All macOS, Windows, and Win32 JavaScript bundles include the runtime bridge. - -### Stage 1 Phase 3: Complete - -- Finalized strict static plan, selector, action, assertion, capability, - platform, result, test, step, and artifact contracts under `/authoring`. -- Added deterministic filtering and sharding plus complete fake-host execution - for clicks, text entry, key/action sequences, scrolling, waits, Storybook - args, screenshots, source, and semantic assertions. -- Added the sanctioned `/wdio` API and typed browser commands for listing, - opening, resetting, asserting, and running story plans without Appium. -- Added confined atomic artifacts, host metadata, `run.json`, and automatic - screenshot, source, and compact-tree failure evidence. -- Added the bounded `/agent` API for listing, explaining, inspecting, finding, - acting, checking, capturing, and running the same story plans. -- Added the JSON `desktop-driver` CLI for fake serving, story list/explain/run, - sharding, evidence, and bounded agent describe/screenshot operations. -- Added manifest-derived fake elements and repeatable per-test state reset so - real component plans can run repeatedly in Stage 1. -- Added typed, statically extractable `desktop-e2e` plans to Button, Checkbox, - and Input. All three extract from real CSF and pass repeatedly through the - sanctioned WebdriverIO runner. -- Evaluated MCP integration and deferred a composed executable adapter until - Stage 2 proves the native command and security model; a schema-only claim is - explicitly insufficient. -- Updated package, Storybook, runtime, app, component, skill, and agent - documentation for the final Stage 1 responsibilities. - -### Remaining work - -Stage 1 is complete; no Phase 1, Phase 2, or Phase 3 deliverables are left -incomplete. - -- Stage 2 remains: implement Windows/Win32 and macOS native host providers and - replace the Stage 1 fake target in native runs. -- Stage 3 remains: release hardening, native artifact ownership, security - review, and CI promotion. -- On-device bridge execution is intentionally deferred to Stage 2; Phase 2 is - validated through the fake host, runtime/server contract tests, live - same-process services, and production bundles. - -## Outcome - -Create a public `@fluentui-react-native/desktop-driver` package that: - -- implements a useful, explicitly documented subset of the W3C WebDriver - Classic protocol without Appium; -- drives React Native macOS, React Native Windows Fabric, and React Native - Win32 Paper applications; -- exposes platform-neutral window, accessibility-tree, input, screenshot, and - diagnostics APIs; -- formalizes the device contracts required by desktop Storybook applications; -- lets component authors declare portable tests alongside stories; -- runs those tests as end-to-end automation or exposes the same operations to - validation agents; -- reuses the existing Storybook channel server for story discovery, selection, - and render events; -- avoids requiring another long-running Node server process for Storybook. - -## Non-goals - -- Do not expose Appium client APIs, Appium capabilities, or the Appium CLI. -- Do not emulate browser-only behavior such as navigation, cookies, frames, - shadow roots, JavaScript execution, prompts, or printing. -- Do not make Storybook a dependency of the generic driver. -- Do not require Jest or a particular agent protocol for authored story tests. - WebdriverIO is the sanctioned high-level automation API. -- Do not use visible text, layout order, or native class names as the stable - selector contract. -- Do not include visual-regression baseline comparison or migration of the - legacy E2E harness in the initial effort; both are future considerations. - -## Architectural decisions - -### Package topology - -Create one new public package: - -```text -@fluentui-react-native/desktop-driver -``` - -Do not initially create either `desktop-driver-server` or -`storybook-desktop-server`. - -`desktop-driver` owns both an embeddable W3C remote end and a standalone CLI. -The server is central to the package rather than an independently useful -product boundary. Keep internal `protocol`, `server`, `client`, and `host` -seams so a server package can be extracted later without changing public -contracts. - -`storybook-desktop` remains the owner of Storybook configuration, the channel -server, Metro and native app lifecycle, platform selection, generated -manifests, and Storybook-specific orchestration. It depends on -`desktop-driver`, registers a Storybook target and orchestration adapter, and -starts the embedded driver listener. - -`storybook-desktop-runtime` remains React Native-only. It exposes the native -story root and sends versioned readiness/error messages over the existing -Storybook channel. It does not host WebDriver or import Node APIs. - -```text -component story - -- type-only --> desktop-driver/authoring - -desktop-driver - -- no dependency --> Storybook, React, React Native, or private apps - -storybook-desktop - --> desktop-driver - -storybook-desktop-runtime - --> Storybook channel only - -apps/storybook - --> storybook-desktop - --> storybook-desktop-runtime -``` - -### Server and process model - -The existing Storybook server remains the Storybook control plane: - -- story index and documentation; -- WebSocket channel; -- story selection; -- Storybook events; -- existing MCP endpoint. - -The W3C remote end uses a separate loopback port because the current upstream -channel server constructs and owns its HTTP server and handles unmatched -requests itself. Mounting WebDriver routes into that listener would couple the -driver to upstream internals and risk conflicting responses. - -Both listeners should run in the same `storybook-desktop` Node process: - -```text -storybook-desktop supervisor process - |- Storybook HTTP/WebSocket/MCP listener - |- WebDriver HTTP listener - |- Metro child process, when needed - |- native host transport, when needed - `- owned application process or attached application lease -``` - -This meets the goal of avoiding another long-running server process while -keeping the two protocols isolated. A non-Storybook application can run the -same WebDriver remote end through the standalone `desktop-driver` CLI. - -Create a separate server package only if one of these triggers occurs: - -1. a consumer needs the server without the client, authoring, and testing APIs; -2. remote host deployment requires a release cadence independent of the - package; -3. native artifacts make the server install materially heavier than the - client; -4. authentication, TLS, or fleet management becomes a separate product - concern. - -### Target registration - -Clients select a server-registered target: - -```json -{ - "capabilities": { - "alwaysMatch": { - "browserName": "furn-native-desktop", - "platformName": "windows", - "furn:target": "agentic-storybook-windows", - "furn:launchMode": "attach" - } - } -} -``` - -Capabilities must not accept arbitrary executable paths, command arguments, -environment variables, manifest paths, or artifact roots. Target definitions -are registered when the server starts and resolve to controlled launch/attach -providers and a confined artifact root. - -An explicit local-development mode may allow ad hoc targets later, but it must -be disabled by default and unavailable to agent-facing APIs. - -### V1 support scope - -V1 supports: - +The platform-neutral service, WebDriver contract, typed clients, WebdriverIO +integration, authored story plans, fake test harness, evidence model, bounded +agent API, native build/cache/verification pipeline, Windows/Win32 helper, and +macOS helper are implemented for the V1 command surface. + +This plan contains only qualification, hardening, release, or explicitly +deferred work that remains. Current behavior and design decisions belong in the +package documentation: + +| Context | Authoritative documentation | +| -------------------------------------- | -------------------------------------------------- | +| Package use and support matrix | [README.md](README.md) | +| Layering and process model | [Architecture](references/architecture.md) | +| W3C service and target registration | [Service integration](references/service.md) | +| Public protocol behavior | [WebDriver contract](references/protocol.md) | +| Native build, cache, and verification | [Native helpers](references/native-helpers.md) | +| FDR1 helper transport | [Native protocol](native/PROTOCOL.md) | +| macOS implementation, signing, and TCC | [macOS provider](native/macos/README.md) | +| Windows/Win32 implementation | [Windows provider](native/windows/README.md) | +| Authored and native tests | [Test integration](references/test-integration.md) | +| Runner and promotion model | [CI integration](references/ci-integration.md) | +| Trust boundary | [Security](references/security.md) | + +When an item below is completed, document its durable behavior in the owning +page and remove it from this plan. + +## Release objective + +Release `@fluentui-react-native/desktop-driver` as a public, +Appium-independent, W3C-compatible driver for React Native macOS Fabric, +React Native Windows Fabric, and React Native Win32 Paper. Tests should use one +portable WebdriverIO or serializable story-plan contract while native providers +report truthful platform capabilities and preserve exact process and input +ownership. + +V1 remains scoped to: + +- macOS 14 or later on Apple Silicon; - Windows 11 x64; -- macOS 14 on Apple Silicon; -- Windows Fabric, Win32 Paper, and macOS Storybook endpoints; -- exactly one active session per physical target. - -Broader operating-system, architecture, and concurrency support is deferred. - -## Package responsibilities - -### `desktop-driver` - -Own: - -- W3C routing, response envelopes, errors, and capability processing; -- session, timeout, window, input, and element state; -- server-side target registry; -- platform-neutral host contract; -- native host transport protocol; -- WebDriver element identity and staleness; -- typed low-level client; -- sanctioned WebdriverIO runner, configuration, matchers, and custom commands; -- generic serializable story-test schema and runner primitives; -- screenshots, artifacts, logs, and diagnostics; -- token-efficient agent operations; -- deterministic fake host and protocol conformance harness. - -Do not depend at runtime on: - -- Appium; -- Storybook; -- React or React Native; -- a private application package. - -### `storybook-desktop` - -Own: - -- a generated platform-specific Story Manifest; -- a Storybook implementation of the driver's `StoryOrchestrator` interface; -- authenticated/correlated channel messages; -- Storybook extension commands; -- one supervisor for channel, Metro, driver, app, and test lifecycle; -- driver port allocation in the existing per-enlistment instance identity; -- Storybook test-plan extraction and digest generation; -- machine-readable readiness output; -- `driver`, `test`, and `agent` CLI flows. - -### `storybook-desktop-runtime` - -Own: - -- a stable native application/root marker; -- a stable native story-canvas marker; -- native-observable current story and preview generation; -- runtime hello, story-ready, story-error, and reset acknowledgements; -- a per-test remount boundary keyed by run ID; -- render-error forwarding. - -### Consuming Storybook app - -Own: - -- target registration and native identity; -- story package discovery and platform exclusions; -- exceptional launch/run commands; -- artifact root; -- concrete `testID` prefix; -- pilot story tests; -- cross-package contract tests. - -## W3C remote-end contract - -Describe the package as a **W3C WebDriver Classic-compatible native desktop -remote end**, not a conforming browser remote end. Unsupported browser commands -return `unsupported operation`; they never return fabricated success values. - -### Initial standard endpoints - -Implement: - -- `GET /status`; -- `POST /session` and `DELETE /session/{id}`; -- `GET|POST /session/{id}/timeouts`; -- current window, window handles, switch window, close window; -- get/set window rectangle where the host reports support; -- find element(s) from the window or an element; -- active element; -- element name/role, text, attributes, properties, rectangle, enabled, and - selected state where supported; -- click, clear, and send keys; -- perform and release actions; -- window screenshot and element screenshot; -- normalized accessibility source. - -Return `unsupported operation` for: - -- navigation and history; -- cookies; -- frames and shadow roots; -- arbitrary JavaScript execution; -- browser prompts; -- printing; -- CSS values; -- new-window creation until native semantics are specified. - -Do not repurpose the standard `pageLoad` timeout for story readiness. Add -namespaced driver timeouts: - -```ts -type DesktopTimeouts = { - appLaunch: number; - nativeCommand: number; - storyRender: number; - stableLayout: number; -}; -``` - -### Capability negotiation - -Implement W3C `alwaysMatch` and ordered `firstMatch` processing, including: - -- extension capability names containing `:`; -- rejection of duplicate keys during merge; -- ordered candidate evaluation; -- `session not created` when no target/provider can satisfy a candidate; -- truthful returned capabilities based on the selected host. - -Use: - -- `platformName: "macos"` or `"windows"` for the operating system; -- `furn:endpoint: "macos" | "windows" | "win32"` for the repository endpoint; -- `furn:renderer: "fabric" | "paper"` for renderer semantics; -- `furn:target` for the registered target; -- `furn:clickMode: "physical" | "accessibility" | "auto"` for environment- - appropriate element-click behavior; -- `furn:features` for negotiated input, tree, state, screenshot, and window - capabilities. - -Return a standard capability only when its semantics are implemented. - -### Errors - -Map native failures to specific WebDriver errors: - -| Condition | WebDriver error | -| --------------------------------------------- | --------------------------- | -| target cannot launch or attach | `session not created` | -| missing/closed session | `invalid session id` | -| missing/closed window | `no such window` | -| lookup does not resolve | `no such element` | -| retained native node is detached/replaced | `stale element reference` | -| malformed locator | `invalid selector` | -| disabled, unfocusable, or empty-bounds target | `element not interactable` | -| another node owns the hit-tested point | `element click intercepted` | -| capture backend fails | `unable to capture screen` | -| deadline expires | `timeout` | -| capability/property/operation is unavailable | `unsupported operation` | - -Error `data` may contain redacted native error codes, operation names, and -artifact IDs. It must not expose environment variables, arbitrary paths, or -private window content. - -### Element identity and staleness - -Expose only session-generated UUIDs under the standard key: - -```text -element-6066-11e4-a52e-4f735466cecf -``` - -Never expose UIA runtime IDs, AX references, React tags, HWNDs, or accessibility -paths as public element IDs. - -Each stored element records: - -- native handle; -- application and window; -- logical scope: `application`, `chrome`, `preview`, or `secondary-window`; -- preview generation, when applicable; -- diagnostic locator fingerprint. - -Every element command performs a cheap liveness check. A story reset increments -the preview generation and invalidates preview elements only. Storybook chrome -and still-live secondary-window elements remain valid. - -Do not reconstruct a missing native object from a role/index path. Re-resolving -to a different object must produce staleness rather than silently changing the -meaning of an existing WebDriver reference. - -### Selectors - -The portable authoring API exposes: - -```ts -by.testId('button-primary'); -by.role('button', { name: 'Save' }); -by.accessibleName('Save'); -by.text('Saved'); -``` - -Wire strategies in the first release: - -- `accessibility id` as a documented extension mapping to `testID`; -- `tag name` mapping to normalized native role; -- `link text` and `partial link text` mapping to accessible name only where - those standard semantics are meaningful. - -Defer CSS and XPath. Do not redefine CSS for a non-DOM tree, and do not add the -cost and brittleness of normalized XML/XPath until a concrete client need is -demonstrated. - -Deterministic authored tests use `testID`. Role and accessible name are -important for accessibility validation and agent exploration, but are not a -replacement for stable IDs. - -### State - -Native platforms expose different state sets. Absence must never become a -false-shaped passing assertion. - -```ts -type SupportedValue = { supported: true; value: T } | { supported: false; reason: string }; -``` - -Normalize, when supported: - -- automation ID; -- accessible name and help; -- role and native role; -- value/text; -- enabled; -- focused/focusable; -- selected, checked/mixed, and expanded; -- visible/offscreen; -- logical rectangle; -- supported accessibility actions/patterns. - -The runner checks declared capabilities before a test. Unsupported required -state produces an explicit skip or unsupported result, not a passing -assertion. - -### Input - -Standard `element.click()` uses the session's negotiated click mode: - -```ts -type ClickMode = 'physical' | 'accessibility' | 'auto'; -``` - -- `physical` performs real pointer input and is the default for local - component validation; -- `accessibility` invokes the native accessibility action and is intended for - environments such as CI where physical input is blocked; -- `auto` prefers physical input and falls back to accessibility activation - only when the host reports that physical input is unavailable. - -The selected mode is returned in `furn:features`. Session creation fails when -the requested mode is unsupported, rather than silently changing interaction -semantics. Accessibility mode is necessarily capability-limited because not -every React Native control projects an activation action. - -Physical click executes: - -1. validate liveness; -2. scroll into view when supported; -3. activate the owning window; -4. refresh bounds; -5. compute an in-view point; -6. hit-test the point; -7. reject interception; -8. send pointer down/up. - -An explicit extension command may invoke accessibility activation regardless -of the session default for accessibility-focused validation. - -Implement W3C Actions with: - -- key, mouse pointer, wheel, and null sources; -- tick grouping and duration; -- viewport, pointer, and element origins; -- depressed key/button tracking; -- Release Actions on normal teardown, timeout, cancellation, and host failure. - -There is one global input mutex per physical desktop. V1 permits exactly one -active session per physical target. - -Public rectangles use logical points/DIPs relative to the current window client -area. Hosts privately convert to screen pixels using window origin, frame -insets, Windows DPI, Retina backing scale, and virtual-desktop origin. Capture -metadata records both logical and pixel dimensions and the scale factor. - -### Screenshots - -Standard screenshot commands return Base64 PNG: - -- session screenshot: current native window content; -- element screenshot: current window capture cropped to the visible element - bounds; -- window decorations excluded by default. - -Extensions may request: - -- window frame inclusion; -- a named artifact; -- all windows, returned as an artifact manifest; -- display capture for diagnostics. - -The platform-neutral stage uses fake captures to establish protocol and -artifact behavior. Real screenshot support arrives with the later native-host -stage. Windows native work includes occlusion-independent HWND capture before -the Windows screenshot capability is advertised. - -### Diagnostics and artifacts - -Provide namespaced commands for: - -- compact JSON accessibility tree; -- full normalized tree/source; -- host and permission diagnostics; -- recent driver, host, story, input, and device events; -- named screenshots and evidence bundles; -- Storybook manifest and current story state. - -Suggested failure bundle: - -```text -artifacts/desktop-driver// - run.json - host.json - sessions// - commands.ndjson - windows.json - source.xml - tree.json - screenshots/ - logs/ - stories/// - result.json - before.png - failure.png -``` - -Result status distinguishes: - -- passed; -- assertion failed; -- skipped unsupported capability; -- timed out; -- cancelled; -- app crashed; -- driver/host failed; -- configuration failed; -- permission failed. - -## Platform-neutral host contract - -The protocol layer depends only on an injected host: - -```ts -interface DesktopHost { - readonly endpoint: 'macos' | 'windows' | 'win32'; - - probe(): Promise; - launch(target: RegisteredTarget): Promise; - attach(target: RegisteredTarget): Promise; - - windows(app: ApplicationLease): Promise; - activate(window: DesktopWindow): Promise; - getWindowRect(window: DesktopWindow): Promise; - setWindowRect(window: DesktopWindow, rect: Partial): Promise; - - find(root: DesktopRoot, selector: NativeSelector, options: FindOptions): Promise; - snapshot(element: NativeElement): Promise; - isAlive(element: NativeElement): Promise; - hitTest(window: DesktopWindow, point: Point): Promise; - - performActions(actions: readonly NativeActionTick[]): Promise; - releaseActions(): Promise; - - captureWindow(window: DesktopWindow): Promise; - captureRect(window: DesktopWindow, rect: Rect): Promise; - - subscribe(listener: DesktopHostEventListener): Disposable; - dispose(): Promise; -} -``` - -`ApplicationLease` records: - -- `launched` or `attached` ownership; -- PID and process creation time; -- target identity; -- known windows; -- graceful close behavior. - -Attached apps are preserved by default. Cleanup uses exact owned resource -records, never process-name matching. - -Host events should include: - -- structure changed; -- focus/property changed; -- window opened/closed; -- app exited; -- host transport failed. - -The host transport begins with a versioned handshake containing protocol and -helper versions, endpoint, architecture, capabilities, and permission state. -A host crash invalidates the session and produces an infrastructure failure; it -is not silently restarted during a test. - -## Native implementation staging - -Keep the transport replaceable behind `DesktopHost`. Do not make an unproven -FFI library or an unsigned native binary a permanent API decision. - -The initial stage contains no Windows or macOS native code. It delivers the -complete protocol, fake host, Storybook integration, WebdriverIO authoring -surface, agent API, and native-host contract using TypeScript/Node only. - -Native platform providers are a separate second delivery stage: - -- Windows 11 x64 and Win32 Paper share a Windows provider built around UI - Automation, configurable physical/accessibility interaction, app/window - ownership, and occlusion-independent Windows Graphics Capture; -- macOS 14 on Apple Silicon receives a provider selected to preserve the - required local and hosted-CI authority while implementing the same host - contract; -- native build, signing, notarization, and artifact distribution are scoped to - that stage rather than prerequisites for the platform-neutral package. - -### Current constraints - -- package installation scripts are disabled; -- new dependencies must satisfy the repository age policy; -- public packages are built and packed on Linux; -- the current publish pipeline does not build, sign, or notarize Windows/macOS - native artifacts; -- current macOS E2E uses a Mac2/XCTest substrate on hosted macOS CI; -- current Win32 Storybook smoke uses in-box Windows UI Automation from - PowerShell on hosted Windows CI. - -### Native-stage feasibility gates - -Evaluate at least these options against the same host contract: - -| Endpoint | Candidate | Purpose | -| ------------- | ---------------------------------------------------- | -------------------------------------------------------- | -| Windows/Win32 | long-lived PowerShell UIA worker with P/Invoke input | zero-published-binary baseline | -| Windows/Win32 | C++/WinRT helper using UIA, SendInput, and WGC | highest-fidelity capture and typed native implementation | -| macOS local | direct AX/CGEvent transport | fast developer attach loop, requires TCC | -| macOS CI | first-party XCTest-based transport | preserve hosted-CI automation without Appium | -| macOS | Swift helper using AX, CGEvent, and ScreenCaptureKit | stable native implementation if build/signing is funded | - -Before native implementation begins, the stage must answer: - -- Can the candidate be built, packaged, and invoked through declared repository - scripts without install-time compilation? -- What identity receives Accessibility and Screen Recording permission? -- Can hosted CI grant or inherit the required authority? -- Can it enumerate and interact with current app windows? -- Does physical pointer/keyboard input reach React Native controls? -- Can it capture composited window content when occluded, scaled, and spread - across monitors? Occlusion-independent capture is required for the Windows - provider in this stage. -- Can it capture secondary Callout windows? -- What is its cold-start and command latency? -- How are native errors and events represented? - -Current direction: - -- keep these platform experiments out of the initial implementation; -- start the native Windows/Win32 stage with a long-lived PowerShell UIA worker - as a contract probe because the current tree proves that substrate can - inspect Win32 on hosted CI; -- implement Windows.Graphics.Capture or equivalent direct HWND capture as part - of that same native stage before advertising full screenshot support; -- retain an XCTest-backed macOS CI transport unless a non-Appium replacement - proves the same hosted-runner authority; -- allow a raw AX/CGEvent macOS provider for local attach workflows; -- introduce signed Swift/C++ helpers only within the native stage and only - after build/sign/notarization and artifact-package ownership are approved. - -Using XCTest as an internal host transport does not make Appium part of the -authoring or wire contract; the package still owns the W3C server, sessions, -capabilities, errors, and public APIs. - -## Storybook device contract - -### Instance manifest - -Extend the current per-enlistment instance model with one generated manifest: - -```ts -type DesktopStorybookDriverManifest = { - schemaVersion: 1; - instanceId: string; - endpoint: 'macos' | 'windows' | 'win32'; - renderer: 'fabric' | 'paper'; - targetId: string; - appName: string; - displayName: string; - testIDPrefix: string; - storybookPort: number; - metroPort: number; - driverPort: number; - platformManifestDigest: string; - portablePlanDigest: string; - bridgeNonce: string; -}; -``` - -This generated projection becomes the single source for the runtime, -supervisor, server target, and smoke/test commands. Its `testIDPrefix` -originates from the consuming app's custom `app.json` Storybook identity, so -the app does not maintain a second identity file or duplicate runtime setting. - -Use two digests: - -1. `platformManifestDigest` covers the exact platform catalog and plans; -2. `portablePlanDigest` covers only explicitly portable stories/tests and - excludes physical paths and platform-only metadata. - -Manifests contain package names and package-relative POSIX paths. Absolute -package roots stay in process memory and are excluded from digests and agent -output. - -### Required native markers - -Every desktop Storybook endpoint must expose: - -1. a stable application/root marker; -2. a stable story canvas/root `testID`; -3. the active story ID in native-observable state; -4. the preview generation or run ID in native-observable state. - -Only the story root is a universal chrome contract. Do not require macOS or -Windows to fork upstream LiteUI merely to expose the Win32-specific sidebar, -addon, or resize-handle IDs. - -### Runtime bridge - -The runtime sends versioned channel events: - -```ts -type DesktopBridgeEvent = - | { - type: 'furn:desktop:hello'; - version: 1; - instanceId: string; - endpoint: 'macos' | 'windows' | 'win32'; - targetId: string; - platformManifestDigest: string; - nonce: string; - } - | { - type: 'furn:desktop:story-ready'; - requestId: string; - runId: string; - storyId: string; - previewGeneration: number; - portablePlanDigest: string; - } - | { - type: 'furn:desktop:story-error'; - requestId: string; - runId: string; - storyId: string; - message: string; - }; -``` - -The supervisor rejects wrong instance, endpoint, target, digest, nonce, -duplicate bridge, and stale reconnect messages. - -Story selection: - -1. authenticate the runtime hello; -2. validate the story against the platform manifest; -3. create request and run IDs; -4. issue selection through the existing Storybook channel; -5. await the correlated runtime `story-ready`; -6. verify the native story marker and preview generation; -7. wait for the native canvas root; -8. invalidate prior preview element references; -9. optionally wait for stable layout. - -Each test receives a fresh run ID and remount boundary. The runtime resets local -story state and, when requested, Storybook args before acknowledging readiness. - -### Story Manifest - -`storybook-desktop` generates a platform-specific manifest from the same story -configuration used by the app. It must preserve data not guaranteed by the -current `/index.json` response: - -- canonical story ID; -- package name and package-relative source path; -- platform membership; -- authored tags; -- extracted `parameters.desktopDriver`; -- capability requirements; -- exact-platform and portable-plan digests. - -Static extraction fails loudly with file and location when a test plan is not -serializable. It must never silently omit an authored plan. - -Pass the manifest to the embedded driver in memory or through an owned -generated file. Do not introduce a fourth manifest HTTP listener. - -## Component-authored story tests - -The primary contract is a versioned, statically serializable plan in story -parameters: - -```tsx -import type { DesktopStoryTests } from '@fluentui-react-native/desktop-driver/authoring'; - -export const Default: Story = { - parameters: { - desktopDriver: { - version: 1, - tests: [ - { - id: 'pointer-and-keyboard-focus', - title: 'Supports pointer and keyboard focus', - requires: ['pointer', 'keyboard', 'focus'], - steps: [ - { - expect: { - target: { testId: 'button-primary' }, - state: 'enabled', - }, - }, - { - action: 'click', - target: { testId: 'button-primary' }, - }, - { - expect: { - target: { testId: 'button-primary' }, - state: 'focused', - }, - }, - { - action: 'keys', - value: ['TAB'], - }, - { - action: 'screenshot', - name: 'keyboard-focus', - }, - ], - }, - ], - } satisfies DesktopStoryTests, - }, -}; -``` - -Initial actions: - -- wait for target/state/stable layout; -- click and double-click; -- clear and type; -- key and W3C action sequences; -- scroll; -- update Storybook args; -- screenshot; -- capture tree/source; -- annotate evidence. - -Initial assertions: - -- exists/count; -- displayed; -- enabled; -- focused/focusable; -- selected/checked/mixed/expanded; -- accessible name/help; -- role; -- value/text; -- bounds; -- active element. - -Rules: - -- no platform branches inside a portable plan; -- differences use declarative `requires`, platform inclusion, and explicit skip - reasons; -- selectors use stable IDs for deterministic interaction; -- plans are hashable, listable before app launch, shardable, and - agent-readable; -- the test runner distinguishes unsupported capability from failed assertion. - -Reserve a later imperative escape hatch for cases the DSL cannot express. It -must be an explicitly referenced React Native-free module, marked nonportable -or less agent-readable, and must not enter component package build/publish -output accidentally. Do not add it until real authored tests demonstrate the -need. - -## Public APIs - -### Typed client - -```ts -const client = await createDesktopDriverClient({ url: ready.webdriverUrl }); -const session = await client.newSession({ - platformName: 'windows', - 'furn:target': 'agentic-storybook-windows', -}); - -const story = await session.storybook.open('components-button--default'); -const button = await story.find(by.testId('button-primary')); - -await button.click(); -await button.waitFor({ focused: true }); -await story.screenshot({ name: 'focused-button' }); -await story.runTest('pointer-and-keyboard-focus'); -await session.delete(); -``` - -The high-level client calls the same W3C and extension routes available to -external clients. - -### WebdriverIO automation API - -WebdriverIO is the sanctioned high-level test API for component authors and -automation suites. The package provides supported configuration, typed custom -commands, selectors, matchers, lifecycle integration, and Storybook commands. -The serializable story-plan DSL runs through this same WebdriverIO integration. - -The W3C server remains client-neutral and does not require Appium. Validate it -with: - -- a raw HTTP protocol suite; -- the low-level typed client; -- the sanctioned WebdriverIO remote client and runner. - -WebdriverIO is a supported dependency of the high-level testing surface, while -the protocol server modules remain independent of it. - -### Agent API - -Expose coarse, JSON-safe operations: - -```ts -agent.listStories(); -agent.openStory(storyId); -agent.describe({ scope: 'canvas', depth: 3 }); -agent.find({ testId: 'button-primary' }); -agent.click({ testId: 'button-primary' }); -agent.type({ testId: 'input', text: 'hello' }); -agent.check({ testId: 'button-primary', role: 'button', enabled: true }); -agent.screenshot({ scope: 'window', name: 'button' }); -agent.runStoryTest(storyId, testId); -agent.getArtifacts(); -agent.dispose(); -``` - -`describe` returns a bounded projection containing role, name, `testID`, -supported state, bounds, and child count. Lookup failures may include bounded -nearest-ID suggestions. - -Ship the typed API and JSON CLI first. MCP requires an actual executable -adapter, not only a tool-schema file. Later, add an MCP route to the driver -listener or a composed adapter owned by `storybook-desktop`; do not create a -separate MCP package. - -### CLI and supervisor - -`desktop-driver`: - -```text -desktop-driver serve -desktop-driver doctor --target --json -desktop-driver tree --session --json -desktop-driver screenshot --session --output -``` - -`storybook-desktop`: - -```text -storybook-desktop driver --windows -storybook-desktop test --windows [--story ] [--tag ] -storybook-desktop agent --windows -storybook-desktop manifest --windows -storybook-desktop instance --windows --json -``` - -The Storybook supervisor: - -1. resolves platform and instance identity; -2. generates manifests; -3. starts the channel server and embedded driver listener; -4. starts Metro when needed; -5. registers the exact target; -6. launches or attaches the app; -7. authenticates the runtime bridge; -8. runs tests or writes agent-ready connection data; -9. releases input and tears down only owned resources. - -## Implemented package shape - -```text -packages/agentic/desktop-driver/ - AGENTS.md - PLAN.md - README.md - SPEC.md - package.json - tsconfig.json - jest.config.cjs - config/ - cli.cjs - src/ - index.ts - authoring/ - index.ts - results.ts - storyTests.ts - artifacts/ - ArtifactManager.ts - index.ts - client/ - DesktopDriverClient.ts - index.ts - cli/ - createDesktopDriverCommand.ts - index.ts - protocol/ - actions.ts - capabilities.ts - constants.ts - errors.ts - timeouts.ts - types.ts - server/ - createDesktopDriverServer.ts - index.ts - SessionManager.ts - TargetRegistry.ts - host/ - types.ts - hosts/ - fake/FakeDesktopHost.ts - runner/ - index.ts - StoryTestRunner.ts - wdio/ - DesktopWebdriver.ts - index.ts - agent/ - DesktopAgent.ts - index.ts - testing/ - FakeStoryOrchestrator.ts - fakeStoryElements.ts - index.ts - protocolHarness.ts -``` - -The later native stage adds `hosts/windows` and `hosts/macos`, plus any -platform artifact packages approved by the native distribution design. - -Potential subpath exports: - -- `.`; -- `./authoring`; -- `./artifacts`; -- `./client`; -- `./cli`; -- `./server`; -- `./agent`; -- `./runner`; -- `./testing`; -- `./wdio`; -- `./package.json`. - -Use explicit named exports. Keep platform code under `hosts`. The package -`AGENTS.md` should require no Storybook imports, no platform branching outside -host providers, exact ownership cleanup, and declared-script validation. - -When implementation starts, conform to repository package rules: - -- `build` is `tsc -b`; -- composite TypeScript output and build info are configured; -- workspace dependencies and project references match; -- the root project references the package; -- dependencies satisfy catalog and package-age policy; -- publishing checks and a changeset are included. - -## Milestones - -### Stage 1: Platform-neutral foundation - -Stage 1 intentionally contains no Windows or macOS native code. - -#### Phase 1: W3C core and fake host - Complete - -Deliver: - -- package skeleton; -- W3C router and response/error model; -- capability negotiation; -- target/session/window/element stores; -- timeouts and input state machine; -- deterministic fake host; -- raw HTTP, typed-client, and WebdriverIO contract tests. - -Exit: - -- a client creates and deletes a session; -- portable element, action, wait, and screenshot tests pass against the fake - host; -- unsupported routes return explicit W3C errors. - -#### Phase 2: Storybook manifests, bridge, and supervisor - Complete - -Deliver: - -- instance/driver manifest and driver port; -- Story Manifest and static test-plan extraction; -- `StoryOrchestrator` adapter; -- correlated runtime bridge and native story marker; -- per-test remount/reset; -- `storybook-desktop` supervisor; -- Storybook extension commands. - -Exit: - -- a fake host can select/reset a story and run a declarative plan; -- stale preview elements are deterministic; -- exact-platform and portable-plan digests are checked; -- no additional Node server process is required. - -#### Phase 3: WebdriverIO, authoring, and agent surface - Complete - -Deliver: - -- public serializable story-plan schema; -- representative plans validated against the fake host; -- sanctioned WebdriverIO configuration, runner, custom commands, and matchers; -- low-level typed client; -- JSON CLI and bounded agent API; -- standardized reports and failure bundles; -- optional real MCP adapter evaluation. - -Exit: - -- an author can declare, list, shard, and run a plan through WebdriverIO against - the fake host; -- an agent can list, explain, execute, and diagnose the same plan; -- protocol, authoring, artifacts, and agent APIs are stable before platform - code is introduced. - -### Stage 2: Native desktop providers - -Stage 2 implements the platform contracts proven in Stage 1. - -#### Phase 4: Windows and Win32 native provider - Not started - -Deliver: - -- selected Windows host transport; -- UI Automation tree and event support; -- configurable physical and accessibility click modes; -- physical keyboard, pointer, and wheel actions; -- app attach/launch leases; -- occlusion-independent HWND capture through Windows Graphics Capture or an - equivalent native implementation; -- multi-window and Callout handling. - -Exit: - -- one unchanged Button, Checkbox, Input, scrolling, screenshot, and - secondary-window suite passes on Windows Fabric and Win32 Paper; -- attached apps survive teardown; -- exact owned resources are cleaned; -- artifacts distinguish assertion, app, and host failures. - -Land the first platform jobs as non-required until reliability and artifact -quality are established. - -#### Phase 5: macOS native provider - Not started - -Deliver: - -- selected local and CI transport(s); -- accessibility tree and events; -- configurable physical and accessibility click modes; -- keyboard and pointer actions; -- bundle-identity launch/attach; -- direct window capture; -- permission diagnostics; -- XCTest-backed provider if required to preserve hosted CI. - -Exit: - -- the unchanged portable WebdriverIO suite passes on macOS 14 Apple Silicon; -- missing authority fails before session creation with actionable diagnostics; -- attached apps survive teardown; -- the chosen CI environment is repeatable. - -### Stage 3: Release hardening - -#### Phase 6: Release readiness - Not started - -Deliver: - -- protocol compatibility suite; -- security review; -- performance/timeout budgets; -- clean-install and package-pack validation; -- package-size review; -- helper signing/notarization and artifact packages if selected; -- documentation, changeset, and CI promotion criteria. - -Exit: - -- public package contents are reproducible; -- native artifacts have an owned build/signing pipeline; -- no required install scripts are needed; -- supported platform jobs are promotable to required gates. - -## Validation matrix - -| Capability | macOS | Windows Fabric | Win32 Paper | -| ------------------------- | ---------------------------- | ---------------------------- | -------------------------------------- | -| attach and preserve | bundle/window identity | process/AUMID/HWND | process/HWND | -| launch and owned cleanup | provider-defined | packaged activation | prebuilt-host provider | -| `testID` lookup | verify AX mapping | verify UIA mapping | current UIA smoke establishes baseline | -| role/name/state | AX/XCTest | UIA | UIA | -| pointer input | CGEvent/XCTest | SendInput | SendInput | -| keyboard/Unicode | CGEvent/XCTest | SendInput | SendInput | -| wheel/scroll | provider capability | SendInput/UIA | SendInput/UIA | -| window screenshot | SCK/XCTest/provider | WGC/provider | WGC/provider | -| element screenshot | crop with scale | crop with DPI | crop with DPI | -| multiple windows | app windows | HWNDs | REX/Callout HWNDs | -| story select/reset | channel bridge | channel bridge | channel bridge | -| stale preview detection | generation + native liveness | generation + native liveness | generation + native liveness | -| app/render error | bridge + process watch | bridge + process watch | bridge + process watch | -| permission/desktop doctor | TCC/test authority | interactive session/UIPI | interactive session/UIPI | - -Also validate: - -- Windows 11 x64; -- macOS 14 on Apple Silicon; -- Windows 100%, 150%, and 200% scaling; -- Retina and non-Retina macOS where supported; -- multiple monitors and non-primary virtual-desktop origins; -- light, dark, and high-contrast themes; -- foreground, background, minimized, and occluded windows; -- denied macOS permissions; -- elevated Windows targets; -- locked/disconnected desktops; -- duplicate matching windows; -- secondary Callout windows; -- channel reconnect, duplicate runtime, stale nonce, and wrong digest; -- app and host failure during a command; -- port collision and parallel enlistments; -- raw HTTP, first-party, and WebdriverIO clients. - -Pilot stories: - -- Button pointer and keyboard focus; -- Checkbox checked/indeterminate state; -- Input type and clear; -- scrollable content; -- a secondary Callout window; -- a controlled failure for artifact verification. - -## Security and reliability - -- Bind loopback only by default. -- Reject browser-origin requests; do not enable permissive CORS. -- Require explicit authentication and configuration for any non-loopback bind. -- Use server-registered targets, not client-supplied commands. -- Scope trees and screenshots to registered target windows. -- Cap request body size, tree depth/node count, screenshot dimensions, command - deadlines, and retained logs. -- Confine artifact paths beneath an owned run root; reject absolute paths, - traversal, Windows device/alternate-stream paths, and symlink escapes. -- Redact environment variables and physical roots from public diagnostics. -- Record PID plus process creation time. -- Never kill by process name or fixed port. -- Preserve attached applications. -- Release all input state on every teardown path. -- Run automation only in an interactive desktop session. -- Treat screenshots and accessibility trees as potentially sensitive evidence. - -## Principal risks - -| Risk | Mitigation | -| ------------------------------------------------------------ | --------------------------------------------------------------- | -| platform state projection differs | capability-gated assertions; unsupported is not false | -| native element identity changes on remount | session UUIDs, liveness checks, preview generations | -| physical input is global and flaky | one input owner, foreground verification, serialized actions | -| macOS authority differs locally and in CI | native-stage dual-transport evaluation and fail-fast doctor | -| composited-window capture is backend-specific | direct capture gate before advertising screenshots | -| native artifacts cannot be built by current publish pipeline | keep binaries off critical path until an owned pipeline exists | -| Storybook channel is broadcast-oriented | nonce/instance/digest handshake plus native marker verification | -| static test extraction misses dynamic values | literal schema and loud file/location errors | -| authored DSL becomes too limited | add an imperative escape hatch only from demonstrated cases | -| agent/server can control a real desktop | loopback, origin rejection, target registry, bounded APIs | -| Storybook upgrades change channel behavior | isolate behind adapter and contract tests | -| sanctioned WebdriverIO surface drifts from raw W3C behavior | run the same contract cases through raw HTTP and WebdriverIO | - -## Open questions - -1. **Imperative test escape hatch:** What concrete scenarios must the initial - serializable DSL support before an executable sidecar is justified? -2. **MCP:** Is typed API plus JSON CLI enough for the initial agent experience, - or is a real MCP endpoint required for the first release? -3. **CI promotion:** What duration and pass-rate threshold should move new - desktop-driver jobs from advisory to required? - -## Future considerations - -### Visual testing - -V1 captures screenshots and complete scale/window/story metadata as evidence. -Baseline storage, image comparison, tolerances, approval workflows, and -cross-platform visual-diff policy are deferred until native capture fidelity -has been proven on all endpoints. - -### Legacy E2E migration - -The initial effort does not migrate or retire the existing Appium E2E harness. -After the new driver reaches platform and scenario parity, evaluate incremental -migration, dual-running duration, and retirement criteria as a separate -project. - -### Additional platforms and concurrency - -V1 supports Windows 11 x64 and macOS 14 on Apple Silicon, with one active -session per physical target. Windows 10, Windows ARM64, Intel/universal macOS, -and concurrent sessions are future expansion work. - -## References - -- [W3C WebDriver](https://www.w3.org/TR/webdriver2/) -- [Apple Accessibility for macOS](https://developer.apple.com/library/archive/documentation/Accessibility/Conceptual/AccessibilityMacOSX/) -- [AXUIElement](https://developer.apple.com/documentation/applicationservices/axuielement) -- [Quartz Event Services](https://developer.apple.com/documentation/coregraphics/quartz-event-services) -- [ScreenCaptureKit](https://developer.apple.com/documentation/screencapturekit) -- [XCUIApplication](https://developer.apple.com/documentation/xcuiautomation/xcuiapplication) -- [Microsoft UI Automation](https://learn.microsoft.com/en-us/windows/win32/winauto/entry-uiauto-win32) -- [UI Automation control patterns](https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-controlpatternsoverview) -- [SendInput](https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-sendinput) -- [Windows screen capture](https://learn.microsoft.com/en-us/windows/apps/develop/media-authoring-processing/screen-capture) +- one active session per physical target; +- loopback-only trusted local operation. + +## Windows and Win32 qualification + +- [ ] Run the physical Button, Checkbox, Input, and wheel plans on an unlocked + interactive desktop for both Windows Fabric and Win32 Paper. Capability + skips on a hosted noninteractive runner are not completion. +- [ ] Complete the full `stories-and-tests` lifecycle on the selected + authoritative Windows CI environment and confirm no app/helper leaks or + unreleased input. +- [ ] Qualify occlusion-independent window capture and element crops at 100%, + 150%, and 200% live DPI on both endpoints. +- [ ] Measure UI Automation events, physical input, Windows Graphics Capture, + cancellation, and forced helper death on the current hosted runner. + Assign unreliable capabilities to an interactive self-hosted runner + rather than assuming support. +- [ ] Bound the long-lived native element table under sustained remount and + multi-window use. +- [ ] Define and test CSS-pixel-to-Windows-wheel-unit conversion. +- [ ] Replace per-mutation detached timeout threads if sustained-provider-stall + qualification demonstrates that the current model can accumulate work. +- [ ] Decide from measured wait latency whether native UI Automation event + notifications add enough value beyond authoritative queries to enter V1. + +Context: +[Windows provider](native/windows/README.md), +[CI integration](references/ci-integration.md), and +[Test integration](references/test-integration.md). + +## macOS qualification + +- [ ] Qualify a real Apple Development identity. Record the identity class, + certificate hash, designated requirement, Xcode/Swift versions, artifact + ID, permission state, and repeated helper/app restart behavior. +- [ ] Complete the explicit TCC grant/revoke/regrant matrix for Accessibility, + PostEvent, and Screen Recording, including required helper or parent + restarts and the responsible process for each result. +- [ ] Prove the authoritative managed CI model on a self-hosted Apple Silicon + Mac with an auto-logged-in Aqua user. +- [ ] Define and deploy the MDM PPPC policy for Accessibility and PostEvent + using the selected stable designated requirement. +- [ ] Confirm capture-required plans skip only when Screen Recording is + explicitly optional; semantic and input plans must fail when required + authority is absent. +- [ ] Exercise first-run guidance, cache reuse, signing-identity rotation, + permission continuity, upgrade, and rollback on a clean managed machine. + +The local reusable-identity runs, Button/Checkbox/Input authored plans, +secondary Callout windows, 2x Retina crops, permission-denial diagnostics, +input recovery, and direct-spawn versus LaunchServices experiment are complete +and documented in [the macOS provider guide](native/macos/README.md). Do not +repeat them as implementation tasks unless a regression is observed. + +## Release hardening + +- [ ] Add and freeze the native wire major/minor/feature compatibility suite + across Node, macOS, Windows, and Win32. +- [ ] Complete a focused security review of loopback serving, target + registration, artifact confinement, helper substitution, signing, + process leases, physical-input recovery, and sensitive evidence. +- [ ] Define performance and timeout budgets for startup, lookup, input, + capture, cancellation, cleanup, and sustained tree traversal. +- [ ] Close remaining cache coverage for stale locks, compatible multi-version + consumers, in-use Windows artifacts, force rebuild, corruption + quarantine, and garbage collection. +- [ ] Review packed size, runtime dependencies, platform import policy, native + source completeness, and first-public-release metadata. +- [ ] Run at least 100 representative executions per endpoint over 14 days with + at least 99% infrastructure success, zero leaked app/helper processes, + zero unreleased-input incidents, and complete helper metadata for every + infrastructure failure. +- [ ] Promote only jobs whose required capabilities are present and whose + failures and evidence are actionable. +- [ ] Create the release changeset and complete repository publishing checks + after all required platform gates pass. + +## Optional distributed macOS prebuilt + +Source builds remain the required complete path. Decide whether the first +release needs an organization-built prebuilt. If it does: + +- [ ] Define the external provisioning and publisher trust policy. The package + must not download or auto-update the artifact. +- [ ] Sign with Developer ID Application, Hardened Runtime, and a secure + timestamp. +- [ ] Notarize and staple the app bundle. +- [ ] Verify the publisher identity, designated requirement, notarization + ticket, source digest, artifact hash, and live helper handshake before + installation. +- [ ] Exercise clean install, first-run permissions, managed install-root + resolution, upgrade, rollback, and signer rotation. + +If no prebuilt is needed, record that decision in +[Native helpers](references/native-helpers.md) and close this section without +adding distribution machinery. + +## Deferred work + +These items are outside V1 and should start only after a concrete requirement +or the stated evidence gate exists: + +- **Persistent macOS broker:** retain direct spawn unless measured authority or + reliability evidence shows that a stable LaunchServices-owned broker is + better. A broker would require an authenticated per-user transport, atomic + install/upgrade, stale-process recovery, and preserved cross-process input + locking. +- **Executable MCP adapter:** the typed agent API and JSON CLI remain the + executable automation surfaces. Add a composed MCP adapter only when a real + client requires it; a schema-only claim is insufficient. +- **Imperative plan sidecar:** keep story plans static and serializable until a + demonstrated portable scenario cannot be expressed by the DSL. +- **Minimized-window capture policy:** continue returning explicit capture + unavailability. Consider restore/capture/restore only as a registered-target + policy after foreground and state-restoration effects are measured. +- **Visual regression:** baseline storage, comparison, tolerances, and approval + workflow begin only after capture fidelity is fully qualified on all + endpoints. +- **Legacy E2E migration:** evaluate migration and dual-running separately + after the new driver reaches platform and scenario parity. +- **Additional platforms and concurrency:** Windows 10, Windows ARM64, + Intel/universal macOS, and concurrent sessions are future expansion work. + +## Completion + +The package is release-ready when: + +1. required native capabilities pass on authoritative interactive environments; +2. unsupported capabilities skip explicitly and cannot produce false green + results; +3. build inputs, artifact identity, signing, and provenance are auditable; +4. source builds work without install scripts or an optional prebuilt; +5. attached apps survive and owned resources always clean up; +6. cancellation and helper failure leave no depressed input; +7. security, reliability, package, and CI promotion gates above are complete; +8. README and reference pages describe the final supported operating model. diff --git a/packages/agentic/desktop-driver/README.md b/packages/agentic/desktop-driver/README.md index 42c425b8d79..cb4023485f1 100644 --- a/packages/agentic/desktop-driver/README.md +++ b/packages/agentic/desktop-driver/README.md @@ -1,223 +1,147 @@ # React Native Desktop Driver -`@fluentui-react-native/desktop-driver` is a W3C WebDriver-compatible remote end -for React Native desktop applications. It does not use Appium. - -Stage 1 provides the complete platform-neutral protocol, sanctioned WebdriverIO -API, serializable story-test contract, deterministic fake host, evidence -reports, JSON CLI, and bounded agent API. Native Windows, Win32, and macOS host -providers are separate Stage 2 work described in [PLAN.md](PLAN.md). - -## Package boundaries - -| Surface | Responsibility | -| --------------------------------------- | ----------------------------------------------------------- | -| `@fluentui-react-native/desktop-driver` | Public types and common APIs | -| `/authoring` | Serializable story plans, selectors, and result types | -| `/wdio` | Sanctioned WebdriverIO connection, commands, and runner | -| `/agent` | Bounded JSON-safe inspection and action API | -| `/client` | Low-level typed W3C client | -| `/server` | Embeddable W3C remote end and target/session state | -| `/artifacts` | Confined atomic evidence persistence | -| `/testing` | Fake host, fake Storybook orchestration, and test harnesses | - -The protocol server remains client-neutral. WebdriverIO is a dependency of the -high-level `/wdio` surface, not an implementation primitive for routing, -capability negotiation, or native hosts. - -## Authoring story tests - -Plans are inline static data under `parameters.desktopDriver`. Storybook can -extract and shard them without importing a React Native story module, and -agents can explain the same steps before execution. - -```tsx -import type { DesktopStoryTests } from '@fluentui-react-native/desktop-driver/authoring'; - -export const Default: Story = { - tags: ['desktop-e2e'], - parameters: { - desktopDriver: { - version: 1, - tests: [ - { - id: 'pointer-focus', - title: 'Responds to activation and receives focus', - requires: ['focus', 'screenshot'], - steps: [ - { action: 'wait', target: { testId: 'my-button' } }, - { expect: { state: 'role', target: { testId: 'my-button' }, value: 'button' } }, - { action: 'click', target: { testId: 'my-button' } }, - { expect: { state: 'focused', target: { testId: 'my-button' }, value: true } }, - { action: 'screenshot', name: 'focused-button', target: { testId: 'my-button' } }, - ], - }, - ], - } satisfies DesktopStoryTests, - }, -}; -``` +`@fluentui-react-native/desktop-driver` is a W3C WebDriver Classic-compatible +remote end for automating React Native desktop applications. It drives macOS, +React Native Windows Fabric, and React Native Win32 Paper without Appium and +provides one portable WebdriverIO and story-test contract across those +endpoints. -Static extraction accepts JSON literals wrapped by TypeScript `satisfies` or -`as`. Do not hide plans behind variables, functions, spreads, computed -properties, or runtime platform branches. Invalid or dynamic plans fail -manifest generation with their source file, story, and line. +The package contains the client-neutral WebDriver service, typed clients, +WebdriverIO integration, serializable test plans, bounded agent operations, +evidence management, native-helper source and build tooling, and deterministic +fake hosts for contract tests. It does not depend on Storybook, React, or React +Native. -Portable selectors are: +## Supported targets -- `{ testId }` for deterministic interaction; -- `{ role, name? }` for semantic lookup; -- `{ accessibleName }`; -- `{ text }`. +| Endpoint | Operating system | Architecture | Native implementation | +| --------------------------- | ----------------- | ----------------------- | ----------------------------------------------------------- | +| React Native macOS Fabric | macOS 14 or later | Apple Silicon (`arm64`) | Swift, AXUIElement, Quartz Event Services, ScreenCaptureKit | +| React Native Windows Fabric | Windows 11 | `x64` | C++20, UI Automation, SendInput, Windows Graphics Capture | +| React Native Win32 Paper | Windows 11 | `x64` | Same Windows helper and protocol | -Supported actions include click, double-click, clear, type, keys, W3C actions, -scroll, Storybook arg updates, waits, screenshots, source capture, and notes. -Assertions cover existence/count, role, accessible name, text/value, displayed, -enabled, focused, selected, checked/mixed, and expanded state. +V1 permits one active WebDriver session per physical target. Native builds run +only on the target operating system. Installation never compiles or downloads a +helper. -Use `requires` for capabilities such as `keyboard`, `focus`, `wheel`, or -`screenshot`. Missing capabilities produce an explicit skipped result rather -than a false pass. Platform divergence belongs in declarative `platforms` or -capability requirements, not branches inside a plan. +## Start here -## Sanctioned WebdriverIO API +For a repository checkout, use Node 22.12 or later and Yarn 4: -```ts -import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; +```sh +yarn +yarn workspace @fluentui-react-native/desktop-driver build +``` -const desktop = await connectDesktopWebdriver({ - platformName: 'windows', - targetId: 'agenticstorybook-windows', - url: 'http://127.0.0.1:39859', -}); +Build the native helper explicitly on the machine that will run it: -try { - const manifest = await desktop.browser.desktopListStories(); - await desktop.browser.desktopOpenStory('components-button--default'); - await desktop.browser.desktopExpect({ - state: 'enabled', - target: { testId: 'agentic-storybook-button' }, - value: true, - }); - const result = await desktop.browser.desktopRunStoryTests({ - artifactsRoot: 'artifacts/windows/desktop-driver', - selection: { tag: 'desktop-e2e', shardCount: 2, shardIndex: 0 }, - }); -} finally { - await desktop.delete(); -} -``` +```sh +# Windows or Win32 +desktop-driver build-driver --platform windows -Registered browser commands are: - -- `desktopListStories()`; -- `desktopOpenStory(storyId, runId?)`; -- `desktopResetStory(storyId, runId?)`; -- `desktopExpect(expectation)`; -- `desktopRunStoryTests(options?)`. - -The runner filters by story, test, and tag, shards the sorted -`storyId/testId` list deterministically, checks required capabilities, resets -the preview for every run, and distinguishes assertion, timeout, cancellation, -skip, and infrastructure outcomes. - -The server serializes commands per session and all input globally. Timeout -paths abort host work before releasing input, while runner cancellation drains -its in-flight request before cleanup, so a later test cannot inherit a late key -or pointer action. - -## Evidence - -Supplying `artifactsRoot` writes atomically beneath that root: - -```text -artifactsRoot/ - host.json - run.json - tests/ - -/ - - failure.png - failure-source.xml - failure-tree.json +# macOS; use a stable identity for repeatable privacy authorization +desktop-driver build-driver --platform macos \ + --macos-signing-identity "Apple Development: Developer Name (TEAMID)" ``` -Artifact names are sanitized and confined beneath the configured root. -Failures attempt screenshot, source, and compact-tree capture; evidence-capture -errors are reported without replacing the original test failure. +Then resolve the verified helper, register a controlled application target, and +start the loopback service: -## JSON CLI +```ts +import { createDesktopDriverServer, NativeDesktopHost, resolveNativeDesktopDriver } from '@fluentui-react-native/desktop-driver'; -The CLI prints structured JSON for automation: +const artifact = await resolveNativeDesktopDriver({ + platform: 'windows', +}); +const host = new NativeDesktopHost({ + application: { + executablePath: 'C:\\apps\\MyReactNativeApp.exe', + windowTitle: 'My React Native App', + }, + artifact, + endpoint: 'windows', +}); +const service = await createDesktopDriverServer({ + targets: [ + { + endpoint: 'windows', + host, + id: 'my-react-native-app', + platformName: 'windows', + renderer: 'fabric', + }, + ], +}); -```sh -desktop-driver serve --manifest story-manifest.windows.json --target fake-windows - -desktop-driver stories list \ - --url http://127.0.0.1:4444 \ - --target fake-windows - -desktop-driver stories explain components-button--default \ - --url http://127.0.0.1:4444 \ - --target fake-windows - -desktop-driver stories run \ - --url http://127.0.0.1:4444 \ - --target fake-windows \ - --tag desktop-e2e \ - --artifacts artifacts/windows/desktop-driver - -desktop-driver agent describe \ - --url http://127.0.0.1:4444 \ - --target fake-windows \ - --scope story \ - --artifacts artifacts/windows/desktop-driver +console.log(service.url); +// Keep the service alive while tests run, then await service.close(). ``` -`serve` is the Stage 1 fake target. It is not a native-provider substitute. - -## Agent API +Connect through the sanctioned WebdriverIO surface: ```ts -import { connectDesktopAgent } from '@fluentui-react-native/desktop-driver/agent'; +import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; -const agent = await connectDesktopAgent({ - artifactsRoot: 'artifacts/windows/desktop-driver', +const desktop = await connectDesktopWebdriver({ platformName: 'windows', - targetId: 'agenticstorybook-windows', - url: 'http://127.0.0.1:39859', + targetId: 'my-react-native-app', + url: service.url, }); try { - await agent.listStories(); - await agent.openStory('components-button--default'); - await agent.describe({ scope: 'story', depth: 3, maxNodes: 100 }); - await agent.click({ testId: 'agentic-storybook-button' }); - await agent.screenshot('button'); - await agent.runStoryTest('components-button--default', 'pointer-focus'); + const button = await desktop.browser.$('~save-button'); + await button.click(); } finally { - await agent.delete(); + await desktop.delete(); + await service.close(); } ``` -The API intentionally exposes coarse operations and bounded tree projections. -It does not reveal native handles or permit arbitrary process, environment, or -artifact-path capabilities. - -## MCP decision - -Phase 3 does not add another MCP listener. The Storybook MCP remains the -documentation and story-metadata surface; the typed agent API and JSON CLI are -the executable native-validation surface. Reconsider a composed MCP adapter -after Stage 2 proves the native commands and their security model. A tool-schema -file without an executable adapter is not considered an MCP integration. - -## Protocol and security - -See [SPEC.md](SPEC.md) for implemented W3C routes, extension commands, -capabilities, device contracts, and unsupported browser behavior. - -The server binds loopback, rejects browser-origin requests, accepts only -server-registered targets, permits one session per physical target, applies -host command deadlines, and preserves attached applications. Never expose it -remotely without a separately designed authentication and transport policy. +The example launches the registered application. Use `launchMode: 'attach'` +only when another trusted owner supplies and maintains the application +lifecycle. The standalone `desktop-driver serve` command intentionally exposes +a fake target for protocol and integration tests; it is not a native service +shortcut. + +See [Getting started](references/getting-started.md) for installation, +toolchains, first builds, and diagnostic commands. + +## Documentation + +| Guide | Use it for | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| [Getting started](references/getting-started.md) | Initial setup, package development, first helper build, and troubleshooting | +| [Architecture](references/architecture.md) | Package layers, process boundaries, request flow, and ownership | +| [Service integration](references/service.md) | Embedding the remote end, registering targets, launch/attach behavior, and clients | +| [Native helpers](references/native-helpers.md) | Build and resolution policy, cache layout, verification, and configuration | +| [macOS native provider](native/macos/README.md) | Swift implementation, signing, TCC permissions, diagnostics, and native qualification | +| [Windows native provider](native/windows/README.md) | C++ implementation, toolchain, application identity, capture, and native qualification | +| [Test integration](references/test-integration.md) | WebdriverIO, authored story plans, fake harnesses, artifacts, and native tests | +| [CI integration](references/ci-integration.md) | Package gates, native jobs, real-app smoke, signing, and artifacts | +| [Security model](references/security.md) | Loopback boundary, registered targets, helper trust, process ownership, and sensitive evidence | +| [WebDriver contract](references/protocol.md) | Supported W3C routes, capabilities, elements, errors, and Storybook extensions | +| [Native wire protocol](native/PROTOCOL.md) | Private framed stdio protocol between Node and the native helper | +| [Remaining work](PLAN.md) | Qualification and release work that is not complete | +| [Changelog](CHANGELOG.md) | Versioned package changes | + +Contributors and coding agents start with [AGENTS.md](AGENTS.md). + +## Public entry points + +| Export | Purpose | +| ------------ | ----------------------------------------------------------------------------------------- | +| Package root | Native-helper build/resolution, `DesktopHost`, server, low-level client, and common types | +| `/authoring` | Serializable story plans, selectors, capabilities, and result types | +| `/wdio` | WebdriverIO connection, custom commands, assertions, and story-plan runner | +| `/client` | Low-level typed W3C client | +| `/server` | Embeddable remote end, session manager, and target registry | +| `/agent` | Bounded JSON-safe inspection, action, and evidence API | +| `/artifacts` | Confined, atomic evidence persistence | +| `/runner` | Plan selection, sharding, execution, and result classification | +| `/storybook` | Story manifest and orchestration contracts, without a Storybook dependency | +| `/testing` | Fake host, fake Storybook orchestrator, and protocol harnesses | +| `/cli` | Embeddable Commander command and CLI runner | + +The service implements native application semantics rather than pretending to +be a browser. Unsupported browser features return `unsupported operation` +instead of fabricated success. diff --git a/packages/agentic/desktop-driver/SPEC.md b/packages/agentic/desktop-driver/SPEC.md deleted file mode 100644 index d9e366f03d0..00000000000 --- a/packages/agentic/desktop-driver/SPEC.md +++ /dev/null @@ -1,105 +0,0 @@ -# Desktop Driver Contract - -## Scope - -Desktop Driver is a W3C WebDriver Classic-compatible native desktop remote end. -It intentionally implements native application semantics rather than -pretending to be a browser. - -V1 platform names are `macos` and `windows`; endpoint names are `macos`, -`windows`, and `win32`. - -## Sessions and targets - -- Targets are registered by the server. -- Capabilities select a target with `furn:target`. -- Capabilities never carry executable paths, environment variables, or output - directories. -- Exactly one active session may own a physical target. -- Commands are serialized per session, and input operations share one global - mutex. -- A target records whether the application was launched or attached. -- Deleting an attached session preserves the application. -- Cleanup uses owned process identity, never process-name matching. - -Supported extension capabilities: - -- `furn:target`; -- `furn:endpoint`; -- `furn:renderer`; -- `furn:clickMode`: `physical`, `accessibility`, or `auto`; -- returned `furn:features`. - -## W3C routes - -Implemented standard behavior: - -- status and new/delete session; -- get/set timeouts; -- current window, handles, switching, closing, and rectangles; -- element lookup from a window or element; -- active element; -- element name, text, attribute, property, rectangle, enabled, selected, and - displayed queries; -- click, clear, and send keys; -- perform/release actions; -- window and element screenshots; -- accessibility source. - -Implemented `furn` extensions: - -- `GET /session/{id}/furn/manifest`; -- `GET|POST /session/{id}/furn/story`; -- `POST /session/{id}/furn/story/reset`; -- `POST /session/{id}/furn/story/args`; -- `GET /session/{id}/furn/tree`. - -Navigation, cookies, frames, shadow roots, prompts, printing, arbitrary -JavaScript execution, and browser CSS behavior return `unsupported operation`. - -## Elements - -Public references use only the standard -`element-6066-11e4-a52e-4f735466cecf` key with session UUIDs. Native handles, -UIA runtime IDs, AX references, React tags, and HWNDs are private. - -Every command validates native liveness. Story reset increments the preview -generation and invalidates preview elements while preserving live application -and Storybook chrome references. - -Pointer and wheel actions may use WebDriver element origins. The server resolves -the public session element UUID, validates preview generation and native -liveness, and passes only `{ elementId: }` to the host contract. -Public WebDriver element references never cross into platform providers. - -Every host operation receives an `AbortSignal`. On timeout, the server aborts -the host operation before input release, session teardown, or a subsequent -command. Providers must stop side effects and settle promptly; they may never -apply late input after cleanup. - -Portable lookup uses accessibility ID, normalized role, accessible name, and -the namespaced `-furn:text` strategy. Deterministic authored interactions use -`testID`. - -## Storybook readiness - -The runtime and server validate: - -- instance, target, endpoint, and catalog identity; -- a private bridge nonce; -- request and run IDs; -- portable-plan digest; -- preview generation; -- the native story-root marker. - -Only the authenticated bridge client can complete or fail a selection. Every -test receives a new run ID and keyed preview remount. - -## Results - -Run status is `passed` or `failed`. Tests distinguish `passed`, `failed`, -`skipped`, `timed-out`, `cancelled`, and `infrastructure-error`. Steps record -status, duration, error, and artifacts. - -Unsupported required capabilities are skipped explicitly. An unavailable -native property never becomes a false-shaped passing assertion. diff --git a/packages/agentic/desktop-driver/native/PROTOCOL.md b/packages/agentic/desktop-driver/native/PROTOCOL.md new file mode 100644 index 00000000000..8c3ffc8fab3 --- /dev/null +++ b/packages/agentic/desktop-driver/native/PROTOCOL.md @@ -0,0 +1,248 @@ +# Desktop Driver native host protocol + +FDR1 is the private protocol between the Node `NativeDesktopHost` adapter and +one long-lived operating-system helper. It is not a public remote API and is +not exposed to WebDriver clients. + +The protocol uses inherited stdin/stdout. The helper opens no listener, receives +no WebDriver capabilities, and cannot choose arbitrary applications or +artifact paths. Stderr is reserved for bounded human-readable diagnostics. + +## Versions and startup + +The checked-in machine-readable version is [protocol.json](protocol.json): + +```json +{ + "wireProtocol": { + "major": 1, + "minor": 0 + } +} +``` + +Before selecting an artifact, Node invokes: + +```text +furn-desktop-driver-host --handshake --json +``` + +That one-shot process writes an unframed JSON `hello` document and exits. Node +verifies provider, architecture, build ID, source digest, features, and +protocol. + +For service operation, Node invokes: + +```text +furn-desktop-driver-host --stdio +``` + +The long-lived process first emits a framed `hello`. Its major version must +equal Node's major version and its minor version must be at least the required +minor. The actual long-lived process must identify the same verified artifact; +a successful one-shot probe cannot substitute for this handshake. + +## Frame format + +Every long-lived frame starts with a 12-byte little-endian header: + +| Offset | Size | Value | +| ------ | ---- | -------------------------------- | +| 0 | 4 | ASCII `FDR1` | +| 4 | 1 | Frame type: `1` JSON, `2` binary | +| 5 | 3 | Reserved zero bytes | +| 8 | 4 | Payload byte length | + +Limits: + +| Payload | Maximum | +| ----------------------------- | ----------------- | +| JSON frame | 8 MiB | +| Binary frame | 64 MiB | +| Binary correlation identifier | 1,024 UTF-8 bytes | + +Invalid magic, nonzero reserved bytes, unknown types, oversize frames, +truncated identifiers, and invalid JSON fail the transport. Provider request +parsers may impose tighter command-specific limits. + +## JSON messages + +All JSON messages are UTF-8 objects with a `type`. + +### `hello` + +Identifies the running helper: + +```json +{ + "type": "hello", + "provider": "windows", + "architecture": "x64", + "buildId": "", + "sourceDigest": "", + "protocol": { "major": 1, "minor": 0 }, + "features": ["..."] +} +``` + +The provider can be `windows` or `macos`. The Windows provider serves both +Windows Fabric and Win32 Paper; the selected endpoint is supplied to `probe` +and target commands. + +### `request` + +Node sends a correlated command: + +```json +{ + "type": "request", + "id": "", + "command": "find", + "params": {} +} +``` + +Parameters are command-specific serializable data already validated by the +Node boundary. Public WebDriver element UUIDs are resolved before dispatch; +only helper-private opaque IDs cross FDR1. + +Request and cancellation correlation IDs are limited to 256 UTF-8 bytes. +Helpers ignore invalid IDs before queuing work or echoing a response. + +### `response` + +A successful response carries `result`: + +```json +{ + "type": "response", + "id": "", + "result": {} +} +``` + +A failed response carries a structured helper error: + +```json +{ + "type": "response", + "id": "", + "error": { + "code": "no-such-element", + "message": "The element is no longer available.", + "data": {} + } +} +``` + +`data` is optional and bounded. It may include diagnostic operation names and +platform error codes, but not native handles or unrestricted environment data. +Node maps helper codes onto the public WebDriver error contract. + +### `event` + +Helpers can publish advisory application, window, structure, and transport +events: + +```json +{ + "type": "event", + "event": "window-opened", + "sequence": 12, + "payload": {} +} +``` + +Queries remain authoritative. Events can invalidate state or wake waits but do +not replace a liveness or state query. + +### `cancel` and `cancelled` + +When a command's `AbortSignal` fires, Node sends: + +```json +{ "type": "cancel", "id": "" } +``` + +The helper stops side effects, unwinds command-local work, and emits: + +```json +{ "type": "cancelled", "id": "" } +``` + +Node retains the session queue and physical-input ownership until the request +settles. Failure to settle before the cancellation deadline is fatal to the +helper and triggers restricted input recovery. + +## Binary messages + +PNG capture data uses a binary frame. Its payload is: + +| Offset | Size | Value | +| ------ | --------- | ---------------------------------- | +| 0 | 4 | Correlation identifier byte length | +| 4 | N | UTF-8 identifier | +| 4 + N | remaining | Raw bytes | + +The corresponding JSON response declares: + +```json +{ + "type": "response", + "id": "", + "result": { + "mimeType": "image/png", + "width": 1280, + "height": 720, + "scaleFactor": 2 + }, + "binary": { + "id": "", + "mimeType": "image/png", + "width": 1280, + "height": 720, + "scaleFactor": 2 + } +} +``` + +Node completes the request only after it has both the JSON response and the +declared binary payload. An undeclared or duplicate binary identifier fails the +transport. + +## Restricted helper modes + +Both providers also implement: + +```text +--self-test +--release-input +--release-input --sweep +``` + +`--self-test` runs provider-local checks without opening FDR1. Normal +`--release-input` reads a bounded JSON ledger from stdin and releases exactly +the keys/buttons mirrored by Node. `--sweep` is an operator diagnostic for a +desktop left in an uncertain state; it is not part of normal session cleanup. + +The macOS provider additionally implements: + +```text +--permissions +--permissions --prompt +``` + +These modes emit versioned unframed diagnostic JSON. Ordinary handshake, +self-test, stdio, build, and resolution paths never prompt for privacy access. + +## Change rules + +When changing FDR1: + +1. update [protocol.json](protocol.json), Node framing/types, both native + implementations, and this document together; +2. preserve major-version rejection and explicit feature negotiation; +3. add Node framing and provider-native self-tests; +4. verify malformed, oversize, cancellation, binary, and handshake behavior; +5. update [the public protocol](../references/protocol.md) only if observable + WebDriver semantics also change. diff --git a/packages/agentic/desktop-driver/native/macos/Package.swift b/packages/agentic/desktop-driver/native/macos/Package.swift new file mode 100644 index 00000000000..4e5b9a23e63 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Package.swift @@ -0,0 +1,25 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +let package = Package( + name: "FurnDesktopDriverHost", + platforms: [ + .macOS(.v14), + ], + products: [ + .executable(name: "furn-desktop-driver-host", targets: ["DesktopDriverHost"]), + ], + targets: [ + .executableTarget( + name: "DesktopDriverHost", + linkerSettings: [ + .linkedFramework("ApplicationServices"), + .linkedFramework("AppKit"), + .linkedFramework("ImageIO"), + .linkedFramework("ScreenCaptureKit"), + .linkedFramework("UniformTypeIdentifiers"), + ] + ), + ] +) diff --git a/packages/agentic/desktop-driver/native/macos/README.md b/packages/agentic/desktop-driver/native/macos/README.md new file mode 100644 index 00000000000..af4d7891791 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/README.md @@ -0,0 +1,368 @@ +# macOS native provider + +The macOS provider is a source-built Swift helper for macOS 14 or later on +Apple Silicon. It implements the private FDR1 protocol over stdio and uses +public macOS APIs for application identity, accessibility, physical input, and +capture. + +The helper is a minimal `LSUIElement` agent app with bundle identifier +`com.microsoft.fluentui-react-native.desktop-driver`. It is unsandboxed because +App Sandbox is incompatible with automating arbitrary applications through +Accessibility. + +## Source layout + +| Source | Responsibility | +| -------------------------------------- | --------------------------------------------------------------------------- | +| `Package.swift` | macOS 14 executable target and framework links | +| `Sources/DesktopDriverHost/main.swift` | Helper modes and top-level error handling | +| `Host.swift` | FDR1 hello, request queue, framing, cancellation, and disposal | +| `Driver.swift` | Command dispatch, capability reporting, and physical-input scopes | +| `Application.swift` | Launch/attach leases, exact process/window identity, and ownership | +| `Accessibility.swift` | AX tree walking, lookup, snapshots, focus, hit testing, actions, and source | +| `Input.swift` | Quartz pointer, keyboard, text, wheel, ledger, and recovery behavior | +| `InputLock.swift` | Per-user cross-process physical-input lock and abandoned-owner recovery | +| `Capture.swift` | ScreenCaptureKit window capture, element crop, scale, and PNG encoding | +| `Permissions.swift` | TCC preflight/prompt diagnostics and Security.framework signing evidence | +| `Support.swift` | Errors, cancellation, JSON helpers, deadlines, and AX utilities | +| `SelfTest.swift` | Provider-local framing, identity, AX, input, capture, and permission checks | + +The Node build coordinator is +`../../src/native/macos/buildMacOSDriver.ts`; runtime adaptation is under +`../../src/hosts/native`. + +## Build + +Requirements: + +- macOS 14 or later; +- Apple Silicon (`arm64`); +- Xcode and its Swift command-line tools; +- Node 22.12 or later for the package CLI. + +Build the package and helper: + +```sh +yarn workspace @fluentui-react-native/desktop-driver build +yarn workspace @fluentui-react-native/desktop-driver exec \ + desktop-driver build-driver --platform macos +``` + +Installed consumers can use the shorter command: + +```sh +desktop-driver build-driver --platform macos +``` + +The coordinator: + +1. hashes only `Package.swift`, optional `Package.resolved`, and Swift files + beneath `Sources`; +2. stages those inputs into an isolated scratch root outside the package; +3. invokes `swift build` for arm64 macOS 14; +4. assembles `FurnDesktopDriverHost.app` with a generated `Info.plist`; +5. strips path-dependent local symbols from release output; +6. signs and verifies the app; +7. performs the one-shot helper handshake; +8. publishes the complete verified bundle to the immutable native cache. + +SwiftPM `.build`, DerivedData, editor files, and other generated output never +participate in source identity or staging. + +## Signing + +### Stable development signing + +Use a stable identity whenever Accessibility or Screen Recording authorization +must survive ordinary source rebuilds: + +```sh +security find-identity -v -p codesigning + +desktop-driver build-driver \ + --platform macos \ + --macos-signing-identity "Apple Development: Developer Name (TEAMID)" +``` + +The option accepts an identity name or SHA-1 hash. It can also be set through: + +```sh +export FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY="Apple Development: Developer Name (TEAMID)" +``` + +An Apple Development identity is preferred. A contributor without an Apple +account can create one reusable local self-signed **Code Signing** certificate +with Keychain Access Certificate Assistant and trust it for code signing. Do +not recreate the certificate for every build: the leaf-certificate hash is part +of the compatibility identity and authorization continuity. + +Certificate-signed builds use Hardened Runtime and a secure timestamp. Artifact +verification compares the actual bundle against the recorded: + +- leaf-certificate SHA-1 hash; +- signing authority and identity; +- team identifier when present; +- designated requirement; +- Hardened Runtime state; +- timestamp policy; +- executable and source identities; +- FDR1 handshake. + +A helper signed by an unexpected identity fails closed. + +### Ad hoc signing + +If no identity is configured, the coordinator applies a deterministic ad hoc +signature with no secure timestamp and emits a warning. This path supports +contributor builds but privacy grants can bind to a changing cdhash and may +need to be granted again after a rebuild. + +Ad hoc signing is not a distribution model. The current source does not +implement a Developer ID distribution or notarization pipeline. If an +organization-distributed helper is added, its external release infrastructure +must apply Developer ID Application signing, Hardened Runtime, a secure +timestamp, notarization, and stapling. That optional work remains in +[PLAN.md](../../PLAN.md). + +## Helper lifecycle + +Supported modes: + +```text +furn-desktop-driver-host --handshake --json +furn-desktop-driver-host --stdio +furn-desktop-driver-host --self-test +furn-desktop-driver-host --permissions [--prompt] +furn-desktop-driver-host --release-input [--sweep] +``` + +The long-lived `--stdio` process: + +1. emits a verified macOS/arm64 FDR1 hello; +2. processes one command at a time on its worker queue; +3. handles correlated cancellation before accepting later side effects; +4. holds a per-user `flock` while physical input is active; +5. releases tracked keys and buttons on failure, cancellation, and disposal; +6. exits after `dispose`. + +The input lock lives beneath +`~/Library/Caches/com.microsoft.fluentui-react-native.desktop-driver`, so +helpers started through different Aqua bootstrap namespaces still serialize +physical input. Restricted `--release-input` accepts only the Node-owned ledger. +`--sweep` is an operator recovery diagnostic, not normal cleanup. + +## Application and window identity + +The helper uses server-owned `bundleIdentifier`, `executablePath`, +`windowTitle`, and optional launch arguments. An owner-generated attach lease +can additionally carry: + +- a nonce; +- endpoint identity; +- PID and process launch time; +- executable and bundle identity; +- exact window identity. + +Attach validates every supplied field against the live application. Lease +timestamps allow only the bounded precision required by +`NSRunningApplication.launchDate`. Zero or ambiguous application/window matches +fail. + +Leases record whether the app was launched or attached. Session deletion +preserves attached apps and closes only an exact process owned by the helper. +Opaque helper IDs represent windows and AX elements; native references never +cross the public WebDriver boundary. + +An AX result must be a real `AXWindow` or `AXSheet`. The provider rejects an +`AXApplication` placeholder even if `AXIsProcessTrusted()` is true. It checks +the windows, main-window, and focused-window attributes, refreshes replaced +window identities, and uses CoreGraphics only as a uniquely correlated geometry +fallback. + +## Accessibility and input + +The AX provider supports: + +- lookup by React Native `testID`, normalized role, accessible name, and text; +- bounded tree/source generation; +- snapshots with explicit supported/unsupported state; +- active element, focus, hit testing, clear, and accessibility activation; +- primary and secondary application windows. + +Quartz Event Services supplies physical pointer, keyboard, text, and wheel +input. Physical input requires Accessibility/PostEvent authority for the +responsible process and an unlocked interactive Aqua session. Missing authority +returns `input-unavailable`; the provider never substitutes unrelated +accessibility activation and reports physical success. + +React Native macOS does not move keyboard focus on ordinary mouse-down. Portable +tests should assert the platform's actual activation behavior rather than +copying a Windows focus expectation. + +## Capture + +ScreenCaptureKit captures a uniquely correlated application window and encodes +PNG output through ImageIO. Element screenshots crop the window image using the +matched AppKit display's backing scale factor. Retina output therefore reports +pixel dimensions and `scaleFactor` separately from point geometry. + +Screen Recording permission is degradable. Without it, the helper does not +advertise screenshot capabilities, while AX and input automation can continue +when their own authority is present. Capture fails explicitly when: + +- Screen Recording permission is absent; +- the target is minimized or off-screen; +- ScreenCaptureKit cannot uniquely correlate the AX window; +- crop geometry is empty or outside the capture. + +The provider does not implicitly restore or move a window to make capture pass. + +## Privacy permissions and diagnostics + +Inspect the exact verified helper without prompting: + +```sh +desktop-driver doctor --platform macos --permissions +``` + +The versioned JSON reports: + +- helper PID, parent PID, executable, and app bundle; +- Accessibility trust; +- PostEvent preflight state; +- Screen Recording preflight state; +- cdhash, signer, team identifier, and designated requirement when + Security.framework exposes them; +- whether each value is available and whether this mode can prompt for it. + +Values that cannot be read are marked unavailable rather than inferred. +`PostEvent` is preflight-only in this mode and cannot be prompted. + +Prompt deliberately and interactively: + +```sh +desktop-driver doctor --platform macos --permissions --prompt +``` + +Only this mode may request Accessibility and Screen Recording access. Build, +resolve, ordinary doctor, handshake, self-test, stdio, and normal service +startup never request privacy permission. Restart the helper after changing +Screen Recording access. + +V1 starts the helper directly as a child of Node. macOS can therefore attribute +privacy-sensitive work to Terminal, an IDE, or another responsible parent in +that process tree. Interpret `parentPid`, signing evidence, and all preflight +fields together. A `true` preflight value alone does not prove that the helper +bundle owns the grant. + +A prior qualification experiment launched the helper app externally through +LaunchServices and gave that experimental process a distinct TCC identity. The +checked-in runtime does not use that path: it starts the helper directly from +Node. The experiment did not make AX window publication more reliable, so V1 +retains direct spawn and defers a persistent broker until evidence demonstrates +an authority or reliability benefit. LaunchServices is still used normally +when the helper launches a target application. + +First use `doctor --permissions` and determine which process macOS treats as the +responsible TCC subject. Only then, for manual reset-and-reprompt diagnosis of a +grant that actually belongs to the helper bundle, macOS supports: + +```sh +tccutil reset Accessibility com.microsoft.fluentui-react-native.desktop-driver +tccutil reset ScreenCapture com.microsoft.fluentui-react-native.desktop-driver +tccutil reset PostEvent com.microsoft.fluentui-react-native.desktop-driver +``` + +Do not run these commands as a generic fix. A bundle-scoped reset affects only +the actual TCC subject; if responsibility belongs to Terminal, Node, or an IDE, +the commands above do not reset that grant. Resetting a parent application's +grant can affect unrelated automation, so follow the owning environment's +policy. Never edit `TCC.db`, disable SIP, or use undocumented responsibility +APIs. + +## Tests + +Run portable package checks: + +```sh +yarn format +yarn lint +yarn build +yarn test +``` + +Run the real native build/cache/handshake/self-test contract: + +```sh +FURN_NATIVE_DRIVER_TEST=1 yarn test --runInBand +``` + +Include stable-signature coverage: + +```sh +FURN_NATIVE_DRIVER_TEST=1 \ +FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY="" \ + yarn test --runInBand +``` + +The Swift self-test covers framing, JSON, input ledger and lock recovery, +capture scaling, snapshot normalization, AX diagnostics, CoreGraphics fallback +classification, permission diagnostics, and hello identity. The Node native +contract additionally builds, verifies, reuses, and directly resolves the app; +starts the actual long-lived process; and exercises malformed-frame recovery. + +Native contract tests do not prove TCC authority against a real app. Real +qualification must attach to a built React Native macOS application and cover +AX identity, authored interactions, secondary windows, Retina crops, +permission denial, helper restart, and ownership-safe cleanup. + +## CI + +Authoritative macOS automation uses a managed, self-hosted Apple Silicon Mac +with: + +- an auto-logged-in Aqua user; +- a stable code-signing identity; +- Accessibility and PostEvent approval for the actual responsible process; +- an MDM PPPC policy where organization policy permits; +- serialized physical input; +- access-controlled evidence retention. + +Do not run the helper through `sudo`, a daemon, or an SSH-only session. +Screen Recording can remain optional for non-capture tests when it cannot be +preauthorized; screenshot-required tests should report a capability skip. +Semantic or input tests must fail when their required authority is absent. + +GitHub-hosted macOS smoke is useful regression signal, but it is not +authoritative proof of first-run or durable TCC authorization. The checked-in +package does not implement Developer ID/notarization release infrastructure or +managed PPPC deployment. Those items, clean-machine +install/upgrade/rollback, and explicit permission-revocation qualification +remain in [PLAN.md](../../PLAN.md). + +## Troubleshooting + +| Symptom | Action | +| --------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `no-verified-prebuilt` | Build once or restore the matching signed cache; `doctor` never builds | +| Signature or designated-requirement mismatch | Select the intended identity, rebuild, and do not mutate cached output | +| Accessibility true but no real AX window | Preserve diagnostics, restart/retry the app and helper, and do not accept an `AXApplication` placeholder | +| Physical input unavailable | Grant the actual responsible process Accessibility/PostEvent access and use a logged-in Aqua session | +| Screen capture unavailable | Grant Screen & System Audio Recording, restart the helper, and rerun noninteractive doctor | +| Ambiguous application or window | Correct the registered identity or owner lease; never choose the first match | +| Input lock reports an abandoned owner | Ensure the prior process is gone, use the verified helper's release mode, then retry | +| Element becomes stale after a Storybook reset | Resolve it again in the new preview generation | + +See [Native helpers](../../references/native-helpers.md) for cache and +resolution policy, [CI integration](../../references/ci-integration.md) for +cross-platform gates, and [Security](../../references/security.md) for the +service trust boundary. + +## Apple references + +- [Inside Code Signing: Requirements](https://developer.apple.com/documentation/technotes/tn3127-inside-code-signing-requirements) +- [Accessibility API](https://developer.apple.com/documentation/applicationservices/axuielement) +- [Quartz Event Services](https://developer.apple.com/documentation/coregraphics/quartz-event-services) +- [ScreenCaptureKit](https://developer.apple.com/documentation/screencapturekit) +- [Resetting access to protected resources](https://developer.apple.com/documentation/xcode/resetting-access-to-protected-resources-in-macos) +- [Privacy Preferences Policy Control](https://support.apple.com/guide/deployment/privacy-preferences-policy-control-payload-dep38df53c2a/web) diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift new file mode 100644 index 00000000000..493aff18941 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift @@ -0,0 +1,913 @@ +import ApplicationServices +import Foundation + +enum ElementScope: String { + case application + case chrome + case preview + case secondaryWindow = "secondary-window" +} + +struct SupportedBool { + let value: Bool? + let reason: String + + static func supported(_ value: Bool) -> SupportedBool { + SupportedBool(value: value, reason: "") + } + + static func unsupported(_ reason: String) -> SupportedBool { + SupportedBool(value: nil, reason: reason) + } + + func json() -> JSONObject { + if let value { + return ["supported": true, "value": value] + } + return ["supported": false, "reason": reason] + } +} + +enum CheckedValue { + case bool(Bool) + case mixed + case unsupported(String) + + func json() -> JSONObject { + switch self { + case let .bool(value): + return ["supported": true, "value": value] + case .mixed: + return ["supported": true, "value": "mixed"] + case let .unsupported(reason): + return ["supported": false, "reason": reason] + } + } +} + +struct ElementSnapshot { + let id: String + let automationID: String? + let checked: CheckedValue + let enabled: SupportedBool + let expanded: SupportedBool + let focused: SupportedBool + let name: String? + let parentID: String? + let rect: CGRect + let screenRect: CGRect + let role: String + let scope: ElementScope + let selected: SupportedBool + let text: String? + let value: String? + let visible: SupportedBool + let windowID: String + + func json() -> JSONObject { + var result: JSONObject = [ + "id": id, + "checked": checked.json(), + "enabled": enabled.json(), + "expanded": expanded.json(), + "focused": focused.json(), + "rect": rectJSON(rect), + "role": role, + "scope": scope.rawValue, + "selected": selected.json(), + "visible": visible.json(), + "windowId": windowID, + ] + if let automationID, !automationID.isEmpty { + result["automationId"] = automationID + } + if let name, !name.isEmpty { + result["name"] = name + } + if let parentID, !parentID.isEmpty { + result["parentId"] = parentID + } + if let text { + result["text"] = text + } + if let value { + result["value"] = value + } + return result + } +} + +final class ElementRecord { + let id: String + let processID: pid_t + var element: AXUIElement + var windowID: String + var parentID: String? + var scope: ElementScope + + init( + processID: pid_t, + element: AXUIElement, + windowID: String, + parentID: String?, + scope: ElementScope + ) { + id = IdentifierGenerator.shared.next("element") + self.processID = processID + self.element = element + self.windowID = windowID + self.parentID = parentID + self.scope = scope + } +} + +final class AccessibilityEngine { + private static let maximumTreeNodes = 5_000 + private static let maximumElementRecords = 10_000 + private static let maximumAncestorDepth = 64 + + private var records: [String: ElementRecord] = [:] + private var recordIDsByHash: [CFHashCode: [String]] = [:] + + func tree( + window: WindowRecord, + lease: ApplicationLeaseRecord, + token: CancellationToken + ) throws -> [ElementSnapshot] { + try requireAccessibility("Accessibility tree traversal") + let rootScope: ElementScope = window.primary ? .application : .secondaryWindow + return try walk( + root: window.element, + window: window, + lease: lease, + parentID: nil, + rootScope: rootScope, + token: token + ) + } + + func find( + window: WindowRecord, + lease: ApplicationLeaseRecord, + rootElementID: String?, + selector: JSONObject, + token: CancellationToken + ) throws -> [ElementSnapshot] { + let strategy = try requireString(selector, "strategy") + let value = selector["value"] as? String ?? "" + let root: AXUIElement + let parentID: String? + let rootScope: ElementScope + var excludedID: String? + if let rootElementID, !rootElementID.isEmpty { + let record = try requireRecord(rootElementID) + guard record.windowID == window.id else { + try fail(ErrorCode.staleElement, "The search root does not belong to the requested window.") + } + _ = try snapshot(elementID: rootElementID, window: window, token: token) + root = record.element + parentID = record.parentID + rootScope = record.scope + excludedID = record.id + } else { + root = window.element + parentID = nil + rootScope = window.primary ? .application : .secondaryWindow + } + let snapshots = try walk( + root: root, + window: window, + lease: lease, + parentID: parentID, + rootScope: rootScope, + token: token + ) + return try snapshots.filter { snapshot in + if snapshot.id == excludedID { + return false + } + return try matches(snapshot: snapshot, strategy: strategy, value: value) + } + } + + func snapshot( + elementID: String, + window: WindowRecord, + token: CancellationToken + ) throws -> ElementSnapshot { + try requireAccessibility("Refreshing an accessibility element") + try token.throwIfCancelled() + let record = try requireRecord(elementID) + guard record.windowID == window.id, record.processID == window.processID else { + try fail(ErrorCode.staleElement, "Element \"\(elementID)\" no longer belongs to its tracked window.") + } + try validateLive(record) + return try readSnapshot(record: record, window: window) + } + + func activeElement( + window: WindowRecord, + token: CancellationToken + ) throws -> ElementSnapshot? { + try requireAccessibility("Reading the focused accessibility element") + try token.throwIfCancelled() + let application = AXUIElementCreateApplication(window.processID) + guard let focused = try copyAXAttribute(application, kAXFocusedUIElementAttribute) else { + return nil + } + guard let element = axElement(focused) else { + return nil + } + guard try belongsToWindow(element, window: window.element, token: token) else { + return nil + } + let scope = scopeForKnownElement(element) + ?? (window.primary ? .chrome : .secondaryWindow) + let record = try register( + element, + processID: window.processID, + windowID: window.id, + parentID: parentIDForKnownElement(element), + scope: scope + ) + return try readSnapshot(record: record, window: window) + } + + func hitTest( + window: WindowRecord, + x: CGFloat, + y: CGFloat, + token: CancellationToken + ) throws -> ElementSnapshot? { + let windowFrame = try window.currentFrame() + let point = CGPoint(x: windowFrame.minX + x, y: windowFrame.minY + y) + guard let hit = try hitElement(at: point) else { + return nil + } + var pid: pid_t = 0 + if AXUIElementGetPid(hit, &pid) != .success || pid != window.processID { + return externalWindowSnapshot(window: window, hit: hit) + } + guard try belongsToWindow(hit, window: window.element, token: token) else { + return externalWindowSnapshot(window: window, hit: hit) + } + let resolved = try resolveKnownAncestor(hit, token: token) + let element = resolved?.element ?? hit + let record = try register( + element, + processID: window.processID, + windowID: window.id, + parentID: resolved?.parentID, + scope: resolved?.scope ?? (window.primary ? .chrome : .secondaryWindow) + ) + return try readSnapshot(record: record, window: window) + } + + func elementOwnsPoint( + elementID: String, + point: CGPoint, + token: CancellationToken + ) throws -> Bool { + let target = try requireRecord(elementID) + guard let hit = try hitElement(at: point) else { + return false + } + var cursor: AXUIElement? = hit + for _ in 0.. Bool { + let record = try requireRecord(elementID) + try validateLive(record) + return axNumber(try copyOptionalAXAttribute(record.element, kAXFocusedAttribute))?.boolValue ?? false + } + + func accessibilityClick(_ elementID: String, token: CancellationToken) throws { + try requireAccessibility("Accessibility click") + try token.throwIfCancelled() + let record = try requireRecord(elementID) + try validateLive(record) + let actions = try copyAXActionNames(record.element) + let candidates = [kAXPressAction, kAXConfirmAction, kAXPickAction] + guard let action = candidates.first(where: { actions.contains($0) }) else { + try fail( + ErrorCode.unsupported, + "The element does not expose an AXPress, AXConfirm, or AXPick action, so accessibility click is unavailable." + ) + } + let error = AXUIElementPerformAction(record.element, action as CFString) + if error == .invalidUIElement { + try fail(ErrorCode.staleElement, "Element \"\(elementID)\" is no longer available.") + } + guard error == .success else { + try fail(ErrorCode.automationFailed, "Performing \(action) failed because \(describeAXError(error)).") + } + } + + func tryClearValue(_ elementID: String) throws -> Bool { + let record = try requireRecord(elementID) + try validateLive(record) + var settable: DarwinBoolean = false + let settableError = AXUIElementIsAttributeSettable(record.element, kAXValueAttribute as CFString, &settable) + if settableError == .attributeUnsupported || settableError == .noValue || !settable.boolValue { + return false + } + if settableError == .invalidUIElement { + try fail(ErrorCode.staleElement, "Element \"\(elementID)\" is no longer available.") + } + guard settableError == .success else { + return false + } + let error = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, "" as CFString) + if error == .invalidUIElement { + try fail(ErrorCode.staleElement, "Element \"\(elementID)\" is no longer available.") + } + return error == .success + } + + func requireRecord(_ elementID: String) throws -> ElementRecord { + guard let record = records[elementID] else { + try fail(ErrorCode.staleElement, "Element \"\(elementID)\" is no longer tracked by the native helper.") + } + return record + } + + func forgetWindow(_ windowID: String) { + let removed = records.values.filter { $0.windowID == windowID } + for record in removed { + records.removeValue(forKey: record.id) + let hash = CFHash(record.element) + recordIDsByHash[hash]?.removeAll { $0 == record.id } + if recordIDsByHash[hash]?.isEmpty == true { + recordIDsByHash.removeValue(forKey: hash) + } + } + } + + func reset() { + records.removeAll() + recordIDsByHash.removeAll() + } + + func source(_ snapshots: [ElementSnapshot]) -> String { + var children: [String: [ElementSnapshot]] = [:] + var roots: [ElementSnapshot] = [] + for snapshot in snapshots { + if let parentID = snapshot.parentID { + children[parentID, default: []].append(snapshot) + } else { + roots.append(snapshot) + } + } + var output = "" + enum Visit { + case open(ElementSnapshot) + case close + } + var stack = roots.reversed().map(Visit.open) + while let visit = stack.popLast() { + switch visit { + case .close: + output += "" + case let .open(snapshot): + output += "" + stack.append(.close) + for child in (children[snapshot.id] ?? []).reversed() { + stack.append(.open(child)) + } + } + } + output += "" + return output + } + + private func walk( + root: AXUIElement, + window: WindowRecord, + lease: ApplicationLeaseRecord, + parentID: String?, + rootScope: ElementScope, + token: CancellationToken + ) throws -> [ElementSnapshot] { + struct Pending { + let element: AXUIElement + let parentID: String? + let scope: ElementScope + } + var pending = [Pending(element: root, parentID: parentID, scope: rootScope)] + var snapshots: [ElementSnapshot] = [] + var visitedElements: [CFHashCode: [AXUIElement]] = [:] + var visited = 0 + let windowFrame = try window.currentFrame() + while let item = pending.popLast() { + let elementHash = CFHash(item.element) + if visitedElements[elementHash]?.contains(where: { sameAXElement($0, item.element) }) == true { + continue + } + visitedElements[elementHash, default: []].append(item.element) + guard visited < Self.maximumTreeNodes else { + try fail(ErrorCode.treeTooLarge, "The native accessibility tree exceeds the 5000-node limit.") + } + if visited % 64 == 0 { + try token.throwIfCancelled() + } + visited += 1 + + var scope = item.scope + let record = try register( + item.element, + processID: window.processID, + windowID: window.id, + parentID: item.parentID, + scope: scope + ) + var snapshot: ElementSnapshot + do { + snapshot = try readSnapshot(record: record, window: window, windowFrame: windowFrame) + } catch let error as HelperError where error.code == ErrorCode.staleElement { + continue + } + var childScope = scope + if + window.primary, + scope != .preview, + let storyRootTestId = lease.storyRootTestId, + snapshot.automationID == storyRootTestId + { + scope = .preview + childScope = .preview + record.scope = .preview + snapshot = try readSnapshot(record: record, window: window, windowFrame: windowFrame) + } else if window.primary, scope == .application { + childScope = .chrome + } + + let children = axElements(try copyOptionalAXAttribute(item.element, kAXChildrenAttribute)) + for child in children.reversed() { + pending.append(Pending(element: child, parentID: record.id, scope: childScope)) + } + snapshots.append(snapshot) + } + return snapshots + } + + private func register( + _ element: AXUIElement, + processID: pid_t, + windowID: String, + parentID: String?, + scope: ElementScope + ) throws -> ElementRecord { + let hash = CFHash(element) + for id in recordIDsByHash[hash] ?? [] { + if let record = records[id], sameAXElement(record.element, element) { + record.element = element + record.windowID = windowID + record.parentID = parentID + record.scope = scope + return record + } + } + guard records.count < Self.maximumElementRecords else { + try fail( + ErrorCode.treeTooLarge, + "The native element table reached its 10000-record safety limit. Close the session and create a new one." + ) + } + let record = ElementRecord( + processID: processID, + element: element, + windowID: windowID, + parentID: parentID, + scope: scope + ) + records[record.id] = record + recordIDsByHash[hash, default: []].append(record.id) + return record + } + + private func validateLive(_ record: ElementRecord) throws { + var pid: pid_t = 0 + guard AXUIElementGetPid(record.element, &pid) == .success, pid == record.processID, processIsAlive(pid) else { + try fail(ErrorCode.staleElement, "Element \"\(record.id)\" is no longer available.") + } + let role = try requiredAXRole(record.element, staleMessage: "Element \"\(record.id)\" is no longer available.") + if role == nil { + try fail(ErrorCode.staleElement, "Element \"\(record.id)\" is no longer available.") + } + } + + private func readSnapshot( + record: ElementRecord, + window: WindowRecord, + windowFrame resolvedWindowFrame: CGRect? = nil + ) throws -> ElementSnapshot { + let element = record.element + let screenRect = try axOptionalFrame(element) + let windowFrame = try resolvedWindowFrame ?? window.currentFrame() + let rect = CGRect( + x: screenRect.minX - windowFrame.minX, + y: screenRect.minY - windowFrame.minY, + width: screenRect.width, + height: screenRect.height + ) + let nativeRole = try requiredAXRole(element, staleMessage: "Element \"\(record.id)\" is no longer available.") ?? "AXUnknown" + let subrole = axString(try copyOptionalAXAttribute(element, kAXSubroleAttribute)) + let automationID = axString(try copyOptionalAXAttribute(element, kAXIdentifierAttribute)) + let title = axString(try copyOptionalAXAttribute(element, kAXTitleAttribute)) + let description = axString(try copyOptionalAXAttribute(element, kAXDescriptionAttribute)) + let help = axString(try copyOptionalAXAttribute(element, kAXHelpAttribute)) + let name = [title, description, help].compactMap { $0 }.first { !$0.isEmpty } + let valueObject = try copyOptionalAXAttribute(element, kAXValueAttribute) + let value = stringValue(valueObject) + let role = try normalizeRole(nativeRole, subrole: subrole, value: value, element: element) + let text: String? + if role == "text" { + text = value ?? name + } else if role == "textbox" || role == "document" { + text = value + } else { + text = nil + } + let enabled = try supportedBool( + element, + attribute: kAXEnabledAttribute, + reason: "The provider does not report an enabled state." + ) + let focused = try supportedBool( + element, + attribute: kAXFocusedAttribute, + reason: "The provider does not report keyboard focus." + ) + let expanded = try supportedBool( + element, + attribute: kAXExpandedAttribute, + reason: "The element does not expose expanded state." + ) + let selected = try supportedBool( + element, + attribute: kAXSelectedAttribute, + reason: "The element does not expose selected state." + ) + let hidden = try optionalBool(element, attribute: "AXHidden") + let visible = SupportedBool.supported(hidden != true && screenRect.width > 0 && screenRect.height > 0) + let checked = checkedValue(role: role, value: valueObject) + + return ElementSnapshot( + id: record.id, + automationID: automationID, + checked: checked, + enabled: enabled, + expanded: expanded, + focused: focused, + name: name, + parentID: record.parentID, + rect: rect, + screenRect: screenRect, + role: role, + scope: record.scope, + selected: selected, + text: text, + value: value, + visible: visible, + windowID: record.windowID + ) + } + + private func supportedBool( + _ element: AXUIElement, + attribute: String, + reason: String + ) throws -> SupportedBool { + guard let value = try optionalBool(element, attribute: attribute) else { + return .unsupported(reason) + } + return .supported(value) + } + + private func optionalBool(_ element: AXUIElement, attribute: String) throws -> Bool? { + guard let value = axNumber(try copyOptionalAXAttribute(element, attribute)) else { + return nil + } + return value.boolValue + } + + func checkedValue(role: String, value: CFTypeRef?) -> CheckedValue { + guard role == "checkbox" || role == "radio" || role == "switch" else { + return .unsupported("The element role does not expose checked state.") + } + if let number = axNumber(value) { + if number.intValue == 2 { + return .mixed + } + return .bool(number.boolValue) + } + if let string = axString(value)?.lowercased() { + let components = string.split(separator: ",").map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + if components.contains("mixed") { + return .mixed + } + if components.contains(where: { ["1", "true", "on", "selected", "checked"].contains($0) }) { + return .bool(true) + } + if components.contains(where: { ["0", "false", "off", "unselected", "unchecked"].contains($0) }) { + return .bool(false) + } + } + return .unsupported("The element does not expose a Boolean or mixed accessibility value.") + } + + private func stringValue(_ value: CFTypeRef?) -> String? { + if let string = axString(value) { + return string + } + if let number = axNumber(value) { + return isBoolean(number) ? (number.boolValue ? "true" : "false") : number.stringValue + } + return nil + } + + func normalizeRole(_ role: String, subrole: String?, value: String? = nil, element: AXUIElement? = nil) throws -> String { + let roles: [String: String] = [ + kAXApplicationRole: "application", + kAXBrowserRole: "document", + kAXButtonRole: "button", + kAXCheckBoxRole: "checkbox", + kAXComboBoxRole: "combobox", + kAXDisclosureTriangleRole: "button", + kAXGroupRole: "group", + kAXImageRole: "image", + "AXLink": "link", + kAXListRole: "list", + kAXMenuRole: "menu", + kAXMenuBarRole: "menubar", + kAXMenuItemRole: "menuitem", + kAXOutlineRole: "tree", + kAXPopUpButtonRole: "combobox", + kAXProgressIndicatorRole: "progressbar", + kAXRadioButtonRole: "radio", + kAXRadioGroupRole: "radiogroup", + kAXRowRole: "row", + kAXScrollAreaRole: "scrollarea", + kAXScrollBarRole: "scrollbar", + kAXSheetRole: "dialog", + kAXSliderRole: "slider", + kAXStaticTextRole: "text", + kAXTabGroupRole: "tablist", + kAXTableRole: "table", + kAXTextAreaRole: "textbox", + kAXTextFieldRole: "textbox", + kAXToolbarRole: "toolbar", + kAXWindowRole: "window", + ] + if subrole == kAXDialogSubrole { + return "dialog" + } + if subrole == "AXSwitch" { + return "switch" + } + let normalized = roles[role] ?? role.replacingOccurrences(of: "AX", with: "").lowercased() + guard normalized == "unknown" else { + return normalized + } + + if let nativeRole = value? + .split(separator: ",") + .first? + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased(), + ["checkbox", "radio", "switch"].contains(nativeRole) + { + return nativeRole + } + + if let element { + var settable: DarwinBoolean = false + let error = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable) + if error == .success, settable.boolValue { + return "textbox" + } + if error != .attributeUnsupported && error != .noValue && error != .failure && error != .illegalArgument { + try throwAX(error, operation: "Checking whether AXValue is writable") + } + } + return normalized + } + + private func matches(snapshot: ElementSnapshot, strategy: String, value: String) throws -> Bool { + switch strategy { + case "accessibility id": + return snapshot.automationID == value + case "tag name": + return snapshot.role == normalizeRoleQuery(value) + case "link text": + return snapshot.name == value + case "partial link text": + return !value.isEmpty && (snapshot.name?.contains(value) ?? false) + case "-furn:text": + return snapshot.text == value || snapshot.value == value || snapshot.name == value + default: + try fail(ErrorCode.invalidParams, "Locator strategy \"\(strategy)\" is not supported.") + } + } + + private func normalizeRoleQuery(_ value: String) -> String { + let normalized = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + let aliases: [String: String] = [ + "edit": "textbox", + "input": "textbox", + "textfield": "textbox", + "hyperlink": "link", + "radiobutton": "radio", + "combo box": "combobox", + "listbox": "list", + "statusbar": "status", + "tabitem": "tab", + "progress": "progressbar", + "spinner": "spinbutton", + ] + return aliases[normalized] ?? normalized + } + + private func parent(of element: AXUIElement) throws -> AXUIElement? { + guard let value = try copyAXAttribute(element, kAXParentAttribute) else { + return nil + } + return axElement(value) + } + + private func belongsToWindow( + _ element: AXUIElement, + window: AXUIElement, + token: CancellationToken + ) throws -> Bool { + var cursor: AXUIElement? = element + for _ in 0.. AXUIElement? { + try requireAccessibility("Accessibility hit testing") + var element: AXUIElement? + let error = AXUIElementCopyElementAtPosition( + AXUIElementCreateSystemWide(), + Float(point.x), + Float(point.y), + &element + ) + if error == .noValue || error == .cannotComplete { + return nil + } + try throwAX(error, operation: "Hit testing the desktop") + return element + } + + private func resolveKnownAncestor( + _ element: AXUIElement, + token: CancellationToken + ) throws -> ElementRecord? { + var cursor: AXUIElement? = element + for _ in 0.. ElementRecord? { + let hash = CFHash(element) + for id in recordIDsByHash[hash] ?? [] { + if let record = records[id], sameAXElement(record.element, element) { + return record + } + } + return nil + } + + private func scopeForKnownElement(_ element: AXUIElement) -> ElementScope? { + knownRecord(element)?.scope + } + + private func parentIDForKnownElement(_ element: AXUIElement) -> String? { + knownRecord(element)?.parentID + } + + private func externalWindowSnapshot( + window: WindowRecord, + hit: AXUIElement + ) -> ElementSnapshot { + let screenRect = (try? axFrame(hit)) ?? .zero + let windowFrame = (try? window.currentFrame()) ?? .zero + return ElementSnapshot( + id: IdentifierGenerator.shared.next("external-window"), + automationID: nil, + checked: .unsupported("An external desktop window does not expose checked state."), + enabled: .supported(true), + expanded: .unsupported("An external desktop window does not expose expanded state."), + focused: .supported(false), + name: "External desktop window", + parentID: nil, + rect: CGRect( + x: screenRect.minX - windowFrame.minX, + y: screenRect.minY - windowFrame.minY, + width: screenRect.width, + height: screenRect.height + ), + screenRect: screenRect, + role: "window", + scope: .application, + selected: .unsupported("An external desktop window does not expose selected state."), + text: nil, + value: nil, + visible: .supported(true), + windowID: window.id + ) + } + + private func escapeXML(_ value: String) -> String { + var output = "" + for scalar in value.unicodeScalars { + switch scalar.value { + case 0x26: + output += "&" + case 0x3C: + output += "<" + case 0x3E: + output += ">" + case 0x22: + output += """ + case 0x27: + output += "'" + case 0..<0x20 where scalar.value != 0x09 && scalar.value != 0x0A && scalar.value != 0x0D: + output.append(" ") + default: + output.unicodeScalars.append(scalar) + } + } + return output + } + + private func formatNumber(_ value: CGFloat) -> String { + let number = rounded(value) + if number.rounded() == number { + return String(Int64(number)) + } + return String(number) + } +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Application.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Application.swift new file mode 100644 index 00000000000..0af415a097f --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Application.swift @@ -0,0 +1,1267 @@ +import AppKit +import ApplicationServices +import Foundation + +// NSRunningApplication launch dates do not guarantee the microsecond precision of proc_pidinfo. +private let externalLeaseStartTimeTolerance: TimeInterval = 1 + +final class ApplicationLeaseRecord { + let id: String + let ownership: String + let endpoint: String + let storyRootTestId: String? + let processID: pid_t + let processStartedAt: Date + let bundleIdentifier: String? + let executablePath: String? + let windowTitle: String + var primaryWindowID: String? + + init( + ownership: String, + endpoint: String, + storyRootTestId: String?, + processID: pid_t, + processStartedAt: Date, + bundleIdentifier: String?, + executablePath: String?, + windowTitle: String + ) { + id = IdentifierGenerator.shared.next("lease") + self.ownership = ownership + self.endpoint = endpoint + self.storyRootTestId = storyRootTestId + self.processID = processID + self.processStartedAt = processStartedAt + self.bundleIdentifier = bundleIdentifier + self.executablePath = executablePath + self.windowTitle = windowTitle + } + + func json() -> JSONObject { + [ + "id": id, + "ownership": ownership, + "processId": Int(processID), + "processStartedAt": formatDate(processStartedAt), + ] + } +} + +final class WindowRecord { + let id: String + let processID: pid_t + var leaseID: String + var primary: Bool + var element: AXUIElement + var lastKnownFrame: CGRect? + var title: String + + init(processID: pid_t, leaseID: String, primary: Bool, element: AXUIElement, frame: CGRect? = nil, title: String) { + id = IdentifierGenerator.shared.next("window") + self.processID = processID + self.leaseID = leaseID + self.primary = primary + self.element = element + lastKnownFrame = frame + self.title = title + } + + func currentFrame() throws -> CGRect { + do { + let frame = try axFrame(element) + if frame.width > 0, frame.height > 0 { + lastKnownFrame = frame + return frame + } + } catch let error as HelperError where error.code == ErrorCode.automationFailed { + if let lastKnownFrame { + return lastKnownFrame + } + throw error + } + return lastKnownFrame ?? .zero + } +} + +private final class LaunchResultBox: @unchecked Sendable { + let lock = NSLock() + var application: NSRunningApplication? + var error: Error? +} + +struct AXWindowAttributeDiagnostic { + let attribute: String + let error: AXError + let returnedType: String + let elementCount: Int + let elements: [AXUIElement] + let valueValid: Bool + + func json() -> JSONObject { + let elementEvidence = elements.prefix(16).map(windowElementDiagnostic) + return [ + "attribute": attribute, + "elementCount": elementCount, + "elements": elementEvidence, + "error": axErrorJSON(error), + "returnedType": returnedType, + "truncated": elementCount > elementEvidence.count, + "valueValid": valueValid, + ] + } +} + +struct AXWindowRoleDiagnostic { + let error: AXError + let role: String? + let valueValid: Bool + + func json() -> JSONObject { + [ + "error": axErrorJSON(error), + "role": role.map { bounded($0, bytes: 128) } ?? NSNull(), + "valueValid": valueValid, + ] + } +} + +private func windowElementDiagnostic(_ element: AXUIElement) -> JSONObject { + var roleValue: CFTypeRef? + let roleError = AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &roleValue) + var titleValue: CFTypeRef? + let titleError = AXUIElementCopyAttributeValue(element, kAXTitleAttribute as CFString, &titleValue) + var names: CFArray? + let namesError = AXUIElementCopyAttributeNames(element, &names) + let allNames = (names as? [String] ?? []).sorted() + let boundedNames = allNames.prefix(32).map { bounded($0, bytes: 128) } + var value: JSONObject = [ + "attributeNameCount": allNames.count, + "attributeNames": boundedNames, + "attributeNamesError": axErrorJSON(namesError), + "attributeNamesTruncated": allNames.count > boundedNames.count, + "roleError": axErrorJSON(roleError), + "titleError": axErrorJSON(titleError), + ] + if let role = axString(roleValue) { + value["role"] = bounded(role, bytes: 128) + } else { + value["role"] = NSNull() + } + if let title = axString(titleValue) { + value["title"] = bounded(title, bytes: 512) + } + return value +} + +struct AXWindowQueryOutcome { + let processID: pid_t + let trusted: Bool + let queries: [AXWindowAttributeDiagnostic] + let roleQueries: [AXWindowRoleDiagnostic] + let realWindows: [AXUIElement] + let placeholderCount: Int + let otherRoleCount: Int + + func diagnostic(reason: String, exactTitle: String? = nil, matchCount: Int? = nil) -> JSONObject { + var value: JSONObject = [ + "otherRoleCount": otherRoleCount, + "placeholderCount": placeholderCount, + "processId": Int(processID), + "queries": queries.map { $0.json() }, + "realWindowCount": realWindows.count, + "reason": reason, + "roleQueries": roleQueries.prefix(16).map { $0.json() }, + "roleQueriesTruncated": roleQueries.count > 16, + "trusted": trusted, + ] + if let exactTitle { + value["exactTitle"] = bounded(exactTitle, bytes: 512) + } + if let matchCount { + value["matchingTitleCount"] = matchCount + } + return value + } +} + +func classifyAXWindowQuery( + queries: [AXWindowAttributeDiagnostic], + realWindowCount: Int, + placeholderCount: Int, + otherRoleCount: Int, + matchingTitleCount: Int?, + roleErrors: [AXError] = [] +) -> String { + let errors = queries.map(\.error) + roleErrors + let softErrors: Set = [.success, .attributeUnsupported, .noValue] + if errors.contains(.apiDisabled) { + return "api-disabled" + } + if realWindowCount == 0, errors.contains(.cannotComplete) { + return "target-non-response" + } + if realWindowCount == 0, errors.contains(.invalidUIElement) { + return "invalid-element" + } + if realWindowCount == 0 { + if errors.contains(where: { !softErrors.contains($0) }) { + return "window-query-failed" + } + if placeholderCount > 0, otherRoleCount == 0 { + return "placeholder-only" + } + if otherRoleCount > 0 { + return "no-real-window" + } + if errors.allSatisfy({ softErrors.contains($0) }) { + return "no-value-or-unsupported" + } + return "window-query-failed" + } + if matchingTitleCount == 0 { + return "no-matching-title" + } + return "matched" +} + +func incompleteAXCandidateDiscovery( + queries: [AXWindowAttributeDiagnostic], + roleQueries: [AXWindowRoleDiagnostic], + titleErrorCount: Int +) -> Bool { + let windowQueryIncomplete = queries.contains { + $0.attribute == kAXWindowsAttribute && ($0.error != .success || !$0.valueValid) + } + let roleQueryIncomplete = roleQueries.contains { + $0.error != .success + || !$0.valueValid + || ($0.role != kAXWindowRole && $0.role != kAXSheetRole) + } + if titleErrorCount > 0 || windowQueryIncomplete || roleQueryIncomplete { + return true + } + guard let authoritative = queries.first(where: { $0.attribute == kAXWindowsAttribute }) else { + return true + } + let fallbackElements = queries + .filter { $0.attribute != kAXWindowsAttribute } + .flatMap(\.elements) + return fallbackElements.contains { fallback in + !authoritative.elements.contains { sameAXElement($0, fallback) } + } +} + +func completeAXWindowTitle(error: AXError, title: String?) -> Bool { + error == .success && title != nil +} + +func incompleteAXMultiProcessDiscovery( + queriesByCandidate: [[AXWindowAttributeDiagnostic]], + roleQueriesByCandidate: [[AXWindowRoleDiagnostic]], + titleErrorCount: Int +) -> Bool { + guard queriesByCandidate.count == roleQueriesByCandidate.count else { + return true + } + if titleErrorCount > 0 { + return true + } + return zip(queriesByCandidate, roleQueriesByCandidate).contains { queries, roleQueries in + incompleteAXCandidateDiscovery(queries: queries, roleQueries: roleQueries, titleErrorCount: 0) + } +} + +func classifyAXAttachment(matchCount: Int, discoveryIncomplete: Bool) -> String { + if matchCount > 1 { + return "ambiguous" + } + if discoveryIncomplete { + return "incomplete" + } + return matchCount == 1 ? "matched" : "missing" +} + +final class ApplicationManager { + private var leases: [String: ApplicationLeaseRecord] = [:] + private var leaseOrder: [String] = [] + private var windows: [String: WindowRecord] = [:] + private var windowIDsByHash: [CFHashCode: [String]] = [:] + + func launch(_ params: JSONObject, token: CancellationToken) throws -> ApplicationLeaseRecord { + let application = try requireObject(params["application"], "The launch request is missing its application descriptor.") + let bundleIdentifier = optionalString(application, "bundleIdentifier") + let executablePath = optionalString(application, "executablePath") + let windowTitle = optionalString(application, "windowTitle") + guard bundleIdentifier != nil || executablePath != nil else { + try fail( + ErrorCode.invalidParams, + "The application descriptor needs a \"bundleIdentifier\" or an \"executablePath\"." + ) + } + guard let windowTitle else { + try fail(ErrorCode.invalidParams, "The application descriptor needs an exact \"windowTitle\" to identify its window.") + } + let arguments = try parseArguments(application["arguments"]) + let preexisting = Set( + NSWorkspace.shared.runningApplications + .filter { bundleIdentifier == nil || $0.bundleIdentifier == bundleIdentifier } + .map(\.processIdentifier) + ) + + let running: NSRunningApplication + if let appURL = try resolveApplicationURL(bundleIdentifier: bundleIdentifier, executablePath: executablePath) { + let bundle = Bundle(url: appURL) + if let bundleIdentifier, bundle?.bundleIdentifier != bundleIdentifier { + try fail( + ErrorCode.launchFailed, + "The selected application bundle does not have the expected bundle identifier \"\(bundleIdentifier)\"." + ) + } + running = try launchApplication(at: appURL, arguments: arguments, token: token) + if preexisting.contains(running.processIdentifier) { + try fail( + ErrorCode.launchFailed, + "Launch Services reused an application process that existed before this request, so the helper refused to claim ownership." + ) + } + } else if let executablePath { + let process = Process() + process.executableURL = URL(fileURLWithPath: executablePath) + process.arguments = arguments + do { + try process.run() + } catch { + try fail(ErrorCode.launchFailed, "Starting the application failed: \(error.localizedDescription)") + } + guard let application = NSRunningApplication(processIdentifier: process.processIdentifier) else { + process.terminate() + try fail(ErrorCode.launchFailed, "The launched process did not become a macOS running application.") + } + running = application + } else { + try fail(ErrorCode.launchFailed, "The application could not be resolved for launch.") + } + + do { + try validateApplication( + running, + expectedBundleIdentifier: bundleIdentifier, + expectedExecutablePath: executablePath, + code: ErrorCode.launchFailed + ) + _ = try waitForUniqueWindow( + processID: running.processIdentifier, + exactTitle: windowTitle, + timeoutMilliseconds: 90_000, + token: token, + code: ErrorCode.launchFailed + ) + return try createLease( + ownership: "launched", + running: running, + windowTitle: windowTitle, + params: params + ) + } catch { + running.terminate() + throw error + } + } + + func attach(_ params: JSONObject, token: CancellationToken) throws -> ApplicationLeaseRecord { + let application = try requireObject(params["application"], "The attach request is missing its application descriptor.") + let endpoint = optionalString(params, "endpoint") ?? "macos" + let configuredBundleIdentifier = optionalString(application, "bundleIdentifier") + let configuredExecutablePath = optionalString(application, "executablePath") + let configuredWindowTitle = optionalString(application, "windowTitle") + + if let leasePath = optionalString(application, "leasePath") { + let lease = try readLease(path: leasePath, token: token) + guard integer(lease, "schemaVersion") == 1 else { + try fail(ErrorCode.leaseInvalid, "The application lease does not use schema version 1.") + } + guard let expectedNonce = optionalString(application, "leaseNonce") else { + try fail(ErrorCode.leaseInvalid, "The application descriptor does not carry the expected lease nonce.") + } + guard optionalString(lease, "nonce") == expectedNonce else { + try fail(ErrorCode.leaseInvalid, "The application lease nonce does not match this session.") + } + guard optionalString(lease, "endpoint") == endpoint else { + try fail(ErrorCode.leaseInvalid, "The application lease was written for a different endpoint.") + } + guard + let processNumber = lease["processId"] as? NSNumber, + !isBoolean(processNumber), + processNumber.doubleValue == Double(processNumber.int32Value), + processNumber.int32Value > 0 + else { + try fail(ErrorCode.leaseInvalid, "The application lease does not name a valid process.") + } + let pid = pid_t(processNumber.int32Value) + guard processIsAlive(pid), let running = NSRunningApplication(processIdentifier: pid) else { + try fail(ErrorCode.leaseInvalid, "The leased application process is no longer running.") + } + guard + let actualStart = processStartTime(pid), + let expectedStartText = optionalString(lease, "processStartedAt"), + let expectedStart = parseDate(expectedStartText) + else { + try fail(ErrorCode.leaseInvalid, "The application lease does not carry a valid process start time.") + } + guard abs(actualStart.timeIntervalSince(expectedStart)) <= externalLeaseStartTimeTolerance else { + try fail(ErrorCode.leaseInvalid, "The leased process identifier was reused by a different process.") + } + + guard let leaseBundleIdentifier = optionalString(lease, "bundleIdentifier") else { + try fail(ErrorCode.leaseInvalid, "The application lease does not carry the target bundle identifier.") + } + if + let configuredBundleIdentifier, + configuredBundleIdentifier != leaseBundleIdentifier + { + try fail(ErrorCode.leaseInvalid, "The application lease names a different bundle identifier than the target.") + } + let expectedBundleIdentifier = leaseBundleIdentifier + + guard let leaseExecutablePath = optionalString(lease, "executablePath") else { + try fail(ErrorCode.leaseInvalid, "The application lease does not carry the target executable path.") + } + if + let configuredExecutablePath, + canonicalPath(configuredExecutablePath) != canonicalPath(leaseExecutablePath) + { + try fail(ErrorCode.leaseInvalid, "The application lease names a different executable than the target.") + } + let expectedExecutablePath = leaseExecutablePath + + let leaseWindowTitle = optionalString(lease, "windowTitle") + if let configuredWindowTitle, let leaseWindowTitle, configuredWindowTitle != leaseWindowTitle { + try fail(ErrorCode.leaseInvalid, "The application lease names a different window title than the target.") + } + guard let windowTitle = leaseWindowTitle ?? configuredWindowTitle else { + try fail(ErrorCode.leaseInvalid, "The application lease does not name the application window.") + } + + try validateApplication( + running, + expectedBundleIdentifier: expectedBundleIdentifier, + expectedExecutablePath: expectedExecutablePath, + code: ErrorCode.leaseInvalid + ) + _ = try waitForUniqueWindow( + processID: pid, + exactTitle: windowTitle, + timeoutMilliseconds: 10_000, + token: token, + code: ErrorCode.attachFailed + ) + return try createLease(ownership: "attached", running: running, windowTitle: windowTitle, params: params) + } + + guard let windowTitle = configuredWindowTitle else { + try fail( + ErrorCode.invalidParams, + "Attaching without a lease requires an exact \"windowTitle\" in the application descriptor." + ) + } + let candidates = NSWorkspace.shared.runningApplications.filter { application in + guard !application.isTerminated else { + return false + } + if let configuredBundleIdentifier, application.bundleIdentifier != configuredBundleIdentifier { + return false + } + if let configuredExecutablePath, !matchesExecutable(application, expected: configuredExecutablePath) { + return false + } + return true + } + var matches: [(NSRunningApplication, AXUIElement)] = [] + var queryOutcomes: [AXWindowQueryOutcome] = [] + var titleErrorCount = 0 + var titleErrors: [JSONObject] = [] + for candidate in candidates { + try token.throwIfCancelled() + let outcome = queryAccessibilityWindows(processID: candidate.processIdentifier) + queryOutcomes.append(outcome) + let reason = classifyAXWindowQuery( + queries: outcome.queries, + realWindowCount: outcome.realWindows.count, + placeholderCount: outcome.placeholderCount, + otherRoleCount: outcome.otherRoleCount, + matchingTitleCount: nil, + roleErrors: outcome.roleQueries.map(\.error) + ) + if reason == "api-disabled" { + try throwCriticalWindowQueryFailure(outcome, code: ErrorCode.attachFailed) + } + let candidateWindows = outcome.realWindows + for window in candidateWindows { + var titleValue: CFTypeRef? + let titleError = AXUIElementCopyAttributeValue(window, kAXTitleAttribute as CFString, &titleValue) + let candidateTitle = titleError == .success ? axString(titleValue) : nil + if titleError == .apiDisabled { + try throwAX(titleError, operation: "Reading \(kAXTitleAttribute)") + } + guard completeAXWindowTitle(error: titleError, title: candidateTitle) else { + titleErrorCount += 1 + if titleErrors.count < 16 { + titleErrors.append([ + "error": axErrorJSON(titleError), + "processId": Int(candidate.processIdentifier), + "returnedType": cfTypeName(titleValue), + ]) + } + continue + } + if candidateTitle == windowTitle { + matches.append((candidate, window)) + } + } + } + let realWindowCount = queryOutcomes.reduce(0) { $0 + $1.realWindows.count } + let placeholderCount = queryOutcomes.reduce(0) { $0 + $1.placeholderCount } + let otherRoleCount = queryOutcomes.reduce(0) { $0 + $1.otherRoleCount } + let queries = queryOutcomes.flatMap(\.queries) + let roleErrors = queryOutcomes.flatMap { $0.roleQueries.map(\.error) } + let discoveryIncomplete = incompleteAXMultiProcessDiscovery( + queriesByCandidate: queryOutcomes.map(\.queries), + roleQueriesByCandidate: queryOutcomes.map(\.roleQueries), + titleErrorCount: titleErrorCount + ) + let attachmentResolution = classifyAXAttachment( + matchCount: matches.count, + discoveryIncomplete: discoveryIncomplete + ) + if attachmentResolution == "ambiguous" { + try fail( + ErrorCode.ambiguousTarget, + "\(matches.count) windows are titled \"\(windowTitle)\", so the target is ambiguous." + ) + } + if attachmentResolution == "incomplete", !matches.isEmpty { + let queryEvidence = queryOutcomes.prefix(16).map { $0.diagnostic(reason: "candidate-query") } + try fail( + ErrorCode.ambiguousTarget, + "A window titled \"\(windowTitle)\" was found, but Accessibility could not inspect every candidate process to prove it is unique.", + data: [ + "candidateApplicationCount": candidates.count, + "candidateQueries": queryEvidence, + "confirmedMatchCount": matches.count, + "exactTitle": bounded(windowTitle, bytes: 512), + "reason": "incomplete-candidate-discovery", + "titleErrorCount": titleErrorCount, + "titleErrors": titleErrors, + "titleErrorsTruncated": titleErrorCount > titleErrors.count, + "truncated": candidates.count > queryEvidence.count, + ] + ) + } + if matches.isEmpty { + let reason: String + if candidates.isEmpty { + reason = "no-matching-application" + } else if discoveryIncomplete { + reason = "incomplete-candidate-discovery" + } else { + reason = classifyAXWindowQuery( + queries: queries, + realWindowCount: realWindowCount, + placeholderCount: placeholderCount, + otherRoleCount: otherRoleCount, + matchingTitleCount: 0, + roleErrors: roleErrors + ) + } + let queryEvidence = queryOutcomes.prefix(16).map { $0.diagnostic(reason: "candidate-query") } + let message: String + switch reason { + case "placeholder-only": + message = "The candidate application returned only AXApplication placeholders instead of a real AXWindow or AXSheet." + case "no-value-or-unsupported": + message = "The candidate applications returned no values for AXWindows, AXMainWindow, or AXFocusedWindow." + case "no-real-window": + message = "The candidate applications returned accessibility elements, but none are AXWindow or AXSheet." + case "target-non-response": + message = "The candidate applications did not complete their accessibility window queries." + case "invalid-element": + message = "The candidate application accessibility objects became invalid during window discovery." + case "window-query-failed": + message = "Accessibility window discovery failed before any candidate returned a usable window." + case "incomplete-candidate-discovery": + message = "Accessibility could not inspect every candidate window, so the requested title could not be ruled out." + default: + message = "No window titled \"\(windowTitle)\" is currently open." + } + try fail( + ErrorCode.attachFailed, + message, + data: [ + "candidateApplicationCount": candidates.count, + "candidateQueries": queryEvidence, + "exactTitle": bounded(windowTitle, bytes: 512), + "reason": reason, + "titleErrorCount": titleErrorCount, + "titleErrors": titleErrors, + "titleErrorsTruncated": titleErrorCount > titleErrors.count, + "truncated": candidates.count > queryEvidence.count, + ] + ) + } + let running = matches[0].0 + try validateApplication( + running, + expectedBundleIdentifier: configuredBundleIdentifier, + expectedExecutablePath: configuredExecutablePath, + code: ErrorCode.attachFailed + ) + return try createLease(ownership: "attached", running: running, windowTitle: windowTitle, params: params) + } + + func closeApplication(_ leaseID: String, token: CancellationToken) throws -> [String] { + guard let lease = leases[leaseID] else { + return [] + } + let windowIDs = windows.values.filter { $0.leaseID == leaseID }.map(\.id) + if lease.ownership == "launched", let running = NSRunningApplication(processIdentifier: lease.processID) { + guard + processIsAlive(lease.processID), + let start = processStartTime(lease.processID), + abs(start.timeIntervalSince(lease.processStartedAt)) <= 0.01 + else { + try fail(ErrorCode.noSuchLease, "The launched application process identity no longer matches its lease.") + } + _ = running.terminate() + let deadline = DispatchTime.now() + .seconds(5) + while processIsAlive(lease.processID), DispatchTime.now() < deadline { + try token.wait(milliseconds: 50) + } + if processIsAlive(lease.processID) { + _ = running.forceTerminate() + } + } + removeLease(leaseID) + return windowIDs + } + + func listWindows(_ leaseID: String, token: CancellationToken) throws -> [WindowRecord] { + guard let lease = leases[leaseID] else { + try fail(ErrorCode.noSuchLease, "The requested application lease is not held by this helper.") + } + guard + processIsAlive(lease.processID), + let start = processStartTime(lease.processID), + abs(start.timeIntervalSince(lease.processStartedAt)) <= 0.01 + else { + try fail(ErrorCode.noSuchLease, "The leased application process has exited or changed identity.") + } + let geometryDeadline = DispatchTime.now() + .seconds(1) + var results: [WindowRecord] = [] + var candidateCount = 0 + var zeroGeometryCount = 0 + var geometryDiagnostics: [JSONObject] = [] + while results.isEmpty { + let trackedElements = windows.values.filter { $0.leaseID == leaseID }.map(\.element) + let elements = uniqueAXElements(trackedElements + (try accessibilityWindows(processID: lease.processID))) + candidateCount = elements.count + zeroGeometryCount = 0 + for element in elements { + try token.throwIfCancelled() + do { + let title = try windowTitleOf(element) + let axGeometry = try axFrame(element) + let frame: CGRect + if axGeometry.width > 0, axGeometry.height > 0 { + frame = axGeometry + } else { + let fallback = coreGraphicsWindowFrame( + processID: lease.processID, + title: title, + authoritativeWindowCount: elements.count + ) + frame = fallback.frame ?? .zero + if geometryDiagnostics.count < 16 { + var diagnostic = fallback.diagnostics + diagnostic["accessibilityTitle"] = bounded(title, bytes: 256) + geometryDiagnostics.append(diagnostic) + } + } + guard frame.width > 0, frame.height > 0 else { + zeroGeometryCount += 1 + continue + } + let primary = title == lease.windowTitle + results.append(registerWindow(element, lease: lease, primary: primary, frame: frame, title: title)) + } catch let error as HelperError + where (error.code == ErrorCode.automationFailed || error.code == ErrorCode.staleElement) + && DispatchTime.now() < geometryDeadline + { + continue + } + } + if !results.isEmpty || DispatchTime.now() >= geometryDeadline { + break + } + try token.wait(milliseconds: 50) + } + if results.isEmpty, candidateCount > 0 { + try fail( + ErrorCode.automationFailed, + "The application exposes \(candidateCount) live accessibility window candidate(s), but \(zeroGeometryCount) have empty geometry.", + data: [ + "candidateCount": candidateCount, + "coreGraphicsQueries": geometryDiagnostics, + "reason": "empty-window-geometry", + "screenCaptureAccess": CGPreflightScreenCaptureAccess(), + "zeroGeometryCount": zeroGeometryCount, + ] + ) + } + results.sort { + if $0.primary != $1.primary { + return $0.primary + } + return $0.id < $1.id + } + if lease.primaryWindowID == nil, let first = results.first { + first.primary = true + lease.primaryWindowID = first.id + } + return results + } + + func requireWindow(_ windowID: String) throws -> WindowRecord { + guard let record = windows[windowID] else { + try fail(ErrorCode.noSuchWindow, "Window \"\(windowID)\" is not tracked by this helper.") + } + guard processIsAlive(record.processID) else { + try fail(ErrorCode.noSuchWindow, "Window \"\(windowID)\" has been closed or changed ownership.") + } + var needsRefresh = pidForAXElement(record.element) != record.processID + if !needsRefresh { + do { + let role = try requiredAXRole(record.element, staleMessage: "Window \"\(windowID)\" is no longer available.") + needsRefresh = role != kAXWindowRole && role != kAXSheetRole + } catch let error as HelperError where error.code == ErrorCode.staleElement { + needsRefresh = true + } + } + if needsRefresh { + let refreshed = try uniqueWindow(processID: record.processID, exactTitle: record.title, missingCode: ErrorCode.noSuchWindow) + replaceWindowElement(record, with: refreshed) + } + return record + } + + func lease(for window: WindowRecord) -> ApplicationLeaseRecord? { + leases[window.leaseID] + } + + func primaryLease() -> ApplicationLeaseRecord? { + leaseOrder.lazy.compactMap { self.leases[$0] }.first + } + + func forgetWindow(_ windowID: String) { + guard let record = windows.removeValue(forKey: windowID) else { + return + } + let hash = CFHash(record.element) + windowIDsByHash[hash]?.removeAll { $0 == windowID } + if windowIDsByHash[hash]?.isEmpty == true { + windowIDsByHash.removeValue(forKey: hash) + } + leases[record.leaseID]?.primaryWindowID = leases[record.leaseID]?.primaryWindowID == windowID + ? nil + : leases[record.leaseID]?.primaryWindowID + } + + func windowIDs(for leaseID: String) -> [String] { + windows.values.filter { $0.leaseID == leaseID }.map(\.id) + } + + func dispose(token: CancellationToken) { + for leaseID in leaseOrder.reversed() { + try? _ = closeApplication(leaseID, token: token) + } + } + + private func createLease( + ownership: String, + running: NSRunningApplication, + windowTitle: String, + params: JSONObject + ) throws -> ApplicationLeaseRecord { + guard let start = processStartTime(running.processIdentifier) else { + try fail( + ownership == "launched" ? ErrorCode.launchFailed : ErrorCode.attachFailed, + "The application process start identity could not be read." + ) + } + let lease = ApplicationLeaseRecord( + ownership: ownership, + endpoint: optionalString(params, "endpoint") ?? "macos", + storyRootTestId: optionalString(params, "storyRootTestId"), + processID: running.processIdentifier, + processStartedAt: start, + bundleIdentifier: running.bundleIdentifier, + executablePath: processExecutablePath(running.processIdentifier), + windowTitle: windowTitle + ) + leases[lease.id] = lease + leaseOrder.append(lease.id) + let primary = try uniqueWindow( + processID: running.processIdentifier, + exactTitle: windowTitle, + missingCode: ownership == "launched" ? ErrorCode.launchFailed : ErrorCode.attachFailed + ) + let record = registerWindow(primary, lease: lease, primary: true, title: windowTitle) + lease.primaryWindowID = record.id + return lease + } + + private func registerWindow( + _ element: AXUIElement, + lease: ApplicationLeaseRecord, + primary: Bool, + frame: CGRect? = nil, + title: String + ) -> WindowRecord { + let hash = CFHash(element) + for id in windowIDsByHash[hash] ?? [] { + if let existing = windows[id], sameAXElement(existing.element, element) { + existing.element = element + existing.leaseID = lease.id + existing.primary = existing.primary || primary + existing.title = title + if let frame { + existing.lastKnownFrame = frame + } + if existing.primary { + lease.primaryWindowID = id + } + return existing + } + } + let record = WindowRecord( + processID: lease.processID, + leaseID: lease.id, + primary: primary, + element: element, + frame: frame, + title: title + ) + windows[record.id] = record + windowIDsByHash[hash, default: []].append(record.id) + if primary { + lease.primaryWindowID = record.id + } + return record + } + + private func replaceWindowElement(_ record: WindowRecord, with element: AXUIElement) { + let previousHash = CFHash(record.element) + windowIDsByHash[previousHash]?.removeAll { $0 == record.id } + if windowIDsByHash[previousHash]?.isEmpty == true { + windowIDsByHash.removeValue(forKey: previousHash) + } + record.element = element + windowIDsByHash[CFHash(element), default: []].append(record.id) + } + + private func removeLease(_ leaseID: String) { + for id in windowIDs(for: leaseID) { + forgetWindow(id) + } + leases.removeValue(forKey: leaseID) + leaseOrder.removeAll { $0 == leaseID } + } + + private func readLease(path: String, token: CancellationToken) throws -> JSONObject { + try token.throwIfCancelled() + let url = URL(fileURLWithPath: path) + let data: Data + do { + data = try Data(contentsOf: url, options: [.mappedIfSafe]) + } catch { + try fail(ErrorCode.leaseInvalid, "Opening the application lease file failed: \(error.localizedDescription)") + } + guard !data.isEmpty else { + try fail(ErrorCode.leaseInvalid, "The application lease file is empty.") + } + guard data.count <= 1024 * 1024 else { + try fail(ErrorCode.leaseInvalid, "The application lease file exceeds the 1 MiB limit.") + } + return try requireObject(try parseJSON(data), "The application lease must be a JSON object.") + } + + private func parseArguments(_ value: Any?) throws -> [String] { + guard let value else { + return [] + } + guard let list = value as? [Any] else { + try fail(ErrorCode.invalidParams, "The application descriptor \"arguments\" field must be an array.") + } + var arguments: [String] = [] + for entry in list { + guard let entry = entry as? String else { + try fail(ErrorCode.invalidParams, "Every application launch argument must be a string.") + } + arguments.append(entry) + } + return arguments + } + + private func resolveApplicationURL( + bundleIdentifier: String?, + executablePath: String? + ) throws -> URL? { + var byIdentifier: URL? + if let bundleIdentifier { + byIdentifier = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundleIdentifier) + if byIdentifier == nil { + try fail(ErrorCode.launchFailed, "No installed application has bundle identifier \"\(bundleIdentifier)\".") + } + } + var byExecutable: URL? + if let executablePath { + var candidate = URL(fileURLWithPath: executablePath).standardizedFileURL + while candidate.path != "/" { + if candidate.pathExtension.lowercased() == "app" { + byExecutable = candidate + break + } + candidate.deleteLastPathComponent() + } + } + if let byIdentifier, let byExecutable { + guard canonicalPath(byIdentifier.path) == canonicalPath(byExecutable.path) else { + try fail(ErrorCode.launchFailed, "The configured bundle identifier and executable resolve to different applications.") + } + } + return byIdentifier ?? byExecutable + } + + private func launchApplication( + at url: URL, + arguments: [String], + token: CancellationToken + ) throws -> NSRunningApplication { + let configuration = NSWorkspace.OpenConfiguration() + configuration.arguments = arguments + configuration.activates = false + configuration.addsToRecentItems = false + configuration.createsNewApplicationInstance = true + let result = LaunchResultBox() + let semaphore = DispatchSemaphore(value: 0) + NSWorkspace.shared.openApplication(at: url, configuration: configuration) { application, error in + result.lock.lock() + result.application = application + result.error = error + result.lock.unlock() + semaphore.signal() + } + while semaphore.wait(timeout: .now() + .milliseconds(50)) == .timedOut { + try token.throwIfCancelled() + } + result.lock.lock() + let application = result.application + let error = result.error + result.lock.unlock() + if let error { + try fail(ErrorCode.launchFailed, "Launch Services could not start the application: \(error.localizedDescription)") + } + guard let application else { + try fail(ErrorCode.launchFailed, "Launch Services did not return the launched application.") + } + return application + } + + private func validateApplication( + _ application: NSRunningApplication, + expectedBundleIdentifier: String?, + expectedExecutablePath: String?, + code: String + ) throws { + guard processIsAlive(application.processIdentifier) else { + try fail(code, "The target application process is not running.") + } + if let expectedBundleIdentifier, application.bundleIdentifier != expectedBundleIdentifier { + try fail(code, "The target process does not have the expected bundle identifier \"\(expectedBundleIdentifier)\".") + } + if let expectedExecutablePath, !matchesExecutable(application, expected: expectedExecutablePath) { + try fail(code, "The target process does not run the expected executable.") + } + } + + private func matchesExecutable(_ application: NSRunningApplication, expected: String) -> Bool { + let expectedURL = URL(fileURLWithPath: expected).standardizedFileURL + if expectedURL.pathExtension.lowercased() == "app", let bundleURL = application.bundleURL { + return canonicalPath(expectedURL.path) == canonicalPath(bundleURL.path) + } + guard let actual = processExecutablePath(application.processIdentifier) else { + return false + } + return canonicalPath(expected) == actual + } + + private func waitForUniqueWindow( + processID: pid_t, + exactTitle: String, + timeoutMilliseconds: Int, + token: CancellationToken, + code: String + ) throws -> AXUIElement { + let deadline = DispatchTime.now() + .milliseconds(timeoutMilliseconds) + while true { + try token.throwIfCancelled() + do { + return try uniqueWindow(processID: processID, exactTitle: exactTitle, missingCode: code) + } catch let error as HelperError where error.code == code { + if !processIsAlive(processID) { + try fail(code, "The application exited before it showed its expected window.") + } + if DispatchTime.now() >= deadline { + throw error + } + } + try token.wait(milliseconds: 100) + } + } + + private func uniqueWindow( + processID: pid_t, + exactTitle: String, + missingCode: String + ) throws -> AXUIElement { + let outcome = queryAccessibilityWindows(processID: processID) + try throwCriticalWindowQueryFailure(outcome, code: missingCode) + var matches: [AXUIElement] = [] + var titleErrorCount = 0 + var titleErrors: [JSONObject] = [] + for window in outcome.realWindows { + var titleValue: CFTypeRef? + let titleError = AXUIElementCopyAttributeValue(window, kAXTitleAttribute as CFString, &titleValue) + let candidateTitle = titleError == .success ? axString(titleValue) : nil + if titleError == .apiDisabled { + try throwAX(titleError, operation: "Reading \(kAXTitleAttribute)") + } + guard completeAXWindowTitle(error: titleError, title: candidateTitle) else { + titleErrorCount += 1 + if titleErrors.count < 16 { + titleErrors.append([ + "error": axErrorJSON(titleError), + "returnedType": cfTypeName(titleValue), + ]) + } + continue + } + if candidateTitle == exactTitle { + matches.append(window) + } + } + let discoveryIncomplete = incompleteAXCandidateDiscovery( + queries: outcome.queries, + roleQueries: outcome.roleQueries, + titleErrorCount: titleErrorCount + ) + let attachmentResolution = classifyAXAttachment( + matchCount: matches.count, + discoveryIncomplete: discoveryIncomplete + ) + if attachmentResolution == "ambiguous" || (attachmentResolution == "incomplete" && !matches.isEmpty) { + var data = outcome.diagnostic( + reason: discoveryIncomplete ? "incomplete-candidate-discovery" : "ambiguous-title", + exactTitle: exactTitle, + matchCount: matches.count + ) + data["titleErrorCount"] = titleErrorCount + data["titleErrors"] = titleErrors + data["titleErrorsTruncated"] = titleErrorCount > titleErrors.count + let message = discoveryIncomplete + ? "A window titled \"\(exactTitle)\" was found, but Accessibility could not inspect every window to prove it is unique." + : "The application shows \(matches.count) windows titled \"\(exactTitle)\"." + try fail(ErrorCode.ambiguousTarget, message, data: data) + } + if matches.isEmpty { + let reason = discoveryIncomplete + ? "incomplete-candidate-discovery" + : classifyAXWindowQuery( + queries: outcome.queries, + realWindowCount: outcome.realWindows.count, + placeholderCount: outcome.placeholderCount, + otherRoleCount: outcome.otherRoleCount, + matchingTitleCount: 0, + roleErrors: outcome.roleQueries.map(\.error) + ) + let message: String + switch reason { + case "placeholder-only": + message = + "The application returned only AXApplication placeholders instead of a real AXWindow or AXSheet while looking for \"\(exactTitle)\"." + case "no-real-window": + message = + "The application returned accessibility elements, but none are AXWindow or AXSheet while looking for \"\(exactTitle)\"." + case "no-value-or-unsupported": + message = + "AXWindows, AXMainWindow, and AXFocusedWindow returned no window values while looking for \"\(exactTitle)\"." + case "incomplete-candidate-discovery": + message = + "Accessibility could not inspect every application window, so the requested title \"\(exactTitle)\" could not be ruled out." + default: + message = "The application does not currently show a window titled \"\(exactTitle)\"." + } + var data = outcome.diagnostic(reason: reason, exactTitle: exactTitle, matchCount: 0) + data["titleErrorCount"] = titleErrorCount + data["titleErrors"] = titleErrors + data["titleErrorsTruncated"] = titleErrorCount > titleErrors.count + try fail( + missingCode, + message, + data: data + ) + } + return matches[0] + } + + private func accessibilityWindows(processID: pid_t) throws -> [AXUIElement] { + let outcome = queryAccessibilityWindows(processID: processID) + try throwCriticalWindowQueryFailure(outcome, code: ErrorCode.automationFailed) + return outcome.realWindows + } + + private func queryAccessibilityWindows(processID: pid_t) -> AXWindowQueryOutcome { + let application = AXUIElementCreateApplication(processID) + let attributes = [kAXWindowsAttribute, kAXMainWindowAttribute, kAXFocusedWindowAttribute] + let queries = attributes.map { queryWindowAttribute(application, attribute: $0) } + let elements = uniqueAXElements(queries.flatMap(\.elements)) + var roleQueries: [AXWindowRoleDiagnostic] = [] + var realWindows: [AXUIElement] = [] + var placeholderCount = 0 + var otherRoleCount = 0 + for element in elements { + var roleValue: CFTypeRef? + let roleError = AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &roleValue) + let role = roleError == .success ? axString(roleValue) : nil + let roleValueValid = roleError == .success && role != nil + roleQueries.append(AXWindowRoleDiagnostic(error: roleError, role: role, valueValid: roleValueValid)) + if role == kAXWindowRole || role == kAXSheetRole { + realWindows.append(element) + } else if role == kAXApplicationRole { + placeholderCount += 1 + } else if roleValueValid { + otherRoleCount += 1 + } + } + return AXWindowQueryOutcome( + processID: processID, + trusted: AXIsProcessTrusted(), + queries: queries, + roleQueries: roleQueries, + realWindows: realWindows, + placeholderCount: placeholderCount, + otherRoleCount: otherRoleCount + ) + } + + private func throwCriticalWindowQueryFailure(_ outcome: AXWindowQueryOutcome, code: String) throws { + let reason = classifyAXWindowQuery( + queries: outcome.queries, + realWindowCount: outcome.realWindows.count, + placeholderCount: outcome.placeholderCount, + otherRoleCount: outcome.otherRoleCount, + matchingTitleCount: nil, + roleErrors: outcome.roleQueries.map(\.error) + ) + if reason == "api-disabled" { + try fail( + ErrorCode.inputUnavailable, + "Application window discovery failed because the Accessibility API is disabled for the helper, even if the trust preflight appears granted. Re-enable Accessibility access and restart the helper.", + data: outcome.diagnostic(reason: reason) + ) + } + if reason == "target-non-response" { + try fail( + code, + "The target application did not complete one or more accessibility window queries.", + data: outcome.diagnostic(reason: reason) + ) + } + if reason == "invalid-element" { + try fail( + code, + "The target application's accessibility object became invalid during window discovery.", + data: outcome.diagnostic(reason: reason) + ) + } + if incompleteAXCandidateDiscovery( + queries: outcome.queries, + roleQueries: outcome.roleQueries, + titleErrorCount: 0 + ) { + try fail( + code, + "Accessibility returned an incomplete or internally inconsistent window enumeration.", + data: outcome.diagnostic(reason: "incomplete-window-enumeration") + ) + } + if reason == "window-query-failed" { + try fail( + code, + "Accessibility window discovery failed before the target returned a usable window.", + data: outcome.diagnostic(reason: reason) + ) + } + } + + private func queryWindowAttribute( + _ application: AXUIElement, + attribute: String + ) -> AXWindowAttributeDiagnostic { + var value: CFTypeRef? + let error = AXUIElementCopyAttributeValue(application, attribute as CFString, &value) + let elements: [AXUIElement] + let elementCount: Int + let valueValid: Bool + if error == .success { + if attribute == kAXWindowsAttribute { + let returnedElements = strictAXElements(value) + valueValid = returnedElements != nil + elementCount = returnedElements?.count ?? 0 + elements = returnedElements ?? [] + } else if let element = axElement(value) { + valueValid = true + elementCount = 1 + elements = [element] + } else { + valueValid = false + elementCount = 0 + elements = [] + } + } else { + valueValid = false + elementCount = 0 + elements = [] + } + return AXWindowAttributeDiagnostic( + attribute: attribute, + error: error, + returnedType: cfTypeName(value), + elementCount: elementCount, + elements: elements, + valueValid: valueValid + ) + } + + private func windowTitleOf(_ element: AXUIElement) throws -> String { + axString(try copyAXAttribute(element, kAXTitleAttribute)) ?? "" + } + + private func pidForAXElement(_ element: AXUIElement) -> pid_t? { + var pid: pid_t = 0 + guard AXUIElementGetPid(element, &pid) == .success else { + return nil + } + return pid + } +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Capture.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Capture.swift new file mode 100644 index 00000000000..f3d6973832d --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Capture.swift @@ -0,0 +1,286 @@ +import ApplicationServices +import AppKit +import Foundation +import ImageIO +import ScreenCaptureKit +import UniformTypeIdentifiers + +struct EncodedImage { + let data: Data + let width: Int + let height: Int + let scaleFactor: Double +} + +func matchingBackingScaleFactor( + displayID: CGDirectDisplayID, + screens: [(displayID: CGDirectDisplayID, scaleFactor: CGFloat)] +) -> CGFloat { + max(1, screens.first(where: { $0.displayID == displayID })?.scaleFactor ?? 1) +} + +private final class ShareableContentBox: @unchecked Sendable { + let lock = NSLock() + var content: SCShareableContent? + var error: Error? +} + +private final class ScreenshotBox: @unchecked Sendable { + let lock = NSLock() + var image: CGImage? + var error: Error? +} + +final class CaptureEngine { + var available: Bool { + if #available(macOS 14.0, *) { + return CGPreflightScreenCaptureAccess() + } + return false + } + + func captureWindow( + window: WindowRecord, + title: String, + token: CancellationToken + ) throws -> EncodedImage { + let windowFrame = try window.currentFrame() + return try capture( + window: window, + title: title, + cropScreenRect: nil, + windowFrame: windowFrame, + token: token + ) + } + + func captureElement( + window: WindowRecord, + title: String, + element: ElementSnapshot, + token: CancellationToken + ) throws -> EncodedImage { + guard element.screenRect.width > 0, element.screenRect.height > 0 else { + try fail(ErrorCode.unsupported, "The element does not occupy a capturable area.") + } + let windowFrame = try window.currentFrame() + return try capture( + window: window, + title: title, + cropScreenRect: element.screenRect, + windowFrame: windowFrame, + token: token + ) + } + + private func capture( + window: WindowRecord, + title: String, + cropScreenRect: CGRect?, + windowFrame: CGRect, + token: CancellationToken + ) throws -> EncodedImage { + guard available else { + try fail(ErrorCode.unsupported, "SCScreenshotManager is unavailable on this version of macOS.") + } + guard CGPreflightScreenCaptureAccess() else { + try fail( + ErrorCode.captureFailed, + "Screen Recording permission is required. Enable Furn Desktop Driver Host in System Settings > Privacy & Security > Screen & System Audio Recording, then restart the helper." + ) + } + try token.throwIfCancelled() + let content = try shareableContent(token: token) + let processCandidates = content.windows.filter { candidate in + candidate.owningApplication?.processID == window.processID + } + let titleCandidates = title.isEmpty ? [] : processCandidates.filter { $0.title == title } + let candidates = titleCandidates.isEmpty ? processCandidates : titleCandidates + guard !candidates.isEmpty else { + try fail( + ErrorCode.captureFailed, + "ScreenCaptureKit could not find any window owned by the tracked application process.", + data: [ + "processCandidateCount": processCandidates.count, + "processId": Int(window.processID), + "reason": "no-process-window", + "requestedTitle": bounded(title, bytes: 512), + ] + ) + } + let selected = candidates.min { + frameDistance($0.frame, windowFrame) < frameDistance($1.frame, windowFrame) + }! + if candidates.count > 1 { + let sorted = candidates.sorted { frameDistance($0.frame, windowFrame) < frameDistance($1.frame, windowFrame) } + if sorted.count > 1 + && abs(frameDistance(sorted[0].frame, windowFrame) - frameDistance(sorted[1].frame, windowFrame)) < 0.5 + { + try fail( + ErrorCode.captureFailed, + "ScreenCaptureKit found multiple indistinguishable windows for the target.", + data: [ + "candidateCount": candidates.count, + "processCandidateCount": processCandidates.count, + "reason": "ambiguous-window-geometry", + "requestedTitle": bounded(title, bytes: 512), + "titleCandidateCount": titleCandidates.count, + "windowFrame": rectJSON(windowFrame), + ] + ) + } + } + guard selected.isOnScreen else { + try fail( + ErrorCode.unsupported, + "Capture is unavailable while the window is minimized or off screen; the helper never restores a window to capture it." + ) + } + let scale = scaleFactor(for: selected.frame, displays: content.displays) + let configuration = SCStreamConfiguration() + configuration.width = max(1, Int((selected.frame.width * scale).rounded())) + configuration.height = max(1, Int((selected.frame.height * scale).rounded())) + configuration.showsCursor = false + configuration.capturesAudio = false + configuration.ignoreShadowsSingleWindow = true + configuration.backgroundColor = CGColor.clear + let filter = SCContentFilter(desktopIndependentWindow: selected) + let captured = try screenshot(filter: filter, configuration: configuration, token: token) + try token.throwIfCancelled() + + let finalImage: CGImage + if let cropScreenRect { + let scaleX = CGFloat(captured.width) / max(1, selected.frame.width) + let scaleY = CGFloat(captured.height) / max(1, selected.frame.height) + var crop = CGRect( + x: (cropScreenRect.minX - selected.frame.minX) * scaleX, + y: (cropScreenRect.minY - selected.frame.minY) * scaleY, + width: cropScreenRect.width * scaleX, + height: cropScreenRect.height * scaleY + ).integral + crop = crop.intersection(CGRect(x: 0, y: 0, width: captured.width, height: captured.height)) + guard !crop.isNull, crop.width > 0, crop.height > 0, let cropped = captured.cropping(to: crop) else { + try fail(ErrorCode.captureFailed, "The requested element capture lies outside the captured window.") + } + finalImage = cropped + } else { + finalImage = captured + } + let encoded = try encodePNG(finalImage) + return EncodedImage( + data: encoded, + width: finalImage.width, + height: finalImage.height, + scaleFactor: rounded(scale) + ) + } + + private func shareableContent(token: CancellationToken) throws -> SCShareableContent { + let box = ShareableContentBox() + let semaphore = DispatchSemaphore(value: 0) + SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: false) { content, error in + box.lock.lock() + box.content = content + box.error = error + box.lock.unlock() + semaphore.signal() + } + try wait(semaphore, token: token, operation: "ScreenCaptureKit window enumeration") + box.lock.lock() + let content = box.content + let error = box.error + box.lock.unlock() + if let error { + try fail(ErrorCode.captureFailed, "ScreenCaptureKit window enumeration failed: \(error.localizedDescription)") + } + guard let content else { + try fail(ErrorCode.captureFailed, "ScreenCaptureKit returned no shareable desktop content.") + } + return content + } + + private func screenshot( + filter: SCContentFilter, + configuration: SCStreamConfiguration, + token: CancellationToken + ) throws -> CGImage { + let box = ScreenshotBox() + let semaphore = DispatchSemaphore(value: 0) + SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration) { image, error in + box.lock.lock() + box.image = image + box.error = error + box.lock.unlock() + semaphore.signal() + } + try wait(semaphore, token: token, operation: "ScreenCaptureKit screenshot") + box.lock.lock() + let image = box.image + let error = box.error + box.lock.unlock() + if let error { + try fail(ErrorCode.captureFailed, "ScreenCaptureKit screenshot failed: \(error.localizedDescription)") + } + guard let image else { + try fail(ErrorCode.captureFailed, "ScreenCaptureKit returned no image for the target window.") + } + return image + } + + private func wait( + _ semaphore: DispatchSemaphore, + token: CancellationToken, + operation: String + ) throws { + let deadline = DispatchTime.now() + .seconds(10) + while semaphore.wait(timeout: .now() + .milliseconds(50)) == .timedOut { + try token.throwIfCancelled() + if DispatchTime.now() >= deadline { + try fail(ErrorCode.captureFailed, "\(operation) did not complete within 10 seconds.") + } + } + try token.throwIfCancelled() + } + + private func scaleFactor(for windowFrame: CGRect, displays: [SCDisplay]) -> CGFloat { + let center = CGPoint(x: windowFrame.midX, y: windowFrame.midY) + if + let display = displays.first(where: { $0.frame.contains(center) }) + { + let screens = NSScreen.screens.compactMap { screen -> (displayID: CGDirectDisplayID, scaleFactor: CGFloat)? in + guard let displayID = (screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value else { + return nil + } + return (displayID, screen.backingScaleFactor) + } + return matchingBackingScaleFactor(displayID: display.displayID, screens: screens) + } + return 1 + } + + private func frameDistance(_ left: CGRect, _ right: CGRect) -> CGFloat { + abs(left.minX - right.minX) + + abs(left.minY - right.minY) + + abs(left.width - right.width) + + abs(left.height - right.height) + } + + private func encodePNG(_ image: CGImage) throws -> Data { + let data = NSMutableData() + guard + let destination = CGImageDestinationCreateWithData( + data, + UTType.png.identifier as CFString, + 1, + nil + ) + else { + try fail(ErrorCode.captureFailed, "Creating the PNG encoder failed.") + } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { + try fail(ErrorCode.captureFailed, "Encoding the captured image as PNG failed.") + } + return data as Data + } +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift new file mode 100644 index 00000000000..f7578b70bb0 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift @@ -0,0 +1,565 @@ +import AppKit +import ApplicationServices +import Foundation + +struct CommandResult { + var result: Any = NSNull() + var binaryID: String? + var binaryData: Data? + var binaryMetadata: JSONObject? +} + +final class Driver { + private let applications = ApplicationManager() + private let accessibility = AccessibilityEngine() + private let input = InputController() + private let capture = CaptureEngine() + private let inputLock = PhysicalInputLock.shared + private var persistentInputLock = false + + func execute(command: String, params: JSONObject, token: CancellationToken) throws -> CommandResult { + try token.throwIfCancelled() + switch command { + case "probe": + return CommandResult(result: probe(params)) + case "launch": + return CommandResult(result: try applications.launch(params, token: token).json()) + case "attach": + return CommandResult(result: try applications.attach(params, token: token).json()) + case "closeApplication": + let lease = try requireObject(params["lease"], "The closeApplication request is missing its lease.") + let leaseID = try requireString(lease, "id") + for windowID in applications.windowIDs(for: leaseID) { + accessibility.forgetWindow(windowID) + } + _ = try applications.closeApplication(leaseID, token: token) + return CommandResult() + case "windows": + let lease = try requireObject(params["lease"], "The windows request is missing its lease.") + let records = try applications.listWindows(try requireString(lease, "id"), token: token) + return CommandResult(result: try records.map(windowJSON)) + case "closeWindow": + let windowID = try requireString(params, "windowId") + let window = try applications.requireWindow(windowID) + try closeWindow(window) + accessibility.forgetWindow(windowID) + applications.forgetWindow(windowID) + return CommandResult() + case "activate": + try activate(try applications.requireWindow(try requireString(params, "windowId")), token: token) + return CommandResult() + case "getWindowRect": + let window = try applications.requireWindow(try requireString(params, "windowId")) + return CommandResult(result: rectJSON(try window.currentFrame())) + case "setWindowRect": + let window = try applications.requireWindow(try requireString(params, "windowId")) + let rect = try requireObject(params["rect"], "The setWindowRect request is missing its rectangle.") + return CommandResult(result: rectJSON(try setWindowRect(window, requested: rect, token: token))) + case "find": + let root = try requireObject(params["root"], "The find request requires a root and a selector.") + let selector = try requireObject(params["selector"], "The find request requires a root and a selector.") + let window = try applications.requireWindow(try requireString(root, "windowId")) + guard let lease = applications.lease(for: window) else { + try fail(ErrorCode.noSuchLease, "The requested window no longer belongs to a held application lease.") + } + let snapshots = try accessibility.find( + window: window, + lease: lease, + rootElementID: optionalString(root, "elementId"), + selector: selector, + token: token + ) + return CommandResult(result: snapshots.map { $0.json() }) + case "snapshot": + let elementID = try requireString(params, "elementId") + let window = try windowForElement(elementID) + return CommandResult(result: try accessibility.snapshot(elementID: elementID, window: window, token: token).json()) + case "activeElement": + let window = try applications.requireWindow(try requireString(params, "windowId")) + let snapshot = try accessibility.activeElement(window: window, token: token) + return CommandResult(result: snapshot?.json() ?? NSNull()) + case "hitTest": + let window = try applications.requireWindow(try requireString(params, "windowId")) + let snapshot = try accessibility.hitTest( + window: window, + x: CGFloat(number(params, "x")), + y: CGFloat(number(params, "y")), + token: token + ) + return CommandResult(result: snapshot?.json() ?? NSNull()) + case "click": + try click( + elementID: try requireString(params, "elementId"), + mode: optionalString(params, "mode") ?? "auto", + token: token + ) + return CommandResult() + case "clear": + try clear(elementID: try requireString(params, "elementId"), token: token) + return CommandResult() + case "sendKeys": + guard let text = params["text"] as? String else { + try fail(ErrorCode.invalidParams, "The sendKeys request requires a string \"text\".") + } + try sendKeys(elementID: try requireString(params, "elementId"), text: text, token: token) + return CommandResult() + case "performActions": + guard let actions = params["actions"] as? [Any] else { + try fail(ErrorCode.invalidParams, "The performActions request requires an array of input sources.") + } + try withPhysicalInput(token: token) { + try performActions(actions, token: token) + } + return CommandResult() + case "releaseActions": + try withPhysicalInput(token: token) { + input.releaseAll() + } + return CommandResult() + case "captureWindow": + let window = try applications.requireWindow(try requireString(params, "windowId")) + return try captureResult( + capture.captureWindow(window: window, title: try windowTitle(window), token: token) + ) + case "captureElement": + let elementID = try requireString(params, "elementId") + let window = try windowForElement(elementID) + let snapshot = try accessibility.snapshot(elementID: elementID, window: window, token: token) + return try captureResult( + capture.captureElement( + window: window, + title: try windowTitle(window), + element: snapshot, + token: token + ) + ) + case "source": + let window = try applications.requireWindow(try requireString(params, "windowId")) + guard let lease = applications.lease(for: window) else { + try fail(ErrorCode.noSuchLease, "The requested window no longer belongs to a held application lease.") + } + return CommandResult(result: try accessibility.source(accessibility.tree(window: window, lease: lease, token: token))) + case "tree": + let window = try applications.requireWindow(try requireString(params, "windowId")) + guard let lease = applications.lease(for: window) else { + try fail(ErrorCode.noSuchLease, "The requested window no longer belongs to a held application lease.") + } + return CommandResult( + result: try accessibility.tree(window: window, lease: lease, token: token).map { $0.json() } + ) + case "dispose": + releaseInput() + applications.dispose(token: token) + accessibility.reset() + return CommandResult() + default: + try fail(ErrorCode.unsupported, "The macOS helper does not implement the \"\(command)\" command.") + } + } + + func releaseInput() { + if persistentInputLock { + input.releaseAll() + inputLock.release() + persistentInputLock = false + return + } + guard input.hasDepressedInput else { + return + } + let idle = CancellationToken() + do { + try inputLock.withLock(token: idle, timeoutMilliseconds: 5_000, policy: .adopt) { + input.releaseAll() + } + } catch { + writeDiagnostic("furn-desktop-driver-host: input cleanup failed: \(error)") + } + } + + private func probe(_ params: JSONObject) -> JSONObject { + let trusted = AXIsProcessTrusted() + let screenshot = capture.available + return [ + "endpoint": optionalString(params, "endpoint") ?? "macos", + "features": [ + "accessibilityClick": trusted, + "elementScreenshot": screenshot, + "focus": trusted, + "keyboard": trusted, + "physicalClick": trusted, + "screenshot": screenshot, + "setWindowRect": trusted, + "wheel": trusted, + ], + "platformName": "macos", + "protocolVersion": 1, + ] + } + + private func windowJSON(_ window: WindowRecord) throws -> JSONObject { + [ + "id": window.id, + "rect": rectJSON(try window.currentFrame()), + "title": try windowTitle(window), + ] + } + + private func windowTitle(_ window: WindowRecord) throws -> String { + axString(try copyAXAttribute(window.element, kAXTitleAttribute)) ?? "" + } + + private func windowForElement(_ elementID: String) throws -> WindowRecord { + let record = try accessibility.requireRecord(elementID) + return try applications.requireWindow(record.windowID) + } + + private func closeWindow(_ window: WindowRecord) throws { + try requireAccessibility("Closing an application window") + guard + let closeButtonValue = try copyAXAttribute(window.element, kAXCloseButtonAttribute), + let closeButton = axElement(closeButtonValue) + else { + try fail(ErrorCode.unsupported, "The application window does not expose an accessibility close button.") + } + let error = AXUIElementPerformAction(closeButton, kAXPressAction as CFString) + guard error == .success else { + try fail(ErrorCode.automationFailed, "Closing the application window failed because \(describeAXError(error)).") + } + } + + private func activate(_ window: WindowRecord, token: CancellationToken) throws { + try requireAccessibility("Activating an application window") + guard let application = NSRunningApplication(processIdentifier: window.processID) else { + try fail(ErrorCode.noSuchWindow, "The requested window's application is no longer running.") + } + _ = application.activate(options: []) + _ = AXUIElementSetAttributeValue(window.element, kAXMainAttribute as CFString, kCFBooleanTrue) + let raised = AXUIElementPerformAction(window.element, kAXRaiseAction as CFString) + if raised != .success && raised != .actionUnsupported { + try fail(ErrorCode.windowActivation, "Raising the application window failed because \(describeAXError(raised)).") + } + let deadline = DispatchTime.now() + .seconds(2) + while DispatchTime.now() < deadline { + try token.throwIfCancelled() + let main = axNumber(try copyAXAttribute(window.element, kAXMainAttribute))?.boolValue == true + if NSWorkspace.shared.frontmostApplication?.processIdentifier == window.processID && (main || !window.primary) { + return + } + try token.wait(milliseconds: 50) + } + try fail( + ErrorCode.windowActivation, + "The target window did not become the active foreground application window." + ) + } + + private func setWindowRect( + _ window: WindowRecord, + requested: JSONObject, + token: CancellationToken + ) throws -> CGRect { + try requireAccessibility("Setting an application window rectangle") + try token.throwIfCancelled() + let current = try window.currentFrame() + var target = CGRect( + x: requested["x"] == nil ? current.minX : CGFloat(number(requested, "x")), + y: requested["y"] == nil ? current.minY : CGFloat(number(requested, "y")), + width: requested["width"] == nil ? current.width : CGFloat(number(requested, "width")), + height: requested["height"] == nil ? current.height : CGFloat(number(requested, "height")) + ) + guard + target.minX.isFinite, + target.minY.isFinite, + target.width.isFinite, + target.height.isFinite, + target.width > 0, + target.height > 0 + else { + try fail(ErrorCode.invalidParams, "The requested window rectangle must contain finite positive dimensions.") + } + try setAXGeometry(window.element, attribute: kAXPositionAttribute, point: target.origin) + try setAXGeometry(window.element, attribute: kAXSizeAttribute, size: target.size) + try token.wait(milliseconds: 16) + let refreshed = try axFrame(window.element) + if refreshed.width > 0, refreshed.height > 0 { + target = refreshed + } + window.lastKnownFrame = target + return target + } + + private func setAXGeometry(_ element: AXUIElement, attribute: String, point: CGPoint) throws { + var settable: DarwinBoolean = false + let check = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) + try throwAX(check, operation: "Checking window position support") + guard settable.boolValue else { + try fail(ErrorCode.unsupported, "The application window position cannot be changed through Accessibility.") + } + var point = point + guard let value = AXValueCreate(.cgPoint, &point) else { + try fail(ErrorCode.internalError, "Creating the accessibility window position failed.") + } + let error = AXUIElementSetAttributeValue(element, attribute as CFString, value) + guard error == .success else { + try fail(ErrorCode.automationFailed, "Setting the window position failed because \(describeAXError(error)).") + } + } + + private func setAXGeometry(_ element: AXUIElement, attribute: String, size: CGSize) throws { + var settable: DarwinBoolean = false + let check = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) + try throwAX(check, operation: "Checking window size support") + guard settable.boolValue else { + try fail(ErrorCode.unsupported, "The application window size cannot be changed through Accessibility.") + } + var size = size + guard let value = AXValueCreate(.cgSize, &size) else { + try fail(ErrorCode.internalError, "Creating the accessibility window size failed.") + } + let error = AXUIElementSetAttributeValue(element, attribute as CFString, value) + guard error == .success else { + try fail(ErrorCode.automationFailed, "Setting the window size failed because \(describeAXError(error)).") + } + } + + private func click(elementID: String, mode: String, token: CancellationToken) throws { + let window = try windowForElement(elementID) + _ = try accessibility.snapshot(elementID: elementID, window: window, token: token) + if mode == "accessibility" { + try accessibility.accessibilityClick(elementID, token: token) + return + } + guard mode == "physical" || mode == "auto" else { + try fail(ErrorCode.invalidParams, "Click mode \"\(mode)\" is not supported.") + } + if !input.physicalInputAvailable() { + if mode == "auto" { + try accessibility.accessibilityClick(elementID, token: token) + return + } + try fail( + ErrorCode.inputUnavailable, + "Physical pointer input requires Accessibility permission in System Settings > Privacy & Security > Accessibility." + ) + } + try withPhysicalInput(token: token) { + try activate(window, token: token) + let snapshot = try accessibility.snapshot(elementID: elementID, window: window, token: token) + guard + snapshot.enabled.value != false, + snapshot.visible.value == true, + snapshot.screenRect.width > 0, + snapshot.screenRect.height > 0 + else { + try fail(ErrorCode.notInteractable, "The element is disabled, hidden, or does not occupy a clickable area.") + } + let point = CGPoint(x: snapshot.screenRect.midX, y: snapshot.screenRect.midY) + guard try accessibility.elementOwnsPoint(elementID: elementID, point: point, token: token) else { + try fail(ErrorCode.notInteractable, "Another desktop element owns the element's physical click point.") + } + try input.click(at: point, token: token) + } + } + + private func clear(elementID: String, token: CancellationToken) throws { + let window = try windowForElement(elementID) + _ = try accessibility.snapshot(elementID: elementID, window: window, token: token) + if try accessibility.tryClearValue(elementID) { + return + } + guard input.physicalInputAvailable() else { + try fail( + ErrorCode.inputUnavailable, + "The element does not expose a writable value and physical input requires Accessibility permission." + ) + } + try withPhysicalInput(token: token) { + try activate(window, token: token) + try accessibility.setFocus(elementID) + try token.wait(milliseconds: 16) + try input.pressCommandADelete(token: token) + } + } + + private func sendKeys(elementID: String, text: String, token: CancellationToken) throws { + guard input.physicalInputAvailable() else { + try fail( + ErrorCode.inputUnavailable, + "Physical keyboard input requires Accessibility permission in System Settings > Privacy & Security > Accessibility." + ) + } + let window = try windowForElement(elementID) + _ = try accessibility.snapshot(elementID: elementID, window: window, token: token) + try withPhysicalInput(token: token) { + try activate(window, token: token) + try accessibility.setFocus(elementID) + for _ in 0..<10 where try !accessibility.hasFocus(elementID) { + try token.wait(milliseconds: 20) + } + guard try accessibility.hasFocus(elementID) else { + try fail(ErrorCode.notInteractable, "The element did not take keyboard focus, so text could not be typed.") + } + try input.typeText(text, token: token) + } + } + + private func withPhysicalInput( + token: CancellationToken, + _ body: () throws -> T + ) throws -> T { + let hadPersistentLock = persistentInputLock + try inputLock.acquire(token: token, timeoutMilliseconds: 30_000, policy: .fail) + input.syncPointer() + do { + let result = try body() + if hadPersistentLock { + inputLock.release() + if !input.hasDepressedInput { + inputLock.release() + persistentInputLock = false + } + } else if input.hasDepressedInput { + persistentInputLock = true + } else { + inputLock.release() + } + return result + } catch { + input.releaseAll() + inputLock.release() + if hadPersistentLock { + inputLock.release() + persistentInputLock = false + } + throw error + } + } + + private func performActions(_ actions: [Any], token: CancellationToken) throws { + let sources = try actions.map { + try requireObject($0, "Every input source must be an object.") + } + let ticks = sources.reduce(0) { maximum, source in + max(maximum, (source["actions"] as? [Any])?.count ?? 0) + } + for tick in 0.. 0 { + try token.wait(milliseconds: pause) + } + } + } + + private func resolveActionPoint(_ action: JSONObject, token: CancellationToken) throws -> CGPoint { + let x = CGFloat(number(action, "x")) + let y = CGFloat(number(action, "y")) + if let origin = action["origin"] as? JSONObject { + let elementID = try requireString(origin, "elementId") + let window = try windowForElement(elementID) + let snapshot = try accessibility.snapshot(elementID: elementID, window: window, token: token) + return CGPoint(x: snapshot.screenRect.midX + x, y: snapshot.screenRect.midY + y) + } + if action["origin"] as? String == "pointer" { + return CGPoint(x: input.pointer.x + x, y: input.pointer.y + y) + } + guard + let lease = applications.primaryLease(), + let primaryWindowID = lease.primaryWindowID + else { + try fail( + ErrorCode.invalidParams, + "The helper does not hold an application window to resolve viewport coordinates against." + ) + } + let window = try applications.requireWindow(primaryWindowID) + let frame = try window.currentFrame() + return CGPoint(x: frame.minX + x, y: frame.minY + y) + } + + private func actionDuration(_ action: JSONObject) -> Int { + max(0, min(30_000, integer(action, "duration"))) + } + + private func actionButton(_ action: JSONObject) -> Int { + integer(action, "button") + } + + private func captureResult(_ image: EncodedImage) throws -> CommandResult { + let binaryID = IdentifierGenerator.shared.next("image") + let metadata: JSONObject = [ + "id": binaryID, + "mimeType": "image/png", + "height": image.height, + "width": image.width, + "scaleFactor": image.scaleFactor, + ] + return CommandResult( + result: [ + "height": image.height, + "mimeType": "image/png", + "scaleFactor": image.scaleFactor, + "width": image.width, + ], + binaryID: binaryID, + binaryData: image.data, + binaryMetadata: metadata + ) + } +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Framing.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Framing.swift new file mode 100644 index 00000000000..ed0dd3853f2 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Framing.swift @@ -0,0 +1,140 @@ +import Foundation + +enum FrameType: UInt8 { + case json = 1 + case binary = 2 +} + +struct Frame { + let type: FrameType + let payload: Data +} + +enum Framing { + static let headerBytes = 12 + static let maximumJSONBytes = 8 * 1024 * 1024 + static let maximumBinaryBytes = 64 * 1024 * 1024 + static let maximumCorrelationIDBytes = 256 + static let maximumBinaryIDBytes = 1024 + static let magic = Data("FDR1".utf8) + + static func encode(type: FrameType, payload: Data) throws -> Data { + let limit = type == .json ? maximumJSONBytes : maximumBinaryBytes + guard payload.count <= limit else { + try fail(ErrorCode.internalError, "Native frame length \(payload.count) exceeds the \(limit)-byte type limit.") + } + var frame = Data() + frame.append(magic) + frame.append(type.rawValue) + frame.append(contentsOf: [0, 0, 0]) + var length = UInt32(payload.count).littleEndian + withUnsafeBytes(of: &length) { frame.append(contentsOf: $0) } + frame.append(payload) + return frame + } + + static func encodeJSON(_ value: Any) throws -> Data { + try encode(type: .json, payload: serializeJSON(value)) + } + + static func encodeBinary(id: String, data: Data) throws -> Data { + let idData = Data(id.utf8) + guard !idData.isEmpty, idData.count <= maximumBinaryIDBytes else { + try fail(ErrorCode.internalError, "The binary frame identifier is invalid.") + } + guard 4 + idData.count + data.count <= maximumBinaryBytes else { + try fail(ErrorCode.captureFailed, "The captured image exceeds the 64 MiB native binary frame limit.") + } + var payload = Data() + var length = UInt32(idData.count).littleEndian + withUnsafeBytes(of: &length) { payload.append(contentsOf: $0) } + payload.append(idData) + payload.append(data) + return try encode(type: .binary, payload: payload) + } + + static func decodeHeader(_ data: Data) -> (FrameType, Int)? { + guard data.count == headerBytes, data.prefix(4) == magic else { + return nil + } + guard data[5] == 0, data[6] == 0, data[7] == 0, let type = FrameType(rawValue: data[4]) else { + return nil + } + let length = data.withUnsafeBytes { + Int(UInt32(littleEndian: $0.loadUnaligned(fromByteOffset: 8, as: UInt32.self))) + } + let limit = type == .json ? maximumJSONBytes : maximumBinaryBytes + guard length <= limit else { + return nil + } + return (type, length) + } +} + +final class FrameReader { + private let input: FileHandle + + init(_ input: FileHandle) { + self.input = input + } + + func read() throws -> Frame? { + guard let header = try readExact(Framing.headerBytes) else { + return nil + } + guard let (type, length) = Framing.decodeHeader(header) else { + try fail(ErrorCode.invalidRequest, "The native helper received an invalid FDR1 frame header.") + } + guard let payload = try readExact(length) else { + try fail(ErrorCode.invalidRequest, "The native helper received a truncated FDR1 frame.") + } + return Frame(type: type, payload: payload) + } + + private func readExact(_ count: Int) throws -> Data? { + if count == 0 { + return Data() + } + var output = Data() + while output.count < count { + let chunk: Data + do { + chunk = try input.read(upToCount: count - output.count) ?? Data() + } catch { + try fail(ErrorCode.invalidRequest, "Reading native protocol input failed: \(error.localizedDescription)") + } + if chunk.isEmpty { + return output.isEmpty ? nil : output + } + output.append(chunk) + } + return output + } +} + +final class FrameWriter: @unchecked Sendable { + private let lock = NSLock() + private let output: FileHandle + + init(_ output: FileHandle) { + self.output = output + } + + func writeJSON(_ value: Any) throws { + try write(Framing.encodeJSON(value)) + } + + func writeBinary(id: String, data: Data) throws { + try write(Framing.encodeBinary(id: id, data: data)) + } + + private func write(_ data: Data) throws { + lock.lock() + defer { lock.unlock() } + do { + try output.write(contentsOf: data) + } catch { + try fail(ErrorCode.internalError, "Writing native protocol output failed: \(error.localizedDescription)") + } + } +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/GeneratedBuildInfo.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/GeneratedBuildInfo.swift new file mode 100644 index 00000000000..72a5fc45603 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/GeneratedBuildInfo.swift @@ -0,0 +1,5 @@ +// The build coordinator overwrites this file in its staged source copy. +enum FurnBuildInfo { + static let buildId = "direct-swift-build" + static let sourceDigest = "direct-swift-build" +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift new file mode 100644 index 00000000000..b65284ea222 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift @@ -0,0 +1,272 @@ +import Foundation +import Darwin + +func createHello() -> JSONObject { + [ + "type": "hello", + "provider": "macos", + "architecture": "arm64", + "buildId": FurnBuildInfo.buildId, + "sourceDigest": FurnBuildInfo.sourceDigest, + "minimumOs": "14.0", + "protocol": [ + "major": 1, + "minor": 0, + ], + "features": [ + "accessibilityClick", + "activeElement", + "attach", + "capture", + "elementCapture", + "find", + "hitTest", + "keyboard", + "launch", + "pointer", + "probe", + "releaseActions", + "setWindowRect", + "source", + "tree", + "wheel", + ], + ] +} + +private struct PendingRequest { + let id: String + let command: String + let params: JSONObject +} + +final class Host { + private let condition = NSCondition() + private let token = CancellationToken() + private let driver = Driver() + private var queue: [PendingRequest] = [] + private var activeID: String? + private var stopping = false + private var writer: FrameWriter? + private let workerDone = DispatchSemaphore(value: 0) + + func run() -> Int32 { + let writer = FrameWriter(.standardOutput) + self.writer = writer + let reader = FrameReader(.standardInput) + do { + try writer.writeJSON(createHello()) + } catch { + writeDiagnostic("furn-desktop-driver-host: writing hello failed: \(error)") + return 1 + } + + let worker = Thread { [weak self] in + self?.workerLoop() + } + worker.name = "furn-desktop-driver-worker" + worker.start() + + var exitCode: Int32 = 0 + do { + while let frame = try reader.read() { + if frame.type != .json { + writeDiagnostic("The native helper ignored an unexpected binary frame on stdin.") + continue + } + let value: Any + do { + value = try parseJSON(frame.payload) + } catch { + writeDiagnostic("The native helper ignored malformed JSON: \(error)") + continue + } + guard let message = value as? JSONObject else { + writeDiagnostic("The native helper ignored a JSON message that is not an object.") + continue + } + let type = message["type"] as? String ?? "" + let id = message["id"] as? String ?? "" + if (type == "cancel" || type == "request") + && (id.isEmpty || id.utf8.count > Framing.maximumCorrelationIDBytes) + { + writeDiagnostic("The native helper ignored a request with an invalid correlation identifier.") + continue + } + if type == "cancel" { + handleCancel(id) + continue + } + guard type == "request" else { + writeDiagnostic("The native helper received a message that is not a correlated request.") + continue + } + let command = message["command"] as? String ?? "" + let params = message["params"] as? JSONObject ?? [:] + condition.lock() + queue.append(PendingRequest(id: id, command: command, params: params)) + condition.signal() + condition.unlock() + } + } catch { + writeDiagnostic("furn-desktop-driver-host: \(error)") + exitCode = 1 + } + + condition.lock() + stopping = true + token.cancel() + condition.broadcast() + condition.unlock() + workerDone.wait() + self.writer = nil + return exitCode + } + + private func handleCancel(_ id: String) { + condition.lock() + if activeID == id { + token.cancel() + condition.unlock() + return + } + if let index = queue.firstIndex(where: { $0.id == id }) { + queue.remove(at: index) + } + condition.unlock() + writeCancelled(id) + } + + private func workerLoop() { + defer { workerDone.signal() } + while true { + condition.lock() + while queue.isEmpty && !stopping { + condition.wait() + } + if stopping && queue.isEmpty { + condition.unlock() + return + } + let request = queue.removeFirst() + activeID = request.id + token.reset() + condition.unlock() + + var cancelled = false + do { + let result = try driver.execute(command: request.command, params: request.params, token: token) + try writeResponse(id: request.id, command: request.command, result: result) + } catch is CancelledError { + cancelled = true + } catch let error as HelperError { + writeError(id: request.id, command: request.command, code: error.code, message: error.message, data: error.data) + } catch { + writeError( + id: request.id, + command: request.command, + code: ErrorCode.internalError, + message: error.localizedDescription + ) + } + + if cancelled { + driver.releaseInput() + writeCancelled(request.id) + } + condition.lock() + activeID = nil + condition.unlock() + + if request.command == "dispose" && !cancelled { + try? FileHandle.standardOutput.synchronize() + Darwin.exit(EXIT_SUCCESS) + } + } + } + + private func writeResponse(id: String, command: String, result: CommandResult) throws { + if let binaryID = result.binaryID, let binaryData = result.binaryData { + guard 4 + binaryID.utf8.count + binaryData.count <= Framing.maximumBinaryBytes else { + writeError( + id: id, + command: command, + code: ErrorCode.captureFailed, + message: "The captured image exceeds the 64 MiB native binary frame limit." + ) + return + } + } + var response: JSONObject = [ + "type": "response", + "id": id, + "result": result.result, + ] + if let metadata = result.binaryMetadata { + response["binary"] = metadata + } + let serialized = try serializeJSON(response) + if serialized.count > Framing.maximumJSONBytes { + let treeCommand = command == "find" || command == "source" || command == "tree" + writeError( + id: id, + command: command, + code: treeCommand ? ErrorCode.treeTooLarge : ErrorCode.internalError, + message: "The native command response exceeds the 8 MiB JSON frame limit." + ) + return + } + try writer?.writeJSON(response) + if let binaryID = result.binaryID, let binaryData = result.binaryData { + try writer?.writeBinary(id: binaryID, data: binaryData) + } + } + + private func writeError( + id: String, + command: String, + code: String, + message: String, + data: JSONObject? = nil + ) { + var errorData = data ?? [:] + errorData["command"] = bounded(command, bytes: 256) + var response: JSONObject = [ + "type": "response", + "id": id, + "error": [ + "code": code, + "message": bounded(message, bytes: 4096), + "data": errorData, + ], + ] + do { + if try serializeJSON(response).count > Framing.maximumJSONBytes { + response["error"] = [ + "code": code, + "message": bounded(message, bytes: 4096), + "data": [ + "command": bounded(command, bytes: 256), + "diagnosticsTruncated": true, + ], + ] + } + try writer?.writeJSON(response) + } catch { + writeDiagnostic("Native error write failed: \(error)") + Darwin.exit(EXIT_FAILURE) + } + } + + private func writeCancelled(_ id: String) { + do { + try writer?.writeJSON([ + "type": "cancelled", + "id": id, + ]) + } catch { + writeDiagnostic("Native cancellation write failed: \(error)") + Darwin.exit(EXIT_FAILURE) + } + } +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Input.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Input.swift new file mode 100644 index 00000000000..faaf87029d0 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Input.swift @@ -0,0 +1,441 @@ +import ApplicationServices +import Foundation + +enum KeyStroke { + case keyCode(CGKeyCode) + case unicode(String) +} + +struct PressedKey { + let value: String + let stroke: KeyStroke +} + +struct ReleaseLedger { + let keys: [String] + let buttons: [Int] +} + +final class InputController { + private let source = CGEventSource(stateID: .hidSystemState) + private var pressedKeys: [PressedKey] = [] + private var pressedButtons: [Int] = [] + private(set) var pointer: CGPoint = .zero + private var pointerInitialized = false + + var hasDepressedInput: Bool { + !pressedKeys.isEmpty || !pressedButtons.isEmpty + } + + var depressedKeyCount: Int { pressedKeys.count } + var depressedButtonCount: Int { pressedButtons.count } + + func physicalInputAvailable() -> Bool { + AXIsProcessTrusted() + } + + func syncPointer() { + if let position = CGEvent(source: nil)?.location { + pointer = position + pointerInitialized = true + } + } + + func movePointer(to target: CGPoint, durationMilliseconds: Int, token: CancellationToken) throws { + try requirePhysicalInput() + if !pointerInitialized { + syncPointer() + } + let start = pointer + let duration = max(0, min(durationMilliseconds, 30_000)) + let steps = duration == 0 ? 1 : max(1, min(60, duration / 16)) + for step in 1...steps { + try token.throwIfCancelled() + let progress = CGFloat(step) / CGFloat(steps) + let position = CGPoint( + x: start.x + (target.x - start.x) * progress, + y: start.y + (target.y - start.y) * progress + ) + try postMouse(type: moveEventType(), point: position, button: pressedButtons.last ?? 0) + pointer = position + if step < steps { + try token.wait(milliseconds: max(1, duration / steps)) + } + } + pointer = target + try token.wait(milliseconds: 16) + if let observed = CGEvent(source: nil)?.location, + abs(observed.x - target.x) > 2 || abs(observed.y - target.y) > 2 + { + try fail(ErrorCode.inputFailed, "macOS did not move the pointer to the requested native element coordinate.") + } + } + + func pointerDown(_ button: Int, token: CancellationToken) throws { + try requirePhysicalInput() + try token.throwIfCancelled() + _ = try mouseButton(button) + if pressedButtons.contains(button) { + return + } + try postMouse(type: mouseEventType(button: button, down: true), point: pointer, button: button) + pressedButtons.append(button) + } + + func pointerUp(_ button: Int, token: CancellationToken) throws { + try token.throwIfCancelled() + _ = try mouseButton(button) + guard let index = pressedButtons.lastIndex(of: button) else { + return + } + try requirePhysicalInput() + try postMouse(type: mouseEventType(button: button, down: false), point: pointer, button: button) + pressedButtons.remove(at: index) + } + + func pointerCancel() { + for button in pressedButtons.reversed() { + try? postMouse(type: mouseEventType(button: button, down: false), point: pointer, button: button) + } + pressedButtons.removeAll() + } + + func click(at point: CGPoint, token: CancellationToken) throws { + try movePointer(to: point, durationMilliseconds: 0, token: token) + try token.wait(milliseconds: 50) + try pointerDown(0, token: token) + try token.wait(milliseconds: 50) + try pointerUp(0, token: token) + } + + func wheel(deltaX: Double, deltaY: Double, token: CancellationToken) throws { + try requirePhysicalInput() + try token.throwIfCancelled() + guard let event = CGEvent( + scrollWheelEvent2Source: source, + units: .pixel, + wheelCount: 2, + wheel1: Int32(clamping: Int((-deltaY).rounded())), + wheel2: Int32(clamping: Int(deltaX.rounded())), + wheel3: 0 + ) else { + try fail(ErrorCode.inputFailed, "Creating the macOS wheel event failed.") + } + event.post(tap: .cghidEventTap) + } + + func keyDown(_ value: String, token: CancellationToken) throws { + try requirePhysicalInput() + try token.throwIfCancelled() + guard isSingleWebDriverKey(value) else { + try fail(ErrorCode.invalidParams, "A keyDown value must contain exactly one Unicode code point.") + } + if value == "\u{E000}" { + releaseAll() + return + } + let stroke = strokeForValue(value) + try postKey(stroke, down: true) + pressedKeys.append(PressedKey(value: value, stroke: stroke)) + } + + func keyUp(_ value: String, token: CancellationToken) throws { + try token.throwIfCancelled() + guard isSingleWebDriverKey(value) else { + try fail(ErrorCode.invalidParams, "A keyUp value must contain exactly one Unicode code point.") + } + if value == "\u{E000}" { + releaseAll() + return + } + try requirePhysicalInput() + if let index = pressedKeys.lastIndex(where: { $0.value == value }) { + let stroke = pressedKeys[index].stroke + try postKey(stroke, down: false) + pressedKeys.remove(at: index) + return + } + try postKey(strokeForValue(value), down: false) + } + + func typeText(_ text: String, token: CancellationToken) throws { + try requirePhysicalInput() + for scalar in text.unicodeScalars { + try token.throwIfCancelled() + let value = String(scalar) + if value == "\r" { + continue + } + let normalized = value == "\n" ? "\u{E006}" : value + let stroke = strokeForValue(normalized) + try postKey(stroke, down: true) + pressedKeys.append(PressedKey(value: normalized, stroke: stroke)) + try postKey(stroke, down: false) + pressedKeys.removeLast() + } + } + + func pressCommandADelete(token: CancellationToken) throws { + try chord([.keyCode(55), .keyCode(0)], token: token) + try chord([.keyCode(51)], token: token) + } + + func releaseAll() { + for key in pressedKeys.reversed() { + try? postKey(key.stroke, down: false) + } + pressedKeys.removeAll() + pointerCancel() + } + + func releaseExact(_ ledger: ReleaseLedger) throws -> (keys: Int, buttons: Int) { + if ledger.keys.isEmpty && ledger.buttons.isEmpty { + return (0, 0) + } + try requirePhysicalInput() + syncPointer() + for key in ledger.keys.reversed() { + try postKey(strokeForValue(key), down: false) + } + for button in ledger.buttons.reversed() { + _ = try mouseButton(button) + try postMouse(type: mouseEventType(button: button, down: false), point: pointer, button: button) + } + return (ledger.keys.count, ledger.buttons.count) + } + + func releaseDesktopModifiers() throws -> (keys: Int, buttons: Int) { + try requirePhysicalInput() + let modifierCodes: [CGKeyCode] = [56, 60, 59, 62, 58, 61, 55, 54] + var keyCount = 0 + for code in modifierCodes where CGEventSource.keyState(.combinedSessionState, key: code) { + try postKey(.keyCode(code), down: false) + keyCount += 1 + } + syncPointer() + var buttonCount = 0 + for button in 0...4 where CGEventSource.buttonState(.combinedSessionState, button: try mouseButton(button)) { + try postMouse(type: mouseEventType(button: button, down: false), point: pointer, button: button) + buttonCount += 1 + } + return (keyCount, buttonCount) + } + + private func chord(_ strokes: [KeyStroke], token: CancellationToken) throws { + try requirePhysicalInput() + let base = pressedKeys.count + for (index, stroke) in strokes.enumerated() { + try token.throwIfCancelled() + try postKey(stroke, down: true) + pressedKeys.append(PressedKey(value: "chord:\(index)", stroke: stroke)) + } + while pressedKeys.count > base { + let stroke = pressedKeys.last!.stroke + try postKey(stroke, down: false) + pressedKeys.removeLast() + } + } + + private func requirePhysicalInput() throws { + guard physicalInputAvailable() else { + try fail( + ErrorCode.inputUnavailable, + "Physical input requires Accessibility permission. Enable Furn Desktop Driver Host in System Settings > Privacy & Security > Accessibility, then restart the helper." + ) + } + } + + private func postKey(_ stroke: KeyStroke, down: Bool) throws { + let event: CGEvent? + switch stroke { + case let .keyCode(code): + event = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: down) + case let .unicode(value): + event = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: down) + if let event { + let units = Array(value.utf16) + units.withUnsafeBufferPointer { + event.keyboardSetUnicodeString(stringLength: $0.count, unicodeString: $0.baseAddress) + } + } + } + guard let event else { + try fail(ErrorCode.inputFailed, "Creating a macOS keyboard event failed.") + } + event.post(tap: .cghidEventTap) + } + + private func postMouse(type: CGEventType, point: CGPoint, button: Int) throws { + let nativeButton = try mouseButton(button) + guard let event = CGEvent( + mouseEventSource: source, + mouseType: type, + mouseCursorPosition: point, + mouseButton: nativeButton + ) else { + try fail(ErrorCode.inputFailed, "Creating a macOS pointer event failed.") + } + event.setIntegerValueField(.mouseEventButtonNumber, value: Int64(nativeButton.rawValue)) + event.post(tap: .cghidEventTap) + } + + private func moveEventType() -> CGEventType { + guard let button = pressedButtons.last else { + return .mouseMoved + } + switch button { + case 0: + return .leftMouseDragged + case 2: + return .rightMouseDragged + default: + return .otherMouseDragged + } + } + + private func mouseEventType(button: Int, down: Bool) -> CGEventType { + switch button { + case 0: + return down ? .leftMouseDown : .leftMouseUp + case 2: + return down ? .rightMouseDown : .rightMouseUp + default: + return down ? .otherMouseDown : .otherMouseUp + } + } + + private func mouseButton(_ button: Int) throws -> CGMouseButton { + switch button { + case 0: + return .left + case 1: + return .center + case 2: + return .right + case 3: + return CGMouseButton(rawValue: 3)! + case 4: + return CGMouseButton(rawValue: 4)! + default: + try fail(ErrorCode.invalidParams, "Pointer button \(button) is not supported on macOS.") + } + } + + private func strokeForValue(_ value: String) -> KeyStroke { + if let code = specialKeyCodes[value] { + return .keyCode(code) + } + return .unicode(value) + } +} + +func isSingleWebDriverKey(_ value: String) -> Bool { + value.unicodeScalars.count == 1 +} + +private let specialKeyCodes: [String: CGKeyCode] = [ + "\u{E001}": 53, + "\u{E002}": 114, + "\u{E003}": 51, + "\u{E004}": 48, + "\u{E005}": 71, + "\u{E006}": 36, + "\u{E007}": 76, + "\u{E008}": 56, + "\u{E009}": 59, + "\u{E00A}": 58, + "\u{E00B}": 113, + "\u{E00C}": 53, + "\u{E00D}": 49, + "\u{E00E}": 116, + "\u{E00F}": 121, + "\u{E010}": 119, + "\u{E011}": 115, + "\u{E012}": 123, + "\u{E013}": 126, + "\u{E014}": 124, + "\u{E015}": 125, + "\u{E016}": 114, + "\u{E017}": 117, + "\u{E018}": 41, + "\u{E019}": 24, + "\u{E01A}": 82, + "\u{E01B}": 83, + "\u{E01C}": 84, + "\u{E01D}": 85, + "\u{E01E}": 86, + "\u{E01F}": 87, + "\u{E020}": 88, + "\u{E021}": 89, + "\u{E022}": 91, + "\u{E023}": 92, + "\u{E024}": 67, + "\u{E025}": 69, + "\u{E026}": 65, + "\u{E027}": 78, + "\u{E028}": 65, + "\u{E029}": 75, + "\u{E031}": 122, + "\u{E032}": 120, + "\u{E033}": 99, + "\u{E034}": 118, + "\u{E035}": 96, + "\u{E036}": 97, + "\u{E037}": 98, + "\u{E038}": 100, + "\u{E039}": 101, + "\u{E03A}": 109, + "\u{E03B}": 103, + "\u{E03C}": 111, + "\u{E03D}": 55, + "\u{E050}": 60, + "\u{E051}": 62, + "\u{E052}": 61, + "\u{E053}": 54, + "\u{E054}": 116, + "\u{E055}": 121, + "\u{E056}": 119, + "\u{E057}": 115, + "\u{E058}": 123, + "\u{E059}": 126, + "\u{E05A}": 124, + "\u{E05B}": 125, + "\u{E05C}": 114, + "\u{E05D}": 117, +] + +func parseReleaseLedger(_ value: Any) throws -> ReleaseLedger { + guard let object = value as? JSONObject else { + try fail(ErrorCode.invalidParams, "The release ledger must be a JSON object.") + } + for key in object.keys where key != "keys" && key != "buttons" { + try fail(ErrorCode.invalidParams, "The release ledger contains the unknown field \"\(key)\".") + } + let rawKeys = object["keys"] ?? [] + let rawButtons = object["buttons"] ?? [] + guard let keyValues = rawKeys as? [Any] else { + try fail(ErrorCode.invalidParams, "The release ledger field \"keys\" must be an array.") + } + guard let buttonValues = rawButtons as? [Any] else { + try fail(ErrorCode.invalidParams, "The release ledger field \"buttons\" must be an array.") + } + var keys: [String] = [] + for value in keyValues { + guard let value = value as? String, isSingleWebDriverKey(value) else { + try fail(ErrorCode.invalidParams, "Every release ledger key must be one WebDriver key value.") + } + keys.append(value) + } + var buttons: [Int] = [] + for value in buttonValues { + guard let number = value as? NSNumber, !isBoolean(number) else { + try fail(ErrorCode.invalidParams, "Every release ledger button must be a number.") + } + let button = number.intValue + guard number.doubleValue == Double(button), (0...4).contains(button) else { + try fail(ErrorCode.invalidParams, "Release ledger buttons must be W3C pointer buttons between 0 and 4.") + } + buttons.append(button) + } + return ReleaseLedger(keys: keys, buttons: buttons) +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/InputLock.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/InputLock.swift new file mode 100644 index 00000000000..d8a4a048327 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/InputLock.swift @@ -0,0 +1,186 @@ +import Foundation +import Darwin + +enum AbandonPolicy { + case fail + case adopt +} + +struct InputLockOwner: Codable { + let pid: Int32 + let processStartedAt: String +} + +final class PhysicalInputLock: @unchecked Sendable { + static let shared = PhysicalInputLock() + + private let stateLock = NSLock() + private var depth = 0 + private var descriptor: Int32 = -1 + private let lockDirectory: URL + private let lockURL: URL + private let ownerURL: URL + + private enum OwnerState { + case missing + case valid(InputLockOwner) + case invalid + } + + init(baseName: String = "furn-desktop-driver-physical-input-v1-\(getuid())") { + lockDirectory = URL(fileURLWithPath: NSHomeDirectory(), isDirectory: true) + .appendingPathComponent("Library/Caches/com.microsoft.fluentui-react-native.desktop-driver", isDirectory: true) + lockURL = lockDirectory.appendingPathComponent(baseName + ".lock", isDirectory: false) + ownerURL = lockDirectory.appendingPathComponent(baseName + ".owner.json", isDirectory: false) + } + + func withLock( + token: CancellationToken, + timeoutMilliseconds: Int = 30_000, + policy: AbandonPolicy = .fail, + _ body: () throws -> T + ) throws -> T { + try acquire(token: token, timeoutMilliseconds: timeoutMilliseconds, policy: policy) + defer { release() } + return try body() + } + + func acquire(token: CancellationToken, timeoutMilliseconds: Int, policy: AbandonPolicy) throws { + stateLock.lock() + if depth > 0 { + depth += 1 + stateLock.unlock() + return + } + stateLock.unlock() + + do { + try FileManager.default.createDirectory( + at: lockDirectory, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: lockDirectory.path) + } catch { + try fail(ErrorCode.inputBusy, "Creating the physical input lock directory failed: \(error.localizedDescription)") + } + let fd = open(lockURL.path, O_CREAT | O_RDWR | O_CLOEXEC, S_IRUSR | S_IWUSR) + guard fd >= 0 else { + try fail(ErrorCode.inputBusy, "Creating the physical input lock failed with errno \(errno).") + } + let deadline = DispatchTime.now() + .milliseconds(timeoutMilliseconds) + do { + while flock(fd, LOCK_EX | LOCK_NB) != 0 { + if errno != EWOULDBLOCK && errno != EAGAIN { + try fail(ErrorCode.inputBusy, "Acquiring the physical input lock failed with errno \(errno).") + } + try token.throwIfCancelled() + if DispatchTime.now() >= deadline { + try fail( + ErrorCode.inputBusy, + "Another desktop driver helper held physical input for longer than \(timeoutMilliseconds) ms. Wait for that session to finish, or run the helper with --release-input if it is gone." + ) + } + try token.wait(milliseconds: 50) + } + + switch readOwner() { + case let .valid(owner): + let ownerStart = parseDate(owner.processStartedAt) + let actualStart = processStartTime(owner.pid) + let sameIdentity = ownerStart != nil && actualStart != nil + && abs(ownerStart!.timeIntervalSince(actualStart!)) <= 0.01 + if processIsAlive(owner.pid) && sameIdentity { + try fail( + ErrorCode.inputBusy, + "The physical input owner record still names a live desktop driver helper. Wait for it to finish before retrying." + ) + } + if policy == .fail { + try fail( + ErrorCode.inputBusy, + "A previous desktop driver helper terminated while holding physical input, so keys or buttons may still be depressed. Run the helper with --release-input to recover, then retry the command." + ) + } + writeDiagnostic("furn-desktop-driver-host: adopting physical input abandoned by a terminated helper.") + case .invalid: + if policy == .fail { + try fail( + ErrorCode.inputBusy, + "The physical input owner record is invalid, so prior input cleanup cannot be proven. Run the helper with --release-input to recover, then retry the command." + ) + } + writeDiagnostic("furn-desktop-driver-host: adopting an invalid physical input owner record for recovery.") + case .missing: + break + } + + try writeOwner() + stateLock.lock() + descriptor = fd + depth = 1 + stateLock.unlock() + } catch { + _ = flock(fd, LOCK_UN) + close(fd) + throw error + } + } + + func release() { + stateLock.lock() + guard depth > 0 else { + stateLock.unlock() + return + } + depth -= 1 + if depth > 0 { + stateLock.unlock() + return + } + let fd = descriptor + descriptor = -1 + stateLock.unlock() + + try? FileManager.default.removeItem(at: ownerURL) + _ = flock(fd, LOCK_UN) + close(fd) + } + + private func readOwner() -> OwnerState { + guard FileManager.default.fileExists(atPath: ownerURL.path) else { + return .missing + } + guard let data = try? Data(contentsOf: ownerURL), !data.isEmpty, data.count <= 4096 else { + return .invalid + } + guard let owner = try? JSONDecoder().decode(InputLockOwner.self, from: data) else { + return .invalid + } + return .valid(owner) + } + + private func writeOwner() throws { + guard let start = processStartTime(getpid()) else { + try fail(ErrorCode.inputBusy, "The helper could not read its process start time for physical input ownership.") + } + let owner = InputLockOwner(pid: getpid(), processStartedAt: formatDate(start)) + do { + let data = try JSONEncoder().encode(owner) + try data.write(to: ownerURL, options: [.atomic]) + try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: ownerURL.path) + } catch { + try fail(ErrorCode.inputBusy, "Writing the physical input owner record failed: \(error.localizedDescription)") + } + } + + func installDeadOwnerForSelfTest() throws { + let owner = InputLockOwner(pid: Int32.max, processStartedAt: "1970-01-01T00:00:00.000Z") + try JSONEncoder().encode(owner).write(to: ownerURL, options: [.atomic]) + } + + func cleanupSelfTestFiles() { + try? FileManager.default.removeItem(at: ownerURL) + try? FileManager.default.removeItem(at: lockURL) + } +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Permissions.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Permissions.swift new file mode 100644 index 00000000000..5951c77a2c5 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Permissions.swift @@ -0,0 +1,155 @@ +import ApplicationServices +import Foundation +import Security + +private func evidenceField(_ value: String?, unavailable: String) -> JSONObject { + if let value, !value.isEmpty { + return [ + "status": "available", + "value": bounded(value, bytes: 4096), + ] + } + return [ + "reason": unavailable, + "status": "unavailable", + ] +} + +private func hexadecimalString(_ data: Data) -> String { + data.map { String(format: "%02x", $0) }.joined() +} + +private func statusDescription(_ status: OSStatus) -> String { + if let message = SecCopyErrorMessageString(status, nil) as String? { + return "\(message) (\(status))" + } + return "Security framework status \(status)" +} + +private func helperBundlePath(executablePath: String) -> String? { + var candidate = URL(fileURLWithPath: executablePath).standardizedFileURL.deletingLastPathComponent() + while candidate.path != "/" { + if candidate.pathExtension.lowercased() == "app" { + return canonicalPath(candidate.path) + } + candidate.deleteLastPathComponent() + } + return nil +} + +private func signingEvidence() -> JSONObject { + let unavailable = "Security framework did not return this value for the running helper." + var dynamicCode: SecCode? + let selfStatus = SecCodeCopySelf(SecCSFlags(), &dynamicCode) + guard selfStatus == errSecSuccess, let dynamicCode else { + let reason = "SecCodeCopySelf failed: \(statusDescription(selfStatus))." + return [ + "cdhash": evidenceField(nil, unavailable: reason), + "designatedRequirement": evidenceField(nil, unavailable: reason), + "signer": evidenceField(nil, unavailable: reason), + "source": "Security.framework", + "teamIdentifier": evidenceField(nil, unavailable: reason), + ] + } + + var staticCode: SecStaticCode? + let staticStatus = SecCodeCopyStaticCode(dynamicCode, SecCSFlags(), &staticCode) + guard staticStatus == errSecSuccess, let staticCode else { + let reason = "SecCodeCopyStaticCode failed: \(statusDescription(staticStatus))." + return [ + "cdhash": evidenceField(nil, unavailable: reason), + "designatedRequirement": evidenceField(nil, unavailable: reason), + "signer": evidenceField(nil, unavailable: reason), + "source": "Security.framework", + "teamIdentifier": evidenceField(nil, unavailable: reason), + ] + } + + var information: CFDictionary? + let informationStatus = SecCodeCopySigningInformation( + staticCode, + SecCSFlags(rawValue: kSecCSSigningInformation), + &information + ) + let values = information as? [CFString: Any] + let informationReason = informationStatus == errSecSuccess + ? unavailable + : "SecCodeCopySigningInformation failed: \(statusDescription(informationStatus))." + let cdhash = (values?[kSecCodeInfoUnique] as? Data).map(hexadecimalString) + let teamIdentifier = values?[kSecCodeInfoTeamIdentifier] as? String + let certificates = values?[kSecCodeInfoCertificates] as? [SecCertificate] + let signer = certificates?.first.flatMap { SecCertificateCopySubjectSummary($0) as String? } + + var requirement: SecRequirement? + let requirementStatus = SecCodeCopyDesignatedRequirement(staticCode, SecCSFlags(), &requirement) + var requirementText: CFString? + let requirementStringStatus: OSStatus + if requirementStatus == errSecSuccess, let requirement { + requirementStringStatus = SecRequirementCopyString(requirement, SecCSFlags(), &requirementText) + } else { + requirementStringStatus = requirementStatus + } + let requirementReason = requirementStringStatus == errSecSuccess + ? unavailable + : "Reading the designated requirement failed: \(statusDescription(requirementStringStatus))." + + return [ + "cdhash": evidenceField(cdhash, unavailable: informationReason), + "designatedRequirement": evidenceField(requirementText as String?, unavailable: requirementReason), + "signer": evidenceField(signer, unavailable: informationReason), + "source": "Security.framework", + "teamIdentifier": evidenceField(teamIdentifier, unavailable: informationReason), + ] +} + +func createPermissionDiagnostic(prompt: Bool) -> JSONObject { + let accessibilityBefore = AXIsProcessTrusted() + let postEventBefore = CGPreflightPostEventAccess() + let screenCaptureBefore = CGPreflightScreenCaptureAccess() + + var accessibilityAfter = accessibilityBefore + var screenCaptureAfter = screenCaptureBefore + var screenCaptureRequestResult: Bool? + if prompt { + let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary + accessibilityAfter = AXIsProcessTrustedWithOptions(options) + screenCaptureRequestResult = CGRequestScreenCaptureAccess() + screenCaptureAfter = CGPreflightScreenCaptureAccess() + } + + let executablePath = processExecutablePath(getpid()) + ?? canonicalPath(CommandLine.arguments.first ?? "") + let bundlePath = helperBundlePath(executablePath: executablePath) + var accessibility: JSONObject = ["preflight": accessibilityBefore] + var screenCapture: JSONObject = ["preflight": screenCaptureBefore] + if prompt { + accessibility["afterPrompt"] = accessibilityAfter + screenCapture["afterPrompt"] = screenCaptureAfter + screenCapture["requestResult"] = screenCaptureRequestResult ?? false + } + + return [ + "notes": [ + "Ordinary helper commands never request privacy permissions.", + "Screen Recording permission changes may require restarting the helper before preflight state and window metadata update.", + ], + "permissions": [ + "accessibility": accessibility, + "postEvent": [ + "preflight": postEventBefore, + "promptSupportedByThisMode": false, + ], + "screenCapture": screenCapture, + ], + "process": [ + "appBundlePath": evidenceField(bundlePath, unavailable: "The running executable is not nested in an .app bundle."), + "executablePath": executablePath, + "parentPid": Int(getppid()), + "pid": Int(getpid()), + ], + "promptRequested": prompt, + "schemaVersion": 1, + "signing": signingEvidence(), + "type": "permissions", + ] +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift new file mode 100644 index 00000000000..d2377d1c5c5 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift @@ -0,0 +1,520 @@ +import ApplicationServices +import Foundation + +private final class SelfTestRunner { + private(set) var checks = 0 + private(set) var failures = 0 + + func check(_ condition: @autoclosure () -> Bool, _ name: String) { + checks += 1 + if condition() { + print("ok - \(name)") + } else { + failures += 1 + print("not ok - \(name)") + } + } + + func run() { + testFraming() + testJSON() + testInputLedger() + testInputLock() + testCaptureScale() + testSnapshotShape() + testAXIdentity() + testAXWindowDiagnostics() + testCoreGraphicsWindowSelection() + testReactNativeAccessibilityFallbacks() + testPermissionDiagnostic() + testHello() + } + + private func testFraming() { + do { + let payload = Data(#"{"type":"response","id":"a"}"#.utf8) + let frame = try Framing.encode(type: .json, payload: payload) + check(frame.count == Framing.headerBytes + payload.count, "json frame length") + let header = frame.prefix(Framing.headerBytes) + let decoded = Framing.decodeHeader(Data(header)) + check(decoded?.0 == .json, "json frame type") + check(decoded?.1 == payload.count, "json frame payload length") + check(frame.dropFirst(Framing.headerBytes) == payload, "json frame payload round trip") + + let binary = try Framing.encodeBinary(id: "image-1", data: Data([1, 2, 3, 4])) + let binaryHeader = Framing.decodeHeader(Data(binary.prefix(Framing.headerBytes))) + check(binaryHeader?.0 == .binary, "binary frame type") + check(binaryHeader?.1 == 4 + 7 + 4, "binary frame payload length") + + var corrupt = Data(repeating: 0, count: Framing.headerBytes) + corrupt.replaceSubrange(0..<4, with: Data("FDR0".utf8)) + check(Framing.decodeHeader(corrupt) == nil, "invalid frame magic is rejected") + var reserved = Data(repeating: 0, count: Framing.headerBytes) + reserved.replaceSubrange(0..<4, with: Framing.magic) + reserved[4] = FrameType.json.rawValue + reserved[5] = 1 + check(Framing.decodeHeader(reserved) == nil, "nonzero reserved frame bytes are rejected") + } catch { + check(false, "framing operations do not throw") + } + } + + private func testJSON() { + do { + let document = Data(#"{"a":1,"b":-2.5,"c":"x\ny\u00e9","d":[true,false,null]}"#.utf8) + let parsed = try parseJSON(document) as? JSONObject + check(parsed != nil, "json object parses") + check((parsed?["a"] as? NSNumber)?.intValue == 1, "json integer field") + check((parsed?["b"] as? NSNumber)?.doubleValue == -2.5, "json fractional field") + check((parsed?["c"] as? String) == "x\nyé", "json escapes decode") + let encoded = try serializeJSON(parsed ?? [:]) + let roundTrip = try parseJSON(encoded) as? JSONObject + check(roundTrip != nil, "json round trip") + } catch { + check(false, "json operations do not throw") + } + } + + private func testInputLedger() { + check(isSingleWebDriverKey("A"), "ASCII key values contain one code point") + check(isSingleWebDriverKey("😀"), "supplementary key values contain one code point") + check(!isSingleWebDriverKey("AB"), "multi-code-point key values are rejected") + do { + let ledger = try parseReleaseLedger([ + "keys": ["a", "\u{E008}"], + "buttons": [0, 2], + ]) + check(ledger.keys.count == 2, "release ledger parses keys") + check(ledger.buttons == [0, 2], "release ledger parses buttons") + } catch { + check(false, "release ledger parses") + } + do { + _ = try parseReleaseLedger(["buttons": [9]]) + check(false, "invalid release ledger button is rejected") + } catch { + check(true, "invalid release ledger button is rejected") + } + } + + private func testInputLock() { + let name = "furn-desktop-driver-self-test-\(getpid())-\(UUID().uuidString)" + let lock = PhysicalInputLock(baseName: name) + let token = CancellationToken() + defer { lock.cleanupSelfTestFiles() } + do { + let value = try lock.withLock(token: token, timeoutMilliseconds: 200) { 42 } + check(value == 42, "idle physical input lock is available") + } catch { + check(false, "idle physical input lock is available") + } + do { + try lock.installDeadOwnerForSelfTest() + _ = try lock.withLock(token: token, timeoutMilliseconds: 200, policy: .fail) { 0 } + check(false, "dead owner record fails closed") + } catch let error as HelperError { + check(error.code == ErrorCode.inputBusy, "dead owner record fails closed") + } catch { + check(false, "dead owner record fails closed") + } + do { + let value = try lock.withLock(token: token, timeoutMilliseconds: 200, policy: .adopt) { 7 } + check(value == 7, "release-only recovery adopts a dead owner record") + let second = try lock.withLock(token: token, timeoutMilliseconds: 200) { 8 } + check(second == 8, "recovery clears the owner record") + } catch { + check(false, "release-only recovery adopts and clears a dead owner record") + } + } + + private func testCaptureScale() { + check( + matchingBackingScaleFactor( + displayID: 42, + screens: [(displayID: 7, scaleFactor: 1), (displayID: 42, scaleFactor: 2)] + ) == 2, + "capture uses the matched display backing scale" + ) + check( + matchingBackingScaleFactor(displayID: 99, screens: [(displayID: 42, scaleFactor: 2)]) == 1, + "capture defaults to one scale when the display is unknown" + ) + } + + private func testSnapshotShape() { + let snapshot = ElementSnapshot( + id: "element-1", + automationID: "button-primary", + checked: .mixed, + enabled: .supported(true), + expanded: .unsupported("leaf"), + focused: .supported(false), + name: "Primary", + parentID: "element-0", + rect: CGRect(x: 10, y: 20, width: 120, height: 40), + screenRect: CGRect(x: 100, y: 200, width: 120, height: 40), + role: "button", + scope: .preview, + selected: .unsupported("no selection"), + text: nil, + value: nil, + visible: .supported(true), + windowID: "window-1" + ) + let json = snapshot.json() + check(json["scope"] as? String == "preview", "snapshot serializes its scope") + check((json["checked"] as? JSONObject)?["value"] as? String == "mixed", "mixed checked state serializes") + check(json["text"] == nil, "absent optional text is omitted") + check((json["rect"] as? JSONObject)?["width"] as? Double == 120, "snapshot serializes logical geometry") + check(json["id"] as? String == "element-1", "snapshot identifiers stay opaque") + } + + private func testAXIdentity() { + let first = AXUIElementCreateApplication(getpid()) + let second = AXUIElementCreateApplication(getpid()) + check(sameAXElement(first, second), "equivalent Accessibility elements compare equal") + check(uniqueAXElements([first, second]).count == 1, "duplicate Accessibility elements collapse to one identity") + check(strictAXElements([first, second] as CFArray)?.count == 2, "strict AX arrays accept only accessibility elements") + check(strictAXElements([first, "invalid"] as CFArray) == nil, "strict AX arrays reject mixed element types") + } + + private func testAXWindowDiagnostics() { + func query( + _ error: AXError, + elementCount: Int = 0, + valueValid: Bool? = nil + ) -> AXWindowAttributeDiagnostic { + AXWindowAttributeDiagnostic( + attribute: kAXWindowsAttribute, + error: error, + returnedType: elementCount == 0 ? "none" : "CFArray", + elementCount: elementCount, + elements: [], + valueValid: valueValid ?? (error == .success) + ) + } + + let manyElements = (0..<65).map { _ in AXUIElementCreateApplication(getpid()) } + let manyWindows = AXWindowAttributeDiagnostic( + attribute: kAXWindowsAttribute, + error: .success, + returnedType: "CFArray", + elementCount: manyElements.count, + elements: manyElements, + valueValid: true + ) + check(manyWindows.elements.count == 65, "AX window diagnostics preserve every functional element") + check( + (manyWindows.json()["elements"] as? [JSONObject])?.count == 16, + "AX window diagnostic evidence remains bounded" + ) + + check( + classifyAXWindowQuery( + queries: [query(.apiDisabled)], + realWindowCount: 0, + placeholderCount: 0, + otherRoleCount: 0, + matchingTitleCount: nil + ) == "api-disabled", + "AX API-disabled window queries remain distinguishable" + ) + check( + classifyAXWindowQuery( + queries: [query(.cannotComplete)], + realWindowCount: 0, + placeholderCount: 0, + otherRoleCount: 0, + matchingTitleCount: nil + ) == "target-non-response", + "AX cannot-complete window queries remain distinguishable" + ) + check( + classifyAXWindowQuery( + queries: [query(.invalidUIElement)], + realWindowCount: 0, + placeholderCount: 0, + otherRoleCount: 0, + matchingTitleCount: nil + ) == "invalid-element", + "invalid AX application elements remain distinguishable" + ) + check( + classifyAXWindowQuery( + queries: [query(.attributeUnsupported), query(.noValue)], + realWindowCount: 0, + placeholderCount: 0, + otherRoleCount: 0, + matchingTitleCount: nil + ) == "no-value-or-unsupported", + "unsupported and no-value AX window attributes remain distinguishable" + ) + check( + classifyAXWindowQuery( + queries: [query(.success, elementCount: 1)], + realWindowCount: 0, + placeholderCount: 1, + otherRoleCount: 0, + matchingTitleCount: nil + ) == "placeholder-only", + "AXApplication placeholders are never accepted as windows" + ) + check( + classifyAXWindowQuery( + queries: [query(.success, elementCount: 1)], + realWindowCount: 0, + placeholderCount: 1, + otherRoleCount: 0, + matchingTitleCount: nil, + roleErrors: [.failure] + ) == "window-query-failed", + "hard AX role errors take precedence over placeholder classification" + ) + check( + classifyAXWindowQuery( + queries: [query(.success, elementCount: 1)], + realWindowCount: 1, + placeholderCount: 0, + otherRoleCount: 0, + matchingTitleCount: 0 + ) == "no-matching-title", + "real windows without the configured title remain distinguishable" + ) + check( + classifyAXWindowQuery( + queries: [query(.success, elementCount: 1)], + realWindowCount: 1, + placeholderCount: 0, + otherRoleCount: 0, + matchingTitleCount: 1 + ) == "matched", + "real titled AXWindow results are accepted" + ) + check( + incompleteAXCandidateDiscovery( + queries: [query(.cannotComplete)], + roleQueries: [], + titleErrorCount: 0 + ), + "failed AXWindows queries make multi-process discovery incomplete" + ) + check( + incompleteAXCandidateDiscovery( + queries: [query(.success, valueValid: false)], + roleQueries: [], + titleErrorCount: 0 + ), + "malformed AXWindows values make multi-process discovery incomplete" + ) + check( + incompleteAXCandidateDiscovery( + queries: [query(.success)], + roleQueries: [], + titleErrorCount: 1 + ), + "failed window title reads make multi-process discovery incomplete" + ) + check( + incompleteAXCandidateDiscovery( + queries: [query(.success)], + roleQueries: [AXWindowRoleDiagnostic(error: .success, role: nil, valueValid: false)], + titleErrorCount: 0 + ), + "malformed successful AX role values make discovery incomplete" + ) + check( + incompleteAXCandidateDiscovery( + queries: [query(.success, elementCount: 1)], + roleQueries: [ + AXWindowRoleDiagnostic(error: .success, role: kAXApplicationRole, valueValid: true), + ], + titleErrorCount: 0 + ), + "AXApplication placeholders make authoritative window discovery incomplete" + ) + let authoritativeElement = AXUIElementCreateApplication(getpid()) + let fallbackElement = AXUIElementCreateSystemWide() + let authoritativeWindows = AXWindowAttributeDiagnostic( + attribute: kAXWindowsAttribute, + error: .success, + returnedType: "CFArray", + elementCount: 1, + elements: [authoritativeElement], + valueValid: true + ) + let inconsistentFallback = AXWindowAttributeDiagnostic( + attribute: kAXMainWindowAttribute, + error: .success, + returnedType: "AXUIElement", + elementCount: 1, + elements: [fallbackElement], + valueValid: true + ) + check( + incompleteAXCandidateDiscovery( + queries: [authoritativeWindows, inconsistentFallback], + roleQueries: [], + titleErrorCount: 0 + ), + "fallback windows absent from AXWindows make discovery incomplete" + ) + check( + !incompleteAXMultiProcessDiscovery( + queriesByCandidate: [ + [authoritativeWindows], + [ + AXWindowAttributeDiagnostic( + attribute: kAXWindowsAttribute, + error: .success, + returnedType: "CFArray", + elementCount: 1, + elements: [fallbackElement], + valueValid: true + ), + ], + ], + roleQueriesByCandidate: [[], []], + titleErrorCount: 0 + ), + "independent candidate window enumerations are not compared across processes" + ) + check( + !completeAXWindowTitle(error: .attributeUnsupported, title: nil), + "unsupported AX window titles are incomplete" + ) + check( + completeAXWindowTitle(error: .success, title: ""), + "successful empty AX window titles remain valid values" + ) + check( + classifyAXAttachment(matchCount: 1, discoveryIncomplete: true) == "incomplete", + "one confirmed match remains ambiguous after incomplete discovery" + ) + check( + classifyAXAttachment(matchCount: 1, discoveryIncomplete: false) == "matched", + "one confirmed match succeeds after complete discovery" + ) + } + + private func testCoreGraphicsWindowSelection() { + let frame = CGRect(x: 10, y: 20, width: 300, height: 200) + let unique = selectCoreGraphicsWindowFrame( + candidates: [CoreGraphicsWindowCandidate(frame: frame, title: nil)], + title: "Expected", + authoritativeWindowCount: 1, + screenCaptureAccess: false + ) + check(unique.frame == frame, "CoreGraphics fallback does not require window names for a unique PID/layer candidate") + check( + unique.diagnostics["reason"] as? String == "unique-candidate", + "CoreGraphics fallback records unique-candidate evidence" + ) + + let duplicateGeometry = selectCoreGraphicsWindowFrame( + candidates: [ + CoreGraphicsWindowCandidate(frame: frame, title: nil), + CoreGraphicsWindowCandidate(frame: frame, title: nil), + ], + title: "Expected", + authoritativeWindowCount: 1, + screenCaptureAccess: false + ) + check( + duplicateGeometry.frame == frame, + "CoreGraphics fallback accepts one unique geometry when Screen Recording hides names" + ) + check( + duplicateGeometry.diagnostics["titleMatchingAvailable"] as? Bool == false, + "CoreGraphics fallback reports unavailable title matching" + ) + let wrongTitle = selectCoreGraphicsWindowFrame( + candidates: [CoreGraphicsWindowCandidate(frame: frame, title: "Different")], + title: "Expected", + authoritativeWindowCount: 1, + screenCaptureAccess: true + ) + check(wrongTitle.frame == nil, "CoreGraphics fallback rejects an explicitly different title") + check( + wrongTitle.diagnostics["reason"] as? String == "no-matching-title", + "CoreGraphics fallback records title mismatches" + ) + let ambiguousUntitled = selectCoreGraphicsWindowFrame( + candidates: [CoreGraphicsWindowCandidate(frame: frame, title: nil)], + title: "Expected", + authoritativeWindowCount: 2, + screenCaptureAccess: false + ) + check( + ambiguousUntitled.frame == nil, + "CoreGraphics fallback rejects unnamed candidates when AX exposes multiple windows" + ) + } + + private func testReactNativeAccessibilityFallbacks() { + let engine = AccessibilityEngine() + do { + let checkboxRole = try engine.normalizeRole("AXUnknown", subrole: nil, value: "checkbox, unchecked") + check( + checkboxRole == "checkbox", + "React Native checkbox values recover the native role" + ) + let checked = engine.checkedValue(role: "checkbox", value: "checkbox, checked" as CFString) + check( + (checked.json()["value"] as? Bool) == true, + "React Native composite checkbox values recover checked state" + ) + let unchecked = engine.checkedValue(role: "checkbox", value: "checkbox, unchecked" as CFString) + check( + (unchecked.json()["value"] as? Bool) == false, + "React Native composite checkbox values recover unchecked state" + ) + } catch { + check(false, "React Native accessibility fallbacks do not throw") + } + } + + private func testPermissionDiagnostic() { + let diagnostic = createPermissionDiagnostic(prompt: false) + check(diagnostic["schemaVersion"] as? Int == 1, "permission diagnostics are versioned") + check(diagnostic["type"] as? String == "permissions", "permission diagnostics identify their type") + check(diagnostic["promptRequested"] as? Bool == false, "self-tests never prompt for privacy permissions") + let process = diagnostic["process"] as? JSONObject + check((process?["pid"] as? Int) == Int(getpid()), "permission diagnostics report the helper PID") + check(process?["executablePath"] as? String != nil, "permission diagnostics report the helper executable") + let permissions = diagnostic["permissions"] as? JSONObject + check(permissions?["accessibility"] is JSONObject, "permission diagnostics report Accessibility preflight") + check(permissions?["postEvent"] is JSONObject, "permission diagnostics report PostEvent preflight") + check(permissions?["screenCapture"] is JSONObject, "permission diagnostics report Screen Recording preflight") + let signing = diagnostic["signing"] as? JSONObject + check(signing?["cdhash"] is JSONObject, "permission diagnostics explicitly report cdhash availability") + check(signing?["signer"] is JSONObject, "permission diagnostics explicitly report signer availability") + check(signing?["teamIdentifier"] is JSONObject, "permission diagnostics explicitly report team availability") + check( + signing?["designatedRequirement"] is JSONObject, + "permission diagnostics explicitly report designated-requirement availability" + ) + } + + private func testHello() { + let hello = createHello() + check(hello["type"] as? String == "hello", "hello announces its type") + check( + hello["provider"] as? String == "macos" && hello["architecture"] as? String == "arm64", + "hello reports the provider identity" + ) + check( + !(hello["buildId"] as? String ?? "").isEmpty && !(hello["sourceDigest"] as? String ?? "").isEmpty, + "hello carries the generated build identity" + ) + check((hello["protocol"] as? JSONObject)?["major"] as? Int == 1, "hello reports the wire protocol") + } +} + +func runSelfTest() -> Int32 { + let runner = SelfTestRunner() + runner.run() + print( + "\(runner.failures == 0 ? "self-test passed" : "self-test failed"): \(runner.checks - runner.failures)/\(runner.checks) checks" + ) + return runner.failures == 0 ? 0 : 1 +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Support.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Support.swift new file mode 100644 index 00000000000..26e99890f09 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Support.swift @@ -0,0 +1,659 @@ +import ApplicationServices +import Foundation +import Darwin + +#if !arch(arm64) + #error("Furn Desktop Driver Host V1 supports only macOS arm64.") +#endif + +typealias JSONObject = [String: Any] + +enum ErrorCode { + static let unsupported = "unsupported-operation" + static let staleElement = "stale-element" + static let invalidRequest = "invalid-request" + static let invalidParams = "invalid-params" + static let noSuchWindow = "no-such-window" + static let noSuchElement = "no-such-element" + static let notInteractable = "element-not-interactable" + static let automationFailed = "automation-failed" + static let inputUnavailable = "input-unavailable" + static let inputBusy = "input-busy" + static let inputFailed = "input-failed" + static let launchFailed = "launch-failed" + static let attachFailed = "attach-failed" + static let leaseInvalid = "lease-invalid" + static let ambiguousTarget = "ambiguous-target" + static let noSuchLease = "no-such-lease" + static let windowActivation = "window-activation-failed" + static let captureFailed = "capture-failed" + static let treeTooLarge = "tree-too-large" + static let internalError = "internal-error" +} + +struct HelperError: Error, CustomStringConvertible { + let code: String + let message: String + let data: JSONObject? + + var description: String { message } + + init(code: String, message: String, data: JSONObject? = nil) { + self.code = code + self.message = message + self.data = data + } +} + +struct CancelledError: Error {} + +@inline(__always) +func fail(_ code: String, _ message: String, data: JSONObject? = nil) throws -> Never { + throw HelperError(code: code, message: message, data: data) +} + +final class CancellationToken: @unchecked Sendable { + private let lock = NSLock() + private var cancelled = false + + func cancel() { + lock.lock() + cancelled = true + lock.unlock() + } + + func reset() { + lock.lock() + cancelled = false + lock.unlock() + } + + func throwIfCancelled() throws { + lock.lock() + let value = cancelled + lock.unlock() + if value { + throw CancelledError() + } + } + + func wait(milliseconds: Int) throws { + let deadline = DispatchTime.now() + .milliseconds(max(0, milliseconds)) + repeat { + try throwIfCancelled() + let remaining = deadline.uptimeNanoseconds > DispatchTime.now().uptimeNanoseconds + ? deadline.uptimeNanoseconds - DispatchTime.now().uptimeNanoseconds + : 0 + if remaining == 0 { + break + } + usleep(useconds_t(min(10_000, max(1_000, remaining / 1_000)))) + } while true + try throwIfCancelled() + } +} + +final class IdentifierGenerator: @unchecked Sendable { + static let shared = IdentifierGenerator() + + private let lock = NSLock() + private let salt = String(UUID().uuidString.prefix(8)).lowercased() + private var nextValue: UInt64 = 1 + + func next(_ prefix: String) -> String { + lock.lock() + let value = nextValue + nextValue += 1 + lock.unlock() + return "\(prefix)-\(salt)-\(String(value, radix: 16))" + } +} + +func requireObject(_ value: Any?, _ message: String) throws -> JSONObject { + guard let object = value as? JSONObject else { + try fail(ErrorCode.invalidParams, message) + } + return object +} + +func requireString(_ object: JSONObject, _ key: String) throws -> String { + guard let value = object[key] as? String, !value.isEmpty else { + try fail(ErrorCode.invalidParams, "The request is missing the required \"\(key)\" parameter.") + } + return value +} + +func optionalString(_ object: JSONObject, _ key: String) -> String? { + guard let value = object[key] as? String, !value.isEmpty else { + return nil + } + return value +} + +func number(_ object: JSONObject, _ key: String, default defaultValue: Double = 0) -> Double { + guard let value = object[key] as? NSNumber, !isBoolean(value) else { + return defaultValue + } + return value.doubleValue +} + +func integer(_ object: JSONObject, _ key: String, default defaultValue: Int = 0) -> Int { + guard let value = object[key] as? NSNumber, !isBoolean(value) else { + return defaultValue + } + return value.intValue +} + +func isBoolean(_ value: NSNumber) -> Bool { + CFGetTypeID(value) == CFBooleanGetTypeID() +} + +func parseJSON(_ data: Data) throws -> Any { + do { + return try JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) + } catch { + try fail(ErrorCode.invalidRequest, "Malformed JSON: \(error.localizedDescription)") + } +} + +func serializeJSON(_ value: Any) throws -> Data { + guard JSONSerialization.isValidJSONObject(value) else { + try fail(ErrorCode.internalError, "The native command produced a value that cannot be encoded as JSON.") + } + do { + return try JSONSerialization.data(withJSONObject: value, options: []) + } catch { + try fail(ErrorCode.internalError, "Encoding JSON failed: \(error.localizedDescription)") + } +} + +func bounded(_ value: String, bytes: Int) -> String { + guard value.utf8.count > bytes else { + return value + } + var output = "" + var count = 0 + for character in value { + let size = String(character).utf8.count + if count + size > bytes { + break + } + output.append(character) + count += size + } + return output + "..." +} + +func writeDiagnostic(_ message: String) { + let line = bounded(message, bytes: 1000) + "\n" + FileHandle.standardError.write(Data(line.utf8)) +} + +func rounded(_ value: CGFloat) -> Double { + guard value.isFinite else { + return 0 + } + return (Double(value) * 100).rounded() / 100 +} + +func rounded(_ value: Double) -> Double { + guard value.isFinite else { + return 0 + } + return (value * 100).rounded() / 100 +} + +func rectJSON(_ rect: CGRect) -> JSONObject { + [ + "height": rounded(rect.height), + "width": rounded(rect.width), + "x": rounded(rect.origin.x), + "y": rounded(rect.origin.y), + ] +} + +func canonicalPath(_ path: String) -> String { + URL(fileURLWithPath: path).standardizedFileURL.resolvingSymlinksInPath().path +} + +func processIsAlive(_ pid: pid_t) -> Bool { + guard pid > 0 else { + return false + } + if kill(pid, 0) == 0 { + return true + } + return errno == EPERM +} + +func processStartTime(_ pid: pid_t) -> Date? { + var info = proc_bsdinfo() + let size = Int32(MemoryLayout.size) + let read = withUnsafeMutablePointer(to: &info) { + proc_pidinfo(pid, PROC_PIDTBSDINFO, 0, $0, size) + } + guard read == size else { + return nil + } + return Date( + timeIntervalSince1970: TimeInterval(info.pbi_start_tvsec) + + TimeInterval(info.pbi_start_tvusec) / 1_000_000 + ) +} + +func processExecutablePath(_ pid: pid_t) -> String? { + var buffer = [CChar](repeating: 0, count: Int(MAXPATHLEN) * 4) + let length = proc_pidpath(pid, &buffer, UInt32(buffer.count)) + guard length > 0 else { + return nil + } + return canonicalPath(String(cString: buffer)) +} + +private let isoFormatter: ISO8601DateFormatter = { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter +}() + +func formatDate(_ date: Date) -> String { + isoFormatter.string(from: date) +} + +func parseDate(_ value: String) -> Date? { + if let date = isoFormatter.date(from: value) { + return date + } + let fallback = ISO8601DateFormatter() + fallback.formatOptions = [.withInternetDateTime] + return fallback.date(from: value) +} + +func requireAccessibility(_ operation: String) throws { + guard AXIsProcessTrusted() else { + try fail( + ErrorCode.inputUnavailable, + "\(operation) requires Accessibility permission. Enable Furn Desktop Driver Host in System Settings > Privacy & Security > Accessibility, then restart the helper." + ) + } +} + +func describeAXError(_ error: AXError) -> String { + switch error { + case .actionUnsupported: + return "the accessibility action is unsupported" + case .apiDisabled: + return "the Accessibility API is disabled" + case .attributeUnsupported: + return "the accessibility attribute is unsupported" + case .cannotComplete: + return "the target application did not complete the accessibility request" + case .failure: + return "the Accessibility API reported a general failure" + case .illegalArgument: + return "the accessibility request contained an illegal argument" + case .invalidUIElement: + return "the accessibility element is no longer valid" + case .invalidUIElementObserver: + return "the accessibility observer is no longer valid" + case .noValue: + return "the accessibility attribute has no value" + case .notEnoughPrecision: + return "the accessibility value cannot be represented precisely" + case .notImplemented: + return "the accessibility operation is not implemented" + case .notificationAlreadyRegistered: + return "the accessibility notification is already registered" + case .notificationNotRegistered: + return "the accessibility notification is not registered" + case .notificationUnsupported: + return "the accessibility notification is unsupported" + case .parameterizedAttributeUnsupported: + return "the parameterized accessibility attribute is unsupported" + case .success: + return "success" + @unknown default: + return "Accessibility error \(error.rawValue)" + } +} + +func axErrorName(_ error: AXError) -> String { + switch error { + case .actionUnsupported: + return "actionUnsupported" + case .apiDisabled: + return "apiDisabled" + case .attributeUnsupported: + return "attributeUnsupported" + case .cannotComplete: + return "cannotComplete" + case .failure: + return "failure" + case .illegalArgument: + return "illegalArgument" + case .invalidUIElement: + return "invalidUIElement" + case .invalidUIElementObserver: + return "invalidUIElementObserver" + case .noValue: + return "noValue" + case .notEnoughPrecision: + return "notEnoughPrecision" + case .notImplemented: + return "notImplemented" + case .notificationAlreadyRegistered: + return "notificationAlreadyRegistered" + case .notificationNotRegistered: + return "notificationNotRegistered" + case .notificationUnsupported: + return "notificationUnsupported" + case .parameterizedAttributeUnsupported: + return "parameterizedAttributeUnsupported" + case .success: + return "success" + @unknown default: + return "unknown" + } +} + +func axErrorJSON(_ error: AXError) -> JSONObject { + [ + "code": Int(error.rawValue), + "description": describeAXError(error), + "name": axErrorName(error), + ] +} + +func cfTypeName(_ value: CFTypeRef?) -> String { + guard let value else { + return "none" + } + return (CFCopyTypeIDDescription(CFGetTypeID(value)) as String?) ?? "unknown" +} + +func throwAX(_ error: AXError, operation: String, staleMessage: String? = nil) throws { + if error == .success { + return + } + if error == .invalidUIElement { + try fail(ErrorCode.staleElement, staleMessage ?? "\(operation) failed because the accessibility element is no longer valid.") + } + if error == .apiDisabled { + try fail( + ErrorCode.inputUnavailable, + "\(operation) failed because the Accessibility API is disabled for the helper. Re-enable Accessibility access and restart it.", + data: [ + "accessibility": [ + "preflight": AXIsProcessTrusted(), + ], + "axError": axErrorJSON(error), + "reason": "api-disabled", + ] + ) + } + try fail(ErrorCode.automationFailed, "\(operation) failed because \(describeAXError(error)).") +} + +func copyAXAttribute(_ element: AXUIElement, _ attribute: String) throws -> CFTypeRef? { + var value: CFTypeRef? + let error = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + if error == .attributeUnsupported || error == .noValue { + return nil + } + try throwAX(error, operation: "Reading \(attribute)") + return value +} + +func copyOptionalAXAttribute(_ element: AXUIElement, _ attribute: String) throws -> CFTypeRef? { + var value: CFTypeRef? + let error = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + // Fabric elements can advertise optional metadata or geometry that disappears during a React remount. + if error == .attributeUnsupported || error == .noValue || error == .failure || error == .illegalArgument { + return nil + } + try throwAX(error, operation: "Reading \(attribute)") + return value +} + +func requiredAXRole(_ element: AXUIElement, staleMessage: String) throws -> String? { + var value: CFTypeRef? + let error = AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &value) + if error == .invalidUIElement || error == .illegalArgument { + try fail(ErrorCode.staleElement, staleMessage) + } + try throwAX(error, operation: "Reading \(kAXRoleAttribute)", staleMessage: staleMessage) + return axString(value) +} + +func copyAXActionNames(_ element: AXUIElement) throws -> [String] { + var names: CFArray? + let error = AXUIElementCopyActionNames(element, &names) + if error == .actionUnsupported { + return [] + } + try throwAX(error, operation: "Reading accessibility actions") + return names as? [String] ?? [] +} + +func axPoint(_ value: CFTypeRef?) -> CGPoint? { + guard let value, CFGetTypeID(value) == AXValueGetTypeID() else { + return nil + } + var point = CGPoint.zero + guard AXValueGetValue(value as! AXValue, .cgPoint, &point) else { + return nil + } + return point +} + +func axSize(_ value: CFTypeRef?) -> CGSize? { + guard let value, CFGetTypeID(value) == AXValueGetTypeID() else { + return nil + } + var size = CGSize.zero + guard AXValueGetValue(value as! AXValue, .cgSize, &size) else { + return nil + } + return size +} + +func axString(_ value: CFTypeRef?) -> String? { + guard let value else { + return nil + } + if CFGetTypeID(value) == CFStringGetTypeID() { + return value as? String + } + return nil +} + +func axNumber(_ value: CFTypeRef?) -> NSNumber? { + guard let value, CFGetTypeID(value) == CFNumberGetTypeID() || CFGetTypeID(value) == CFBooleanGetTypeID() else { + return nil + } + return value as? NSNumber +} + +func axElements(_ value: CFTypeRef?) -> [AXUIElement] { + guard let value, CFGetTypeID(value) == CFArrayGetTypeID() else { + return [] + } + return value as? [AXUIElement] ?? [] +} + +func strictAXElements(_ value: CFTypeRef?) -> [AXUIElement]? { + guard let value, CFGetTypeID(value) == CFArrayGetTypeID(), let items = value as? [AnyObject] else { + return nil + } + var elements: [AXUIElement] = [] + elements.reserveCapacity(items.count) + for item in items { + guard CFGetTypeID(item) == AXUIElementGetTypeID() else { + return nil + } + elements.append(item as! AXUIElement) + } + return elements +} + +func axElement(_ value: CFTypeRef?) -> AXUIElement? { + guard let value, CFGetTypeID(value) == AXUIElementGetTypeID() else { + return nil + } + return (value as! AXUIElement) +} + +func axFrame(_ element: AXUIElement) throws -> CGRect { + let position = axPoint(try copyAXAttribute(element, kAXPositionAttribute)) + let size = axSize(try copyAXAttribute(element, kAXSizeAttribute)) + guard let position, let size else { + return .zero + } + return CGRect(origin: position, size: size) +} + +func axOptionalFrame(_ element: AXUIElement) throws -> CGRect { + let position = axPoint(try copyOptionalAXAttribute(element, kAXPositionAttribute)) + let size = axSize(try copyOptionalAXAttribute(element, kAXSizeAttribute)) + guard let position, let size else { + return .zero + } + return CGRect(origin: position, size: size) +} + +struct CoreGraphicsWindowCandidate { + let frame: CGRect + let title: String? +} + +struct CoreGraphicsWindowFrameMatch { + let frame: CGRect? + let diagnostics: JSONObject +} + +func selectCoreGraphicsWindowFrame( + candidates: [CoreGraphicsWindowCandidate], + title: String, + authoritativeWindowCount: Int, + screenCaptureAccess: Bool +) -> CoreGraphicsWindowFrameMatch { + let uniqueFrames = candidates.reduce(into: [CGRect]()) { frames, candidate in + if !frames.contains(where: { $0.equalTo(candidate.frame) }) { + frames.append(candidate.frame) + } + } + var reason = "ambiguous" + var selected: CGRect? + let titleMatchingAvailable = candidates.contains(where: { $0.title != nil }) + if titleMatchingAvailable { + let titleMatches = candidates.filter { candidate in + title.isEmpty ? candidate.title?.isEmpty == true : candidate.title == title + } + if titleMatches.count == 1 { + reason = "unique-title" + selected = titleMatches[0].frame + } else if titleMatches.count > 1 { + let titleFrames = titleMatches.reduce(into: [CGRect]()) { frames, candidate in + if !frames.contains(where: { $0.equalTo(candidate.frame) }) { + frames.append(candidate.frame) + } + } + if titleFrames.count == 1 { + reason = "unique-title-geometry" + selected = titleFrames[0] + } + } else { + reason = "no-matching-title" + } + } else if authoritativeWindowCount == 1, candidates.count == 1 { + reason = "unique-candidate" + selected = candidates[0].frame + } else if authoritativeWindowCount == 1, uniqueFrames.count == 1, let frame = uniqueFrames.first { + reason = "unique-geometry" + selected = frame + } else { + reason = "title-unavailable" + } + let boundedCandidates = candidates.prefix(16).map { candidate -> JSONObject in + var value: JSONObject = ["frame": rectJSON(candidate.frame)] + if let title = candidate.title { + value["title"] = bounded(title, bytes: 256) + } + return value + } + return CoreGraphicsWindowFrameMatch( + frame: selected, + diagnostics: [ + "candidateCount": candidates.count, + "candidates": boundedCandidates, + "authoritativeWindowCount": authoritativeWindowCount, + "reason": reason, + "screenCaptureAccess": screenCaptureAccess, + "titleMatchingAvailable": titleMatchingAvailable, + "truncated": candidates.count > boundedCandidates.count, + "uniqueGeometryCount": uniqueFrames.count, + ] + ) +} + +func coreGraphicsWindowFrame( + processID: pid_t, + title: String, + authoritativeWindowCount: Int +) -> CoreGraphicsWindowFrameMatch { + let screenCaptureAccess = CGPreflightScreenCaptureAccess() + guard + let descriptions = CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) + as? [[String: Any]] + else { + return CoreGraphicsWindowFrameMatch( + frame: nil, + diagnostics: [ + "candidateCount": 0, + "reason": "enumeration-failed", + "screenCaptureAccess": screenCaptureAccess, + "titleMatchingAvailable": false, + ] + ) + } + let candidates = descriptions.compactMap { description -> CoreGraphicsWindowCandidate? in + guard + (description[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == processID, + (description[kCGWindowLayer as String] as? NSNumber)?.intValue == 0, + let bounds = description[kCGWindowBounds as String] as? [String: Any], + let frame = CGRect(dictionaryRepresentation: bounds as CFDictionary), + frame.width > 0, + frame.height > 0 + else { + return nil + } + return CoreGraphicsWindowCandidate( + frame: frame, + title: description[kCGWindowName as String] as? String + ) + } + return selectCoreGraphicsWindowFrame( + candidates: candidates, + title: title, + authoritativeWindowCount: authoritativeWindowCount, + screenCaptureAccess: screenCaptureAccess + ) +} + +func sameAXElement(_ left: AXUIElement, _ right: AXUIElement) -> Bool { + CFEqual(left, right) +} + +func uniqueAXElements(_ elements: [AXUIElement]) -> [AXUIElement] { + var elementsByHash: [CFHashCode: [AXUIElement]] = [:] + var result: [AXUIElement] = [] + for element in elements { + let hash = CFHash(element) + if elementsByHash[hash]?.contains(where: { sameAXElement($0, element) }) == true { + continue + } + elementsByHash[hash, default: []].append(element) + result.append(element) + } + return result +} diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/main.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/main.swift new file mode 100644 index 00000000000..3fa22130a25 --- /dev/null +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/main.swift @@ -0,0 +1,128 @@ +import Foundation +import Darwin + +private let maximumLedgerBytes = 16 * 1024 * 1024 + +private func readReleaseLedgerInput() throws -> Data { + var data = Data() + while true { + let chunk = try FileHandle.standardInput.read(upToCount: 4096) ?? Data() + if chunk.isEmpty { + return data + } + data.append(chunk) + if data.count > maximumLedgerBytes { + try fail(ErrorCode.invalidRequest, "The release ledger on standard input is too large.") + } + } +} + +private func printUsage() -> Int32 { + writeDiagnostic( + """ + Usage: furn-desktop-driver-host --handshake --json | --self-test | --stdio + furn-desktop-driver-host --permissions [--prompt] + furn-desktop-driver-host --release-input (reads {"keys":[],"buttons":[]} from stdin) + furn-desktop-driver-host --release-input --sweep (diagnostics: release held modifiers/buttons) + """ + ) + return 2 +} + +private func releaseInput(sweep: Bool) -> Int32 { + let input = InputController() + let token = CancellationToken() + do { + if sweep { + let counts = try PhysicalInputLock.shared.withLock( + token: token, + timeoutMilliseconds: 5_000, + policy: .adopt + ) { + try input.releaseDesktopModifiers() + } + print(#"{"releasedKeys":\#(counts.keys),"releasedButtons":\#(counts.buttons),"sweep":true}"#) + return 0 + } + guard isatty(STDIN_FILENO) == 0 else { + try fail( + ErrorCode.invalidRequest, + #"Standard input did not carry a release ledger such as {"keys":[],"buttons":[]}."# + ) + } + let data = try readReleaseLedgerInput() + guard !data.isEmpty else { + try fail( + ErrorCode.invalidRequest, + #"Standard input did not carry a release ledger such as {"keys":[],"buttons":[]}."# + ) + } + let ledger = try parseReleaseLedger(parseJSON(data)) + let counts = try PhysicalInputLock.shared.withLock( + token: token, + timeoutMilliseconds: 5_000, + policy: .adopt + ) { + try input.releaseExact(ledger) + } + print(#"{"releasedKeys":\#(counts.keys),"releasedButtons":\#(counts.buttons)}"#) + return 0 + } catch let error as HelperError { + writeDiagnostic("furn-desktop-driver-host: \(error.code): \(error.message)") + return error.code == ErrorCode.inputUnavailable ? 3 : 1 + } catch { + writeDiagnostic("furn-desktop-driver-host: \(error.localizedDescription)") + return 1 + } +} + +private func printPermissions(prompt: Bool) -> Int32 { + do { + FileHandle.standardOutput.write(try serializeJSON(createPermissionDiagnostic(prompt: prompt))) + FileHandle.standardOutput.write(Data([0x0A])) + return 0 + } catch let error as HelperError { + writeDiagnostic("furn-desktop-driver-host: \(error.code): \(error.message)") + return 1 + } catch { + writeDiagnostic("furn-desktop-driver-host: \(error.localizedDescription)") + return 1 + } +} + +let arguments = Array(CommandLine.arguments.dropFirst()) +let result: Int32 +switch arguments.first { +case "--handshake": + do { + FileHandle.standardOutput.write(try serializeJSON(createHello())) + FileHandle.standardOutput.write(Data([0x0A])) + result = 0 + } catch { + writeDiagnostic("furn-desktop-driver-host: \(error)") + result = 1 + } +case "--self-test": + result = arguments.count == 1 ? runSelfTest() : printUsage() +case "--permissions": + if arguments.count == 1 { + result = printPermissions(prompt: false) + } else if arguments.count == 2 && arguments[1] == "--prompt" { + result = printPermissions(prompt: true) + } else { + result = printUsage() + } +case "--release-input": + if arguments.count == 1 { + result = releaseInput(sweep: false) + } else if arguments.count == 2 && arguments[1] == "--sweep" { + result = releaseInput(sweep: true) + } else { + result = printUsage() + } +case "--stdio": + result = arguments.count == 1 ? Host().run() : printUsage() +default: + result = printUsage() +} +Darwin.exit(result) diff --git a/packages/agentic/desktop-driver/native/protocol.json b/packages/agentic/desktop-driver/native/protocol.json new file mode 100644 index 00000000000..089da6c656b --- /dev/null +++ b/packages/agentic/desktop-driver/native/protocol.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": 1, + "wireProtocol": { + "major": 1, + "minor": 0 + }, + "frame": { + "magic": "FDR1", + "headerBytes": 12, + "jsonType": 1, + "binaryType": 2 + } +} diff --git a/packages/agentic/desktop-driver/native/windows/DesktopDriverHost.vcxproj b/packages/agentic/desktop-driver/native/windows/DesktopDriverHost.vcxproj new file mode 100644 index 00000000000..ed8a6334d0c --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/DesktopDriverHost.vcxproj @@ -0,0 +1,111 @@ + + + + + Debug + x64 + + + Release + x64 + + + + {D8954580-603D-4836-9007-A6B302DBF3E4} + Win32Proj + FurnDesktopDriverHost + 10.0.22000.0 + 10.0 + + + + Application + v143 + true + Unicode + + + Application + v143 + false + true + Unicode + + + + + + + furn-desktop-driver-host + true + + + + true + stdcpp20 + MultiThreadedDebug + Level4 + $(FurnGeneratedDir);%(AdditionalIncludeDirectories) + UNICODE;_UNICODE;NOMINMAX;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) + + + windowsapp.lib;uiautomationcore.lib;d3d11.lib;dxgi.lib;windowscodecs.lib;dwmapi.lib;%(AdditionalDependencies) + Console + + + + + true + true + true + stdcpp20 + MultiThreaded + Level4 + $(FurnGeneratedDir);%(AdditionalIncludeDirectories) + UNICODE;_UNICODE;NOMINMAX;WIN32_LEAN_AND_MEAN;NDEBUG;%(PreprocessorDefinitions) + + + windowsapp.lib;uiautomationcore.lib;d3d11.lib;dxgi.lib;windowscodecs.lib;dwmapi.lib;%(AdditionalDependencies) + true + true + Console + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/agentic/desktop-driver/native/windows/README.md b/packages/agentic/desktop-driver/native/windows/README.md new file mode 100644 index 00000000000..2fdc9cdcb7c --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/README.md @@ -0,0 +1,131 @@ +# Windows native provider + +The Windows provider is a source-built C++20 x64 helper shared by React Native +Windows Fabric and React Native Win32 Paper. It implements the private FDR1 +protocol over stdio and has no listener, managed runtime, Windows App SDK, or +third-party native dependency. + +## Source layout + +| Source | Responsibility | +| -------------------------------- | ------------------------------------------------------------------------- | +| `DesktopDriverHost.vcxproj` | Warning-clean MSBuild project, C++20, static CRT | +| `src/host.*` | FDR1 request loop, cancellation, and command dispatch | +| `src/application.*` | Launch/attach leases, process and window ownership | +| `src/automation.*` | UI Automation tree, lookup, state, and opaque identities | +| `src/input.*` | Physical/accessibility activation, keyboard, pointer, wheel, and recovery | +| `src/inputlock.*` | Session-local cross-process physical-input mutex | +| `src/capture.*` | HWND Windows Graphics Capture, D3D11, WIC PNG output | +| `src/framing.*` and `src/json.*` | Bounded FDR1 framing and JSON | +| `src/driver.*` | Command surface and normalized provider results | +| `src/selftest.*` | Native parsing, state, cancellation, geometry, and recovery checks | + +The Node build coordinator is +`../../src/native/windows/buildWindowsDriver.ts`; runtime adaptation is under +`../../src/hosts/native`. + +## Toolchain and build + +Requirements: + +- Windows 11 x64; +- Visual Studio 2022 with **Desktop development with C++**; +- an installed MSVC x64 platform toolset; +- a Windows 10 or Windows 11 SDK. + +Build from any consumer that can resolve the package: + +```powershell +desktop-driver build-driver --platform windows +desktop-driver build-driver --platform win32 +``` + +Both endpoints resolve the same provider. The coordinator uses `vswhere.exe` to +locate Visual Studio, selects the newest installed MSVC toolset and Windows SDK, +and invokes the checked-in project with: + +- x64 platform; +- Debug or Release configuration; +- `/MT` static runtime; +- no restore; +- external intermediate and output directories; +- generated source/build identity outside the package. + +No CMake, NuGet restore, .NET runtime, or Windows App SDK is required by the +helper. + +## Application identity + +Launch requires: + +- `aumid` for a packaged app or `executablePath` for an unpackaged app; +- an exact `windowTitle`; +- optional fixed, server-owned arguments. + +Attach accepts an owner-generated nonce-bound lease, or an exact title only +when it resolves to one live process/window. Zero or multiple matches fail. +Leases record ownership, PID, process start, executable, and the primary window. +Attached apps survive WebDriver deletion; launched apps are closed only by +their exact recorded process identity. + +Unpackaged arguments use the standard `CommandLineToArgvW` quoting rules. +Launch ownership requires a non-preexisting window from the exact created +process or a verified descendant, plus a matching canonical executable or +AUMID and a process start after the launch request. Reused packaged processes +fail with guidance to use attach mode. Cancellation, launch failure, session +cleanup, and helper disposal close only causally owned processes and retain +the lease when forced termination cannot be proven. + +## Automation behavior + +The helper provides: + +- UI Automation lookup by `testID`, role, accessible name, and text; +- normalized element state with explicit unsupported values; +- live opaque window and element identity; +- physical SendInput click, keyboard, pointer, and wheel actions; +- accessibility activation where the control exposes a supported pattern; +- window activation and hit testing; +- complete source/tree output; +- occlusion-independent window capture and scaled element crops through + Windows Graphics Capture and WIC; +- exact input release on cancellation, teardown, or restricted crash recovery. + +Physical input requires an unlocked interactive desktop and compatible user +session/integrity. The helper advertises capability unavailability instead of +fabricating a pass. + +Windows and Win32 share implementation but retain distinct endpoint and +renderer capabilities. Minimized-window capture remains unsupported rather +than changing app state through an implicit restore. + +## Verification and tests + +Run package tests first, then the opt-in native contract: + +```powershell +yarn test +$env:FURN_NATIVE_DRIVER_TEST = '1' +yarn test --runInBand +``` + +The contract builds and resolves the helper, validates cache reuse and +handshakes, runs the native self-test, exercises malformed-frame recovery, and +verifies shared Windows/Win32 selection. + +Real qualification must run against the target React Native app. Use an +interactive runner for physical click, keyboard, wheel, multi-window, capture, +and 100/150/200 percent DPI checks. Hosted capability skips are not a physical +input pass. + +CI must build this helper explicitly into a job-local cache, run the opt-in +native contract once for the shared Windows/Win32 provider, verify each +endpoint with `desktop-driver doctor`, and set +`FURN_DESKTOP_DRIVER_BUILD_POLICY=never` before Storybook prep or smoke. This +keeps compilation outside the app lifecycle and turns compatibility drift into +an actionable cache failure. + +See [Native helpers](../../references/native-helpers.md) for cache and +verification policy, [CI integration](../../references/ci-integration.md) for +runner requirements, and [PLAN.md](../../PLAN.md) for unfinished Windows +qualification. diff --git a/packages/agentic/desktop-driver/native/windows/src/app.manifest b/packages/agentic/desktop-driver/native/windows/src/app.manifest new file mode 100644 index 00000000000..c92a74207e2 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/app.manifest @@ -0,0 +1,9 @@ + + + + + PerMonitorV2 + true + + + diff --git a/packages/agentic/desktop-driver/native/windows/src/application.cpp b/packages/agentic/desktop-driver/native/windows/src/application.cpp new file mode 100644 index 00000000000..90045141ae6 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/application.cpp @@ -0,0 +1,1240 @@ +#include "application.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "geometry.h" + +namespace furn { + +std::wstring QuoteWindowsCommandLineArgument(std::wstring_view value) { + std::wstring quoted(1, L'"'); + std::size_t backslashes = 0; + for (const wchar_t character : value) { + if (character == L'\\') { + backslashes += 1; + continue; + } + if (character == L'"') { + quoted.append(backslashes * 2 + 1, L'\\'); + quoted += L'"'; + backslashes = 0; + continue; + } + quoted.append(backslashes, L'\\'); + backslashes = 0; + quoted += character; + } + quoted.append(backslashes * 2, L'\\'); + quoted += L'"'; + return quoted; +} + +namespace { + +constexpr unsigned int kLaunchTimeoutMs = 90000; +constexpr unsigned int kLaunchGraceMs = 15000; +constexpr unsigned int kCloseTimeoutMs = 5000; +constexpr unsigned int kActivationTimeoutMs = 2000; +constexpr DWORD kLaunchCleanupGraceMs = 250; +constexpr DWORD kLaunchCleanupForcedMs = 1000; +constexpr unsigned int kDisposeGraceMs = 3000; +constexpr unsigned int kDisposeForcedMs = 3000; +constexpr DWORD kOwnedProcessAccess = PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE | PROCESS_TERMINATE; +// PowerShell writes the lease with 100-nanosecond precision; allow a small +// tolerance so rounding never rejects the correct process. +constexpr std::uint64_t kStartTimeToleranceTicks = 100000; // 10 milliseconds + +struct EnumerationContext { + DWORD processId{0}; + std::vector windows; +}; + +BOOL CALLBACK CollectWindow(HWND window, LPARAM parameter) { + auto* context = reinterpret_cast(parameter); + DWORD processId = 0; + GetWindowThreadProcessId(window, &processId); + if (context->processId != 0 && processId != context->processId) { + return TRUE; + } + if (IsWindowVisible(window) == FALSE) { + return TRUE; + } + if (GetAncestor(window, GA_ROOT) != window) { + return TRUE; + } + const LONG_PTR exStyle = GetWindowLongPtrW(window, GWL_EXSTYLE); + if ((exStyle & WS_EX_TOOLWINDOW) != 0 && ReadWindowTitle(window).empty()) { + return TRUE; + } + RECT bounds{}; + if (GetWindowRect(window, &bounds) == FALSE || IsEmptyRect(bounds)) { + return TRUE; + } + context->windows.push_back(window); + return TRUE; +} + +std::string ReadFileWithRetry(const std::wstring& path, const CancellationToken& token) { + for (int attempt = 0; attempt < 20; attempt += 1) { + token.ThrowIfCancelled(); + const HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) { + const DWORD error = GetLastError(); + if (error == ERROR_SHARING_VIOLATION) { + token.Wait(25); + continue; + } + FailLastError(kErrorLeaseInvalid, "Opening the application lease file", error); + } + std::string contents; + char buffer[4096]; + DWORD read = 0; + while (ReadFile(file, buffer, sizeof(buffer), &read, nullptr) != FALSE && read > 0) { + contents.append(buffer, read); + } + CloseHandle(file); + if (contents.size() >= 3 && static_cast(contents[0]) == 0xEF && + static_cast(contents[1]) == 0xBB && static_cast(contents[2]) == 0xBF) { + contents.erase(0, 3); + } + return contents; + } + Fail(kErrorLeaseInvalid, "The application lease file stayed locked by another writer."); +} + +winrt::handle OpenProcessForIdentity(DWORD processId, std::string_view failureCode) { + winrt::handle process{OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE, FALSE, processId)}; + if (!process) { + FailLastError(std::string(failureCode), "Opening the target application process", GetLastError()); + } + return process; +} + +bool IsProcessAlive(HANDLE process) { + if (process == nullptr) { + return false; + } + return WaitForSingleObject(process, 0) == WAIT_TIMEOUT; +} + +std::wstring NormalizeExecutablePath(std::wstring path) { + if (path.starts_with(L"\\\\?\\UNC\\")) { + return L"\\\\" + path.substr(8); + } + if (path.starts_with(L"\\\\?\\")) { + return path.substr(4); + } + return path; +} + +std::wstring CanonicalizeExecutablePath(const std::wstring& executablePath) { + const HANDLE rawFile = + CreateFileW(executablePath.c_str(), FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (rawFile == INVALID_HANDLE_VALUE) { + FailLastError(kErrorInvalidParams, "Opening the configured application executable", GetLastError()); + } + winrt::handle file; + file.attach(rawFile); + const DWORD required = + GetFinalPathNameByHandleW(file.get(), nullptr, 0, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); + if (required == 0) { + FailLastError(kErrorInvalidParams, "Resolving the configured application executable", GetLastError()); + } + std::wstring resolved(required, L'\0'); + const DWORD written = GetFinalPathNameByHandleW(file.get(), resolved.data(), required, + FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); + if (written == 0 || written >= required) { + FailLastError(kErrorInvalidParams, "Resolving the configured application executable", GetLastError()); + } + resolved.resize(written); + return NormalizeExecutablePath(std::move(resolved)); +} + +std::wstring ReadProcessApplicationUserModelId(HANDLE process) { + UINT32 length = 0; + const LONG measured = GetApplicationUserModelId(process, &length, nullptr); + if (measured != ERROR_INSUFFICIENT_BUFFER || length == 0) { + return {}; + } + std::wstring value(length, L'\0'); + if (GetApplicationUserModelId(process, &length, value.data()) != ERROR_SUCCESS || length == 0) { + return {}; + } + value.resize(length - 1); + return value; +} + +bool ProcessStartedForLaunch(HANDLE process, const FILETIME& launchStartedAt) { + FILETIME processStartedAt{}; + if (!TryReadProcessStartTime(process, processStartedAt)) { + return false; + } + return FileTimeTicks(processStartedAt) >= FileTimeTicks(launchStartedAt); +} + +bool ProcessMatchesLaunchIdentity(HANDLE process, const std::wstring& aumid, + const std::wstring& executablePath) { + if (!aumid.empty()) { + const std::wstring actualAumid = ReadProcessApplicationUserModelId(process); + return !actualAumid.empty() && _wcsicmp(actualAumid.c_str(), aumid.c_str()) == 0; + } + const std::wstring actualPath = NormalizeExecutablePath(ReadProcessImagePath(process)); + return !actualPath.empty() && _wcsicmp(actualPath.c_str(), executablePath.c_str()) == 0; +} + +winrt::handle DuplicateProcessHandle(HANDLE process) { + HANDLE duplicated = nullptr; + if (DuplicateHandle(GetCurrentProcess(), process, GetCurrentProcess(), &duplicated, 0, + FALSE, DUPLICATE_SAME_ACCESS) == FALSE) { + return {}; + } + winrt::handle result; + result.attach(duplicated); + return result; +} + +void PostCloseToExactProcessWindows(DWORD processId, const FILETIME& expectedStartTime) { + if (processId == 0 || FileTimeTicks(expectedStartTime) == 0) { + return; + } + for (const HWND window : EnumerateTopLevelWindows(processId)) { + DWORD owner = 0; + GetWindowThreadProcessId(window, &owner); + if (owner != processId) { + continue; + } + winrt::handle currentProcess{ + OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE, FALSE, owner)}; + FILETIME currentStartTime{}; + if (!currentProcess || + !TryReadProcessStartTime(currentProcess.get(), currentStartTime) || + FileTimeTicks(currentStartTime) != FileTimeTicks(expectedStartTime)) { + continue; + } + DWORD revalidatedOwner = 0; + GetWindowThreadProcessId(window, &revalidatedOwner); + if (revalidatedOwner == owner) { + PostMessageW(window, WM_CLOSE, 0, 0); + } + } +} + +struct TrackedLaunchProcess { + DWORD processId{0}; + FILETIME startTime{}; + winrt::handle process; +}; + +class LaunchProcessGuard { + public: + ~LaunchProcessGuard() { + try { + const std::string failure = Cleanup(); + if (!failure.empty()) { + std::cerr << "furn-desktop-driver-host: launch cleanup failed: " << failure << '\n'; + } + } catch (...) { + std::cerr << "furn-desktop-driver-host: launch cleanup failed with an unknown error.\n"; + } + } + + void Track(DWORD processId, winrt::handle process) { + if (!process || Find(processId) != nullptr) { + return; + } + FILETIME startTime{}; + TryReadProcessStartTime(process.get(), startTime); + processes_.push_back(TrackedLaunchProcess{processId, startTime, std::move(process)}); + } + + HANDLE Find(DWORD processId) const { + const auto entry = std::find_if(processes_.begin(), processes_.end(), + [processId](const TrackedLaunchProcess& candidate) { + return candidate.processId == processId; + }); + return entry == processes_.end() ? nullptr : entry->process.get(); + } + + winrt::handle Take(DWORD processId) { + const auto entry = std::find_if(processes_.begin(), processes_.end(), + [processId](const TrackedLaunchProcess& candidate) { + return candidate.processId == processId; + }); + if (entry == processes_.end()) { + return {}; + } + winrt::handle process = std::move(entry->process); + processes_.erase(entry); + return process; + } + + std::string Cleanup() { + std::string failures; + std::unordered_map terminationErrors; + std::vector handles; + for (TrackedLaunchProcess& tracked : processes_) { + if (!IsProcessAlive(tracked.process.get())) { + continue; + } + PostCloseToExactProcessWindows(tracked.processId, tracked.startTime); + handles.push_back(tracked.process.get()); + } + if (!handles.empty()) { + if (handles.size() <= MAXIMUM_WAIT_OBJECTS) { + WaitForMultipleObjects(static_cast(handles.size()), handles.data(), TRUE, + kLaunchCleanupGraceMs); + } else { + Sleep(kLaunchCleanupGraceMs); + } + } + for (TrackedLaunchProcess& tracked : processes_) { + if (!IsProcessAlive(tracked.process.get())) { + continue; + } + if (TerminateProcess(tracked.process.get(), 0) == FALSE && + IsProcessAlive(tracked.process.get())) { + terminationErrors.emplace(tracked.process.get(), GetLastError()); + } + } + if (!handles.empty()) { + if (handles.size() <= MAXIMUM_WAIT_OBJECTS) { + WaitForMultipleObjects(static_cast(handles.size()), handles.data(), TRUE, + kLaunchCleanupForcedMs); + } else { + Sleep(kLaunchCleanupForcedMs); + } + } + std::vector remaining; + for (TrackedLaunchProcess& tracked : processes_) { + if (!IsProcessAlive(tracked.process.get())) { + continue; + } + if (!failures.empty()) { + failures += "; "; + } + const auto terminationError = terminationErrors.find(tracked.process.get()); + failures += terminationError == terminationErrors.end() + ? "Process " + std::to_string(tracked.processId) + + " did not exit during bounded launch cleanup." + : "Terminating process " + std::to_string(tracked.processId) + + " failed: " + + DescribeHresult(HRESULT_FROM_WIN32(terminationError->second)); + remaining.push_back(std::move(tracked)); + } + processes_ = std::move(remaining); + return failures; + } + + std::vector ReleaseRemaining() { + return std::exchange(processes_, {}); + } + + private: + std::vector processes_; +}; + +struct LaunchRootIdentity { + DWORD processId{0}; + FILETIME startTime{}; + winrt::handle process; + bool trusted{false}; + + bool valid() const noexcept { return trusted && processId != 0 && process; } +}; + +std::unordered_map CaptureParentProcessIds() { + std::unordered_map parents; + const HANDLE rawSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (rawSnapshot == INVALID_HANDLE_VALUE) { + return parents; + } + winrt::handle snapshot; + snapshot.attach(rawSnapshot); + PROCESSENTRY32W entry{}; + entry.dwSize = sizeof(entry); + if (Process32FirstW(snapshot.get(), &entry) == FALSE) { + return parents; + } + do { + parents.emplace(entry.th32ProcessID, entry.th32ParentProcessID); + } while (Process32NextW(snapshot.get(), &entry) != FALSE); + return parents; +} + +bool IsCausallyRelatedProcess(DWORD candidateProcessId, + const FILETIME& candidateStartTime, + const LaunchRootIdentity& root, + const std::unordered_map& parents) { + if (candidateProcessId == 0 || !root.valid()) { + return false; + } + DWORD current = candidateProcessId; + std::uint64_t currentStartTicks = FileTimeTicks(candidateStartTime); + const std::uint64_t rootStartTicks = FileTimeTicks(root.startTime); + std::unordered_set visited; + for (std::size_t depth = 0; depth < 64 && current != 0 && visited.insert(current).second; + depth += 1) { + if (current == root.processId) { + return currentStartTicks == rootStartTicks; + } + const auto parent = parents.find(current); + if (parent == parents.end()) { + return false; + } + const DWORD parentProcessId = parent->second; + if (parentProcessId == root.processId) { + winrt::handle currentRoot{ + OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE, FALSE, + root.processId)}; + FILETIME currentRootStartTime{}; + if (!currentRoot) { + if (GetLastError() != ERROR_INVALID_PARAMETER) { + return false; + } + } else { + if (!TryReadProcessStartTime(currentRoot.get(), currentRootStartTime) || + FileTimeTicks(currentRootStartTime) != rootStartTicks) { + return false; + } + } + return rootStartTicks <= currentStartTicks; + } + winrt::handle parentProcess{ + OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE, FALSE, + parentProcessId)}; + FILETIME parentStartTime{}; + if (!parentProcess || !TryReadProcessStartTime(parentProcess.get(), parentStartTime)) { + return false; + } + const std::uint64_t parentStartTicks = FileTimeTicks(parentStartTime); + if (parentStartTicks < rootStartTicks || parentStartTicks > currentStartTicks) { + return false; + } + current = parentProcessId; + currentStartTicks = parentStartTicks; + } + return false; +} + +void TrackRelatedLaunchProcesses(const LaunchRootIdentity& root, + const std::wstring& aumid, + const std::wstring& executablePath, + const FILETIME& launchStartedAt, + LaunchProcessGuard& launchProcesses) { + if (!root.valid()) { + return; + } + const std::unordered_map parents = CaptureParentProcessIds(); + for (const auto& [processId, _parentProcessId] : parents) { + winrt::handle process{OpenProcess(kOwnedProcessAccess, FALSE, processId)}; + FILETIME processStartTime{}; + if (!process || !TryReadProcessStartTime(process.get(), processStartTime) || + !ProcessStartedForLaunch(process.get(), launchStartedAt) || + !IsCausallyRelatedProcess(processId, processStartTime, root, parents) || + !ProcessMatchesLaunchIdentity(process.get(), aumid, executablePath)) { + continue; + } + launchProcesses.Track(processId, std::move(process)); + } +} + +[[noreturn]] void RethrowWithCleanup(std::exception_ptr original, const std::string& cleanupFailure) { + if (cleanupFailure.empty()) { + std::rethrow_exception(original); + } + try { + std::rethrow_exception(original); + } catch (const std::exception& error) { + Fail(kErrorCleanupFailed, std::string(error.what()) + " Launch cleanup also failed: " + cleanupFailure); + } catch (...) { + Fail(kErrorCleanupFailed, "Application launch failed and cleanup also failed: " + cleanupFailure); + } +} + +struct ActivationOutcome { + bool activated{false}; + bool sawForeground{false}; +}; + +class ForegroundInputAttachment { + public: + explicit ForegroundInputAttachment(DWORD selfThread) noexcept : selfThread_(selfThread) {} + + ~ForegroundInputAttachment() { + if (otherThread_ != 0) { + AttachThreadInput(selfThread_, otherThread_, FALSE); + } + } + + bool Attach(DWORD otherThread) { + if (otherThread_ != 0 || otherThread == 0 || otherThread == selfThread_) { + return otherThread_ != 0; + } + if (AttachThreadInput(selfThread_, otherThread, TRUE) == FALSE) { + return false; + } + otherThread_ = otherThread; + return true; + } + + bool attached() const noexcept { return otherThread_ != 0; } + + private: + DWORD selfThread_{0}; + DWORD otherThread_{0}; +}; + +ActivationOutcome ActivateWindowInternal(HWND window, const CancellationToken& token) { + if (window == nullptr || IsWindow(window) == FALSE) { + Fail(kErrorNoSuchWindow, "The requested window is no longer available."); + } + if (IsIconic(window) != FALSE) { + ShowWindow(window, SW_RESTORE); + } + AllowSetForegroundWindow(ASFW_ANY); + const DWORD currentThread = GetCurrentThreadId(); + ForegroundInputAttachment inputAttachment(currentThread); + const Deadline deadline(kActivationTimeoutMs); + ActivationOutcome outcome; + while (true) { + token.ThrowIfCancelled(); + const HWND foreground = GetForegroundWindow(); + outcome.sawForeground = outcome.sawForeground || foreground != nullptr; + if (foreground == window || (foreground != nullptr && GetAncestor(foreground, GA_ROOT) == window)) { + outcome.activated = true; + break; + } + if (deadline.Expired()) { + break; + } + if (!inputAttachment.attached()) { + // Foreground ownership only transfers between threads that share an + // input queue, so borrow the current foreground thread's queue. + const DWORD candidate = GetWindowThreadProcessId(foreground != nullptr ? foreground : window, nullptr); + inputAttachment.Attach(candidate); + } + BringWindowToTop(window); + SetForegroundWindow(window); + SetActiveWindow(window); + token.Wait(50); + } + return outcome; +} + +} // namespace + +std::vector EnumerateTopLevelWindows(DWORD processId) { + EnumerationContext context; + context.processId = processId; + EnumWindows(CollectWindow, reinterpret_cast(&context)); + return context.windows; +} + +std::vector FindWindowsWithTitle(const std::wstring& title) { + std::vector matches; + for (const HWND window : EnumerateTopLevelWindows(0)) { + if (ReadWindowTitle(window) == title) { + matches.push_back(window); + } + } + return matches; +} + +std::wstring ReadWindowTitle(HWND window) { + const int length = GetWindowTextLengthW(window); + if (length <= 0) { + return {}; + } + std::wstring title(static_cast(length) + 1, L'\0'); + const int written = GetWindowTextW(window, title.data(), length + 1); + title.resize(written < 0 ? 0 : static_cast(written)); + return title; +} + +bool TryReadProcessStartTime(HANDLE process, FILETIME& startTime) { + FILETIME exitTime{}; + FILETIME kernelTime{}; + FILETIME userTime{}; + return GetProcessTimes(process, &startTime, &exitTime, &kernelTime, &userTime) != FALSE; +} + +std::wstring ReadProcessImagePath(HANDLE process) { + for (DWORD capacity = 1024; capacity <= 32768; capacity *= 2) { + std::wstring buffer(capacity, L'\0'); + DWORD size = capacity; + if (QueryFullProcessImageNameW(process, 0, buffer.data(), &size) != FALSE) { + buffer.resize(size); + return buffer; + } + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + return {}; + } + } + return {}; +} + +bool TryActivateWindow(HWND window, const CancellationToken& token) { + return ActivateWindowInternal(window, token).activated; +} + +void ActivateWindow(HWND window, const CancellationToken& token) { + const ActivationOutcome outcome = ActivateWindowInternal(window, token); + if (outcome.activated) { + return; + } + if (!outcome.sawForeground) { + Fail(kErrorWindowActivation, + "The session has no interactive foreground desktop, so the window could not be activated."); + } + Fail(kErrorWindowActivation, "The target window did not become the foreground window."); +} + +ApplicationManager::~ApplicationManager() { + try { + const CancellationToken idle; + Dispose(idle); + } catch (const std::exception& error) { + std::cerr << "furn-desktop-driver-host: application cleanup during shutdown failed: " + << error.what() << '\n'; + } catch (...) { + std::cerr << "furn-desktop-driver-host: application cleanup during shutdown failed.\n"; + } + for (auto& entry : leases_) { + if (entry.second.process != nullptr) { + CloseHandle(entry.second.process); + entry.second.process = nullptr; + } + } + for (PendingCleanupProcess& pending : pendingCleanupProcesses_) { + if (pending.process != nullptr) { + CloseHandle(pending.process); + pending.process = nullptr; + } + } +} + +void ApplicationManager::TrackPendingCleanupProcess(DWORD processId, HANDLE process) { + if (process == nullptr) { + return; + } + FILETIME startTime{}; + if (!TryReadProcessStartTime(process, startTime)) { + CloseHandle(process); + return; + } + try { + pendingCleanupProcesses_.push_back(PendingCleanupProcess{processId, startTime, process}); + } catch (...) { + CloseHandle(process); + throw; + } +} + +std::string ApplicationManager::RegisterWindow(HWND window, DWORD processId, const std::string& leaseId, + bool primary) { + const auto existing = windowIdsByHandle_.find(window); + if (existing != windowIdsByHandle_.end()) { + WindowRecord& record = windowsById_[existing->second]; + record.primary = primary || record.primary; + record.processId = processId; + record.leaseId = leaseId; + return existing->second; + } + WindowRecord record; + record.id = NextIdentifier("window"); + record.window = window; + record.processId = processId; + record.leaseId = leaseId; + record.primary = primary; + windowIdsByHandle_.emplace(window, record.id); + const std::string id = record.id; + windowsById_.emplace(id, std::move(record)); + return id; +} + +ApplicationLease& ApplicationManager::CreateLease(std::string ownership, DWORD processId, HANDLE process, + HWND primaryWindow, const std::wstring& title, + const json::Value& params) { + ApplicationLease lease; + lease.id = NextIdentifier("lease"); + lease.ownership = std::move(ownership); + lease.endpoint = params.StringField("endpoint", "windows"); + lease.storyRootTestId = params.StringField("storyRootTestId"); + lease.processId = processId; + lease.process = process; + lease.primaryWindow = primaryWindow; + lease.windowTitle = title; + if (process != nullptr) { + TryReadProcessStartTime(process, lease.startTime); + } + const std::string id = lease.id; + leases_.emplace(id, std::move(lease)); + leaseOrder_.push_back(id); + ApplicationLease& stored = leases_[id]; + if (primaryWindow != nullptr) { + RegisterWindow(primaryWindow, processId, id, true); + } + return stored; +} + +const ApplicationLease& ApplicationManager::Launch(const json::Value& params, const CancellationToken& token) { + const json::Value* application = params.Find("application"); + if (application == nullptr || !application->IsObject()) { + Fail(kErrorInvalidParams, "The launch request is missing its application descriptor."); + } + const std::wstring aumid = application->WideField("aumid"); + const std::wstring configuredExecutablePath = application->WideField("executablePath"); + const std::wstring windowTitle = application->WideField("windowTitle"); + if (aumid.empty() && configuredExecutablePath.empty()) { + Fail(kErrorInvalidParams, "The application descriptor needs an \"aumid\" or an \"executablePath\"."); + } + if (windowTitle.empty()) { + Fail(kErrorInvalidParams, "The application descriptor needs an exact \"windowTitle\" to identify its window."); + } + + const std::wstring executablePath = + aumid.empty() ? CanonicalizeExecutablePath(configuredExecutablePath) : std::wstring(); + std::wstring arguments; + if (const json::Value* list = application->Find("arguments"); list != nullptr) { + if (!list->IsArray()) { + Fail(kErrorInvalidParams, "The application descriptor \"arguments\" field must be an array."); + } + for (const json::Value& argument : list->Items()) { + if (!argument.IsString()) { + Fail(kErrorInvalidParams, "Every application argument must be a string."); + } + if (!arguments.empty()) { + arguments += L' '; + } + arguments += QuoteWindowsCommandLineArgument(argument.AsWide()); + } + } + + const std::vector preexisting = FindWindowsWithTitle(windowTitle); + FILETIME launchStartedAt{}; + GetSystemTimeAsFileTime(&launchStartedAt); + LaunchProcessGuard launchProcesses; + LaunchRootIdentity launchRoot; + try { + if (!aumid.empty()) { + winrt::com_ptr manager; + CheckHresult(kErrorLaunchFailed, "Creating the application activation manager", + CoCreateInstance(CLSID_ApplicationActivationManager, nullptr, CLSCTX_LOCAL_SERVER, + IID_PPV_ARGS(manager.put()))); + CoAllowSetForegroundWindow(manager.get(), nullptr); + CheckHresult(kErrorLaunchFailed, "Activating the packaged application", + manager->ActivateApplication(aumid.c_str(), arguments.empty() ? nullptr : arguments.c_str(), + AO_NONE, &launchRoot.processId)); + launchRoot.process.attach( + OpenProcess(kOwnedProcessAccess, FALSE, launchRoot.processId)); + if (!launchRoot.process) { + FailLastError(kErrorLaunchFailed, "Opening the activated application process", + GetLastError()); + } + if (!TryReadProcessStartTime(launchRoot.process.get(), launchRoot.startTime)) { + Fail(kErrorLaunchFailed, + "The activated application process identity could not be read."); + } + if (!ProcessStartedForLaunch(launchRoot.process.get(), launchStartedAt)) { + Fail(kErrorLaunchFailed, + "Packaged activation reused a pre-existing process. Use attach mode instead."); + } + launchRoot.trusted = true; + winrt::handle cleanupProcess = DuplicateProcessHandle(launchRoot.process.get()); + if (!cleanupProcess) { + const DWORD error = GetLastError(); + launchRoot.trusted = false; + launchProcesses.Track(launchRoot.processId, std::move(launchRoot.process)); + FailLastError(kErrorLaunchFailed, + "Duplicating the activated process handle for owned cleanup", + error); + } + launchProcesses.Track(launchRoot.processId, std::move(cleanupProcess)); + } else { + std::wstring commandLine = QuoteWindowsCommandLineArgument(executablePath); + if (!arguments.empty()) { + commandLine += L' ' + arguments; + } + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + PROCESS_INFORMATION information{}; + if (CreateProcessW(executablePath.c_str(), commandLine.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, + &startup, &information) == FALSE) { + FailLastError(kErrorLaunchFailed, "Starting the unpackaged application", GetLastError()); + } + CloseHandle(information.hThread); + winrt::handle created; + created.attach(information.hProcess); + launchRoot.processId = information.dwProcessId; + launchRoot.process = DuplicateProcessHandle(created.get()); + const DWORD duplicateError = launchRoot.process ? ERROR_SUCCESS : GetLastError(); + launchProcesses.Track(launchRoot.processId, std::move(created)); + if (!launchRoot.process) { + FailLastError(kErrorLaunchFailed, + "Duplicating the created process handle for identity tracking", + duplicateError); + } + if (!TryReadProcessStartTime(launchRoot.process.get(), launchRoot.startTime)) { + Fail(kErrorLaunchFailed, + "The created application process identity could not be read."); + } + launchRoot.trusted = true; + } + + // Packaged activation can report a short-lived broker. Only a new, + // identity-matched process with a non-preexisting window can become owned. + const Deadline deadline(kLaunchTimeoutMs); + std::optional graceDeadline; + HWND matched = nullptr; + DWORD matchedProcessId = 0; + while (matched == nullptr) { + token.ThrowIfCancelled(); + std::vector> candidates; + const std::unordered_map parents = CaptureParentProcessIds(); + for (const HWND window : FindWindowsWithTitle(windowTitle)) { + if (std::find(preexisting.begin(), preexisting.end(), window) != preexisting.end()) { + continue; + } + DWORD owner = 0; + GetWindowThreadProcessId(window, &owner); + winrt::handle candidate{OpenProcess(kOwnedProcessAccess, FALSE, owner)}; + FILETIME candidateStartTime{}; + if (!candidate || + !TryReadProcessStartTime(candidate.get(), candidateStartTime) || + !ProcessStartedForLaunch(candidate.get(), launchStartedAt) || + !IsCausallyRelatedProcess(owner, candidateStartTime, launchRoot, parents) || + !ProcessMatchesLaunchIdentity(candidate.get(), aumid, executablePath)) { + continue; + } + DWORD revalidatedOwner = 0; + GetWindowThreadProcessId(window, &revalidatedOwner); + if (revalidatedOwner != owner || ReadWindowTitle(window) != windowTitle) { + continue; + } + launchProcesses.Track(owner, std::move(candidate)); + candidates.emplace_back(window, owner); + } + if (candidates.size() > 1) { + Fail(kErrorAmbiguousTarget, + "The launch created multiple matching windows titled \"" + ToUtf8(windowTitle) + "\"."); + } + if (candidates.size() == 1) { + matched = candidates.front().first; + matchedProcessId = candidates.front().second; + break; + } + const HANDLE activatedProcess = launchProcesses.Find(launchRoot.processId); + if (activatedProcess != nullptr && !IsProcessAlive(activatedProcess)) { + if (!graceDeadline) { + graceDeadline.emplace(kLaunchGraceMs); + } else if (graceDeadline->Expired()) { + Fail(kErrorLaunchFailed, "The launched application exited before it showed its window."); + } + } + if (deadline.Expired()) { + Fail(kErrorLaunchFailed, + "The launched application did not present a new identity-matched window titled \"" + + ToUtf8(windowTitle) + "\"."); + } + token.Wait(100); + } + + winrt::handle owned = launchProcesses.Take(matchedProcessId); + if (!owned || !IsProcessAlive(owned.get())) { + Fail(kErrorLaunchFailed, "The launched application process exited before its lease was created."); + } + const std::string extraCleanupFailure = launchProcesses.Cleanup(); + if (!extraCleanupFailure.empty()) { + launchProcesses.Track(matchedProcessId, std::move(owned)); + Fail(kErrorCleanupFailed, "A secondary launch process could not be cleaned up: " + extraCleanupFailure); + } + return CreateLease("launched", matchedProcessId, owned.detach(), matched, windowTitle, params); + } catch (...) { + const std::exception_ptr original = std::current_exception(); + TrackRelatedLaunchProcesses(launchRoot, aumid, executablePath, launchStartedAt, + launchProcesses); + const std::string cleanupFailure = launchProcesses.Cleanup(); + for (TrackedLaunchProcess& tracked : launchProcesses.ReleaseRemaining()) { + TrackPendingCleanupProcess(tracked.processId, tracked.process.detach()); + } + RethrowWithCleanup(original, cleanupFailure); + } +} + +const ApplicationLease& ApplicationManager::Attach(const json::Value& params, const CancellationToken& token) { + const json::Value* application = params.Find("application"); + if (application == nullptr || !application->IsObject()) { + Fail(kErrorInvalidParams, "The attach request is missing its application descriptor."); + } + const std::string endpoint = params.StringField("endpoint", "windows"); + const std::wstring leasePath = application->WideField("leasePath"); + std::wstring windowTitle = application->WideField("windowTitle"); + + if (!leasePath.empty()) { + const std::string contents = ReadFileWithRetry(leasePath, token); + if (TrimAscii(contents).empty()) { + Fail(kErrorLeaseInvalid, "The application lease file is empty."); + } + const json::Value lease = json::Value::Parse(contents); + if (!lease.IsObject() || lease.NumberField("schemaVersion", 0) != 1) { + Fail(kErrorLeaseInvalid, "The application lease does not use schema version 1."); + } + const std::string expectedNonce = application->StringField("leaseNonce"); + if (expectedNonce.empty()) { + Fail(kErrorLeaseInvalid, "The application descriptor does not carry the expected lease nonce."); + } + if (lease.StringField("nonce") != expectedNonce) { + Fail(kErrorLeaseInvalid, "The application lease nonce does not match this session."); + } + if (lease.StringField("endpoint") != endpoint) { + Fail(kErrorLeaseInvalid, "The application lease was written for a different endpoint."); + } + const DWORD processId = static_cast(lease.NumberField("processId", 0)); + if (processId == 0) { + Fail(kErrorLeaseInvalid, "The application lease does not name a process."); + } + winrt::handle process = OpenProcessForIdentity(processId, kErrorLeaseInvalid); + if (!IsProcessAlive(process.get())) { + Fail(kErrorLeaseInvalid, "The leased application process is no longer running."); + } + FILETIME actualStart{}; + if (!TryReadProcessStartTime(process.get(), actualStart)) { + Fail(kErrorLeaseInvalid, "The leased application process identity could not be read."); + } + FILETIME expectedStart{}; + if (!TryParseIso8601(lease.StringField("processStartedAt"), expectedStart)) { + Fail(kErrorLeaseInvalid, "The application lease does not carry a valid process start time."); + } + const std::uint64_t actualTicks = FileTimeTicks(actualStart); + const std::uint64_t expectedTicks = FileTimeTicks(expectedStart); + const std::uint64_t difference = actualTicks > expectedTicks ? actualTicks - expectedTicks + : expectedTicks - actualTicks; + if (difference > kStartTimeToleranceTicks) { + Fail(kErrorLeaseInvalid, "The leased process identifier was reused by a different process."); + } + const std::wstring leaseImage = lease.WideField("executablePath"); + if (!leaseImage.empty()) { + const std::wstring actualImage = ReadProcessImagePath(process.get()); + if (!actualImage.empty() && _wcsicmp(actualImage.c_str(), leaseImage.c_str()) != 0) { + Fail(kErrorLeaseInvalid, "The leased process does not run the expected executable."); + } + } + const std::wstring leaseTitle = lease.WideField("windowTitle"); + if (!leaseTitle.empty()) { + if (!windowTitle.empty() && windowTitle != leaseTitle) { + Fail(kErrorLeaseInvalid, "The application lease names a different window title than the target."); + } + windowTitle = leaseTitle; + } + if (windowTitle.empty()) { + Fail(kErrorLeaseInvalid, "The application lease does not name the application window."); + } + + std::vector matches; + for (const HWND window : EnumerateTopLevelWindows(processId)) { + if (ReadWindowTitle(window) == windowTitle) { + matches.push_back(window); + } + } + if (matches.empty()) { + Fail(kErrorAttachFailed, + "The leased application does not currently show a window titled \"" + ToUtf8(windowTitle) + "\"."); + } + if (matches.size() > 1) { + Fail(kErrorAmbiguousTarget, + "The leased application shows " + std::to_string(matches.size()) + " windows titled \"" + + ToUtf8(windowTitle) + "\"."); + } + return CreateLease("attached", processId, process.detach(), matches.front(), windowTitle, params); + } + + if (windowTitle.empty()) { + Fail(kErrorInvalidParams, + "Attaching without a lease requires an exact \"windowTitle\" in the application descriptor."); + } + std::vector matches; + for (const HWND window : FindWindowsWithTitle(windowTitle)) { + token.ThrowIfCancelled(); + matches.push_back(window); + } + if (matches.empty()) { + Fail(kErrorAttachFailed, "No window titled \"" + ToUtf8(windowTitle) + "\" is currently open."); + } + if (matches.size() > 1) { + Fail(kErrorAmbiguousTarget, + std::to_string(matches.size()) + " windows are titled \"" + ToUtf8(windowTitle) + + "\", so the target is ambiguous."); + } + DWORD processId = 0; + GetWindowThreadProcessId(matches.front(), &processId); + winrt::handle process = OpenProcessForIdentity(processId, kErrorAttachFailed); + return CreateLease("attached", processId, process.detach(), matches.front(), windowTitle, params); +} + +std::vector ApplicationManager::Windows(const std::string& leaseId, const CancellationToken& token) { + const auto entry = leases_.find(leaseId); + if (entry == leases_.end()) { + Fail(kErrorNoSuchLease, "The requested application lease is not held by this helper."); + } + ApplicationLease& lease = entry->second; + token.ThrowIfCancelled(); + if (!IsProcessAlive(lease.process)) { + Fail(kErrorNoSuchLease, "The leased application process has exited."); + } + + std::vector results; + for (const HWND window : EnumerateTopLevelWindows(lease.processId)) { + WindowInfo info; + info.window = window; + info.processId = lease.processId; + info.title = ReadWindowTitle(window); + info.primary = window == lease.primaryWindow || (lease.primaryWindow == nullptr && info.title == lease.windowTitle); + info.id = RegisterWindow(window, lease.processId, leaseId, info.primary); + results.push_back(std::move(info)); + } + if (lease.primaryWindow != nullptr && IsWindow(lease.primaryWindow) == FALSE) { + lease.primaryWindow = nullptr; + } + std::stable_sort(results.begin(), results.end(), + [](const WindowInfo& left, const WindowInfo& right) { return left.primary && !right.primary; }); + if (lease.primaryWindow == nullptr && !results.empty()) { + lease.primaryWindow = results.front().window; + windowsById_[results.front().id].primary = true; + } + return results; +} + +void ApplicationManager::CloseApplication(const std::string& leaseId, const CancellationToken& token) { + const auto entry = leases_.find(leaseId); + if (entry == leases_.end()) { + // Closing an unknown lease is a no-op so teardown stays idempotent. + return; + } + ApplicationLease& lease = entry->second; + + // The lease and its window records stay intact until the close sequence + // finishes, so a cancelled teardown remains retryable instead of orphaning an + // owned process together with its only handle. + if (lease.ownership == "launched" && lease.process != nullptr && + IsProcessAlive(lease.process)) { + PostCloseToExactProcessWindows(lease.processId, lease.startTime); + const Deadline deadline(kCloseTimeoutMs); + while (IsProcessAlive(lease.process) && !deadline.Expired()) { + token.Wait(50); + } + if (IsProcessAlive(lease.process)) { + if (TerminateProcess(lease.process, 0) == FALSE && IsProcessAlive(lease.process)) { + FailLastError(kErrorCleanupFailed, "Terminating the owned application process", GetLastError()); + } + const DWORD waited = WaitForSingleObject(lease.process, 2000); + if (waited == WAIT_FAILED && IsProcessAlive(lease.process)) { + FailLastError(kErrorCleanupFailed, "Waiting for the owned application process", GetLastError()); + } + if (waited != WAIT_OBJECT_0 && IsProcessAlive(lease.process)) { + Fail(kErrorCleanupFailed, "The owned application process did not exit after forced termination."); + } + } + } + + if (lease.process != nullptr) { + CloseHandle(lease.process); + lease.process = nullptr; + } + for (auto iterator = windowsById_.begin(); iterator != windowsById_.end();) { + if (iterator->second.leaseId == leaseId) { + windowIdsByHandle_.erase(iterator->second.window); + iterator = windowsById_.erase(iterator); + continue; + } + ++iterator; + } + leases_.erase(leaseId); + leaseOrder_.erase(std::remove(leaseOrder_.begin(), leaseOrder_.end(), leaseId), leaseOrder_.end()); +} + +void ApplicationManager::Dispose(const CancellationToken& token) { + const std::vector leaseIds = leaseOrder_; + std::string failures; + std::vector ownedProcesses; + for (PendingCleanupProcess& pending : pendingCleanupProcesses_) { + if (IsProcessAlive(pending.process)) { + PostCloseToExactProcessWindows(pending.processId, pending.startTime); + ownedProcesses.push_back(pending.process); + } + } + for (const std::string& leaseId : leaseIds) { + const auto entry = leases_.find(leaseId); + if (entry == leases_.end()) { + continue; + } + ApplicationLease& lease = entry->second; + if (lease.ownership == "launched" && IsProcessAlive(lease.process)) { + PostCloseToExactProcessWindows(lease.processId, lease.startTime); + ownedProcesses.push_back(lease.process); + } + } + + const auto waitForOwnedProcesses = [&](unsigned int timeoutMs) { + const Deadline deadline(timeoutMs); + while (!deadline.Expired()) { + token.ThrowIfCancelled(); + if (std::none_of(ownedProcesses.begin(), ownedProcesses.end(), IsProcessAlive)) { + return; + } + token.Wait(50); + } + }; + + waitForOwnedProcesses(kDisposeGraceMs); + std::unordered_map terminationErrors; + for (const HANDLE process : ownedProcesses) { + if (!IsProcessAlive(process)) { + continue; + } + if (TerminateProcess(process, 0) == FALSE && IsProcessAlive(process)) { + terminationErrors.emplace(process, GetLastError()); + } + } + waitForOwnedProcesses(kDisposeForcedMs); + + for (auto iterator = pendingCleanupProcesses_.begin(); + iterator != pendingCleanupProcesses_.end();) { + if (IsProcessAlive(iterator->process)) { + if (!failures.empty()) { + failures += "; "; + } + const auto terminationError = terminationErrors.find(iterator->process); + failures += terminationError == terminationErrors.end() + ? "Pending owned process " + std::to_string(iterator->processId) + + " did not exit during disposal." + : "Terminating pending owned process " + + std::to_string(iterator->processId) + " failed: " + + DescribeHresult(HRESULT_FROM_WIN32(terminationError->second)); + ++iterator; + continue; + } + CloseHandle(iterator->process); + iterator = pendingCleanupProcesses_.erase(iterator); + } + for (const std::string& leaseId : leaseIds) { + const auto entry = leases_.find(leaseId); + if (entry == leases_.end()) { + continue; + } + ApplicationLease& lease = entry->second; + if (lease.ownership == "launched" && IsProcessAlive(lease.process)) { + if (!failures.empty()) { + failures += "; "; + } + const auto terminationError = terminationErrors.find(lease.process); + failures += terminationError == terminationErrors.end() + ? "Owned application process " + std::to_string(lease.processId) + + " did not exit during disposal." + : "Terminating owned application process " + + std::to_string(lease.processId) + " failed: " + + DescribeHresult(HRESULT_FROM_WIN32(terminationError->second)); + continue; + } + CloseApplication(leaseId, token); + } + if (!failures.empty()) { + Fail(kErrorCleanupFailed, "Disposing native application leases failed: " + failures); + } +} + +WindowInfo ApplicationManager::RequireWindow(const std::string& windowId) { + const auto entry = windowsById_.find(windowId); + if (entry == windowsById_.end()) { + Fail(kErrorNoSuchWindow, "Window \"" + windowId + "\" is not tracked by this helper."); + } + const WindowRecord& record = entry->second; + if (record.window == nullptr || IsWindow(record.window) == FALSE) { + Fail(kErrorNoSuchWindow, "Window \"" + windowId + "\" has been closed."); + } + DWORD processId = 0; + GetWindowThreadProcessId(record.window, &processId); + if (record.processId != 0 && processId != record.processId) { + Fail(kErrorNoSuchWindow, "Window \"" + windowId + "\" now belongs to a different process."); + } + WindowInfo info; + info.id = record.id; + info.window = record.window; + info.processId = record.processId; + info.primary = record.primary; + info.title = ReadWindowTitle(record.window); + return info; +} + +const ApplicationLease* ApplicationManager::FindLease(const std::string& leaseId) const { + const auto entry = leases_.find(leaseId); + return entry == leases_.end() ? nullptr : &entry->second; +} + +const ApplicationLease* ApplicationManager::LeaseForWindow(HWND window) const { + const auto id = windowIdsByHandle_.find(window); + if (id == windowIdsByHandle_.end()) { + return nullptr; + } + const auto record = windowsById_.find(id->second); + if (record == windowsById_.end()) { + return nullptr; + } + return FindLease(record->second.leaseId); +} + +const ApplicationLease* ApplicationManager::PrimaryLease() const { + for (const std::string& id : leaseOrder_) { + const auto entry = leases_.find(id); + if (entry != leases_.end()) { + return &entry->second; + } + } + return nullptr; +} + +std::vector ApplicationManager::WindowIdsForLease(const std::string& leaseId) const { + std::vector ids; + for (const auto& entry : windowsById_) { + if (entry.second.leaseId == leaseId) { + ids.push_back(entry.first); + } + } + return ids; +} + +void ApplicationManager::ForgetWindow(const std::string& windowId) { + const auto entry = windowsById_.find(windowId); + if (entry == windowsById_.end()) { + return; + } + windowIdsByHandle_.erase(entry->second.window); + windowsById_.erase(entry); +} + +json::Value ApplicationManager::SerializeLease(const ApplicationLease& lease) const { + json::Value value = json::Value::Object(); + value.Set("id", json::Value::String(lease.id)); + value.Set("ownership", json::Value::String(lease.ownership)); + value.Set("processId", json::Value::Integer(static_cast(lease.processId))); + const std::string started = FormatFileTimeIso8601(lease.startTime); + if (!started.empty()) { + value.Set("processStartedAt", json::Value::String(started)); + } + return value; +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/application.h b/packages/agentic/desktop-driver/native/windows/src/application.h new file mode 100644 index 00000000000..30d3f04256d --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/application.h @@ -0,0 +1,92 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include "common.h" +#include "json.h" + +namespace furn { + +struct WindowInfo { + std::string id; + HWND window{nullptr}; + DWORD processId{0}; + std::wstring title; + bool primary{false}; +}; + +struct ApplicationLease { + std::string id; + std::string ownership; // "attached" or "launched" + std::string endpoint; + std::string storyRootTestId; + DWORD processId{0}; + FILETIME startTime{}; + std::wstring windowTitle; + HANDLE process{nullptr}; + HWND primaryWindow{nullptr}; +}; + +// Owns application identity, launch/attach leases, and the opaque window table. +// Attached applications are never terminated; owned launches close only the +// exact windows and process that this helper created. +class ApplicationManager { + public: + ~ApplicationManager(); + + const ApplicationLease& Launch(const json::Value& params, const CancellationToken& token); + const ApplicationLease& Attach(const json::Value& params, const CancellationToken& token); + void CloseApplication(const std::string& leaseId, const CancellationToken& token); + void Dispose(const CancellationToken& token); + std::vector Windows(const std::string& leaseId, const CancellationToken& token); + + WindowInfo RequireWindow(const std::string& windowId); + const ApplicationLease* FindLease(const std::string& leaseId) const; + const ApplicationLease* LeaseForWindow(HWND window) const; + const ApplicationLease* PrimaryLease() const; + std::vector WindowIdsForLease(const std::string& leaseId) const; + void ForgetWindow(const std::string& windowId); + json::Value SerializeLease(const ApplicationLease& lease) const; + + private: + std::string RegisterWindow(HWND window, DWORD processId, const std::string& leaseId, bool primary); + ApplicationLease& CreateLease(std::string ownership, DWORD processId, HANDLE process, HWND primaryWindow, + const std::wstring& title, const json::Value& params); + void TrackPendingCleanupProcess(DWORD processId, HANDLE process); + + struct WindowRecord { + std::string id; + HWND window{nullptr}; + DWORD processId{0}; + std::string leaseId; + bool primary{false}; + }; + + struct PendingCleanupProcess { + DWORD processId{0}; + FILETIME startTime{}; + HANDLE process{nullptr}; + }; + + std::unordered_map leases_; + std::vector leaseOrder_; + std::unordered_map windowsById_; + std::unordered_map windowIdsByHandle_; + std::vector pendingCleanupProcesses_; +}; + +std::vector EnumerateTopLevelWindows(DWORD processId); +std::vector FindWindowsWithTitle(const std::wstring& title); +std::wstring ReadWindowTitle(HWND window); +bool TryReadProcessStartTime(HANDLE process, FILETIME& startTime); +std::wstring ReadProcessImagePath(HANDLE process); +std::wstring QuoteWindowsCommandLineArgument(std::wstring_view value); +void ActivateWindow(HWND window, const CancellationToken& token); +bool TryActivateWindow(HWND window, const CancellationToken& token); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/automation.cpp b/packages/agentic/desktop-driver/native/windows/src/automation.cpp new file mode 100644 index 00000000000..3c5cb1ac5ee --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/automation.cpp @@ -0,0 +1,1108 @@ +#include "automation.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace furn { +namespace { + +constexpr std::size_t kMaximumAncestorWalk = 64; +// Provider mutations run on their application's UI thread, which a suspended or +// busy application can stall indefinitely. Every mutation therefore runs as a +// bounded step that reports an explicit failure instead of wedging the helper. +constexpr DWORD kMutationTimeoutMs = 5000; + +struct BoundedState { + std::atomic result{E_PENDING}; + HANDLE done{nullptr}; + + ~BoundedState() { + if (done != nullptr) { + CloseHandle(done); + } + } +}; + +HRESULT RunBounded(std::function call, DWORD timeoutMs, bool& completed) { + const auto state = std::make_shared(); + state->done = CreateEventW(nullptr, TRUE, FALSE, nullptr); + if (state->done == nullptr) { + completed = true; + return call(); + } + std::thread worker([state, call]() { + const HRESULT initialized = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + state->result.store(call()); + SetEvent(state->done); + if (SUCCEEDED(initialized)) { + CoUninitialize(); + } + }); + worker.detach(); + completed = WaitForSingleObject(state->done, timeoutMs) == WAIT_OBJECT_0; + return completed ? state->result.load() : E_PENDING; +} + +template +HRESULT BoundedCall(Fn&& call, std::string_view operation, std::string code) { + bool completed = false; + const HRESULT result = RunBounded(std::function(std::forward(call)), kMutationTimeoutMs, completed); + if (!completed) { + Fail(std::move(code), + std::string(operation) + " did not complete within " + std::to_string(kMutationTimeoutMs) + " ms."); + } + return result; +} + +// Small RAII wrapper so every cached property read clears its VARIANT. +class Variant { + public: + Variant() { VariantInit(&value_); } + ~Variant() { VariantClear(&value_); } + Variant(const Variant&) = delete; + Variant& operator=(const Variant&) = delete; + + VARIANT* put() { + VariantClear(&value_); + VariantInit(&value_); + return &value_; + } + + const VARIANT& get() const noexcept { return value_; } + VARTYPE type() const noexcept { return value_.vt; } + + private: + VARIANT value_{}; +}; + +struct RoleMapping { + CONTROLTYPEID controlType; + const char* role; +}; + +constexpr RoleMapping kRoleMappings[] = { + {UIA_ButtonControlTypeId, "button"}, {UIA_CalendarControlTypeId, "calendar"}, + {UIA_CheckBoxControlTypeId, "checkbox"}, {UIA_ComboBoxControlTypeId, "combobox"}, + {UIA_EditControlTypeId, "textbox"}, {UIA_HyperlinkControlTypeId, "link"}, + {UIA_ImageControlTypeId, "image"}, {UIA_ListItemControlTypeId, "listitem"}, + {UIA_ListControlTypeId, "list"}, {UIA_MenuControlTypeId, "menu"}, + {UIA_MenuBarControlTypeId, "menubar"}, {UIA_MenuItemControlTypeId, "menuitem"}, + {UIA_ProgressBarControlTypeId, "progressbar"}, {UIA_RadioButtonControlTypeId, "radio"}, + {UIA_ScrollBarControlTypeId, "scrollbar"}, {UIA_SliderControlTypeId, "slider"}, + {UIA_SpinnerControlTypeId, "spinbutton"}, {UIA_StatusBarControlTypeId, "status"}, + {UIA_TabControlTypeId, "tablist"}, {UIA_TabItemControlTypeId, "tab"}, + {UIA_TextControlTypeId, "text"}, {UIA_ToolBarControlTypeId, "toolbar"}, + {UIA_ToolTipControlTypeId, "tooltip"}, {UIA_TreeControlTypeId, "tree"}, + {UIA_TreeItemControlTypeId, "treeitem"}, {UIA_CustomControlTypeId, "custom"}, + {UIA_GroupControlTypeId, "group"}, {UIA_ThumbControlTypeId, "thumb"}, + {UIA_DataGridControlTypeId, "grid"}, {UIA_DataItemControlTypeId, "dataitem"}, + {UIA_DocumentControlTypeId, "document"}, {UIA_SplitButtonControlTypeId, "splitbutton"}, + {UIA_WindowControlTypeId, "window"}, {UIA_PaneControlTypeId, "pane"}, + {UIA_HeaderControlTypeId, "header"}, {UIA_HeaderItemControlTypeId, "headeritem"}, + {UIA_TableControlTypeId, "table"}, {UIA_TitleBarControlTypeId, "titlebar"}, + {UIA_SeparatorControlTypeId, "separator"}, {UIA_SemanticZoomControlTypeId, "semanticzoom"}, + {UIA_AppBarControlTypeId, "appbar"}, +}; + +struct RoleAlias { + const char* query; + const char* role; +}; + +constexpr RoleAlias kRoleAliases[] = { + {"edit", "textbox"}, {"input", "textbox"}, {"textfield", "textbox"}, {"hyperlink", "link"}, + {"radiobutton", "radio"},{"checkbox", "checkbox"},{"combo box", "combobox"},{"listbox", "list"}, + {"statusbar", "status"}, {"tabitem", "tab"}, {"tablist", "tablist"}, {"progress", "progressbar"}, + {"spinner", "spinbutton"},{"application", "window"}, +}; + +std::string BstrToUtf8(BSTR value) { + if (value == nullptr) { + return {}; + } + return ToUtf8(std::wstring_view(value, SysStringLen(value))); +} + +std::string VariantString(const VARIANT& value) { + if (value.vt != VT_BSTR || value.bstrVal == nullptr) { + return {}; + } + return BstrToUtf8(value.bstrVal); +} + +std::string EscapeXml(std::string_view value) { + std::string output; + output.reserve(value.size()); + for (const char character : value) { + switch (character) { + case '&': + output += "&"; + break; + case '<': + output += "<"; + break; + case '>': + output += ">"; + break; + case '"': + output += """; + break; + case '\'': + output += "'"; + break; + default: + if (static_cast(character) < 0x20 && character != '\t' && character != '\n' && + character != '\r') { + output += ' '; + } else { + output.push_back(character); + } + break; + } + } + return output; +} + +json::Value SerializeSupported(const SupportedBool& state) { + json::Value value = json::Value::Object(); + if (state.supported) { + value.Set("supported", json::Value::Bool(true)); + value.Set("value", json::Value::Bool(state.value)); + } else { + value.Set("supported", json::Value::Bool(false)); + value.Set("reason", json::Value::String(state.reason)); + } + return value; +} + +json::Value SerializeChecked(const SupportedChecked& state) { + json::Value value = json::Value::Object(); + if (!state.supported) { + value.Set("supported", json::Value::Bool(false)); + value.Set("reason", json::Value::String(state.reason)); + return value; + } + value.Set("supported", json::Value::Bool(true)); + if (state.value == CheckedValue::Mixed) { + value.Set("value", json::Value::String(std::string("mixed"))); + } else { + value.Set("value", json::Value::Bool(state.value == CheckedValue::True)); + } + return value; +} + +SupportedBool Unsupported(std::string reason) { + SupportedBool state; + state.supported = false; + state.reason = std::move(reason); + return state; +} + +std::string FormatNumber(double value) { + return json::Value::Number(RoundToHundredths(value)).Serialize(); +} + +} // namespace + +std::string RoleForControlType(CONTROLTYPEID controlType) { + for (const RoleMapping& mapping : kRoleMappings) { + if (mapping.controlType == controlType) { + return mapping.role; + } + } + return "unknown"; +} + +std::string NormalizeRoleQuery(std::string_view value) { + std::string normalized = ToLowerAscii(TrimAscii(value)); + for (const RoleAlias& alias : kRoleAliases) { + if (normalized == alias.query) { + return alias.role; + } + } + return normalized; +} + +const char* ScopeName(ElementScope scope) { + switch (scope) { + case ElementScope::Application: + return "application"; + case ElementScope::Preview: + return "preview"; + case ElementScope::SecondaryWindow: + return "secondary-window"; + case ElementScope::Chrome: + default: + return "chrome"; + } +} + +bool MatchesSelector(const ElementSnapshot& snapshot, const Selector& selector) { + if (selector.strategy == "accessibility id") { + return !snapshot.automationId.empty() && snapshot.automationId == selector.value; + } + if (selector.strategy == "tag name") { + return snapshot.role == NormalizeRoleQuery(selector.value); + } + if (selector.strategy == "link text") { + return snapshot.name == selector.value; + } + if (selector.strategy == "partial link text") { + return !selector.value.empty() && snapshot.name.find(selector.value) != std::string::npos; + } + if (selector.strategy == "-furn:text") { + return (snapshot.hasText && snapshot.text == selector.value) || + (snapshot.hasValue && snapshot.value == selector.value) || snapshot.name == selector.value; + } + Fail(kErrorInvalidParams, "Locator strategy \"" + selector.strategy + "\" is not supported."); +} + +json::Value SerializeSnapshot(const ElementSnapshot& snapshot) { + json::Value value = json::Value::Object(); + value.Set("id", json::Value::String(snapshot.id)); + if (!snapshot.automationId.empty()) { + value.Set("automationId", json::Value::String(snapshot.automationId)); + } + value.Set("checked", SerializeChecked(snapshot.checked)); + value.Set("enabled", SerializeSupported(snapshot.enabled)); + value.Set("expanded", SerializeSupported(snapshot.expanded)); + value.Set("focused", SerializeSupported(snapshot.focused)); + if (!snapshot.name.empty()) { + value.Set("name", json::Value::String(snapshot.name)); + } + if (!snapshot.parentId.empty()) { + value.Set("parentId", json::Value::String(snapshot.parentId)); + } + json::Value rect = json::Value::Object(); + rect.Set("height", json::Value::Number(snapshot.rect.height)); + rect.Set("width", json::Value::Number(snapshot.rect.width)); + rect.Set("x", json::Value::Number(snapshot.rect.x)); + rect.Set("y", json::Value::Number(snapshot.rect.y)); + value.Set("rect", rect); + value.Set("role", json::Value::String(snapshot.role)); + value.Set("scope", json::Value::String(std::string(ScopeName(snapshot.scope)))); + value.Set("selected", SerializeSupported(snapshot.selected)); + if (snapshot.hasText) { + value.Set("text", json::Value::String(snapshot.text)); + } + if (snapshot.hasValue) { + value.Set("value", json::Value::String(snapshot.value)); + } + value.Set("visible", SerializeSupported(snapshot.visible)); + value.Set("windowId", json::Value::String(snapshot.windowId)); + return value; +} + +void Automation::Initialize() { + if (automation_) { + return; + } + winrt::com_ptr automation; + HRESULT result = CoCreateInstance(CLSID_CUIAutomation8, nullptr, CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(automation.put())); + if (FAILED(result)) { + result = CoCreateInstance(CLSID_CUIAutomation, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(automation.put())); + } + CheckHresult(kErrorAutomationFailed, "Creating the UI Automation client", result); + + winrt::com_ptr automation2; + if (SUCCEEDED(automation->QueryInterface(IID_PPV_ARGS(automation2.put())))) { + // Bound every cross-process transaction so a hung provider cannot stall the + // command worker indefinitely. + automation2->put_ConnectionTimeout(2000); + automation2->put_TransactionTimeout(5000); + } + + CheckHresult(kErrorAutomationFailed, "Reading the UI Automation control view", + automation->get_ControlViewCondition(controlViewCondition_.put())); + CheckHresult(kErrorAutomationFailed, "Reading the UI Automation raw view", + automation->get_RawViewCondition(rawViewCondition_.put())); + CheckHresult(kErrorAutomationFailed, "Reading the UI Automation raw walker", + automation->get_RawViewWalker(rawWalker_.put())); + CheckHresult(kErrorAutomationFailed, "Reading the UI Automation not-supported sentinel", + UiaGetReservedNotSupportedValue(notSupportedValue_.put())); + automation_ = automation; +} + +winrt::com_ptr Automation::ParentOf(IUIAutomationElement* element) const { + winrt::com_ptr parent; + if (!rawWalker_ || FAILED(rawWalker_->GetParentElement(element, parent.put()))) { + return nullptr; + } + return parent; +} + +bool Automation::IsNotSupported(const VARIANT& value) const { + return value.vt == VT_UNKNOWN && value.punkVal != nullptr && notSupportedValue_ && + value.punkVal == notSupportedValue_.get(); +} + +winrt::com_ptr Automation::CreateCacheRequest(TreeScope scope, bool controlViewOnly) const { + winrt::com_ptr request; + CheckHresult(kErrorAutomationFailed, "Creating a UI Automation cache request", + automation_->CreateCacheRequest(request.put())); + constexpr PROPERTYID properties[] = { + UIA_RuntimeIdPropertyId, + UIA_AutomationIdPropertyId, + UIA_NamePropertyId, + UIA_ControlTypePropertyId, + UIA_BoundingRectanglePropertyId, + UIA_IsEnabledPropertyId, + UIA_HasKeyboardFocusPropertyId, + UIA_IsKeyboardFocusablePropertyId, + UIA_IsOffscreenPropertyId, + UIA_IsControlElementPropertyId, + UIA_NativeWindowHandlePropertyId, + UIA_ProcessIdPropertyId, + UIA_ValueValuePropertyId, + UIA_ValueIsReadOnlyPropertyId, + UIA_LegacyIAccessibleStatePropertyId, + UIA_LegacyIAccessibleValuePropertyId, + UIA_ToggleToggleStatePropertyId, + UIA_ExpandCollapseExpandCollapseStatePropertyId, + UIA_SelectionItemIsSelectedPropertyId, + UIA_IsInvokePatternAvailablePropertyId, + UIA_IsLegacyIAccessiblePatternAvailablePropertyId, + UIA_IsTogglePatternAvailablePropertyId, + UIA_IsSelectionItemPatternAvailablePropertyId, + UIA_IsExpandCollapsePatternAvailablePropertyId, + UIA_IsValuePatternAvailablePropertyId, + }; + for (const PROPERTYID property : properties) { + CheckHresult(kErrorAutomationFailed, "Adding a cached UI Automation property", request->AddProperty(property)); + } + CheckHresult(kErrorAutomationFailed, "Configuring the cache element mode", + request->put_AutomationElementMode(AutomationElementMode_Full)); + CheckHresult(kErrorAutomationFailed, "Configuring the cache tree scope", request->put_TreeScope(scope)); + CheckHresult(kErrorAutomationFailed, "Configuring the cache tree filter", + request->put_TreeFilter(controlViewOnly ? controlViewCondition_.get() : rawViewCondition_.get())); + return request; +} + +winrt::com_ptr Automation::RootForWindow(HWND window) const { + if (window == nullptr || IsWindow(window) == FALSE) { + Fail(kErrorNoSuchWindow, "The requested window is no longer available."); + } + winrt::com_ptr element; + const HRESULT result = automation_->ElementFromHandle(window, element.put()); + if (FAILED(result) || !element) { + FailHresult(kErrorNoSuchWindow, "Reading the window automation element", result); + } + return element; +} + +std::wstring Automation::RuntimeKeyFromCache(IUIAutomationElement* element) const { + Variant variant; + if (FAILED(element->GetCachedPropertyValue(UIA_RuntimeIdPropertyId, variant.put()))) { + return {}; + } + if (variant.type() != (VT_ARRAY | VT_I4) || variant.get().parray == nullptr) { + return {}; + } + SAFEARRAY* array = variant.get().parray; + LONG lower = 0; + LONG upper = 0; + if (FAILED(SafeArrayGetLBound(array, 1, &lower)) || FAILED(SafeArrayGetUBound(array, 1, &upper))) { + return {}; + } + std::wstring key; + for (LONG index = lower; index <= upper; index += 1) { + int component = 0; + if (FAILED(SafeArrayGetElement(array, &index, &component))) { + return {}; + } + key += std::to_wstring(component); + key += L'.'; + } + return key; +} + +std::wstring Automation::RuntimeKeyLive(IUIAutomationElement* element) const { + SAFEARRAY* array = nullptr; + if (FAILED(element->GetRuntimeId(&array)) || array == nullptr) { + return {}; + } + LONG lower = 0; + LONG upper = 0; + std::wstring key; + if (SUCCEEDED(SafeArrayGetLBound(array, 1, &lower)) && SUCCEEDED(SafeArrayGetUBound(array, 1, &upper))) { + for (LONG index = lower; index <= upper; index += 1) { + int component = 0; + if (FAILED(SafeArrayGetElement(array, &index, &component))) { + key.clear(); + break; + } + key += std::to_wstring(component); + key += L'.'; + } + } + SafeArrayDestroy(array); + return key; +} + +std::string Automation::RegisterElement(IUIAutomationElement* element, const std::wstring& runtimeKey, + const std::string& windowId, HWND window, const std::string& parentId, + ElementScope scope) { + std::string id; + if (!runtimeKey.empty()) { + const auto existing = idsByRuntimeKey_.find(runtimeKey); + if (existing != idsByRuntimeKey_.end()) { + id = existing->second; + } + } + if (id.empty()) { + id = NextIdentifier("element"); + if (!runtimeKey.empty()) { + idsByRuntimeKey_.emplace(runtimeKey, id); + } + } + ElementRecord& record = recordsById_[id]; + record.id = id; + record.element = nullptr; + record.element.copy_from(element); + record.windowId = windowId; + record.window = window; + record.parentId = parentId; + record.scope = scope; + return id; +} + +const ElementRecord& Automation::RequireRecord(const std::string& elementId) const { + const auto record = recordsById_.find(elementId); + if (record == recordsById_.end()) { + Fail(kErrorStaleElement, "Element \"" + elementId + "\" is no longer tracked by the native helper."); + } + return record->second; +} + +ElementSnapshot Automation::ReadSnapshot(IUIAutomationElement* element, const std::string& windowId, + const std::string& parentId, ElementScope scope, + const std::string& knownId) { + ElementSnapshot snapshot; + snapshot.id = knownId; + snapshot.windowId = windowId; + snapshot.parentId = parentId; + snapshot.scope = scope; + + Variant variant; + if (SUCCEEDED(element->GetCachedPropertyValue(UIA_AutomationIdPropertyId, variant.put()))) { + snapshot.automationId = VariantString(variant.get()); + } + if (SUCCEEDED(element->GetCachedPropertyValue(UIA_NamePropertyId, variant.put()))) { + snapshot.name = VariantString(variant.get()); + } + + CONTROLTYPEID controlType = UIA_CustomControlTypeId; + if (FAILED(element->get_CachedControlType(&controlType))) { + controlType = UIA_CustomControlTypeId; + } + snapshot.role = RoleForControlType(controlType); + + RECT bounds{}; + if (SUCCEEDED(element->get_CachedBoundingRectangle(&bounds))) { + snapshot.physicalRect = bounds; + } + + const auto readBool = [&](PROPERTYID property, const char* reason) { + SupportedBool state; + if (FAILED(element->GetCachedPropertyValueEx(property, TRUE, variant.put()))) { + return Unsupported(reason); + } + if (IsNotSupported(variant.get()) || variant.type() != VT_BOOL) { + return Unsupported(reason); + } + state.supported = true; + state.value = variant.get().boolVal != VARIANT_FALSE; + return state; + }; + + snapshot.enabled = readBool(UIA_IsEnabledPropertyId, "The provider does not report an enabled state."); + snapshot.focused = readBool(UIA_HasKeyboardFocusPropertyId, "The provider does not report keyboard focus."); + const SupportedBool focusable = + readBool(UIA_IsKeyboardFocusablePropertyId, "The provider does not report keyboard focusability."); + snapshot.keyboardFocusable = focusable.supported && focusable.value; + + const SupportedBool offscreen = readBool(UIA_IsOffscreenPropertyId, "The provider does not report visibility."); + const bool hasBounds = bounds.right > bounds.left && bounds.bottom > bounds.top; + snapshot.visible.supported = true; + if (offscreen.supported) { + snapshot.visible.value = !offscreen.value && hasBounds; + } else { + // The provider is silent about occlusion, so fall back to measured bounds + // rather than inventing a default state. + snapshot.visible.value = hasBounds; + } + + const auto patternAvailable = [&](PROPERTYID property) { + if (FAILED(element->GetCachedPropertyValueEx(property, TRUE, variant.put()))) { + return false; + } + return !IsNotSupported(variant.get()) && variant.type() == VT_BOOL && variant.get().boolVal != VARIANT_FALSE; + }; + + snapshot.invokePattern = patternAvailable(UIA_IsInvokePatternAvailablePropertyId); + snapshot.legacyPattern = patternAvailable(UIA_IsLegacyIAccessiblePatternAvailablePropertyId); + snapshot.togglePattern = patternAvailable(UIA_IsTogglePatternAvailablePropertyId); + snapshot.selectionItemPattern = patternAvailable(UIA_IsSelectionItemPatternAvailablePropertyId); + snapshot.expandCollapsePattern = patternAvailable(UIA_IsExpandCollapsePatternAvailablePropertyId); + snapshot.valuePattern = patternAvailable(UIA_IsValuePatternAvailablePropertyId); + if (snapshot.role == "group" && snapshot.valuePattern && snapshot.keyboardFocusable) { + snapshot.role = "textbox"; + } + + if (snapshot.valuePattern) { + if (SUCCEEDED(element->GetCachedPropertyValueEx(UIA_ValueValuePropertyId, TRUE, variant.put())) && + !IsNotSupported(variant.get()) && variant.type() == VT_BSTR) { + snapshot.value = VariantString(variant.get()); + snapshot.hasValue = true; + } + if (SUCCEEDED(element->GetCachedPropertyValueEx(UIA_ValueIsReadOnlyPropertyId, TRUE, variant.put())) && + !IsNotSupported(variant.get()) && variant.type() == VT_BOOL) { + snapshot.valueReadOnly = variant.get().boolVal != VARIANT_FALSE; + } + } + if (!snapshot.hasValue && + SUCCEEDED(element->GetCachedPropertyValueEx(UIA_LegacyIAccessibleValuePropertyId, TRUE, variant.put())) && + !IsNotSupported(variant.get()) && variant.type() == VT_BSTR) { + const std::string legacy = VariantString(variant.get()); + if (!legacy.empty()) { + snapshot.value = legacy; + snapshot.hasValue = true; + } + } + + if (snapshot.role == "text") { + snapshot.text = snapshot.name; + snapshot.hasText = !snapshot.text.empty(); + } else if ((snapshot.role == "textbox" || snapshot.role == "document") && snapshot.hasValue) { + snapshot.text = snapshot.value; + snapshot.hasText = true; + } + + if (snapshot.togglePattern && + SUCCEEDED(element->GetCachedPropertyValueEx(UIA_ToggleToggleStatePropertyId, TRUE, variant.put())) && + !IsNotSupported(variant.get()) && (variant.type() == VT_I4 || variant.type() == VT_INT)) { + snapshot.checked.supported = true; + switch (variant.get().lVal) { + case ToggleState_On: + snapshot.checked.value = CheckedValue::True; + break; + case ToggleState_Indeterminate: + snapshot.checked.value = CheckedValue::Mixed; + break; + default: + snapshot.checked.value = CheckedValue::False; + break; + } + } else if ( + snapshot.role == "checkbox" && snapshot.legacyPattern && + SUCCEEDED(element->GetCachedPropertyValueEx(UIA_LegacyIAccessibleStatePropertyId, TRUE, variant.put())) && + !IsNotSupported(variant.get()) && (variant.type() == VT_I4 || variant.type() == VT_INT)) { + snapshot.checked.supported = true; + const LONG state = variant.get().lVal; + if ((state & STATE_SYSTEM_MIXED) != 0) { + snapshot.checked.value = CheckedValue::Mixed; + } else { + snapshot.checked.value = (state & STATE_SYSTEM_CHECKED) != 0 ? CheckedValue::True : CheckedValue::False; + } + } else { + snapshot.checked.supported = false; + snapshot.checked.reason = + "The element does not expose Toggle or LegacyIAccessible checked state."; + } + + if (snapshot.selectionItemPattern && + SUCCEEDED(element->GetCachedPropertyValueEx(UIA_SelectionItemIsSelectedPropertyId, TRUE, variant.put())) && + !IsNotSupported(variant.get()) && variant.type() == VT_BOOL) { + snapshot.selected.supported = true; + snapshot.selected.value = variant.get().boolVal != VARIANT_FALSE; + } else { + snapshot.selected = Unsupported("The element does not implement the UI Automation SelectionItem pattern."); + } + + if (snapshot.expandCollapsePattern && + SUCCEEDED(element->GetCachedPropertyValueEx(UIA_ExpandCollapseExpandCollapseStatePropertyId, TRUE, + variant.put())) && + !IsNotSupported(variant.get()) && (variant.type() == VT_I4 || variant.type() == VT_INT)) { + if (variant.get().lVal == ExpandCollapseState_LeafNode) { + snapshot.expanded = Unsupported("The element is a leaf node that cannot expand or collapse."); + } else { + snapshot.expanded.supported = true; + snapshot.expanded.value = variant.get().lVal == ExpandCollapseState_Expanded || + variant.get().lVal == ExpandCollapseState_PartiallyExpanded; + } + } else { + snapshot.expanded = Unsupported("The element does not implement the UI Automation ExpandCollapse pattern."); + } + + return snapshot; +} + +std::vector Automation::WalkCachedSubtree(IUIAutomationElement* root, const WindowContext& context, + ElementScope rootScope, const std::string& rootParentId, + const CancellationToken& token, bool controlViewOnly) { + struct PendingNode { + winrt::com_ptr element; + std::string parentId; + ElementScope scope; + }; + + winrt::com_ptr cachedRoot; + const HRESULT result = root->BuildUpdatedCache(CreateCacheRequest(TreeScope_Subtree, controlViewOnly).get(), + cachedRoot.put()); + if (result == UIA_E_ELEMENTNOTAVAILABLE) { + Fail(kErrorStaleElement, "The native element is no longer available."); + } + CheckHresult(kErrorAutomationFailed, "Reading the cached automation subtree", result); + + const WindowMetrics metrics = MeasureWindow(context.window); + const std::string storyRootTestId = ToUtf8(context.storyRootTestId); + std::vector stack; + stack.push_back(PendingNode{cachedRoot, rootParentId, rootScope}); + + std::vector snapshots; + std::size_t visited = 0; + while (!stack.empty()) { + if (visited >= kMaximumTreeNodes) { + Fail(kErrorTreeTooLarge, "The native accessibility tree exceeds the 5000-node limit."); + } + if ((visited % 64) == 0) { + token.ThrowIfCancelled(); + } + visited += 1; + const PendingNode node = stack.back(); + stack.pop_back(); + + ElementScope scope = node.scope; + const std::wstring runtimeKey = RuntimeKeyFromCache(node.element.get()); + const std::string id = + RegisterElement(node.element.get(), runtimeKey, context.windowId, context.window, node.parentId, scope); + ElementSnapshot snapshot = ReadSnapshot(node.element.get(), context.windowId, node.parentId, scope, id); + + ElementScope childScope = scope; + if (context.primary && scope != ElementScope::Preview && !storyRootTestId.empty() && + snapshot.automationId == storyRootTestId) { + scope = ElementScope::Preview; + childScope = ElementScope::Preview; + snapshot.scope = ElementScope::Preview; + recordsById_[id].scope = ElementScope::Preview; + } else if (context.primary && scope == ElementScope::Application) { + childScope = ElementScope::Chrome; + } + + winrt::com_ptr children; + if (SUCCEEDED(node.element->GetCachedChildren(children.put())) && children) { + int count = 0; + if (SUCCEEDED(children->get_Length(&count))) { + for (int index = count - 1; index >= 0; index -= 1) { + winrt::com_ptr child; + if (SUCCEEDED(children->GetElement(index, child.put())) && child) { + stack.push_back(PendingNode{child, id, childScope}); + } + } + } + } + + snapshot.rect = PhysicalRectToClientDips(snapshot.physicalRect, metrics); + snapshots.push_back(std::move(snapshot)); + } + return snapshots; +} + +std::vector Automation::SnapshotWindowTree(const WindowContext& context, + const CancellationToken& token) { + Initialize(); + token.ThrowIfCancelled(); + const winrt::com_ptr root = RootForWindow(context.window); + const ElementScope rootScope = context.primary ? ElementScope::Application : ElementScope::SecondaryWindow; + return WalkCachedSubtree(root.get(), context, rootScope, std::string(), token, true); +} + +std::vector Automation::Find(const WindowContext& context, const std::string& rootElementId, + const Selector& selector, const CancellationToken& token) { + Initialize(); + token.ThrowIfCancelled(); + + winrt::com_ptr searchRoot; + std::string rootId; + std::string rootParentId; + ElementScope rootScope = context.primary ? ElementScope::Application : ElementScope::SecondaryWindow; + if (!rootElementId.empty()) { + const ElementRecord& record = RequireRecord(rootElementId); + searchRoot = record.element; + rootId = record.id; + rootParentId = record.parentId; + rootScope = record.scope; + } else { + searchRoot = RootForWindow(context.window); + } + + // A test identifier can live on a node that the control view hides, so a + // miss retries once through the raw view before reporting no matches. + const bool allowRawView = selector.strategy == "accessibility id"; + std::vector matches; + for (int pass = 0; pass < (allowRawView ? 2 : 1); pass += 1) { + const std::vector snapshots = + WalkCachedSubtree(searchRoot.get(), context, rootScope, rootParentId, token, pass == 0); + for (const ElementSnapshot& snapshot : snapshots) { + // A scoped search stays strictly beneath its root element. + if (!rootId.empty() && snapshot.id == rootId) { + continue; + } + if (MatchesSelector(snapshot, selector)) { + matches.push_back(snapshot); + } + } + if (!matches.empty()) { + break; + } + } + return matches; +} + +ElementSnapshot Automation::SnapshotElement(const std::string& elementId, const CancellationToken& token) { + Initialize(); + token.ThrowIfCancelled(); + const ElementRecord& record = RequireRecord(elementId); + const std::string windowId = record.windowId; + const std::string parentId = record.parentId; + const ElementScope scope = record.scope; + HWND window = record.window; + winrt::com_ptr refreshed; + const HRESULT result = + record.element->BuildUpdatedCache(CreateCacheRequest(TreeScope_Element, true).get(), refreshed.put()); + if (result == UIA_E_ELEMENTNOTAVAILABLE || (SUCCEEDED(result) && !refreshed)) { + Fail(kErrorStaleElement, "Element \"" + elementId + "\" is no longer available."); + } + CheckHresult(kErrorAutomationFailed, "Refreshing the cached element", result); + + if (window == nullptr || IsWindow(window) == FALSE) { + UIA_HWND nativeWindow = nullptr; + if (SUCCEEDED(refreshed->get_CachedNativeWindowHandle(&nativeWindow))) { + window = static_cast(nativeWindow); + } + } + ElementSnapshot snapshot = ReadSnapshot(refreshed.get(), windowId, parentId, scope, elementId); + recordsById_[elementId].element = refreshed; + + const WindowMetrics metrics = MeasureWindow(window); + snapshot.rect = PhysicalRectToClientDips(snapshot.physicalRect, metrics); + return snapshot; +} + +std::optional Automation::ActiveElement(const WindowContext& context, const CancellationToken& token) { + Initialize(); + token.ThrowIfCancelled(); + winrt::com_ptr focused; + const HRESULT result = + automation_->GetFocusedElementBuildCache(CreateCacheRequest(TreeScope_Element, true).get(), focused.put()); + if (FAILED(result) || !focused) { + return std::nullopt; + } + winrt::com_ptr cursor = focused; + bool owned = false; + for (std::size_t depth = 0; depth < kMaximumAncestorWalk && cursor; depth += 1) { + token.ThrowIfCancelled(); + UIA_HWND handle = nullptr; + if (SUCCEEDED(cursor->get_CurrentNativeWindowHandle(&handle)) && static_cast(handle) == context.window) { + owned = true; + break; + } + const winrt::com_ptr parent = ParentOf(cursor.get()); + if (!parent) { + break; + } + cursor = parent; + } + if (!owned) { + return std::nullopt; + } + const std::wstring runtimeKey = RuntimeKeyLive(focused.get()); + std::string parentId; + ElementScope scope = context.primary ? ElementScope::Chrome : ElementScope::SecondaryWindow; + if (!runtimeKey.empty()) { + const auto existing = idsByRuntimeKey_.find(runtimeKey); + if (existing != idsByRuntimeKey_.end()) { + const auto record = recordsById_.find(existing->second); + if (record != recordsById_.end()) { + parentId = record->second.parentId; + scope = record->second.scope; + } + } + } + const std::string id = + RegisterElement(focused.get(), runtimeKey, context.windowId, context.window, parentId, scope); + ElementSnapshot snapshot = ReadSnapshot(focused.get(), context.windowId, parentId, scope, id); + const WindowMetrics metrics = MeasureWindow(context.window); + snapshot.rect = PhysicalRectToClientDips(snapshot.physicalRect, metrics); + return snapshot; +} + +std::optional Automation::HitTest(const WindowContext& context, POINT screenPoint, + const CancellationToken& token) { + Initialize(); + token.ThrowIfCancelled(); + DWORD targetProcessId = 0; + GetWindowThreadProcessId(context.window, &targetProcessId); + const HWND pointWindow = WindowFromPoint(screenPoint); + DWORD pointProcessId = 0; + if (pointWindow != nullptr) { + GetWindowThreadProcessId(pointWindow, &pointProcessId); + } + if (pointProcessId != 0 && targetProcessId != 0 && pointProcessId != targetProcessId) { + ElementSnapshot intercepted; + intercepted.id = NextIdentifier("external-window"); + intercepted.windowId = context.windowId; + intercepted.scope = ElementScope::Application; + intercepted.role = "window"; + intercepted.name = "External desktop window"; + intercepted.enabled.supported = true; + intercepted.enabled.value = true; + intercepted.focused.supported = true; + intercepted.focused.value = false; + intercepted.visible.supported = true; + intercepted.visible.value = true; + intercepted.checked.supported = false; + intercepted.checked.reason = "An external desktop window does not expose checked state."; + intercepted.selected = Unsupported("An external desktop window does not expose selected state."); + intercepted.expanded = Unsupported("An external desktop window does not expose expanded state."); + RECT bounds{}; + if (GetWindowRect(GetAncestor(pointWindow, GA_ROOT), &bounds) != FALSE) { + intercepted.physicalRect = bounds; + intercepted.rect = PhysicalRectToClientDips(bounds, MeasureWindow(context.window)); + } + return intercepted; + } + winrt::com_ptr hit; + const HRESULT result = + automation_->ElementFromPointBuildCache(screenPoint, CreateCacheRequest(TreeScope_Element, true).get(), + hit.put()); + if (FAILED(result) || !hit) { + return std::nullopt; + } + + // The point belongs to the deepest identity the client already knows about: + // an inner presentation node of a known element never intercepts its own + // owner, while an unrelated node still reports its own identity. + winrt::com_ptr resolved = hit; + std::string resolvedId; + std::string parentId; + ElementScope scope = context.primary ? ElementScope::Chrome : ElementScope::SecondaryWindow; + winrt::com_ptr cursor = hit; + for (std::size_t depth = 0; depth < kMaximumAncestorWalk && cursor; depth += 1) { + token.ThrowIfCancelled(); + const std::wstring key = RuntimeKeyLive(cursor.get()); + if (!key.empty()) { + const auto existing = idsByRuntimeKey_.find(key); + if (existing != idsByRuntimeKey_.end()) { + const auto record = recordsById_.find(existing->second); + if (record != recordsById_.end()) { + resolved = cursor; + resolvedId = existing->second; + parentId = record->second.parentId; + scope = record->second.scope; + break; + } + } + } + const winrt::com_ptr parent = ParentOf(cursor.get()); + if (!parent) { + break; + } + cursor = parent; + } + + winrt::com_ptr cached; + const HRESULT refresh = + resolved->BuildUpdatedCache(CreateCacheRequest(TreeScope_Element, true).get(), cached.put()); + if (FAILED(refresh) || !cached) { + return std::nullopt; + } + const std::wstring runtimeKey = RuntimeKeyFromCache(cached.get()); + const std::string id = + resolvedId.empty() + ? RegisterElement(cached.get(), runtimeKey, context.windowId, context.window, parentId, scope) + : resolvedId; + recordsById_[id].element = cached; + ElementSnapshot snapshot = ReadSnapshot(cached.get(), context.windowId, parentId, scope, id); + const WindowMetrics metrics = MeasureWindow(context.window); + snapshot.rect = PhysicalRectToClientDips(snapshot.physicalRect, metrics); + return snapshot; +} + +void Automation::SetFocus(const std::string& elementId) { + const ElementRecord& record = RequireRecord(elementId); + const winrt::com_ptr element = record.element; + const HRESULT result = + BoundedCall([element]() { return element->SetFocus(); }, "Focusing the element", kErrorAutomationFailed); + if (result == UIA_E_ELEMENTNOTAVAILABLE) { + Fail(kErrorStaleElement, "Element \"" + elementId + "\" is no longer available."); + } + if (FAILED(result)) { + FailHresult(kErrorNotInteractable, "Focusing the element", result); + } +} + +bool Automation::HasKeyboardFocus(const std::string& elementId) { + const ElementRecord& record = RequireRecord(elementId); + const winrt::com_ptr element = record.element; + const auto focused = std::make_shared(FALSE); + const HRESULT result = BoundedCall([element, focused]() { return element->get_CurrentHasKeyboardFocus(focused.get()); }, + "Reading the element focus state", kErrorAutomationFailed); + return SUCCEEDED(result) && *focused != FALSE; +} + +void Automation::AccessibilityClick(const std::string& elementId, const CancellationToken& token) { + const ElementRecord& record = RequireRecord(elementId); + const winrt::com_ptr element = record.element; + token.ThrowIfCancelled(); + + winrt::com_ptr invoke; + if (SUCCEEDED(element->GetCurrentPatternAs(UIA_InvokePatternId, IID_PPV_ARGS(invoke.put()))) && invoke) { + CheckHresult(kErrorAutomationFailed, "Invoking the element", + BoundedCall([invoke]() { return invoke->Invoke(); }, "Invoking the element", kErrorAutomationFailed)); + return; + } + winrt::com_ptr toggle; + if (SUCCEEDED(element->GetCurrentPatternAs(UIA_TogglePatternId, IID_PPV_ARGS(toggle.put()))) && toggle) { + CheckHresult(kErrorAutomationFailed, "Toggling the element", + BoundedCall([toggle]() { return toggle->Toggle(); }, "Toggling the element", kErrorAutomationFailed)); + return; + } + winrt::com_ptr selection; + if (SUCCEEDED(element->GetCurrentPatternAs(UIA_SelectionItemPatternId, IID_PPV_ARGS(selection.put()))) && + selection) { + CheckHresult( + kErrorAutomationFailed, "Selecting the element", + BoundedCall([selection]() { return selection->Select(); }, "Selecting the element", kErrorAutomationFailed)); + return; + } + winrt::com_ptr expand; + if (SUCCEEDED(element->GetCurrentPatternAs(UIA_ExpandCollapsePatternId, IID_PPV_ARGS(expand.put()))) && expand) { + const auto state = std::make_shared(ExpandCollapseState_Collapsed); + CheckHresult(kErrorAutomationFailed, "Reading the expand state", + BoundedCall([expand, state]() { return expand->get_CurrentExpandCollapseState(state.get()); }, + "Reading the expand state", kErrorAutomationFailed)); + if (*state == ExpandCollapseState_LeafNode) { + Fail(kErrorUnsupported, "The element is a leaf node and cannot be activated through accessibility."); + } + const bool collapse = *state == ExpandCollapseState_Expanded; + CheckHresult(kErrorAutomationFailed, "Expanding or collapsing the element", + BoundedCall([expand, collapse]() { return collapse ? expand->Collapse() : expand->Expand(); }, + "Expanding or collapsing the element", kErrorAutomationFailed)); + return; + } + winrt::com_ptr legacy; + if (SUCCEEDED(element->GetCurrentPatternAs(UIA_LegacyIAccessiblePatternId, IID_PPV_ARGS(legacy.put()))) && legacy) { + CheckHresult( + kErrorAutomationFailed, "Invoking the legacy accessibility default action", + BoundedCall([legacy]() { return legacy->DoDefaultAction(); }, "Invoking the legacy accessibility default action", + kErrorAutomationFailed)); + return; + } + Fail(kErrorUnsupported, + "The element does not implement an Invoke, Toggle, SelectionItem, ExpandCollapse, or LegacyIAccessible pattern, so an " + "accessibility click is unavailable."); +} + +bool Automation::TryClearValue(const std::string& elementId) { + const ElementRecord& record = RequireRecord(elementId); + winrt::com_ptr value; + if (FAILED(record.element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(value.put()))) || !value) { + return false; + } + const auto readOnly = std::make_shared(TRUE); + const HRESULT state = BoundedCall([value, readOnly]() { return value->get_CurrentIsReadOnly(readOnly.get()); }, + "Reading the element value state", kErrorAutomationFailed); + if (FAILED(state) || *readOnly != FALSE) { + return false; + } + const HRESULT result = BoundedCall( + [value]() { + const BSTR empty = SysAllocString(L""); + const HRESULT assigned = value->SetValue(empty); + SysFreeString(empty); + return assigned; + }, + "Clearing the element value", kErrorAutomationFailed); + return SUCCEEDED(result); +} + +void Automation::ForgetWindow(const std::string& windowId) { + for (auto iterator = recordsById_.begin(); iterator != recordsById_.end();) { + if (iterator->second.windowId == windowId) { + for (auto key = idsByRuntimeKey_.begin(); key != idsByRuntimeKey_.end();) { + key = key->second == iterator->first ? idsByRuntimeKey_.erase(key) : std::next(key); + } + iterator = recordsById_.erase(iterator); + continue; + } + ++iterator; + } +} + +void Automation::Reset() { + recordsById_.clear(); + idsByRuntimeKey_.clear(); +} + +std::string Automation::SerializeSourceXml(const std::vector& snapshots) const { + std::unordered_map> children; + std::vector roots; + for (const ElementSnapshot& snapshot : snapshots) { + if (snapshot.parentId.empty()) { + roots.push_back(&snapshot); + } else { + children[snapshot.parentId].push_back(&snapshot); + } + } + + std::string output = ""; + struct Frame { + const ElementSnapshot* snapshot; + bool open; + }; + std::vector stack; + for (auto iterator = roots.rbegin(); iterator != roots.rend(); ++iterator) { + stack.push_back(Frame{*iterator, true}); + } + while (!stack.empty()) { + const Frame frame = stack.back(); + stack.pop_back(); + if (!frame.open) { + output += ""; + continue; + } + const ElementSnapshot& snapshot = *frame.snapshot; + output += "second.rbegin(); child != entry->second.rend(); ++child) { + stack.push_back(Frame{*child, true}); + } + } + } + output += ""; + return output; +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/automation.h b/packages/agentic/desktop-driver/native/windows/src/automation.h new file mode 100644 index 00000000000..f1bcee7a450 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/automation.h @@ -0,0 +1,147 @@ +#pragma once + +#include "common.h" + +#include + +#include +#include +#include +#include + +#include + +#include "geometry.h" +#include "json.h" + +namespace furn { + +inline constexpr std::size_t kMaximumTreeNodes = 5000; + +enum class ElementScope { Application, Chrome, Preview, SecondaryWindow }; + +enum class CheckedValue { False, True, Mixed }; + +// Mirrors the SupportedValue contract: an unavailable native state is never +// flattened into false. +struct SupportedBool { + bool supported{true}; + bool value{false}; + std::string reason; +}; + +struct SupportedChecked { + bool supported{true}; + CheckedValue value{CheckedValue::False}; + std::string reason; +}; + +struct ElementSnapshot { + std::string id; + std::string automationId; + std::string name; + std::string value; + std::string text; + bool hasValue{false}; + bool hasText{false}; + std::string role; + std::string parentId; + std::string windowId; + ElementScope scope{ElementScope::Chrome}; + RectD rect; + RECT physicalRect{}; + SupportedBool enabled; + SupportedBool focused; + SupportedBool visible; + SupportedBool expanded; + SupportedBool selected; + SupportedChecked checked; + bool invokePattern{false}; + bool legacyPattern{false}; + bool togglePattern{false}; + bool selectionItemPattern{false}; + bool expandCollapsePattern{false}; + bool valuePattern{false}; + bool valueReadOnly{false}; + bool keyboardFocusable{false}; +}; + +struct Selector { + std::string strategy; + std::string value; +}; + +struct WindowContext { + std::string windowId; + HWND window{nullptr}; + bool primary{false}; + std::wstring storyRootTestId; +}; + +struct ElementRecord { + std::string id; + winrt::com_ptr element; + std::string windowId; + HWND window{nullptr}; + std::string parentId; + ElementScope scope{ElementScope::Chrome}; +}; + +std::string RoleForControlType(CONTROLTYPEID controlType); +std::string NormalizeRoleQuery(std::string_view value); +const char* ScopeName(ElementScope scope); +bool MatchesSelector(const ElementSnapshot& snapshot, const Selector& selector); + +// Owns the UI Automation client, the opaque element table, and every read of +// the native tree. All members must be used from the dedicated MTA worker. +class Automation { + public: + void Initialize(); + + std::vector SnapshotWindowTree(const WindowContext& context, const CancellationToken& token); + std::vector Find(const WindowContext& context, const std::string& rootElementId, + const Selector& selector, const CancellationToken& token); + ElementSnapshot SnapshotElement(const std::string& elementId, const CancellationToken& token); + std::optional ActiveElement(const WindowContext& context, const CancellationToken& token); + std::optional HitTest(const WindowContext& context, POINT screenPoint, + const CancellationToken& token); + + const ElementRecord& RequireRecord(const std::string& elementId) const; + void SetFocus(const std::string& elementId); + void AccessibilityClick(const std::string& elementId, const CancellationToken& token); + bool TryClearValue(const std::string& elementId); + bool HasKeyboardFocus(const std::string& elementId); + + void ForgetWindow(const std::string& windowId); + void Reset(); + + std::string SerializeSourceXml(const std::vector& snapshots) const; + + private: + winrt::com_ptr CreateCacheRequest(TreeScope scope, bool controlViewOnly) const; + winrt::com_ptr RootForWindow(HWND window) const; + winrt::com_ptr ParentOf(IUIAutomationElement* element) const; + std::vector WalkCachedSubtree(IUIAutomationElement* root, const WindowContext& context, + ElementScope rootScope, const std::string& rootParentId, + const CancellationToken& token, bool controlViewOnly); + ElementSnapshot ReadSnapshot(IUIAutomationElement* element, const std::string& windowId, const std::string& parentId, + ElementScope scope, const std::string& knownId); + std::string RegisterElement(IUIAutomationElement* element, const std::wstring& runtimeKey, + const std::string& windowId, HWND window, const std::string& parentId, + ElementScope scope); + std::wstring RuntimeKeyFromCache(IUIAutomationElement* element) const; + std::wstring RuntimeKeyLive(IUIAutomationElement* element) const; + bool IsNotSupported(const VARIANT& value) const; + + winrt::com_ptr automation_; + winrt::com_ptr controlViewCondition_; + winrt::com_ptr rawViewCondition_; + winrt::com_ptr rawWalker_; + winrt::com_ptr notSupportedValue_; + std::unordered_map recordsById_; + std::unordered_map idsByRuntimeKey_; +}; + +json::Value SerializeSnapshot(const ElementSnapshot& snapshot); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/capture.cpp b/packages/agentic/desktop-driver/native/windows/src/capture.cpp new file mode 100644 index 00000000000..f5b501c34cf --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/capture.cpp @@ -0,0 +1,305 @@ +#include "capture.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "geometry.h" + +namespace furn { +namespace { + +using winrt::Windows::Graphics::SizeInt32; +using winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame; +using winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool; +using winrt::Windows::Graphics::Capture::GraphicsCaptureItem; +using winrt::Windows::Graphics::Capture::GraphicsCaptureSession; +using winrt::Windows::Graphics::DirectX::DirectXPixelFormat; +using winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice; + +constexpr unsigned int kFrameTimeoutMs = 4000; + +} // namespace + +POINT ResolveCaptureOrigin(HWND window, UINT capturedWidth, UINT capturedHeight) { + RECT windowRect{}; + if (GetWindowRect(window, &windowRect) == FALSE) { + FailLastError(kErrorCaptureFailed, "Reading the captured window rectangle", GetLastError()); + } + RECT frameBounds = windowRect; + if (FAILED(DwmGetWindowAttribute(window, DWMWA_EXTENDED_FRAME_BOUNDS, &frameBounds, sizeof(frameBounds)))) { + frameBounds = windowRect; + } + const auto distance = [&](const RECT& candidate) { + const long width = candidate.right - candidate.left; + const long height = candidate.bottom - candidate.top; + return std::abs(width - static_cast(capturedWidth)) + std::abs(height - static_cast(capturedHeight)); + }; + const RECT& chosen = distance(frameBounds) <= distance(windowRect) ? frameBounds : windowRect; + POINT origin{chosen.left, chosen.top}; + return origin; +} + +struct CaptureEngine::Impl { + winrt::com_ptr device; + winrt::com_ptr context; + IDirect3DDevice runtimeDevice{nullptr}; + winrt::com_ptr imaging; + + void EnsureDevice() { + if (device && runtimeDevice) { + return; + } + const D3D_FEATURE_LEVEL levels[] = {D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0}; + HRESULT result = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, + levels, ARRAYSIZE(levels), D3D11_SDK_VERSION, device.put(), nullptr, + context.put()); + if (FAILED(result)) { + device = nullptr; + context = nullptr; + result = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_WARP, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, levels, + ARRAYSIZE(levels), D3D11_SDK_VERSION, device.put(), nullptr, context.put()); + } + CheckHresult(kErrorCaptureFailed, "Creating the Direct3D 11 capture device", result); + + winrt::com_ptr dxgiDevice; + CheckHresult(kErrorCaptureFailed, "Reading the DXGI capture device", + device->QueryInterface(IID_PPV_ARGS(dxgiDevice.put()))); + winrt::com_ptr<::IInspectable> inspectable; + CheckHresult(kErrorCaptureFailed, "Wrapping the Direct3D device for Windows Graphics Capture", + CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.get(), inspectable.put())); + runtimeDevice = inspectable.as(); + } + + void EnsureImaging() { + if (imaging) { + return; + } + CheckHresult(kErrorCaptureFailed, "Creating the WIC imaging factory", + CoCreateInstance(CLSID_WICImagingFactory, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(imaging.put()))); + } +}; + +CaptureEngine::CaptureEngine() : impl_(std::make_unique()) {} + +CaptureEngine::~CaptureEngine() = default; + +bool CaptureEngine::Available() { + try { + return GraphicsCaptureSession::IsSupported(); + } catch (const winrt::hresult_error&) { + return false; + } +} + +RawFrame CaptureEngine::CaptureWindow(HWND window, const CancellationToken& token) { + if (window == nullptr || IsWindow(window) == FALSE) { + Fail(kErrorNoSuchWindow, "The requested window is no longer available."); + } + if (IsIconic(window) != FALSE) { + Fail(kErrorUnsupported, + "Capture is unavailable while the window is minimized; the helper never restores a window to capture it."); + } + if (!Available()) { + Fail(kErrorUnsupported, "Windows Graphics Capture is unavailable on this system."); + } + impl_->EnsureDevice(); + token.ThrowIfCancelled(); + + GraphicsCaptureItem item{nullptr}; + try { + const auto interop = winrt::get_activation_factory(); + winrt::check_hresult( + interop->CreateForWindow(window, winrt::guid_of(), winrt::put_abi(item))); + } catch (const winrt::hresult_error& error) { + FailHresult(kErrorCaptureFailed, "Creating the window capture item", error.code()); + } + if (!item) { + Fail(kErrorCaptureFailed, "Windows Graphics Capture did not provide a capture item for the window."); + } + + const SizeInt32 size = item.Size(); + if (size.Width <= 0 || size.Height <= 0) { + Fail(kErrorCaptureFailed, "The window reported an empty capture size."); + } + + const auto ready = std::make_shared(CreateEventW(nullptr, TRUE, FALSE, nullptr)); + if (!*ready) { + FailLastError(kErrorCaptureFailed, "Creating the capture synchronization event", GetLastError()); + } + const HANDLE readyHandle = ready->get(); + + Direct3D11CaptureFramePool framePool{nullptr}; + GraphicsCaptureSession session{nullptr}; + winrt::event_token frameToken{}; + try { + framePool = Direct3D11CaptureFramePool::CreateFreeThreaded( + impl_->runtimeDevice, DirectXPixelFormat::B8G8R8A8UIntNormalized, 2, size); + // The frame pool is free-threaded, so the delegate shares ownership of the + // event and a late callback can never signal a recycled handle. + frameToken = framePool.FrameArrived( + [ready](const Direct3D11CaptureFramePool&, const winrt::Windows::Foundation::IInspectable&) { + SetEvent(ready->get()); + }); + session = framePool.CreateCaptureSession(item); + try { + session.IsCursorCaptureEnabled(false); + } catch (const winrt::hresult_error&) { + // Older builds do not expose cursor control; the cursor stays visible. + } + try { + session.IsBorderRequired(false); + } catch (const winrt::hresult_error&) { + // Border removal needs a capability that this helper does not request. + } + session.StartCapture(); + } catch (const winrt::hresult_error& error) { + if (framePool) { + framePool.Close(); + } + FailHresult(kErrorCaptureFailed, "Starting the window capture session", error.code()); + } + + RawFrame result; + try { + Direct3D11CaptureFrame frame{nullptr}; + const Deadline deadline(kFrameTimeoutMs); + while (!frame) { + token.ThrowIfCancelled(); + const DWORD waited = WaitForSingleObject(readyHandle, 50); + if (waited == WAIT_OBJECT_0) { + ResetEvent(readyHandle); + } + frame = framePool.TryGetNextFrame(); + if (frame) { + break; + } + if (deadline.Expired()) { + Fail(kErrorCaptureFailed, "Windows Graphics Capture did not deliver a frame for the window."); + } + } + + const auto access = frame.Surface().as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>(); + winrt::com_ptr texture; + CheckHresult(kErrorCaptureFailed, "Reading the captured Direct3D texture", + access->GetInterface(IID_PPV_ARGS(texture.put()))); + + D3D11_TEXTURE2D_DESC description{}; + texture->GetDesc(&description); + D3D11_TEXTURE2D_DESC staging = description; + staging.Usage = D3D11_USAGE_STAGING; + staging.BindFlags = 0; + staging.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + staging.MiscFlags = 0; + winrt::com_ptr readback; + CheckHresult(kErrorCaptureFailed, "Creating the capture readback texture", + impl_->device->CreateTexture2D(&staging, nullptr, readback.put())); + impl_->context->CopyResource(readback.get(), texture.get()); + + D3D11_MAPPED_SUBRESOURCE mapped{}; + CheckHresult(kErrorCaptureFailed, "Mapping the capture readback texture", + impl_->context->Map(readback.get(), 0, D3D11_MAP_READ, 0, &mapped)); + result.width = description.Width; + result.height = description.Height; + result.stride = result.width * 4; + result.pixels.resize(static_cast(result.stride) * result.height); + const auto* source = static_cast(mapped.pData); + for (UINT row = 0; row < result.height; row += 1) { + std::copy_n(source + static_cast(row) * mapped.RowPitch, result.stride, + result.pixels.begin() + static_cast(row) * result.stride); + } + impl_->context->Unmap(readback.get(), 0); + frame.Close(); + } catch (...) { + if (frameToken) { + framePool.FrameArrived(frameToken); + } + if (session) { + session.Close(); + } + if (framePool) { + framePool.Close(); + } + throw; + } + + if (frameToken) { + framePool.FrameArrived(frameToken); + } + session.Close(); + framePool.Close(); + + result.origin = ResolveCaptureOrigin(window, result.width, result.height); + const UINT dpi = GetDpiForWindow(window); + result.scaleFactor = static_cast(dpi == 0 ? kDefaultDpi : dpi) / static_cast(kDefaultDpi); + return result; +} + +EncodedImage CaptureEngine::Encode(const RawFrame& frame, const RECT& cropInFrame) { + impl_->EnsureImaging(); + RECT bounds{0, 0, static_cast(frame.width), static_cast(frame.height)}; + const RECT crop = IntersectRects(bounds, cropInFrame); + if (IsEmptyRect(crop)) { + Fail(kErrorCaptureFailed, "The requested capture region lies outside the captured window."); + } + const UINT width = static_cast(crop.right - crop.left); + const UINT height = static_cast(crop.bottom - crop.top); + const UINT stride = width * 4; + std::vector pixels(static_cast(stride) * height); + for (UINT row = 0; row < height; row += 1) { + const std::size_t sourceOffset = + (static_cast(crop.top) + row) * frame.stride + static_cast(crop.left) * 4; + std::copy_n(frame.pixels.begin() + static_cast(sourceOffset), stride, + pixels.begin() + static_cast(row) * stride); + } + + winrt::com_ptr stream; + CheckHresult(kErrorCaptureFailed, "Creating the PNG encoding stream", + CreateStreamOnHGlobal(nullptr, TRUE, stream.put())); + winrt::com_ptr encoder; + CheckHresult(kErrorCaptureFailed, "Creating the PNG encoder", + impl_->imaging->CreateEncoder(GUID_ContainerFormatPng, nullptr, encoder.put())); + CheckHresult(kErrorCaptureFailed, "Initializing the PNG encoder", + encoder->Initialize(stream.get(), WICBitmapEncoderNoCache)); + winrt::com_ptr encoded; + winrt::com_ptr properties; + CheckHresult(kErrorCaptureFailed, "Creating the PNG frame", encoder->CreateNewFrame(encoded.put(), properties.put())); + CheckHresult(kErrorCaptureFailed, "Initializing the PNG frame", encoded->Initialize(properties.get())); + CheckHresult(kErrorCaptureFailed, "Sizing the PNG frame", encoded->SetSize(width, height)); + GUID format = GUID_WICPixelFormat32bppBGRA; + CheckHresult(kErrorCaptureFailed, "Selecting the PNG pixel format", encoded->SetPixelFormat(&format)); + CheckHresult(kErrorCaptureFailed, "Writing the PNG pixels", + encoded->WritePixels(height, stride, static_cast(pixels.size()), pixels.data())); + CheckHresult(kErrorCaptureFailed, "Committing the PNG frame", encoded->Commit()); + CheckHresult(kErrorCaptureFailed, "Committing the PNG image", encoder->Commit()); + + STATSTG statistics{}; + CheckHresult(kErrorCaptureFailed, "Measuring the PNG stream", stream->Stat(&statistics, STATFLAG_NONAME)); + const ULONG size = static_cast(statistics.cbSize.QuadPart); + LARGE_INTEGER origin{}; + CheckHresult(kErrorCaptureFailed, "Rewinding the PNG stream", stream->Seek(origin, STREAM_SEEK_SET, nullptr)); + EncodedImage image; + image.png.resize(size); + ULONG read = 0; + CheckHresult(kErrorCaptureFailed, "Reading the PNG stream", stream->Read(image.png.data(), size, &read)); + image.png.resize(read); + image.width = width; + image.height = height; + image.scaleFactor = frame.scaleFactor; + return image; +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/capture.h b/packages/agentic/desktop-driver/native/windows/src/capture.h new file mode 100644 index 00000000000..d3a1d3dd1a2 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/capture.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include +#include +#include + +#include "common.h" + +namespace furn { + +// A single captured window frame in top-down BGRA order. The origin records the +// physical screen coordinate of pixel (0, 0) so element crops can be derived +// from client-relative geometry. +struct RawFrame { + std::vector pixels; + UINT width{0}; + UINT height{0}; + UINT stride{0}; + POINT origin{}; + double scaleFactor{1.0}; +}; + +struct EncodedImage { + std::vector png; + UINT width{0}; + UINT height{0}; + double scaleFactor{1.0}; +}; + +class CaptureEngine { + public: + CaptureEngine(); + ~CaptureEngine(); + CaptureEngine(const CaptureEngine&) = delete; + CaptureEngine& operator=(const CaptureEngine&) = delete; + + bool Available(); + RawFrame CaptureWindow(HWND window, const CancellationToken& token); + EncodedImage Encode(const RawFrame& frame, const RECT& cropInFrame); + + private: + struct Impl; + std::unique_ptr impl_; +}; + +POINT ResolveCaptureOrigin(HWND window, UINT capturedWidth, UINT capturedHeight); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/common.cpp b/packages/agentic/desktop-driver/native/windows/src/common.cpp new file mode 100644 index 00000000000..85bc14c5b46 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/common.cpp @@ -0,0 +1,282 @@ +#include "common.h" + +#include +#include +#include +#include +#include +#include + +namespace furn { +namespace { + +std::string CreateSessionSalt() { + LARGE_INTEGER counter{}; + QueryPerformanceCounter(&counter); + std::mt19937_64 generator(static_cast(counter.QuadPart) ^ + (static_cast(GetCurrentProcessId()) << 32)); + std::array buffer{}; + std::snprintf(buffer.data(), buffer.size(), "%08x", static_cast(generator() & 0xffffffffu)); + return std::string(buffer.data()); +} + +const std::string& SessionSalt() { + static const std::string salt = CreateSessionSalt(); + return salt; +} + +} // namespace + +std::string ToUtf8(std::wstring_view value) { + if (value.empty()) { + return {}; + } + const int length = WideCharToMultiByte(CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0, nullptr, + nullptr); + if (length <= 0) { + Fail(kErrorInternal, "Could not convert UTF-16 text to UTF-8."); + } + std::string output(static_cast(length), '\0'); + WideCharToMultiByte(CP_UTF8, 0, value.data(), static_cast(value.size()), output.data(), length, nullptr, + nullptr); + return output; +} + +std::wstring ToWide(std::string_view value) { + if (value.empty()) { + return {}; + } + const int length = MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0); + if (length <= 0) { + Fail(kErrorInternal, "Could not convert UTF-8 text to UTF-16."); + } + std::wstring output(static_cast(length), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), output.data(), length); + return output; +} + +std::string TrimAscii(std::string_view value) { + const auto isSpace = [](unsigned char character) { return character == ' ' || character == '\t' || character == '\r' || character == '\n'; }; + std::size_t start = 0; + while (start < value.size() && isSpace(static_cast(value[start]))) { + start += 1; + } + std::size_t end = value.size(); + while (end > start && isSpace(static_cast(value[end - 1]))) { + end -= 1; + } + return std::string(value.substr(start, end - start)); +} + +std::string ToLowerAscii(std::string_view value) { + std::string output(value); + std::transform(output.begin(), output.end(), output.begin(), [](unsigned char character) { + return static_cast(character >= 'A' && character <= 'Z' ? character + ('a' - 'A') : character); + }); + return output; +} + +std::wstring TrimWide(std::wstring_view value) { + std::size_t start = 0; + while (start < value.size() && (value[start] == L' ' || value[start] == L'\t')) { + start += 1; + } + std::size_t end = value.size(); + while (end > start && (value[end - 1] == L' ' || value[end - 1] == L'\t')) { + end -= 1; + } + return std::wstring(value.substr(start, end - start)); +} + +void Fail(std::string code, std::string message) { + throw HelperError(std::move(code), message); +} + +std::string DescribeHresult(HRESULT result) { + wchar_t* text = nullptr; + const DWORD length = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, + static_cast(result), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), reinterpret_cast(&text), 0, + nullptr); + std::string description; + if (length > 0 && text != nullptr) { + description = TrimAscii(ToUtf8(std::wstring_view(text, length))); + } + if (text != nullptr) { + LocalFree(text); + } + std::array code{}; + std::snprintf(code.data(), code.size(), "0x%08X", static_cast(result)); + return description.empty() ? std::string(code.data()) : description + " (" + code.data() + ")"; +} + +void FailHresult(std::string code, std::string_view operation, HRESULT result) { + Fail(std::move(code), std::string(operation) + " failed: " + DescribeHresult(result)); +} + +void CheckHresult(std::string code, std::string_view operation, HRESULT result) { + if (FAILED(result)) { + FailHresult(std::move(code), operation, result); + } +} + +void FailLastError(std::string code, std::string_view operation, DWORD error) { + Fail(std::move(code), std::string(operation) + " failed: " + DescribeHresult(HRESULT_FROM_WIN32(error))); +} + +void CancellationToken::Wait(unsigned int milliseconds) const { + // Sleep against a high-resolution deadline so an action pause is never + // shorter than requested and never accumulates system timer rounding. + constexpr long long slice = 10; + LARGE_INTEGER frequency{}; + LARGE_INTEGER start{}; + if (QueryPerformanceFrequency(&frequency) == FALSE || frequency.QuadPart <= 0 || + QueryPerformanceCounter(&start) == FALSE) { + Sleep(milliseconds); + ThrowIfCancelled(); + return; + } + const long long deadline = + start.QuadPart + static_cast((static_cast(milliseconds) / 1000.0) * + static_cast(frequency.QuadPart)); + while (true) { + ThrowIfCancelled(); + LARGE_INTEGER now{}; + if (QueryPerformanceCounter(&now) == FALSE || now.QuadPart >= deadline) { + break; + } + const long long remaining = ((deadline - now.QuadPart) * 1000) / frequency.QuadPart; + Sleep(static_cast(remaining < slice ? (remaining > 0 ? remaining : 1) : slice)); + } + ThrowIfCancelled(); +} + +std::string NextIdentifier(std::string_view prefix) { + static std::atomic counter{1}; + const std::uint64_t value = counter.fetch_add(1, std::memory_order_relaxed); + std::array suffix{}; + std::snprintf(suffix.data(), suffix.size(), "%llx", static_cast(value)); + return std::string(prefix) + "-" + SessionSalt() + "-" + suffix.data(); +} + +std::uint64_t FileTimeTicks(const FILETIME& value) { + ULARGE_INTEGER large{}; + large.LowPart = value.dwLowDateTime; + large.HighPart = value.dwHighDateTime; + return large.QuadPart; +} + +std::string FormatFileTimeIso8601(const FILETIME& value) { + SYSTEMTIME system{}; + if (!FileTimeToSystemTime(&value, &system)) { + return {}; + } + const std::uint64_t ticks = FileTimeTicks(value); + const unsigned int fraction = static_cast(ticks % 10000000ull); + std::array buffer{}; + std::snprintf(buffer.data(), buffer.size(), "%04u-%02u-%02uT%02u:%02u:%02u.%07uZ", system.wYear, system.wMonth, + system.wDay, system.wHour, system.wMinute, system.wSecond, fraction); + return std::string(buffer.data()); +} + +bool TryParseIso8601(std::string_view value, FILETIME& parsed) { + const std::string text = TrimAscii(value); + if (text.size() < 19) { + return false; + } + const auto number = [&text](std::size_t offset, std::size_t length, unsigned int& output) { + if (offset + length > text.size()) { + return false; + } + unsigned int result = 0; + for (std::size_t index = 0; index < length; index += 1) { + const char character = text[offset + index]; + if (character < '0' || character > '9') { + return false; + } + result = result * 10 + static_cast(character - '0'); + } + output = result; + return true; + }; + unsigned int year = 0; + unsigned int month = 0; + unsigned int day = 0; + unsigned int hour = 0; + unsigned int minute = 0; + unsigned int second = 0; + if (!number(0, 4, year) || text[4] != '-' || !number(5, 2, month) || text[7] != '-' || !number(8, 2, day) || + (text[10] != 'T' && text[10] != ' ') || !number(11, 2, hour) || text[13] != ':' || !number(14, 2, minute) || + text[16] != ':' || !number(17, 2, second)) { + return false; + } + std::uint64_t fraction = 0; + std::size_t cursor = 19; + if (cursor < text.size() && text[cursor] == '.') { + cursor += 1; + unsigned int digits = 0; + while (cursor < text.size() && text[cursor] >= '0' && text[cursor] <= '9') { + if (digits < 7) { + fraction = fraction * 10 + static_cast(text[cursor] - '0'); + digits += 1; + } + cursor += 1; + } + while (digits < 7) { + fraction *= 10; + digits += 1; + } + } + long long offsetMinutes = 0; + if (cursor < text.size()) { + const char marker = text[cursor]; + if (marker == 'Z' || marker == 'z') { + cursor += 1; + } else if (marker == '+' || marker == '-') { + unsigned int offsetHour = 0; + unsigned int offsetMinute = 0; + if (!number(cursor + 1, 2, offsetHour)) { + return false; + } + std::size_t minuteOffset = cursor + 3; + if (minuteOffset < text.size() && text[minuteOffset] == ':') { + minuteOffset += 1; + } + if (!number(minuteOffset, 2, offsetMinute)) { + return false; + } + offsetMinutes = static_cast(offsetHour) * 60 + offsetMinute; + if (marker == '+') { + offsetMinutes = -offsetMinutes; + } + cursor = minuteOffset + 2; + } + } + SYSTEMTIME system{}; + system.wYear = static_cast(year); + system.wMonth = static_cast(month); + system.wDay = static_cast(day); + system.wHour = static_cast(hour); + system.wMinute = static_cast(minute); + system.wSecond = static_cast(second); + FILETIME base{}; + if (!SystemTimeToFileTime(&system, &base)) { + return false; + } + const std::uint64_t ticks = + FileTimeTicks(base) + fraction + static_cast(offsetMinutes * 60ll * 10000000ll); + ULARGE_INTEGER large{}; + large.QuadPart = ticks; + parsed.dwLowDateTime = large.LowPart; + parsed.dwHighDateTime = large.HighPart; + return true; +} + +double RoundToHundredths(double value) { + if (!std::isfinite(value)) { + return 0.0; + } + return std::round(value * 100.0) / 100.0; +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/common.h b/packages/agentic/desktop-driver/native/windows/src/common.h new file mode 100644 index 00000000000..fb6f6a85895 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/common.h @@ -0,0 +1,109 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace furn { + +// Wire error codes shared with the Node adapter. "unsupported-operation" and +// "stale-element" carry protocol meaning; the remaining codes are diagnostic. +inline constexpr char kErrorUnsupported[] = "unsupported-operation"; +inline constexpr char kErrorStaleElement[] = "stale-element"; +inline constexpr char kErrorInvalidRequest[] = "invalid-request"; +inline constexpr char kErrorInvalidParams[] = "invalid-params"; +inline constexpr char kErrorNoSuchWindow[] = "no-such-window"; +inline constexpr char kErrorNoSuchElement[] = "no-such-element"; +inline constexpr char kErrorNotInteractable[] = "element-not-interactable"; +inline constexpr char kErrorAutomationFailed[] = "automation-failed"; +inline constexpr char kErrorInputUnavailable[] = "input-unavailable"; +inline constexpr char kErrorInputBusy[] = "input-busy"; +inline constexpr char kErrorInputFailed[] = "input-failed"; +inline constexpr char kErrorLaunchFailed[] = "launch-failed"; +inline constexpr char kErrorAttachFailed[] = "attach-failed"; +inline constexpr char kErrorLeaseInvalid[] = "lease-invalid"; +inline constexpr char kErrorAmbiguousTarget[] = "ambiguous-target"; +inline constexpr char kErrorNoSuchLease[] = "no-such-lease"; +inline constexpr char kErrorWindowActivation[] = "window-activation-failed"; +inline constexpr char kErrorCaptureFailed[] = "capture-failed"; +inline constexpr char kErrorCleanupFailed[] = "cleanup-failed"; +inline constexpr char kErrorTreeTooLarge[] = "tree-too-large"; +inline constexpr char kErrorInternal[] = "internal-error"; + +std::string ToUtf8(std::wstring_view value); +std::wstring ToWide(std::string_view value); +std::string TrimAscii(std::string_view value); +std::string ToLowerAscii(std::string_view value); +std::wstring TrimWide(std::wstring_view value); + +// Structured failure that maps directly onto a protocol error response. +class HelperError : public std::runtime_error { + public: + HelperError(std::string code, const std::string& message) + : std::runtime_error(message), code_(std::move(code)) {} + + const std::string& code() const noexcept { return code_; } + + private: + std::string code_; +}; + +[[noreturn]] void Fail(std::string code, std::string message); +[[noreturn]] void FailHresult(std::string code, std::string_view operation, HRESULT result); +void CheckHresult(std::string code, std::string_view operation, HRESULT result); +[[noreturn]] void FailLastError(std::string code, std::string_view operation, DWORD error); +std::string DescribeHresult(HRESULT result); + +// Raised when a command observes cancellation between two bounded steps. +class CancelledError : public std::exception { + public: + const char* what() const noexcept override { return "The native command was cancelled."; } +}; + +class CancellationToken { + public: + void Cancel() noexcept { cancelled_.store(true, std::memory_order_release); } + void Reset() noexcept { cancelled_.store(false, std::memory_order_release); } + bool IsCancelled() const noexcept { return cancelled_.load(std::memory_order_acquire); } + + void ThrowIfCancelled() const { + if (IsCancelled()) { + throw CancelledError{}; + } + } + + // Sleeps in bounded slices so cancellation is observed promptly. + void Wait(unsigned int milliseconds) const; + + private: + std::atomic cancelled_{false}; +}; + +class Deadline { + public: + explicit Deadline(unsigned int timeoutMs) noexcept : expiresAt_(GetTickCount64() + timeoutMs) {} + + bool Expired() const noexcept { return GetTickCount64() >= expiresAt_; } + + private: + ULONGLONG expiresAt_{0}; +}; + +// Opaque, helper-generated identifiers. Native handles never reach the wire. +std::string NextIdentifier(std::string_view prefix); + +std::string FormatFileTimeIso8601(const FILETIME& value); +bool TryParseIso8601(std::string_view value, FILETIME& parsed); +std::uint64_t FileTimeTicks(const FILETIME& value); + +double RoundToHundredths(double value); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/driver.cpp b/packages/agentic/desktop-driver/native/windows/src/driver.cpp new file mode 100644 index 00000000000..f83607efa24 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/driver.cpp @@ -0,0 +1,555 @@ +#include "driver.h" + +#include +#include + +namespace furn { +namespace { + +json::Value SerializeRect(const RectD& rect) { + json::Value value = json::Value::Object(); + value.Set("height", json::Value::Number(rect.height)); + value.Set("width", json::Value::Number(rect.width)); + value.Set("x", json::Value::Number(rect.x)); + value.Set("y", json::Value::Number(rect.y)); + return value; +} + +std::string RequireString(const json::Value& params, std::string_view key) { + const json::Value* value = params.Find(key); + if (value == nullptr || !value->IsString() || value->AsString().empty()) { + Fail(kErrorInvalidParams, "The request is missing the required \"" + std::string(key) + "\" parameter."); + } + return value->AsString(); +} + +int ActionButton(const json::Value& action) { + const json::Value* button = action.Find("button"); + if (button == nullptr || !button->IsNumber()) { + return 0; + } + return static_cast(button->AsInteger(0)); +} + +unsigned int ActionDuration(const json::Value& action) { + const json::Value* duration = action.Find("duration"); + if (duration == nullptr || !duration->IsNumber()) { + return 0; + } + const std::int64_t value = duration->AsInteger(0); + return value <= 0 ? 0u : static_cast(std::min(value, 30000)); +} + +} // namespace + +WindowContext Driver::ContextForWindow(const std::string& windowId, HWND& window) { + const WindowInfo info = applications_.RequireWindow(windowId); + window = info.window; + WindowContext context; + context.windowId = info.id; + context.window = info.window; + context.primary = info.primary; + if (const ApplicationLease* lease = applications_.LeaseForWindow(info.window); lease != nullptr) { + context.storyRootTestId = ToWide(lease->storyRootTestId); + context.primary = info.primary || info.window == lease->primaryWindow; + } + return context; +} + +HWND Driver::ReferenceWindow() { + const ApplicationLease* lease = applications_.PrimaryLease(); + if (lease != nullptr && lease->primaryWindow != nullptr && IsWindow(lease->primaryWindow) != FALSE) { + return lease->primaryWindow; + } + Fail(kErrorInvalidParams, "The helper does not hold an application window to resolve viewport coordinates against."); +} + +json::Value Driver::ProbeResult(const json::Value& params) { + const bool physical = input_.PhysicalInputAvailable(); + const bool screenshot = capture_.Available(); + json::Value features = json::Value::Object(); + features.Set("accessibilityClick", json::Value::Bool(true)); + features.Set("elementScreenshot", json::Value::Bool(screenshot)); + features.Set("focus", json::Value::Bool(true)); + features.Set("keyboard", json::Value::Bool(physical)); + features.Set("physicalClick", json::Value::Bool(physical)); + features.Set("screenshot", json::Value::Bool(screenshot)); + features.Set("setWindowRect", json::Value::Bool(true)); + features.Set("wheel", json::Value::Bool(physical)); + + json::Value result = json::Value::Object(); + result.Set("endpoint", json::Value::String(params.StringField("endpoint", "windows"))); + result.Set("features", features); + result.Set("platformName", json::Value::String(std::string("windows"))); + result.Set("protocolVersion", json::Value::Integer(1)); + return result; +} + +json::Value Driver::WindowSummary(const WindowInfo& info) { + const WindowMetrics metrics = MeasureWindow(info.window); + json::Value value = json::Value::Object(); + value.Set("id", json::Value::String(info.id)); + value.Set("rect", SerializeRect(ClientRectInScreenDips(metrics))); + value.Set("title", json::Value::String(info.title)); + return value; +} + +ElementSnapshot Driver::RequireLiveElement(const std::string& elementId, const CancellationToken& token) { + return automation_.SnapshotElement(elementId, token); +} + +POINT Driver::ResolveActionPoint(const json::Value& action, const CancellationToken& token) { + const double x = action.NumberField("x", 0.0); + const double y = action.NumberField("y", 0.0); + const json::Value* origin = action.Find("origin"); + if (origin != nullptr && origin->IsObject()) { + const std::string elementId = origin->StringField("elementId"); + if (elementId.empty()) { + Fail(kErrorInvalidParams, "An action element origin is missing its element identifier."); + } + const ElementSnapshot snapshot = RequireLiveElement(elementId, token); + const ElementRecord& record = automation_.RequireRecord(elementId); + const WindowMetrics metrics = MeasureWindow(record.window); + POINT point{}; + point.x = (snapshot.physicalRect.left + snapshot.physicalRect.right) / 2 + + RoundToPixel(PhysicalFromDips(x, metrics.dpi)); + point.y = (snapshot.physicalRect.top + snapshot.physicalRect.bottom) / 2 + + RoundToPixel(PhysicalFromDips(y, metrics.dpi)); + return point; + } + const HWND window = ReferenceWindow(); + const WindowMetrics metrics = MeasureWindow(window); + if (origin != nullptr && origin->IsString() && origin->AsString() == "pointer") { + POINT point = input_.pointer(); + point.x += RoundToPixel(PhysicalFromDips(x, metrics.dpi)); + point.y += RoundToPixel(PhysicalFromDips(y, metrics.dpi)); + return point; + } + return ClientDipsToPhysicalPoint(x, y, metrics); +} + +void Driver::PerformActions(const json::Value& actions, const CancellationToken& token) { + if (!actions.IsArray()) { + Fail(kErrorInvalidParams, "The performActions request requires an array of input sources."); + } + std::size_t ticks = 0; + for (const json::Value& source : actions.Items()) { + const json::Value* list = source.Find("actions"); + if (list != nullptr && list->IsArray()) { + ticks = std::max(ticks, list->Items().size()); + } + } + for (std::size_t tick = 0; tick < ticks; tick += 1) { + token.ThrowIfCancelled(); + unsigned int pause = 0; + for (const json::Value& source : actions.Items()) { + const json::Value* list = source.Find("actions"); + if (list == nullptr || !list->IsArray() || tick >= list->Items().size()) { + continue; + } + const json::Value& action = list->Items()[tick]; + const std::string type = action.StringField("type"); + const std::string sourceType = source.StringField("type", "none"); + if (type == "pause") { + pause = std::max(pause, ActionDuration(action)); + continue; + } + if (sourceType == "key") { + const std::wstring value = action.WideField("value"); + if (value.empty()) { + Fail(kErrorInvalidParams, "A key action is missing its \"value\"."); + } + if (!IsSingleWebDriverKeyValue(value)) { + Fail(kErrorInvalidParams, "A key action value must contain exactly one Unicode code point."); + } + if (type == "keyDown") { + input_.KeyDown(value, token); + } else if (type == "keyUp") { + input_.KeyUp(value, token); + } else { + Fail(kErrorInvalidParams, "Key input sources support only keyDown, keyUp, and pause."); + } + continue; + } + if (sourceType == "pointer") { + if (type == "pointerMove") { + input_.MovePointer(ResolveActionPoint(action, token), ActionDuration(action), token); + } else if (type == "pointerDown") { + input_.PointerDown(ActionButton(action), token); + } else if (type == "pointerUp") { + input_.PointerUp(ActionButton(action), token); + } else if (type == "pointerCancel") { + input_.PointerCancel(); + } else { + Fail(kErrorInvalidParams, "Pointer input sources support only move, down, up, cancel, and pause."); + } + continue; + } + if (sourceType == "wheel") { + if (type != "scroll") { + Fail(kErrorInvalidParams, "Wheel input sources support only scroll and pause."); + } + input_.MovePointer(ResolveActionPoint(action, token), 0, token); + input_.Wheel(action.NumberField("deltaX", 0.0), action.NumberField("deltaY", 0.0), token); + continue; + } + if (sourceType != "none") { + Fail(kErrorInvalidParams, "Input source type \"" + sourceType + "\" is not supported."); + } + } + if (pause > 0) { + token.Wait(pause); + } + } +} + +CommandResult Driver::EncodeCapture(HWND window, const RECT& cropScreen, bool cropToElement, + const CancellationToken& token) { + const RawFrame frame = capture_.CaptureWindow(window, token); + token.ThrowIfCancelled(); + RECT crop{0, 0, static_cast(frame.width), static_cast(frame.height)}; + if (cropToElement) { + crop.left = cropScreen.left - frame.origin.x; + crop.top = cropScreen.top - frame.origin.y; + crop.right = cropScreen.right - frame.origin.x; + crop.bottom = cropScreen.bottom - frame.origin.y; + } + const EncodedImage image = capture_.Encode(frame, crop); + + CommandResult result; + result.result = json::Value::Object(); + result.result.Set("height", json::Value::Integer(image.height)); + result.result.Set("mimeType", json::Value::String(std::string("image/png"))); + result.result.Set("scaleFactor", json::Value::Number(RoundToHundredths(image.scaleFactor))); + result.result.Set("width", json::Value::Integer(image.width)); + result.hasBinary = true; + result.binaryId = NextIdentifier("image"); + result.binaryData = image.png; + result.binaryMetadata = json::Value::Object(); + result.binaryMetadata.Set("id", json::Value::String(result.binaryId)); + result.binaryMetadata.Set("mimeType", json::Value::String(std::string("image/png"))); + result.binaryMetadata.Set("height", json::Value::Integer(image.height)); + result.binaryMetadata.Set("width", json::Value::Integer(image.width)); + result.binaryMetadata.Set("scaleFactor", json::Value::Number(RoundToHundredths(image.scaleFactor))); + return result; +} + +void Driver::ReleaseInput() noexcept { + try { + if (!input_.HasDepressedInput()) { + return; + } + const CancellationToken idle; + try { + const PhysicalInputScope scope(idle, kPhysicalInputCleanupWaitMs, AbandonPolicy::Adopt); + input_.ReleaseAll(); + return; + } catch (const HelperError&) { + // Depressed input must never outlive a cancelled command, so recovery + // proceeds even when the cross-process lock could not be acquired. + input_.ReleaseAll(); + } + } catch (...) { + // Releasing input must never mask the original failure. + } +} + +CommandResult Driver::Execute(const std::string& command, const json::Value& params, const CancellationToken& token) { + CommandResult result; + token.ThrowIfCancelled(); + + if (command == "probe") { + result.result = ProbeResult(params); + return result; + } + if (command == "launch" || command == "attach") { + const ApplicationLease& lease = + command == "launch" ? applications_.Launch(params, token) : applications_.Attach(params, token); + result.result = applications_.SerializeLease(lease); + return result; + } + if (command == "closeApplication") { + const json::Value* lease = params.Find("lease"); + if (lease == nullptr || !lease->IsObject()) { + Fail(kErrorInvalidParams, "The closeApplication request is missing its lease."); + } + const std::string leaseId = lease->StringField("id"); + for (const std::string& windowId : applications_.WindowIdsForLease(leaseId)) { + automation_.ForgetWindow(windowId); + } + applications_.CloseApplication(leaseId, token); + result.result = json::Value::Null(); + return result; + } + if (command == "windows") { + const json::Value* lease = params.Find("lease"); + if (lease == nullptr || !lease->IsObject()) { + Fail(kErrorInvalidParams, "The windows request is missing its lease."); + } + result.result = json::Value::Array(); + for (const WindowInfo& info : applications_.Windows(lease->StringField("id"), token)) { + result.result.Append(WindowSummary(info)); + } + return result; + } + if (command == "closeWindow") { + const std::string windowId = RequireString(params, "windowId"); + HWND window = nullptr; + ContextForWindow(windowId, window); + PostMessageW(window, WM_CLOSE, 0, 0); + automation_.ForgetWindow(windowId); + applications_.ForgetWindow(windowId); + result.result = json::Value::Null(); + return result; + } + if (command == "activate") { + HWND window = nullptr; + ContextForWindow(RequireString(params, "windowId"), window); + ActivateWindow(window, token); + result.result = json::Value::Null(); + return result; + } + if (command == "getWindowRect") { + HWND window = nullptr; + ContextForWindow(RequireString(params, "windowId"), window); + result.result = SerializeRect(ClientRectInScreenDips(MeasureWindow(window))); + return result; + } + if (command == "setWindowRect") { + HWND window = nullptr; + ContextForWindow(RequireString(params, "windowId"), window); + const json::Value* rect = params.Find("rect"); + if (rect == nullptr || !rect->IsObject()) { + Fail(kErrorInvalidParams, "The setWindowRect request is missing its rectangle."); + } + if (IsZoomed(window) != FALSE || IsIconic(window) != FALSE) { + ShowWindow(window, SW_RESTORE); + } + WindowMetrics metrics = MeasureWindow(window); + const RectD current = ClientRectInScreenDips(metrics); + RECT target{}; + target.left = RoundToPixel(PhysicalFromDips(rect->NumberField("x", current.x), metrics.dpi)); + target.top = RoundToPixel(PhysicalFromDips(rect->NumberField("y", current.y), metrics.dpi)); + target.right = target.left + RoundToPixel(PhysicalFromDips(rect->NumberField("width", current.width), metrics.dpi)); + target.bottom = + target.top + RoundToPixel(PhysicalFromDips(rect->NumberField("height", current.height), metrics.dpi)); + const DWORD style = static_cast(GetWindowLongPtrW(window, GWL_STYLE)); + const DWORD exStyle = static_cast(GetWindowLongPtrW(window, GWL_EXSTYLE)); + RECT frame = target; + if (AdjustWindowRectExForDpi(&frame, style, GetMenu(window) != nullptr ? TRUE : FALSE, exStyle, metrics.dpi) == + FALSE) { + FailLastError(kErrorInvalidParams, "Computing the window frame for the requested client size", GetLastError()); + } + // DWM frames, custom title bars, and per-window minimums make the computed + // frame an estimate, so converge on the requested client area by measuring. + for (int attempt = 0; attempt < 4; attempt += 1) { + token.ThrowIfCancelled(); + if (SetWindowPos(window, nullptr, frame.left, frame.top, frame.right - frame.left, frame.bottom - frame.top, + SWP_NOZORDER | SWP_NOACTIVATE) == FALSE) { + FailLastError(kErrorInvalidParams, "Moving the window", GetLastError()); + } + token.Wait(16); + metrics = MeasureWindow(window); + const long deltaX = target.left - metrics.clientOrigin.x; + const long deltaY = target.top - metrics.clientOrigin.y; + const long deltaWidth = (target.right - target.left) - metrics.clientSize.cx; + const long deltaHeight = (target.bottom - target.top) - metrics.clientSize.cy; + if (deltaX == 0 && deltaY == 0 && deltaWidth == 0 && deltaHeight == 0) { + break; + } + const RECT adjusted{frame.left + deltaX, frame.top + deltaY, frame.right + deltaX + deltaWidth, + frame.bottom + deltaY + deltaHeight}; + if (adjusted.left == frame.left && adjusted.top == frame.top && adjusted.right == frame.right && + adjusted.bottom == frame.bottom) { + break; + } + frame = adjusted; + } + result.result = SerializeRect(ClientRectInScreenDips(MeasureWindow(window))); + return result; + } + if (command == "find") { + const json::Value* root = params.Find("root"); + const json::Value* selector = params.Find("selector"); + if (root == nullptr || !root->IsObject() || selector == nullptr || !selector->IsObject()) { + Fail(kErrorInvalidParams, "The find request requires a root and a selector."); + } + HWND window = nullptr; + const WindowContext context = ContextForWindow(RequireString(*root, "windowId"), window); + Selector parsed; + parsed.strategy = selector->StringField("strategy"); + parsed.value = selector->StringField("value"); + result.result = json::Value::Array(); + for (const ElementSnapshot& snapshot : + automation_.Find(context, root->StringField("elementId"), parsed, token)) { + result.result.Append(SerializeSnapshot(snapshot)); + } + return result; + } + if (command == "snapshot") { + result.result = SerializeSnapshot(RequireLiveElement(RequireString(params, "elementId"), token)); + return result; + } + if (command == "activeElement") { + HWND window = nullptr; + const WindowContext context = ContextForWindow(RequireString(params, "windowId"), window); + const std::optional snapshot = automation_.ActiveElement(context, token); + result.result = snapshot ? SerializeSnapshot(*snapshot) : json::Value::Null(); + return result; + } + if (command == "hitTest") { + HWND window = nullptr; + const WindowContext context = ContextForWindow(RequireString(params, "windowId"), window); + const WindowMetrics metrics = MeasureWindow(window); + const POINT point = + ClientDipsToPhysicalPoint(params.NumberField("x", 0.0), params.NumberField("y", 0.0), metrics); + const std::optional snapshot = automation_.HitTest(context, point, token); + result.result = snapshot ? SerializeSnapshot(*snapshot) : json::Value::Null(); + return result; + } + if (command == "click") { + const std::string elementId = RequireString(params, "elementId"); + const std::string mode = params.StringField("mode", "auto"); + RequireLiveElement(elementId, token); + if (mode == "accessibility") { + automation_.AccessibilityClick(elementId, token); + result.result = json::Value::Null(); + return result; + } + if (mode != "physical" && mode != "auto") { + Fail(kErrorInvalidParams, "Click mode \"" + mode + "\" is not supported."); + } + if (!input_.PhysicalInputAvailable()) { + if (mode == "auto") { + automation_.AccessibilityClick(elementId, token); + result.result = json::Value::Null(); + return result; + } + Fail(kErrorInputUnavailable, "Physical pointer input is unavailable on this desktop."); + } + RunPhysical(token, [&]() { + const ElementRecord& record = automation_.RequireRecord(elementId); + if (record.window != nullptr) { + // The server activates the window explicitly before a click; this is a + // best-effort refresh that must not mask the click failure itself. + TryActivateWindow(record.window, token); + } + const ElementSnapshot snapshot = RequireLiveElement(elementId, token); + if (IsEmptyRect(snapshot.physicalRect)) { + Fail(kErrorNotInteractable, "The element does not occupy a clickable area."); + } + POINT center{}; + center.x = (snapshot.physicalRect.left + snapshot.physicalRect.right) / 2; + center.y = (snapshot.physicalRect.top + snapshot.physicalRect.bottom) / 2; + input_.ClickAt(center, token); + }); + result.result = json::Value::Null(); + return result; + } + if (command == "clear") { + const std::string elementId = RequireString(params, "elementId"); + RequireLiveElement(elementId, token); + if (!automation_.TryClearValue(elementId)) { + // Only take focus when the physical fallback can actually run, so a + // failed clear never leaves the application in a changed focus state. + if (!input_.PhysicalInputAvailable()) { + Fail(kErrorInputUnavailable, + "The element does not expose a writable value and physical input is unavailable on this desktop."); + } + RunPhysical(token, [&]() { + automation_.SetFocus(elementId); + token.Wait(16); + input_.PressChord({VK_CONTROL, 'A'}, token); + input_.PressChord({VK_DELETE}, token); + }); + } + result.result = json::Value::Null(); + return result; + } + if (command == "sendKeys") { + const std::string elementId = RequireString(params, "elementId"); + const json::Value* text = params.Find("text"); + if (text == nullptr || !text->IsString()) { + Fail(kErrorInvalidParams, "The sendKeys request requires a string \"text\"."); + } + if (!input_.PhysicalInputAvailable()) { + Fail(kErrorInputUnavailable, "Physical keyboard input is unavailable on this desktop."); + } + RequireLiveElement(elementId, token); + RunPhysical(token, [&]() { + const ElementRecord& record = automation_.RequireRecord(elementId); + if (record.window != nullptr) { + TryActivateWindow(record.window, token); + } + automation_.SetFocus(elementId); + for (int attempt = 0; attempt < 10 && !automation_.HasKeyboardFocus(elementId); attempt += 1) { + token.Wait(20); + } + if (!automation_.HasKeyboardFocus(elementId)) { + Fail(kErrorNotInteractable, "The element did not take keyboard focus, so text could not be typed."); + } + input_.TypeText(text->AsWide(), token); + }); + result.result = json::Value::Null(); + return result; + } + if (command == "performActions") { + const json::Value* actions = params.Find("actions"); + if (actions == nullptr) { + Fail(kErrorInvalidParams, "The performActions request is missing its actions."); + } + // One command owns physical input for its entire tick sequence. + RunPhysical(token, [&]() { PerformActions(*actions, token); }); + result.result = json::Value::Null(); + return result; + } + if (command == "releaseActions") { + RunPhysical(token, [&]() { input_.ReleaseAll(); }); + result.result = json::Value::Null(); + return result; + } + if (command == "captureWindow") { + HWND window = nullptr; + ContextForWindow(RequireString(params, "windowId"), window); + return EncodeCapture(window, RECT{}, false, token); + } + if (command == "captureElement") { + const std::string elementId = RequireString(params, "elementId"); + const ElementSnapshot snapshot = RequireLiveElement(elementId, token); + const ElementRecord& record = automation_.RequireRecord(elementId); + if (record.window == nullptr || IsWindow(record.window) == FALSE) { + Fail(kErrorStaleElement, "The element no longer belongs to a live window."); + } + if (IsEmptyRect(snapshot.physicalRect)) { + Fail(kErrorUnsupported, "The element does not occupy a capturable area."); + } + return EncodeCapture(record.window, snapshot.physicalRect, true, token); + } + if (command == "source") { + HWND window = nullptr; + const WindowContext context = ContextForWindow(RequireString(params, "windowId"), window); + result.result = + json::Value::String(automation_.SerializeSourceXml(automation_.SnapshotWindowTree(context, token))); + return result; + } + if (command == "tree") { + HWND window = nullptr; + const WindowContext context = ContextForWindow(RequireString(params, "windowId"), window); + result.result = json::Value::Array(); + for (const ElementSnapshot& snapshot : automation_.SnapshotWindowTree(context, token)) { + result.result.Append(SerializeSnapshot(snapshot)); + } + return result; + } + if (command == "dispose") { + ReleaseInput(); + applications_.Dispose(token); + automation_.Reset(); + result.result = json::Value::Null(); + return result; + } + + Fail(kErrorUnsupported, "The Windows helper does not implement the \"" + command + "\" command."); +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/driver.h b/packages/agentic/desktop-driver/native/windows/src/driver.h new file mode 100644 index 00000000000..63db73e092a --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/driver.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include + +#include "application.h" +#include "automation.h" +#include "capture.h" +#include "common.h" +#include "input.h" +#include "inputlock.h" +#include "json.h" + +namespace furn { + +struct CommandResult { + json::Value result; + bool hasBinary{false}; + std::string binaryId; + std::vector binaryData; + json::Value binaryMetadata; +}; + +// Executes one protocol command at a time on the dedicated UI Automation +// worker. Every entry point observes cancellation between bounded native steps. +class Driver { + public: + CommandResult Execute(const std::string& command, const json::Value& params, const CancellationToken& token); + + void ReleaseInput() noexcept; + bool HasDepressedInput() const noexcept { return input_.HasDepressedInput(); } + + private: + // Holds the cross-process physical input mutex for a whole action chain and + // guarantees that a failed or cancelled chain releases its depressed input + // before the next owner acquires the lock. + template + void RunPhysical(const CancellationToken& token, Fn&& body) { + const PhysicalInputScope scope(token); + input_.SyncPointerFromCursor(); + try { + body(); + } catch (...) { + if (input_.HasDepressedInput()) { + try { + input_.ReleaseAll(); + } catch (...) { + // Preserve the original failure. + } + } + throw; + } + } + + WindowContext ContextForWindow(const std::string& windowId, HWND& window); + HWND ReferenceWindow(); + json::Value ProbeResult(const json::Value& params); + json::Value WindowSummary(const WindowInfo& info); + void PerformActions(const json::Value& actions, const CancellationToken& token); + POINT ResolveActionPoint(const json::Value& action, const CancellationToken& token); + ElementSnapshot RequireLiveElement(const std::string& elementId, const CancellationToken& token); + CommandResult EncodeCapture(HWND window, const RECT& cropScreen, bool cropToElement, const CancellationToken& token); + + ApplicationManager applications_; + Automation automation_; + InputController input_; + CaptureEngine capture_; +}; + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/framing.cpp b/packages/agentic/desktop-driver/native/windows/src/framing.cpp new file mode 100644 index 00000000000..df198d59ef4 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/framing.cpp @@ -0,0 +1,143 @@ +#include "framing.h" + +#include + +#include "common.h" + +namespace furn { +namespace { + +constexpr std::uint8_t kFrameMagic[4] = {'F', 'D', 'R', '1'}; + +void WriteLittleEndian(std::uint8_t* target, std::uint32_t value) { + target[0] = static_cast(value & 0xffu); + target[1] = static_cast((value >> 8) & 0xffu); + target[2] = static_cast((value >> 16) & 0xffu); + target[3] = static_cast((value >> 24) & 0xffu); +} + +std::uint32_t ReadLittleEndian(const std::uint8_t* source) { + return static_cast(source[0]) | (static_cast(source[1]) << 8) | + (static_cast(source[2]) << 16) | (static_cast(source[3]) << 24); +} + +} // namespace + +std::vector EncodeFrame(std::uint8_t type, const std::uint8_t* payload, std::size_t size) { + const std::size_t maximumSize = + type == kJsonFrameType ? kMaximumJsonFramePayload + : type == kBinaryFrameType ? kMaximumBinaryFramePayload + : 0; + if (maximumSize == 0 || size > maximumSize) { + Fail(kErrorInternal, "The native helper attempted to write an oversized frame."); + } + std::vector frame(kFrameHeaderBytes + size); + std::copy(std::begin(kFrameMagic), std::end(kFrameMagic), frame.begin()); + frame[4] = type; + frame[5] = 0; + frame[6] = 0; + frame[7] = 0; + WriteLittleEndian(frame.data() + 8, static_cast(size)); + if (size > 0) { + std::copy(payload, payload + size, frame.begin() + static_cast(kFrameHeaderBytes)); + } + return frame; +} + +std::vector EncodeJsonFrame(std::string_view json) { + return EncodeFrame(kJsonFrameType, reinterpret_cast(json.data()), json.size()); +} + +std::vector EncodeBinaryFrame(std::string_view identifier, const std::vector& data) { + std::vector payload(4 + identifier.size() + data.size()); + WriteLittleEndian(payload.data(), static_cast(identifier.size())); + std::copy(identifier.begin(), identifier.end(), reinterpret_cast(payload.data()) + 4); + std::copy(data.begin(), data.end(), payload.begin() + static_cast(4 + identifier.size())); + return EncodeFrame(kBinaryFrameType, payload.data(), payload.size()); +} + +bool DecodeFrameHeader(const std::uint8_t* header, std::uint8_t& type, std::uint32_t& length) { + if (!std::equal(std::begin(kFrameMagic), std::end(kFrameMagic), header)) { + return false; + } + type = header[4]; + if (header[5] != 0 || header[6] != 0 || header[7] != 0) { + return false; + } + length = ReadLittleEndian(header + 8); + if (type == kJsonFrameType) { + return length <= kMaximumJsonFramePayload; + } + if (type == kBinaryFrameType) { + return length <= kMaximumBinaryFramePayload; + } + return false; +} + +bool FrameReader::ReadExact(void* buffer, std::size_t length) { + auto* cursor = static_cast(buffer); + std::size_t remaining = length; + while (remaining > 0) { + DWORD read = 0; + if (!ReadFile(input_, cursor, static_cast(remaining), &read, nullptr)) { + const DWORD error = GetLastError(); + if (error == ERROR_BROKEN_PIPE || error == ERROR_HANDLE_EOF) { + return false; + } + FailLastError(kErrorInternal, "Reading the native helper input stream", error); + } + if (read == 0) { + return false; + } + cursor += read; + remaining -= read; + } + return true; +} + +bool FrameReader::Read(Frame& frame) { + std::uint8_t header[kFrameHeaderBytes]{}; + if (!ReadExact(header, sizeof(header))) { + return false; + } + std::uint32_t length = 0; + if (!DecodeFrameHeader(header, frame.type, length)) { + Fail(kErrorInvalidRequest, "The native helper received a malformed frame header."); + } + frame.payload.assign(length, 0); + if (length > 0 && !ReadExact(frame.payload.data(), frame.payload.size())) { + Fail(kErrorInvalidRequest, "The native helper input stream ended inside a frame."); + } + return true; +} + +void FrameWriter::Write(const std::vector& bytes) { + const std::lock_guard guard(mutex_); + const std::uint8_t* cursor = bytes.data(); + std::size_t remaining = bytes.size(); + while (remaining > 0) { + DWORD written = 0; + if (!WriteFile(output_, cursor, static_cast(remaining), &written, nullptr)) { + FailLastError(kErrorInternal, "Writing the native helper output stream", GetLastError()); + } + if (written == 0) { + Fail(kErrorInternal, "Writing the native helper output stream made no progress."); + } + cursor += written; + remaining -= written; + } +} + +void FrameWriter::WriteJson(const json::Value& message) { + WriteJson(message.Serialize()); +} + +void FrameWriter::WriteJson(std::string_view message) { + Write(EncodeJsonFrame(message)); +} + +void FrameWriter::WriteBinary(std::string_view identifier, const std::vector& data) { + Write(EncodeBinaryFrame(identifier, data)); +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/framing.h b/packages/agentic/desktop-driver/native/windows/src/framing.h new file mode 100644 index 00000000000..f497284e053 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/framing.h @@ -0,0 +1,62 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "json.h" + +namespace furn { + +inline constexpr std::uint8_t kJsonFrameType = 1; +inline constexpr std::uint8_t kBinaryFrameType = 2; +inline constexpr std::size_t kFrameHeaderBytes = 12; +inline constexpr std::size_t kMaximumCorrelationIdBytes = 256; +inline constexpr std::uint32_t kMaximumJsonFramePayload = 8u * 1024u * 1024u; +inline constexpr std::uint32_t kMaximumBinaryFramePayload = 64u * 1024u * 1024u; + +struct Frame { + std::uint8_t type{kJsonFrameType}; + std::vector payload; +}; + +std::vector EncodeFrame(std::uint8_t type, const std::uint8_t* payload, std::size_t size); +std::vector EncodeJsonFrame(std::string_view json); +std::vector EncodeBinaryFrame(std::string_view identifier, const std::vector& data); +bool DecodeFrameHeader(const std::uint8_t* header, std::uint8_t& type, std::uint32_t& length); + +// Blocking framed reader over an inherited pipe or file handle. +class FrameReader { + public: + explicit FrameReader(HANDLE input) : input_(input) {} + + bool Read(Frame& frame); + + private: + bool ReadExact(void* buffer, std::size_t length); + + HANDLE input_; +}; + +// Serializes every outbound frame so the worker and reader threads never +// interleave partial writes on stdout. +class FrameWriter { + public: + explicit FrameWriter(HANDLE output) : output_(output) {} + + void WriteJson(const json::Value& message); + void WriteJson(std::string_view message); + void WriteBinary(std::string_view identifier, const std::vector& data); + + private: + void Write(const std::vector& bytes); + + std::mutex mutex_; + HANDLE output_; +}; + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/geometry.cpp b/packages/agentic/desktop-driver/native/windows/src/geometry.cpp new file mode 100644 index 00000000000..dae4ab1c38f --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/geometry.cpp @@ -0,0 +1,111 @@ +#include "geometry.h" + +#include +#include + +#include "common.h" + +namespace furn { + +double DipsFromPhysical(double physical, UINT dpi) { + const UINT effective = dpi == 0 ? kDefaultDpi : dpi; + return physical * static_cast(kDefaultDpi) / static_cast(effective); +} + +double PhysicalFromDips(double dips, UINT dpi) { + const UINT effective = dpi == 0 ? kDefaultDpi : dpi; + return dips * static_cast(effective) / static_cast(kDefaultDpi); +} + +long RoundToPixel(double value) { + if (!std::isfinite(value)) { + return 0; + } + return static_cast(std::llround(value)); +} + +RectD PhysicalRectToClientDips(const RECT& physical, const WindowMetrics& metrics) { + RectD result; + result.x = RoundToHundredths(DipsFromPhysical(static_cast(physical.left - metrics.clientOrigin.x), metrics.dpi)); + result.y = RoundToHundredths(DipsFromPhysical(static_cast(physical.top - metrics.clientOrigin.y), metrics.dpi)); + result.width = RoundToHundredths(DipsFromPhysical(static_cast(physical.right - physical.left), metrics.dpi)); + result.height = RoundToHundredths(DipsFromPhysical(static_cast(physical.bottom - physical.top), metrics.dpi)); + return result; +} + +POINT ClientDipsToPhysicalPoint(double x, double y, const WindowMetrics& metrics) { + POINT point{}; + point.x = metrics.clientOrigin.x + RoundToPixel(PhysicalFromDips(x, metrics.dpi)); + point.y = metrics.clientOrigin.y + RoundToPixel(PhysicalFromDips(y, metrics.dpi)); + return point; +} + +RECT ClientDipsToPhysicalRect(const RectD& rect, const WindowMetrics& metrics) { + const POINT origin = ClientDipsToPhysicalPoint(rect.x, rect.y, metrics); + RECT result{}; + result.left = origin.x; + result.top = origin.y; + result.right = origin.x + RoundToPixel(PhysicalFromDips(rect.width, metrics.dpi)); + result.bottom = origin.y + RoundToPixel(PhysicalFromDips(rect.height, metrics.dpi)); + return result; +} + +RectD ClientRectInScreenDips(const WindowMetrics& metrics) { + RectD result; + result.x = RoundToHundredths(DipsFromPhysical(static_cast(metrics.clientOrigin.x), metrics.dpi)); + result.y = RoundToHundredths(DipsFromPhysical(static_cast(metrics.clientOrigin.y), metrics.dpi)); + result.width = RoundToHundredths(DipsFromPhysical(static_cast(metrics.clientSize.cx), metrics.dpi)); + result.height = RoundToHundredths(DipsFromPhysical(static_cast(metrics.clientSize.cy), metrics.dpi)); + return result; +} + +WindowMetrics MeasureWindow(HWND window) { + if (window == nullptr || IsWindow(window) == FALSE) { + Fail(kErrorNoSuchWindow, "The requested window is no longer available."); + } + WindowMetrics metrics; + metrics.dpi = GetDpiForWindow(window); + if (metrics.dpi == 0) { + metrics.dpi = kDefaultDpi; + } + RECT client{}; + if (GetClientRect(window, &client) == FALSE) { + FailLastError(kErrorNoSuchWindow, "Reading the window client rectangle", GetLastError()); + } + POINT origin{client.left, client.top}; + if (ClientToScreen(window, &origin) == FALSE) { + FailLastError(kErrorNoSuchWindow, "Mapping the window client origin", GetLastError()); + } + metrics.clientOrigin = origin; + metrics.clientSize.cx = client.right - client.left; + metrics.clientSize.cy = client.bottom - client.top; + if (GetWindowRect(window, &metrics.windowRect) == FALSE) { + FailLastError(kErrorNoSuchWindow, "Reading the window rectangle", GetLastError()); + } + return metrics; +} + +bool RectContainsPoint(const RECT& rect, POINT point) { + return point.x >= rect.left && point.x < rect.right && point.y >= rect.top && point.y < rect.bottom; +} + +RECT IntersectRects(const RECT& left, const RECT& right) { + RECT result{}; + result.left = std::max(left.left, right.left); + result.top = std::max(left.top, right.top); + result.right = std::min(left.right, right.right); + result.bottom = std::min(left.bottom, right.bottom); + if (result.right < result.left) { + result.right = result.left; + } + if (result.bottom < result.top) { + result.bottom = result.top; + } + return result; +} + +bool IsEmptyRect(const RECT& rect) { + return rect.right <= rect.left || rect.bottom <= rect.top; +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/geometry.h b/packages/agentic/desktop-driver/native/windows/src/geometry.h new file mode 100644 index 00000000000..cfc14b54662 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/geometry.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +namespace furn { + +// Every public rectangle is expressed in logical device-independent pixels that +// are relative to the top-level client area. UI Automation and the window +// manager both report physical screen pixels, so all conversions funnel through +// this module. +struct RectD { + double x{0.0}; + double y{0.0}; + double width{0.0}; + double height{0.0}; +}; + +struct WindowMetrics { + UINT dpi{96}; + POINT clientOrigin{}; // physical screen coordinates of the client origin + SIZE clientSize{}; // physical client size + RECT windowRect{}; // physical window rectangle +}; + +constexpr UINT kDefaultDpi = 96; + +double DipsFromPhysical(double physical, UINT dpi); +double PhysicalFromDips(double dips, UINT dpi); +long RoundToPixel(double value); + +RectD PhysicalRectToClientDips(const RECT& physical, const WindowMetrics& metrics); +POINT ClientDipsToPhysicalPoint(double x, double y, const WindowMetrics& metrics); +RECT ClientDipsToPhysicalRect(const RectD& rect, const WindowMetrics& metrics); +RectD ClientRectInScreenDips(const WindowMetrics& metrics); + +WindowMetrics MeasureWindow(HWND window); +bool RectContainsPoint(const RECT& rect, POINT point); +RECT IntersectRects(const RECT& left, const RECT& right); +bool IsEmptyRect(const RECT& rect); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/host.cpp b/packages/agentic/desktop-driver/native/windows/src/host.cpp new file mode 100644 index 00000000000..0ef52ad7fc9 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/host.cpp @@ -0,0 +1,302 @@ +#include "host.h" + +#include +#include + +#include +#include +#include + +#include + +#include "build_info.h" + +namespace furn { +namespace { + +void WriteDiagnostic(const std::string& message) { + const std::string bounded = message.size() > 1000 ? message.substr(0, 1000) + "..." : message; + std::cerr << bounded << '\n'; +} + +void WriteFailureDiagnostic(std::string_view operation, const char* message) noexcept { + try { + WriteDiagnostic(std::string(operation) + ": " + message); + } catch (...) { + // A diagnostic failure must never escape a noexcept pipe-failure path. + } +} + +std::string BoundedField(std::string_view value, std::size_t maximumBytes) { + return value.size() <= maximumBytes ? std::string(value) : std::string(value.substr(0, maximumBytes)) + "..."; +} + +} // namespace + +json::Value CreateHello() { + json::Value hello = json::Value::Object(); + hello.Set("type", json::Value::String(std::string("hello"))); + hello.Set("provider", json::Value::String(std::string("windows"))); + hello.Set("architecture", json::Value::String(std::string("x64"))); + hello.Set("buildId", json::Value::String(std::wstring_view(FurnDesktopDriverBuildId))); + hello.Set("sourceDigest", json::Value::String(std::wstring_view(FurnDesktopDriverSourceDigest))); + hello.Set("minimumOs", json::Value::String(std::string("10.0.22000.0"))); + + json::Value protocol = json::Value::Object(); + protocol.Set("major", json::Value::Integer(1)); + protocol.Set("minor", json::Value::Integer(0)); + hello.Set("protocol", protocol); + + json::Value features = json::Value::Array(); + for (const char* feature : {"accessibilityClick", "activeElement", "capture", "elementCapture", "find", "hitTest", + "keyboard", "launch", "attach", "pointer", "probe", "releaseActions", "setWindowRect", + "source", "tree", "wheel"}) { + features.Append(json::Value::String(std::string(feature))); + } + hello.Set("features", features); + return hello; +} + +void Host::RecordWriteFailure(std::string_view operation, const char* message) noexcept { + writeFailed_.store(true, std::memory_order_release); + if (readerThread_ != nullptr) { + CancelSynchronousIo(readerThread_); + } + WriteFailureDiagnostic(operation, message); +} + +bool Host::WriteResponse(const std::string& id, const std::string& command, const CommandResult& result) noexcept { + try { + if (result.hasBinary && + (result.binaryId.size() > kMaximumBinaryFramePayload - 4 || + result.binaryData.size() > kMaximumBinaryFramePayload - 4 - result.binaryId.size())) { + return WriteError(id, command, kErrorCaptureFailed, + "The captured image exceeds the 64 MiB native binary frame limit."); + } + json::Value response = json::Value::Object(); + response.Set("type", json::Value::String(std::string("response"))); + response.Set("id", json::Value::String(id)); + response.Set("result", result.result); + if (result.hasBinary) { + response.Set("binary", result.binaryMetadata); + } + const std::string serialized = response.Serialize(); + if (serialized.size() > kMaximumJsonFramePayload) { + const bool treeCommand = command == "find" || command == "source" || command == "tree"; + return WriteError(id, command, treeCommand ? kErrorTreeTooLarge : kErrorInternal, + "The native command response exceeds the 8 MiB JSON frame limit."); + } + writer_->WriteJson(serialized); + if (result.hasBinary) { + writer_->WriteBinary(result.binaryId, result.binaryData); + } + return true; + } catch (const std::exception& error) { + RecordWriteFailure("Native response write failed", error.what()); + return false; + } catch (...) { + RecordWriteFailure("Native response write failed", "unknown error"); + return false; + } +} + +bool Host::WriteError(const std::string& id, const std::string& command, const std::string& code, + const std::string& message) noexcept { + try { + json::Value error = json::Value::Object(); + error.Set("code", json::Value::String(code)); + error.Set("message", json::Value::String(BoundedField(message, 4096))); + json::Value data = json::Value::Object(); + data.Set("command", json::Value::String(BoundedField(command, 256))); + error.Set("data", data); + + json::Value response = json::Value::Object(); + response.Set("type", json::Value::String(std::string("response"))); + response.Set("id", json::Value::String(id)); + response.Set("error", error); + writer_->WriteJson(response); + return true; + } catch (const std::exception& error) { + RecordWriteFailure("Native error write failed", error.what()); + return false; + } catch (...) { + RecordWriteFailure("Native error write failed", "unknown error"); + return false; + } +} + +bool Host::WriteCancelled(const std::string& id) noexcept { + try { + json::Value cancelled = json::Value::Object(); + cancelled.Set("type", json::Value::String(std::string("cancelled"))); + cancelled.Set("id", json::Value::String(id)); + writer_->WriteJson(cancelled); + return true; + } catch (const std::exception& error) { + RecordWriteFailure("Native cancellation write failed", error.what()); + return false; + } catch (...) { + RecordWriteFailure("Native cancellation write failed", "unknown error"); + return false; + } +} + +void Host::HandleCancel(const std::string& id) { + { + const std::lock_guard guard(mutex_); + if (activeId_ == id) { + // The worker owns the acknowledgement: it must stop side effects and + // release depressed input before the cancellation is reported. + token_.Cancel(); + return; + } + const auto queued = std::find_if(queue_.begin(), queue_.end(), + [&id](const PendingRequest& request) { return request.id == id; }); + if (queued != queue_.end()) { + queue_.erase(queued); + } + } + // A queued or already-settled request has no running side effects, so the + // acknowledgement can be written immediately. + WriteCancelled(id); +} + +void Host::WorkerLoop() { + winrt::init_apartment(winrt::apartment_type::multi_threaded); + while (true) { + PendingRequest request; + { + std::unique_lock guard(mutex_); + signal_.wait(guard, [this] { return stopping_ || !queue_.empty(); }); + if (stopping_ && queue_.empty()) { + break; + } + request = std::move(queue_.front()); + queue_.pop_front(); + token_.Reset(); + activeId_ = request.id; + } + + bool cancelled = false; + bool commandSucceeded = false; + bool wroteResult = true; + try { + const CommandResult result = driver_.Execute(request.command, request.params, token_); + commandSucceeded = true; + wroteResult = WriteResponse(request.id, request.command, result); + } catch (const CancelledError&) { + cancelled = true; + } catch (const HelperError& error) { + wroteResult = WriteError(request.id, request.command, error.code(), error.what()); + } catch (const winrt::hresult_error& error) { + wroteResult = WriteError(request.id, request.command, kErrorInternal, ToUtf8(error.message().c_str())); + } catch (const std::exception& error) { + wroteResult = WriteError(request.id, request.command, kErrorInternal, error.what()); + } + + if (cancelled) { + driver_.ReleaseInput(); + wroteResult = WriteCancelled(request.id); + } + + { + const std::lock_guard guard(mutex_); + activeId_.clear(); + } + if (!wroteResult || writeFailed_.load(std::memory_order_acquire)) { + break; + } + if (request.command == "dispose" && commandSucceeded) { + // The session is over: flush the acknowledged response and end the + // process instead of leaving the reader blocked on a dead stream. + FlushFileBuffers(GetStdHandle(STD_OUTPUT_HANDLE)); + ExitProcess(0); + } + } + winrt::uninit_apartment(); +} + +int Host::Run() { + _setmode(_fileno(stdin), _O_BINARY); + _setmode(_fileno(stdout), _O_BINARY); + FrameWriter writer(GetStdHandle(STD_OUTPUT_HANDLE)); + writer_ = &writer; + FrameReader reader(GetStdHandle(STD_INPUT_HANDLE)); + + writer.WriteJson(CreateHello()); + DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), &readerThread_, 0, FALSE, + DUPLICATE_SAME_ACCESS); + std::thread worker(&Host::WorkerLoop, this); + + int exitCode = 0; + try { + Frame frame; + while (!writeFailed_.load(std::memory_order_acquire) && reader.Read(frame)) { + if (frame.type != kJsonFrameType) { + WriteDiagnostic("The native helper ignored an unexpected binary frame on stdin."); + continue; + } + json::Value message; + try { + message = json::Value::Parse( + std::string_view(reinterpret_cast(frame.payload.data()), frame.payload.size())); + } catch (const HelperError& error) { + WriteDiagnostic(std::string("The native helper ignored malformed JSON: ") + error.what()); + continue; + } + const std::string type = message.StringField("type"); + const std::string id = message.StringField("id"); + if ((type == "cancel" || type == "request") && + (id.empty() || id.size() > kMaximumCorrelationIdBytes)) { + WriteDiagnostic("The native helper ignored a request with an invalid correlation identifier."); + continue; + } + if (type == "cancel") { + HandleCancel(id); + if (writeFailed_.load(std::memory_order_acquire)) { + break; + } + continue; + } + if (type != "request") { + WriteDiagnostic("The native helper received a message that is not a correlated request."); + continue; + } + PendingRequest request; + request.id = id; + request.command = message.StringField("command"); + if (const json::Value* params = message.Find("params"); params != nullptr) { + request.params = *params; + } else { + request.params = json::Value::Object(); + } + { + const std::lock_guard guard(mutex_); + queue_.push_back(std::move(request)); + } + signal_.notify_one(); + } + } catch (const HelperError& error) { + WriteDiagnostic(std::string("furn-desktop-driver-host: ") + error.what()); + exitCode = 1; + } catch (const std::exception& error) { + WriteDiagnostic(std::string("furn-desktop-driver-host: ") + error.what()); + exitCode = 1; + } + + { + const std::lock_guard guard(mutex_); + stopping_ = true; + token_.Cancel(); + } + signal_.notify_all(); + worker.join(); + if (readerThread_ != nullptr) { + CloseHandle(readerThread_); + readerThread_ = nullptr; + } + writer_ = nullptr; + return exitCode; +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/host.h b/packages/agentic/desktop-driver/native/windows/src/host.h new file mode 100644 index 00000000000..95ae70e5ac5 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/host.h @@ -0,0 +1,54 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "driver.h" +#include "framing.h" +#include "json.h" + +namespace furn { + +json::Value CreateHello(); + +// Long-lived stdio host. The reader loop stays responsive while one command +// executes on the dedicated UI Automation worker, so a cancel can always be +// observed mid-command. +class Host { + public: + int Run(); + + private: + struct PendingRequest { + std::string id; + std::string command; + json::Value params; + }; + + void WorkerLoop(); + void HandleCancel(const std::string& id); + bool WriteResponse(const std::string& id, const std::string& command, const CommandResult& result) noexcept; + bool WriteError(const std::string& id, const std::string& command, const std::string& code, + const std::string& message) noexcept; + bool WriteCancelled(const std::string& id) noexcept; + void RecordWriteFailure(std::string_view operation, const char* message) noexcept; + + FrameWriter* writer_{nullptr}; + HANDLE readerThread_{nullptr}; + Driver driver_; + CancellationToken token_; + std::mutex mutex_; + std::condition_variable signal_; + std::deque queue_; + std::string activeId_; + std::atomic writeFailed_{false}; + bool stopping_{false}; +}; + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/input.cpp b/packages/agentic/desktop-driver/native/windows/src/input.cpp new file mode 100644 index 00000000000..b396e7e1323 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/input.cpp @@ -0,0 +1,540 @@ +#include "input.h" + +#include +#include + +#include "geometry.h" + +namespace furn { +namespace { + +struct SpecialKey { + wchar_t value; + WORD virtualKey; + bool extended; +}; + +// W3C WebDriver key code table, restricted to keys the desktop server can +// actually deliver on Windows. +constexpr SpecialKey kSpecialKeys[] = { + {0xE001, VK_CANCEL, false}, {0xE002, VK_HELP, false}, {0xE003, VK_BACK, false}, + {0xE004, VK_TAB, false}, {0xE005, VK_CLEAR, false}, {0xE006, VK_RETURN, false}, + {0xE007, VK_RETURN, true}, {0xE008, VK_SHIFT, false}, {0xE009, VK_CONTROL, false}, + {0xE00A, VK_MENU, false}, {0xE00B, VK_PAUSE, false}, {0xE00C, VK_ESCAPE, false}, + {0xE00D, VK_SPACE, false}, {0xE00E, VK_PRIOR, true}, {0xE00F, VK_NEXT, true}, + {0xE010, VK_END, true}, {0xE011, VK_HOME, true}, {0xE012, VK_LEFT, true}, + {0xE013, VK_UP, true}, {0xE014, VK_RIGHT, true}, {0xE015, VK_DOWN, true}, + {0xE016, VK_INSERT, true}, {0xE017, VK_DELETE, true}, {0xE018, VK_OEM_1, false}, + {0xE019, VK_OEM_PLUS, false}, {0xE01A, VK_NUMPAD0, false}, {0xE01B, VK_NUMPAD1, false}, + {0xE01C, VK_NUMPAD2, false}, {0xE01D, VK_NUMPAD3, false}, {0xE01E, VK_NUMPAD4, false}, + {0xE01F, VK_NUMPAD5, false}, {0xE020, VK_NUMPAD6, false}, {0xE021, VK_NUMPAD7, false}, + {0xE022, VK_NUMPAD8, false}, {0xE023, VK_NUMPAD9, false}, {0xE024, VK_MULTIPLY, false}, + {0xE025, VK_ADD, false}, {0xE026, VK_SEPARATOR, false}, {0xE027, VK_SUBTRACT, false}, + {0xE028, VK_DECIMAL, false}, {0xE029, VK_DIVIDE, true}, {0xE031, VK_F1, false}, + {0xE032, VK_F2, false}, {0xE033, VK_F3, false}, {0xE034, VK_F4, false}, + {0xE035, VK_F5, false}, {0xE036, VK_F6, false}, {0xE037, VK_F7, false}, + {0xE038, VK_F8, false}, {0xE039, VK_F9, false}, {0xE03A, VK_F10, false}, + {0xE03B, VK_F11, false}, {0xE03C, VK_F12, false}, {0xE03D, VK_LWIN, true}, + {0xE050, VK_RSHIFT, false}, {0xE051, VK_RCONTROL, true}, {0xE052, VK_RMENU, true}, + {0xE053, VK_RWIN, true}, {0xE054, VK_PRIOR, false}, {0xE055, VK_NEXT, false}, + {0xE056, VK_END, false}, {0xE057, VK_HOME, false}, {0xE058, VK_LEFT, false}, + {0xE059, VK_UP, false}, {0xE05A, VK_RIGHT, false}, {0xE05B, VK_DOWN, false}, + {0xE05C, VK_INSERT, false}, {0xE05D, VK_DELETE, false}, +}; + +RECT VirtualDesktopRect() { + RECT rect{}; + rect.left = GetSystemMetrics(SM_XVIRTUALSCREEN); + rect.top = GetSystemMetrics(SM_YVIRTUALSCREEN); + rect.right = rect.left + GetSystemMetrics(SM_CXVIRTUALSCREEN); + rect.bottom = rect.top + GetSystemMetrics(SM_CYVIRTUALSCREEN); + if (rect.right <= rect.left || rect.bottom <= rect.top) { + rect.left = 0; + rect.top = 0; + rect.right = std::max(1, GetSystemMetrics(SM_CXSCREEN)); + rect.bottom = std::max(1, GetSystemMetrics(SM_CYSCREEN)); + } + return rect; +} + +void ButtonFlags(int button, DWORD& downFlag, DWORD& upFlag, DWORD& mouseData) { + downFlag = MOUSEEVENTF_LEFTDOWN; + upFlag = MOUSEEVENTF_LEFTUP; + mouseData = 0; + switch (button) { + case 0: + return; + case 1: + downFlag = MOUSEEVENTF_MIDDLEDOWN; + upFlag = MOUSEEVENTF_MIDDLEUP; + return; + case 2: + downFlag = MOUSEEVENTF_RIGHTDOWN; + upFlag = MOUSEEVENTF_RIGHTUP; + return; + case 3: + case 4: + downFlag = MOUSEEVENTF_XDOWN; + upFlag = MOUSEEVENTF_XUP; + mouseData = button == 3 ? XBUTTON1 : XBUTTON2; + return; + default: + Fail(kErrorInvalidParams, "Pointer button " + std::to_string(button) + " is not supported on Windows."); + } +} + +} // namespace + +// A WebDriver key value names exactly one code point, either a printable +// character or a private-use special key. +bool IsSingleWebDriverKeyValue(std::wstring_view value) { + if (value.size() == 1) { + return value[0] < 0xd800 || value[0] > 0xdfff; + } + if (value.size() == 2) { + return value[0] >= 0xd800 && value[0] <= 0xdbff && value[1] >= 0xdc00 && value[1] <= 0xdfff; + } + return false; +} + +ReleaseLedger ParseReleaseLedger(const json::Value& document) { + if (!document.IsObject()) { + Fail(kErrorInvalidParams, "The release ledger must be a JSON object."); + } + for (const auto& member : document.Members()) { + if (member.first != "keys" && member.first != "buttons") { + Fail(kErrorInvalidParams, "The release ledger contains the unknown field \"" + member.first + "\"."); + } + } + + ReleaseLedger ledger; + if (const json::Value* keys = document.Find("keys"); keys != nullptr) { + if (!keys->IsArray()) { + Fail(kErrorInvalidParams, "The release ledger field \"keys\" must be an array."); + } + for (const json::Value& key : keys->Items()) { + if (!key.IsString()) { + Fail(kErrorInvalidParams, "Every release ledger key must be a string."); + } + const std::wstring value = ToWide(key.AsString()); + if (!IsSingleWebDriverKeyValue(value)) { + Fail(kErrorInvalidParams, + "Release ledger key \"" + key.AsString() + "\" is not a single WebDriver key value."); + } + ledger.keys.push_back(value); + } + } + if (const json::Value* buttons = document.Find("buttons"); buttons != nullptr) { + if (!buttons->IsArray()) { + Fail(kErrorInvalidParams, "The release ledger field \"buttons\" must be an array."); + } + for (const json::Value& button : buttons->Items()) { + if (!button.IsNumber()) { + Fail(kErrorInvalidParams, "Every release ledger button must be a number."); + } + const double raw = button.AsNumber(); + const std::int64_t value = button.AsInteger(-1); + if (static_cast(value) != raw || value < 0 || value > 4) { + Fail(kErrorInvalidParams, "Release ledger buttons must be W3C pointer buttons between 0 and 4."); + } + ledger.buttons.push_back(static_cast(value)); + } + } + return ledger; +} + +POINT NormalizeToVirtualDesktop(POINT screenPoint, RECT virtualDesktop) { + const double width = static_cast(virtualDesktop.right - virtualDesktop.left); + const double height = static_cast(virtualDesktop.bottom - virtualDesktop.top); + POINT normalized{}; + normalized.x = RoundToPixel((static_cast(screenPoint.x - virtualDesktop.left) * 65535.0) / + std::max(1.0, width - 1.0)); + normalized.y = RoundToPixel((static_cast(screenPoint.y - virtualDesktop.top) * 65535.0) / + std::max(1.0, height - 1.0)); + normalized.x = std::clamp(normalized.x, 0, 65535); + normalized.y = std::clamp(normalized.y, 0, 65535); + return normalized; +} + +INPUT MakeAbsoluteMouseInput(POINT screenPoint, DWORD flags, DWORD mouseData) { + const POINT normalized = NormalizeToVirtualDesktop(screenPoint, VirtualDesktopRect()); + INPUT input{}; + input.type = INPUT_MOUSE; + input.mi.dx = normalized.x; + input.mi.dy = normalized.y; + input.mi.mouseData = mouseData; + input.mi.dwFlags = flags | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK; + return input; +} + +bool TryMapWebDriverKey(std::wstring_view value, WORD& virtualKey, bool& extended) { + if (value.size() != 1) { + return false; + } + for (const SpecialKey& key : kSpecialKeys) { + if (key.value == value[0]) { + virtualKey = key.virtualKey; + extended = key.extended; + return true; + } + } + return false; +} + +bool InputController::PhysicalInputAvailable() { + if (physicalInput_ == 1) { + return true; + } + const HDESK desktop = OpenInputDesktop(0, FALSE, DESKTOP_READOBJECTS); + if (desktop != nullptr) { + CloseDesktop(desktop); + physicalInput_ = 1; + return true; + } + // Some hosted and packaged launch contexts deny an input-desktop handle even + // while the process shares the active interactive session. A live foreground + // window or active-console session is the non-mutating fallback; SendInput + // remains authoritative and still fails explicitly if UIPI or the desktop + // blocks it. + DWORD sessionId = 0; + return GetForegroundWindow() != nullptr || + (ProcessIdToSessionId(GetCurrentProcessId(), &sessionId) != FALSE && + sessionId == WTSGetActiveConsoleSessionId()); +} + +void InputController::RequirePhysicalInput() { + if (!PhysicalInputAvailable()) { + Fail(kErrorInputUnavailable, + "Physical input is unavailable because the helper is not attached to an interactive input desktop."); + } +} + +void InputController::SyncPointerFromCursor() { + POINT position{}; + if (GetCursorPos(&position) != FALSE) { + pointer_ = position; + pointerInitialized_ = true; + } +} + +void InputController::Send(std::vector& inputs) { + if (inputs.empty()) { + return; + } + const InputSender send = sender_ != nullptr ? sender_ : &SendInput; + const UINT sent = send(static_cast(inputs.size()), inputs.data(), sizeof(INPUT)); + if (sent != inputs.size()) { + FailLastError(kErrorInputFailed, "Injecting physical input", GetLastError()); + } +} + +void InputController::MovePointer(POINT target, unsigned int durationMs, const CancellationToken& token) { + RequirePhysicalInput(); + if (!pointerInitialized_) { + SyncPointerFromCursor(); + pointerInitialized_ = true; + } + const POINT start = pointer_; + const unsigned int steps = durationMs == 0 ? 1 : std::max(1u, std::min(60u, durationMs / 16u)); + for (unsigned int step = 1; step <= steps; step += 1) { + token.ThrowIfCancelled(); + const double progress = static_cast(step) / static_cast(steps); + POINT position{}; + position.x = start.x + RoundToPixel(static_cast(target.x - start.x) * progress); + position.y = start.y + RoundToPixel(static_cast(target.y - start.y) * progress); + std::vector inputs{MakeAbsoluteMouseInput(position, MOUSEEVENTF_MOVE, 0)}; + Send(inputs); + pointer_ = position; + if (step < steps) { + token.Wait(durationMs / steps); + } + } + pointer_ = target; + token.Wait(16); + POINT observed{}; + if (GetCursorPos(&observed) == FALSE || std::abs(observed.x - target.x) > 2 || std::abs(observed.y - target.y) > 2) { + Fail(kErrorInputFailed, + "Windows did not move the pointer to the requested native element coordinate."); + } +} + +void InputController::PointerDown(int button, const CancellationToken& token) { + RequirePhysicalInput(); + token.ThrowIfCancelled(); + DWORD downFlag = 0; + DWORD upFlag = 0; + DWORD mouseData = 0; + ButtonFlags(button, downFlag, upFlag, mouseData); + if (std::find(pressedButtons_.begin(), pressedButtons_.end(), button) != pressedButtons_.end()) { + return; + } + std::vector inputs{MakeAbsoluteMouseInput(pointer_, downFlag, mouseData)}; + Send(inputs); + pressedButtons_.push_back(button); +} + +void InputController::PointerUp(int button, const CancellationToken& token) { + token.ThrowIfCancelled(); + const auto position = std::find(pressedButtons_.begin(), pressedButtons_.end(), button); + if (position == pressedButtons_.end()) { + return; + } + DWORD downFlag = 0; + DWORD upFlag = 0; + DWORD mouseData = 0; + ButtonFlags(button, downFlag, upFlag, mouseData); + std::vector inputs{MakeAbsoluteMouseInput(pointer_, upFlag, mouseData)}; + Send(inputs); + pressedButtons_.erase(position); +} + +void InputController::PointerCancel() { + for (auto iterator = pressedButtons_.rbegin(); iterator != pressedButtons_.rend(); ++iterator) { + DWORD downFlag = 0; + DWORD upFlag = 0; + DWORD mouseData = 0; + ButtonFlags(*iterator, downFlag, upFlag, mouseData); + std::vector inputs{MakeAbsoluteMouseInput(pointer_, upFlag, mouseData)}; + try { + Send(inputs); + } catch (const HelperError&) { + // Release as much as possible; the remaining ledger entries still drop. + } + } + pressedButtons_.clear(); +} + +void InputController::ClickAt(POINT target, const CancellationToken& token) { + MovePointer(target, 0, token); + token.Wait(50); + PointerDown(0, token); + token.Wait(50); + PointerUp(0, token); +} + +void InputController::Wheel(double deltaX, double deltaY, const CancellationToken& token) { + RequirePhysicalInput(); + token.ThrowIfCancelled(); + std::vector inputs; + if (deltaY != 0.0) { + // WebDriver scrolls down with a positive delta; Windows scrolls up with a + // positive wheel value. + inputs.push_back(MakeAbsoluteMouseInput(pointer_, MOUSEEVENTF_WHEEL, + static_cast(static_cast(-RoundToPixel(deltaY))))); + } + if (deltaX != 0.0) { + inputs.push_back(MakeAbsoluteMouseInput(pointer_, MOUSEEVENTF_HWHEEL, + static_cast(static_cast(RoundToPixel(deltaX))))); + } + Send(inputs); +} + +std::vector InputController::StrokesForValue(std::wstring_view value) const { + std::vector strokes; + WORD virtualKey = 0; + bool extended = false; + if (TryMapWebDriverKey(value, virtualKey, extended)) { + strokes.push_back(KeyStroke{false, virtualKey, extended}); + return strokes; + } + for (const wchar_t unit : value) { + strokes.push_back(KeyStroke{true, static_cast(unit), false}); + } + return strokes; +} + +void InputController::SendStroke(const KeyStroke& stroke, bool down) { + INPUT input{}; + input.type = INPUT_KEYBOARD; + if (stroke.unicode) { + input.ki.wVk = 0; + input.ki.wScan = stroke.code; + input.ki.dwFlags = KEYEVENTF_UNICODE | (down ? 0u : KEYEVENTF_KEYUP); + } else { + input.ki.wVk = stroke.code; + input.ki.wScan = static_cast(MapVirtualKeyW(stroke.code, MAPVK_VK_TO_VSC)); + input.ki.dwFlags = (stroke.extended ? KEYEVENTF_EXTENDEDKEY : 0u) | (down ? 0u : KEYEVENTF_KEYUP); + } + std::vector inputs{input}; + Send(inputs); +} + +void InputController::KeyDown(std::wstring_view value, const CancellationToken& token) { + RequirePhysicalInput(); + token.ThrowIfCancelled(); + if (!IsSingleWebDriverKeyValue(value)) { + Fail(kErrorInvalidParams, "A keyDown value must contain exactly one Unicode code point."); + } + if (value == L"\uE000") { + ReleaseAll(); + return; + } + const std::vector strokes = StrokesForValue(value); + // The ledger records the press before injection so a partially injected + // sequence is still released by the ledger. + pressedKeys_.emplace_back(std::wstring(value), strokes); + for (const KeyStroke& stroke : strokes) { + SendStroke(stroke, true); + } +} + +void InputController::KeyUp(std::wstring_view value, const CancellationToken& token) { + token.ThrowIfCancelled(); + if (!IsSingleWebDriverKeyValue(value)) { + Fail(kErrorInvalidParams, "A keyUp value must contain exactly one Unicode code point."); + } + if (value == L"\uE000") { + ReleaseAll(); + return; + } + for (auto iterator = pressedKeys_.rbegin(); iterator != pressedKeys_.rend(); ++iterator) { + if (iterator->first != value) { + continue; + } + RequirePhysicalInput(); + for (auto stroke = iterator->second.rbegin(); stroke != iterator->second.rend(); ++stroke) { + SendStroke(*stroke, false); + } + pressedKeys_.erase(std::next(iterator).base()); + return; + } + // A key that this helper never pressed still produces the physical release so + // the application observes a well-formed key sequence. + RequirePhysicalInput(); + const std::vector strokes = StrokesForValue(value); + for (auto stroke = strokes.rbegin(); stroke != strokes.rend(); ++stroke) { + SendStroke(*stroke, false); + } +} + +void InputController::TypeText(std::wstring_view text, const CancellationToken& token) { + RequirePhysicalInput(); + for (std::size_t index = 0; index < text.size(); index += 1) { + token.ThrowIfCancelled(); + const std::wstring_view unit = text.substr(index, 1); + WORD virtualKey = 0; + bool extended = false; + if (TryMapWebDriverKey(unit, virtualKey, extended)) { + PressAndRelease(std::wstring(unit), {KeyStroke{false, virtualKey, extended}}, token); + continue; + } + if (text[index] == L'\n') { + PressAndRelease(L"\uE006", {KeyStroke{false, VK_RETURN, false}}, token); + continue; + } + if (text[index] == L'\r') { + continue; + } + PressAndRelease(std::wstring(unit), {KeyStroke{true, static_cast(text[index]), false}}, token); + } +} + +void InputController::PressAndRelease(std::wstring ledgerKey, const std::vector& strokes, + const CancellationToken& token) { + token.ThrowIfCancelled(); + pressedKeys_.emplace_back(std::move(ledgerKey), strokes); + for (const KeyStroke& stroke : strokes) { + SendStroke(stroke, true); + } + const std::vector& recorded = pressedKeys_.back().second; + for (auto stroke = recorded.rbegin(); stroke != recorded.rend(); ++stroke) { + SendStroke(*stroke, false); + } + pressedKeys_.pop_back(); +} + +void InputController::PressChord(const std::vector& virtualKeys, const CancellationToken& token) { + RequirePhysicalInput(); + token.ThrowIfCancelled(); + const std::size_t base = pressedKeys_.size(); + for (const WORD key : virtualKeys) { + const KeyStroke stroke{false, key, false}; + // Chord keys enter the ledger under a synthetic name that can never + // collide with a WebDriver key value, so a failed chord still releases. + pressedKeys_.emplace_back(L"chord:" + std::to_wstring(key), std::vector{stroke}); + SendStroke(stroke, true); + } + while (pressedKeys_.size() > base) { + const std::vector& strokes = pressedKeys_.back().second; + for (auto stroke = strokes.rbegin(); stroke != strokes.rend(); ++stroke) { + SendStroke(*stroke, false); + } + pressedKeys_.pop_back(); + } +} + +ReleaseCounts InputController::ReleaseExact(const ReleaseLedger& ledger) { + ReleaseCounts counts; + if (ledger.empty()) { + return counts; + } + RequirePhysicalInput(); + SyncPointerFromCursor(); + for (auto key = ledger.keys.rbegin(); key != ledger.keys.rend(); ++key) { + const std::vector strokes = StrokesForValue(*key); + for (auto stroke = strokes.rbegin(); stroke != strokes.rend(); ++stroke) { + SendStroke(*stroke, false); + } + counts.keys += 1; + } + for (auto button = ledger.buttons.rbegin(); button != ledger.buttons.rend(); ++button) { + DWORD downFlag = 0; + DWORD upFlag = 0; + DWORD mouseData = 0; + ButtonFlags(*button, downFlag, upFlag, mouseData); + std::vector inputs{MakeAbsoluteMouseInput(pointer_, upFlag, mouseData)}; + Send(inputs); + counts.buttons += 1; + } + return counts; +} + +std::size_t InputController::ReleaseDesktopModifiers() { + static constexpr WORD kStickyKeys[] = {VK_LSHIFT, VK_RSHIFT, VK_LCONTROL, VK_RCONTROL, + VK_LMENU, VK_RMENU, VK_LWIN, VK_RWIN}; + static constexpr WORD kStickyButtons[] = {VK_LBUTTON, VK_RBUTTON, VK_MBUTTON, VK_XBUTTON1, VK_XBUTTON2}; + std::size_t released = 0; + for (const WORD key : kStickyKeys) { + if ((GetAsyncKeyState(key) & 0x8000) == 0) { + continue; + } + const bool extended = key == VK_RCONTROL || key == VK_RMENU || key == VK_LWIN || key == VK_RWIN; + SendStroke(KeyStroke{false, key, extended}, false); + released += 1; + } + for (const WORD button : kStickyButtons) { + if ((GetAsyncKeyState(button) & 0x8000) == 0) { + continue; + } + DWORD downFlag = 0; + DWORD upFlag = 0; + DWORD mouseData = 0; + const int index = button == VK_LBUTTON ? 0 + : button == VK_MBUTTON ? 1 + : button == VK_RBUTTON ? 2 + : button == VK_XBUTTON1 ? 3 + : 4; + ButtonFlags(index, downFlag, upFlag, mouseData); + SyncPointerFromCursor(); + std::vector inputs{MakeAbsoluteMouseInput(pointer_, upFlag, mouseData)}; + Send(inputs); + released += 1; + } + return released; +} + +void InputController::ReleaseAll() { + for (auto iterator = pressedKeys_.rbegin(); iterator != pressedKeys_.rend(); ++iterator) { + for (auto stroke = iterator->second.rbegin(); stroke != iterator->second.rend(); ++stroke) { + try { + SendStroke(*stroke, false); + } catch (const HelperError&) { + // Continue releasing the remaining ledger entries. + } + } + } + pressedKeys_.clear(); + PointerCancel(); +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/input.h b/packages/agentic/desktop-driver/native/windows/src/input.h new file mode 100644 index 00000000000..7704297d38d --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/input.h @@ -0,0 +1,103 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include "common.h" +#include "json.h" + +namespace furn { + +// One physical key stroke exactly as it was injected, so the release ledger can +// undo it with the matching flags instead of guessing. +struct KeyStroke { + bool unicode{false}; + WORD code{0}; + bool extended{false}; +}; + +using InputSender = UINT(WINAPI*)(UINT, LPINPUT, int); + +// The exact set of depressed input that another helper instance reported. The +// Node adapter sends it on stdin so recovery releases those inputs and nothing +// else. +struct ReleaseLedger { + std::vector keys; + std::vector buttons; + + bool empty() const noexcept { return keys.empty() && buttons.empty(); } +}; + +struct ReleaseCounts { + std::size_t keys{0}; + std::size_t buttons{0}; +}; + +bool IsSingleWebDriverKeyValue(std::wstring_view value); +ReleaseLedger ParseReleaseLedger(const json::Value& document); + +// Owns every depressed key and pointer button that this helper injected. The +// ledger is the only sanctioned way to release input after failure, +// cancellation, or an explicit release request. +class InputController { + public: + bool PhysicalInputAvailable(); + + POINT pointer() const noexcept { return pointer_; } + void SyncPointerFromCursor(); + + void MovePointer(POINT target, unsigned int durationMs, const CancellationToken& token); + void PointerDown(int button, const CancellationToken& token); + void PointerUp(int button, const CancellationToken& token); + void PointerCancel(); + void ClickAt(POINT target, const CancellationToken& token); + void Wheel(double deltaX, double deltaY, const CancellationToken& token); + + void KeyDown(std::wstring_view value, const CancellationToken& token); + void KeyUp(std::wstring_view value, const CancellationToken& token); + void TypeText(std::wstring_view text, const CancellationToken& token); + void PressChord(const std::vector& virtualKeys, const CancellationToken& token); + + void ReleaseAll(); + // Releases exactly the listed keys and buttons in reverse order, using the + // same injection flags that pressed them. + ReleaseCounts ReleaseExact(const ReleaseLedger& ledger); + // Releases modifiers and buttons that the operating system still reports as + // depressed. This is a diagnostics sweep, not the adapter recovery path. + std::size_t ReleaseDesktopModifiers(); + bool HasDepressedInput() const noexcept { return !pressedKeys_.empty() || !pressedButtons_.empty(); } + std::size_t DepressedKeyCount() const noexcept { return pressedKeys_.size(); } + std::size_t DepressedButtonCount() const noexcept { return pressedButtons_.size(); } + + // Redirects injection so the built-in self-test can exercise the ledger + // without touching the real input desktop. + void UseTestSender(InputSender sender) { + sender_ = sender; + physicalInput_ = 1; + } + + private: + void RequirePhysicalInput(); + void PressAndRelease(std::wstring ledgerKey, const std::vector& strokes, + const CancellationToken& token); + void Send(std::vector& inputs); + void SendStroke(const KeyStroke& stroke, bool down); + std::vector StrokesForValue(std::wstring_view value) const; + + std::vector>> pressedKeys_; + std::vector pressedButtons_; + POINT pointer_{}; + bool pointerInitialized_{false}; + int physicalInput_{-1}; + InputSender sender_{nullptr}; +}; + +bool TryMapWebDriverKey(std::wstring_view value, WORD& virtualKey, bool& extended); +INPUT MakeAbsoluteMouseInput(POINT screenPoint, DWORD flags, DWORD mouseData); +POINT NormalizeToVirtualDesktop(POINT screenPoint, RECT virtualDesktop); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/inputlock.cpp b/packages/agentic/desktop-driver/native/windows/src/inputlock.cpp new file mode 100644 index 00000000000..490ee394978 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/inputlock.cpp @@ -0,0 +1,137 @@ +#include "inputlock.h" + +#include +#include +#include +#include + +namespace furn { +namespace { + +// The named mutex handle lives for the life of the process; ownership is +// tracked per thread so nested command scopes never wait twice. +std::mutex stateMutex; +HANDLE inputMutex = nullptr; +DWORD ownerThread = 0; +unsigned int ownerDepth = 0; +std::wstring inputMutexName = kPhysicalInputMutexName; + +HANDLE EnsureMutex() { + if (inputMutex == nullptr) { + inputMutex = CreateMutexW(nullptr, FALSE, inputMutexName.c_str()); + if (inputMutex == nullptr) { + FailLastError(kErrorInputBusy, "Creating the physical input mutex", GetLastError()); + } + } + return inputMutex; +} + +void WriteDiagnostic(const std::string& message) { + std::cerr << "furn-desktop-driver-host: " << message << '\n'; +} + +} // namespace + +const wchar_t* PhysicalInputMutexName() noexcept { + return inputMutexName.c_str(); +} + +void SetPhysicalInputMutexNameForTesting(std::wstring name) { + const std::lock_guard guard(stateMutex); + if (name.empty() || ownerDepth != 0) { + Fail(kErrorInvalidParams, "The physical input mutex test name is invalid or the mutex is currently owned."); + } + if (inputMutex != nullptr) { + CloseHandle(inputMutex); + inputMutex = nullptr; + } + inputMutexName = std::move(name); + ownerThread = 0; +} + +PhysicalInputScope::PhysicalInputScope(const CancellationToken& token, unsigned int timeoutMs, AbandonPolicy policy) { + HANDLE handle = nullptr; + { + const std::lock_guard guard(stateMutex); + handle = EnsureMutex(); + if (ownerDepth > 0 && ownerThread == GetCurrentThreadId()) { + ownerDepth += 1; + owns_ = true; + nested_ = true; + return; + } + } + + const Deadline deadline(timeoutMs); + while (true) { + token.ThrowIfCancelled(); + const DWORD result = WaitForSingleObject(handle, 50); + if (result == WAIT_OBJECT_0) { + break; + } + if (result == WAIT_ABANDONED) { + if (policy == AbandonPolicy::Adopt) { + WriteDiagnostic("adopting physical input abandoned by a terminated helper."); + break; + } + ReleaseMutex(handle); + Fail(kErrorInputBusy, + "A previous desktop driver helper terminated while holding physical input, so keys or buttons may still " + "be depressed. Run the helper with --release-input to recover, then retry the command."); + } + if (result != WAIT_TIMEOUT) { + FailLastError(kErrorInputBusy, "Waiting for the physical input mutex", GetLastError()); + } + if (deadline.Expired()) { + Fail(kErrorInputBusy, "Another desktop driver helper held physical input for longer than " + + std::to_string(timeoutMs) + + " ms. Wait for the other session to finish its input command, or run the helper with " + "--release-input if that session is gone."); + } + } + + const std::lock_guard guard(stateMutex); + ownerThread = GetCurrentThreadId(); + ownerDepth = 1; + owns_ = true; +} + +PhysicalInputScope::PhysicalInputScope(PhysicalInputScope&& other) noexcept + : owns_(other.owns_), nested_(other.nested_) { + other.owns_ = false; + other.nested_ = false; +} + +PhysicalInputScope& PhysicalInputScope::operator=(PhysicalInputScope&& other) noexcept { + if (this != &other) { + Release(); + owns_ = other.owns_; + nested_ = other.nested_; + other.owns_ = false; + other.nested_ = false; + } + return *this; +} + +PhysicalInputScope::~PhysicalInputScope() { + Release(); +} + +void PhysicalInputScope::Release() noexcept { + if (!owns_) { + return; + } + owns_ = false; + nested_ = false; + const std::lock_guard guard(stateMutex); + if (ownerDepth == 0 || ownerThread != GetCurrentThreadId()) { + return; + } + ownerDepth -= 1; + if (ownerDepth == 0) { + ownerThread = 0; + ReleaseMutex(inputMutex); + } +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/inputlock.h b/packages/agentic/desktop-driver/native/windows/src/inputlock.h new file mode 100644 index 00000000000..ca4a9f797df --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/inputlock.h @@ -0,0 +1,53 @@ +#pragma once + +#include + +#include + +#include "common.h" + +namespace furn { + +// Physical input is a single machine-wide resource. Independent helper +// processes in one interactive session serialize on this named mutex, which +// lives in the session-local namespace so no privilege is required. +inline constexpr wchar_t kPhysicalInputMutexName[] = L"Local\\FurnDesktopDriverPhysicalInput-v1"; +inline constexpr unsigned int kPhysicalInputWaitMs = 30000; +inline constexpr unsigned int kPhysicalInputCleanupWaitMs = 5000; + +enum class AbandonPolicy { + // A helper that died holding physical input may have left keys depressed, so + // the next owner reports it instead of silently continuing. + Fail, + // The recovery path exists precisely to clean that state up. + Adopt, +}; + +// RAII ownership of the cross-process physical input mutex. Nested scopes on +// the owning thread are counted, so a command boundary can hold the lock while +// inner helpers run without a second operating-system wait. +class PhysicalInputScope { + public: + PhysicalInputScope() noexcept = default; + explicit PhysicalInputScope(const CancellationToken& token, unsigned int timeoutMs = kPhysicalInputWaitMs, + AbandonPolicy policy = AbandonPolicy::Fail); + ~PhysicalInputScope(); + + PhysicalInputScope(const PhysicalInputScope&) = delete; + PhysicalInputScope& operator=(const PhysicalInputScope&) = delete; + PhysicalInputScope(PhysicalInputScope&& other) noexcept; + PhysicalInputScope& operator=(PhysicalInputScope&& other) noexcept; + + bool owns() const noexcept { return owns_; } + bool nested() const noexcept { return nested_; } + void Release() noexcept; + + private: + bool owns_{false}; + bool nested_{false}; +}; + +const wchar_t* PhysicalInputMutexName() noexcept; +void SetPhysicalInputMutexNameForTesting(std::wstring name); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/json.cpp b/packages/agentic/desktop-driver/native/windows/src/json.cpp new file mode 100644 index 00000000000..38c53b78a48 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/json.cpp @@ -0,0 +1,516 @@ +#include "json.h" + +#include +#include +#include +#include + +#include "common.h" + +namespace furn::json { +namespace { + +constexpr int kMaximumDepth = 64; + +class Parser { + public: + explicit Parser(std::string_view text) : text_(text) {} + + Value Parse() { + SkipWhitespace(); + Value value = ParseValue(0); + SkipWhitespace(); + if (cursor_ != text_.size()) { + Fail(kErrorInvalidRequest, "Trailing content followed a JSON document."); + } + return value; + } + + private: + [[noreturn]] void Reject(std::string_view reason) const { + Fail(kErrorInvalidRequest, "Invalid JSON at offset " + std::to_string(cursor_) + ": " + std::string(reason)); + } + + void SkipWhitespace() { + while (cursor_ < text_.size()) { + const char character = text_[cursor_]; + if (character == ' ' || character == '\t' || character == '\n' || character == '\r') { + cursor_ += 1; + continue; + } + break; + } + } + + char Peek() const { + if (cursor_ >= text_.size()) { + Reject("unexpected end of document"); + } + return text_[cursor_]; + } + + void Expect(char character) { + if (Peek() != character) { + Reject(std::string("expected '") + character + "'"); + } + cursor_ += 1; + } + + bool Consume(std::string_view literal) { + if (text_.compare(cursor_, literal.size(), literal) == 0) { + cursor_ += literal.size(); + return true; + } + return false; + } + + Value ParseValue(int depth) { + if (depth > kMaximumDepth) { + Reject("document is nested too deeply"); + } + switch (Peek()) { + case '{': + return ParseObject(depth); + case '[': + return ParseArray(depth); + case '"': + return Value::String(ParseString()); + case 't': + if (Consume("true")) { + return Value::Bool(true); + } + Reject("expected true"); + case 'f': + if (Consume("false")) { + return Value::Bool(false); + } + Reject("expected false"); + case 'n': + if (Consume("null")) { + return Value::Null(); + } + Reject("expected null"); + default: + return ParseNumber(); + } + } + + Value ParseObject(int depth) { + Expect('{'); + Value object = Value::Object(); + SkipWhitespace(); + if (Peek() == '}') { + cursor_ += 1; + return object; + } + while (true) { + SkipWhitespace(); + std::string key = ParseString(); + SkipWhitespace(); + Expect(':'); + SkipWhitespace(); + object.Set(std::move(key), ParseValue(depth + 1)); + SkipWhitespace(); + const char separator = Peek(); + cursor_ += 1; + if (separator == '}') { + return object; + } + if (separator != ',') { + cursor_ -= 1; + Reject("expected ',' or '}'"); + } + } + } + + Value ParseArray(int depth) { + Expect('['); + Value array = Value::Array(); + SkipWhitespace(); + if (Peek() == ']') { + cursor_ += 1; + return array; + } + while (true) { + SkipWhitespace(); + array.Append(ParseValue(depth + 1)); + SkipWhitespace(); + const char separator = Peek(); + cursor_ += 1; + if (separator == ']') { + return array; + } + if (separator != ',') { + cursor_ -= 1; + Reject("expected ',' or ']'"); + } + } + } + + void AppendCodepoint(std::string& output, std::uint32_t codepoint) const { + if (codepoint <= 0x7f) { + output.push_back(static_cast(codepoint)); + } else if (codepoint <= 0x7ff) { + output.push_back(static_cast(0xc0 | (codepoint >> 6))); + output.push_back(static_cast(0x80 | (codepoint & 0x3f))); + } else if (codepoint <= 0xffff) { + output.push_back(static_cast(0xe0 | (codepoint >> 12))); + output.push_back(static_cast(0x80 | ((codepoint >> 6) & 0x3f))); + output.push_back(static_cast(0x80 | (codepoint & 0x3f))); + } else { + output.push_back(static_cast(0xf0 | (codepoint >> 18))); + output.push_back(static_cast(0x80 | ((codepoint >> 12) & 0x3f))); + output.push_back(static_cast(0x80 | ((codepoint >> 6) & 0x3f))); + output.push_back(static_cast(0x80 | (codepoint & 0x3f))); + } + } + + std::uint32_t ParseHexQuad() { + if (cursor_ + 4 > text_.size()) { + Reject("truncated \\u escape"); + } + std::uint32_t value = 0; + for (int index = 0; index < 4; index += 1) { + const char character = text_[cursor_ + static_cast(index)]; + value *= 16; + if (character >= '0' && character <= '9') { + value += static_cast(character - '0'); + } else if (character >= 'a' && character <= 'f') { + value += static_cast(character - 'a' + 10); + } else if (character >= 'A' && character <= 'F') { + value += static_cast(character - 'A' + 10); + } else { + Reject("invalid \\u escape"); + } + } + cursor_ += 4; + return value; + } + + std::string ParseString() { + Expect('"'); + std::string output; + while (true) { + if (cursor_ >= text_.size()) { + Reject("unterminated string"); + } + const char character = text_[cursor_]; + cursor_ += 1; + if (character == '"') { + return output; + } + if (character != '\\') { + output.push_back(character); + continue; + } + if (cursor_ >= text_.size()) { + Reject("unterminated escape"); + } + const char escape = text_[cursor_]; + cursor_ += 1; + switch (escape) { + case '"': + output.push_back('"'); + break; + case '\\': + output.push_back('\\'); + break; + case '/': + output.push_back('/'); + break; + case 'b': + output.push_back('\b'); + break; + case 'f': + output.push_back('\f'); + break; + case 'n': + output.push_back('\n'); + break; + case 'r': + output.push_back('\r'); + break; + case 't': + output.push_back('\t'); + break; + case 'u': { + std::uint32_t codepoint = ParseHexQuad(); + if (codepoint >= 0xd800 && codepoint <= 0xdbff && cursor_ + 1 < text_.size() && text_[cursor_] == '\\' && + text_[cursor_ + 1] == 'u') { + const std::size_t restore = cursor_; + cursor_ += 2; + const std::uint32_t low = ParseHexQuad(); + if (low >= 0xdc00 && low <= 0xdfff) { + codepoint = 0x10000 + ((codepoint - 0xd800) << 10) + (low - 0xdc00); + } else { + cursor_ = restore; + } + } + if (codepoint >= 0xd800 && codepoint <= 0xdfff) { + codepoint = 0xfffd; + } + AppendCodepoint(output, codepoint); + break; + } + default: + Reject("unsupported escape sequence"); + } + } + } + + Value ParseNumber() { + const std::size_t start = cursor_; + if (cursor_ < text_.size() && (text_[cursor_] == '-' || text_[cursor_] == '+')) { + cursor_ += 1; + } + while (cursor_ < text_.size()) { + const char character = text_[cursor_]; + if ((character >= '0' && character <= '9') || character == '.' || character == 'e' || character == 'E' || + character == '-' || character == '+') { + cursor_ += 1; + continue; + } + break; + } + if (start == cursor_) { + Reject("expected a value"); + } + double parsed = 0.0; + const char* begin = text_.data() + start; + const char* end = text_.data() + cursor_; + const auto result = std::from_chars(begin, end, parsed); + if (result.ec != std::errc{} || result.ptr != end) { + Reject("invalid number"); + } + return Value::Number(parsed); + } + + std::string_view text_; + std::size_t cursor_{0}; +}; + +void SerializeInto(const Value& value, std::string& output) { + switch (value.kind()) { + case Kind::Null: + output += "null"; + return; + case Kind::Bool: + output += value.AsBool() ? "true" : "false"; + return; + case Kind::Number: { + const double number = value.AsNumber(); + if (!std::isfinite(number)) { + output += "null"; + return; + } + if (number == std::floor(number) && std::fabs(number) < 9007199254740992.0) { + output += std::to_string(static_cast(number)); + return; + } + std::array buffer{}; + const auto result = std::to_chars(buffer.data(), buffer.data() + buffer.size(), number); + output.append(buffer.data(), result.ptr); + return; + } + case Kind::String: + output += EscapeString(value.AsString()); + return; + case Kind::Array: { + output.push_back('['); + bool first = true; + for (const Value& item : value.Items()) { + if (!first) { + output.push_back(','); + } + first = false; + SerializeInto(item, output); + } + output.push_back(']'); + return; + } + case Kind::Object: { + output.push_back('{'); + bool first = true; + for (const auto& member : value.Members()) { + if (!first) { + output.push_back(','); + } + first = false; + output += EscapeString(member.first); + output.push_back(':'); + SerializeInto(member.second, output); + } + output.push_back('}'); + return; + } + } +} + +} // namespace + +Value Value::Null() { + return Value{}; +} + +Value Value::Bool(bool value) { + Value result; + result.kind_ = Kind::Bool; + result.boolean_ = value; + return result; +} + +Value Value::Number(double value) { + Value result; + result.kind_ = Kind::Number; + result.number_ = value; + return result; +} + +Value Value::Integer(std::int64_t value) { + return Number(static_cast(value)); +} + +Value Value::String(std::string value) { + Value result; + result.kind_ = Kind::String; + result.string_ = std::move(value); + return result; +} + +Value Value::String(std::wstring_view value) { + return String(ToUtf8(value)); +} + +Value Value::Array() { + Value result; + result.kind_ = Kind::Array; + return result; +} + +Value Value::Object() { + Value result; + result.kind_ = Kind::Object; + return result; +} + +std::int64_t Value::AsInteger(std::int64_t fallback) const noexcept { + if (kind_ != Kind::Number || !std::isfinite(number_)) { + return fallback; + } + return static_cast(std::llround(number_)); +} + +const std::string& Value::AsString() const noexcept { + static const std::string empty; + return kind_ == Kind::String ? string_ : empty; +} + +std::wstring Value::AsWide() const { + return ToWide(AsString()); +} + +void Value::Append(Value value) { + if (kind_ != Kind::Array) { + kind_ = Kind::Array; + } + items_.push_back(std::move(value)); +} + +void Value::Set(std::string key, Value value) { + if (kind_ != Kind::Object) { + kind_ = Kind::Object; + } + for (auto& member : members_) { + if (member.first == key) { + member.second = std::move(value); + return; + } + } + members_.emplace_back(std::move(key), std::move(value)); +} + +const Value* Value::Find(std::string_view key) const noexcept { + if (kind_ != Kind::Object) { + return nullptr; + } + for (const auto& member : members_) { + if (member.first == key) { + return &member.second; + } + } + return nullptr; +} + +std::string Value::StringField(std::string_view key, std::string fallback) const { + const Value* member = Find(key); + return member != nullptr && member->IsString() ? member->AsString() : fallback; +} + +std::wstring Value::WideField(std::string_view key, std::wstring fallback) const { + const Value* member = Find(key); + return member != nullptr && member->IsString() ? ToWide(member->AsString()) : fallback; +} + +double Value::NumberField(std::string_view key, double fallback) const { + const Value* member = Find(key); + return member != nullptr && member->IsNumber() ? member->AsNumber() : fallback; +} + +bool Value::BoolField(std::string_view key, bool fallback) const { + const Value* member = Find(key); + return member != nullptr && member->IsBool() ? member->AsBool() : fallback; +} + +std::string Value::Serialize() const { + std::string output; + SerializeInto(*this, output); + return output; +} + +Value Value::Parse(std::string_view text) { + return Parser(text).Parse(); +} + +std::string EscapeString(std::string_view value) { + std::string output; + output.reserve(value.size() + 2); + output.push_back('"'); + for (const char character : value) { + switch (character) { + case '"': + output += "\\\""; + break; + case '\\': + output += "\\\\"; + break; + case '\b': + output += "\\b"; + break; + case '\f': + output += "\\f"; + break; + case '\n': + output += "\\n"; + break; + case '\r': + output += "\\r"; + break; + case '\t': + output += "\\t"; + break; + default: + if (static_cast(character) < 0x20) { + std::array buffer{}; + std::snprintf(buffer.data(), buffer.size(), "\\u%04x", static_cast(character) & 0xffu); + output += buffer.data(); + } else { + output.push_back(character); + } + break; + } + } + output.push_back('"'); + return output; +} + +} // namespace furn::json diff --git a/packages/agentic/desktop-driver/native/windows/src/json.h b/packages/agentic/desktop-driver/native/windows/src/json.h new file mode 100644 index 00000000000..c6dd77c7206 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/json.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace furn::json { + +enum class Kind { Null, Bool, Number, String, Array, Object }; + +// Minimal ordered JSON document model. Insertion order is preserved so every +// response, XML dump, and diagnostic payload is byte-for-byte deterministic. +class Value { + public: + Value() = default; + + static Value Null(); + static Value Bool(bool value); + static Value Number(double value); + static Value Integer(std::int64_t value); + static Value String(std::string value); + static Value String(std::wstring_view value); + static Value Array(); + static Value Object(); + + Kind kind() const noexcept { return kind_; } + bool IsNull() const noexcept { return kind_ == Kind::Null; } + bool IsBool() const noexcept { return kind_ == Kind::Bool; } + bool IsNumber() const noexcept { return kind_ == Kind::Number; } + bool IsString() const noexcept { return kind_ == Kind::String; } + bool IsArray() const noexcept { return kind_ == Kind::Array; } + bool IsObject() const noexcept { return kind_ == Kind::Object; } + + bool AsBool(bool fallback = false) const noexcept { return kind_ == Kind::Bool ? boolean_ : fallback; } + double AsNumber(double fallback = 0.0) const noexcept { return kind_ == Kind::Number ? number_ : fallback; } + std::int64_t AsInteger(std::int64_t fallback = 0) const noexcept; + const std::string& AsString() const noexcept; + std::wstring AsWide() const; + + const std::vector& Items() const noexcept { return items_; } + const std::vector>& Members() const noexcept { return members_; } + + void Append(Value value); + void Set(std::string key, Value value); + const Value* Find(std::string_view key) const noexcept; + bool Has(std::string_view key) const noexcept { return Find(key) != nullptr; } + + std::string StringField(std::string_view key, std::string fallback = {}) const; + std::wstring WideField(std::string_view key, std::wstring fallback = {}) const; + double NumberField(std::string_view key, double fallback = 0.0) const; + bool BoolField(std::string_view key, bool fallback = false) const; + + std::string Serialize() const; + static Value Parse(std::string_view text); + + private: + Kind kind_{Kind::Null}; + bool boolean_{false}; + double number_{0.0}; + std::string string_; + std::vector items_; + std::vector> members_; +}; + +std::string EscapeString(std::string_view value); + +} // namespace furn::json diff --git a/packages/agentic/desktop-driver/native/windows/src/main.cpp b/packages/agentic/desktop-driver/native/windows/src/main.cpp new file mode 100644 index 00000000000..b8d2982fcc2 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/main.cpp @@ -0,0 +1,157 @@ +#include + +#include +#include + +#include +#include +#include + +#include + +#include "common.h" +#include "host.h" +#include "input.h" +#include "inputlock.h" +#include "json.h" +#include "selftest.h" + +namespace { + +// A key ledger is a set of single Unicode code points. Sixteen MiB exceeds the +// JSON encoding of the complete Unicode code-point space while remaining +// bounded independently from ordinary framed commands. +constexpr std::size_t kMaximumLedgerBytes = 16 * 1024 * 1024; + +// Reads the whole standard input stream. A stream that was never redirected is +// reported as missing so the restricted mode fails closed instead of blocking +// on an interactive console. +std::string ReadStandardInput(bool& redirected) { + const HANDLE input = GetStdHandle(STD_INPUT_HANDLE); + redirected = input != nullptr && input != INVALID_HANDLE_VALUE && GetFileType(input) != FILE_TYPE_CHAR; + if (!redirected) { + return {}; + } + _setmode(_fileno(stdin), _O_BINARY); + std::string contents; + char buffer[4096]; + DWORD read = 0; + while (ReadFile(input, buffer, sizeof(buffer), &read, nullptr) != FALSE && read > 0) { + contents.append(buffer, read); + if (contents.size() > kMaximumLedgerBytes) { + furn::Fail(furn::kErrorInvalidRequest, "The release ledger on standard input is too large."); + } + } + if (contents.size() >= 3 && static_cast(contents[0]) == 0xEF && + static_cast(contents[1]) == 0xBB && static_cast(contents[2]) == 0xBF) { + contents.erase(0, 3); + } + return contents; +} + +// Restricted mode: release exactly the depressed input that the adapter reports +// on standard input, without touching applications, windows, or any other key. +int ReleaseInputFromLedger() { + try { + bool redirected = false; + const std::string document = ReadStandardInput(redirected); + if (!redirected || furn::TrimAscii(document).empty()) { + furn::Fail(furn::kErrorInvalidRequest, + "Standard input did not carry a release ledger such as {\"keys\":[],\"buttons\":[]}."); + } + const furn::ReleaseLedger ledger = furn::ParseReleaseLedger(furn::json::Value::Parse(document)); + furn::InputController input; + const furn::CancellationToken idle; + // Recovery owns physical input for the whole release chain and adopts a + // mutex abandoned by the helper it is recovering from. + const furn::PhysicalInputScope scope = ledger.empty() + ? furn::PhysicalInputScope() + : furn::PhysicalInputScope(idle, furn::kPhysicalInputCleanupWaitMs, + furn::AbandonPolicy::Adopt); + const furn::ReleaseCounts released = input.ReleaseExact(ledger); + furn::json::Value result = furn::json::Value::Object(); + result.Set("releasedKeys", furn::json::Value::Integer(static_cast(released.keys))); + result.Set("releasedButtons", furn::json::Value::Integer(static_cast(released.buttons))); + std::cout << result.Serialize() << '\n'; + return 0; + } catch (const furn::HelperError& error) { + std::cerr << "furn-desktop-driver-host: " << error.code() << ": " << error.what() << '\n'; + return error.code() == furn::kErrorInputUnavailable ? 3 : 1; + } +} + +// Diagnostics mode: report and release whatever the operating system still +// holds down, for operators recovering a desktop by hand. +int ReleaseInputSweep() { + furn::InputController input; + if (!input.PhysicalInputAvailable()) { + std::cerr << "furn-desktop-driver-host: physical input is unavailable on this desktop.\n"; + return 3; + } + try { + const furn::CancellationToken idle; + const furn::PhysicalInputScope scope(idle, furn::kPhysicalInputCleanupWaitMs, furn::AbandonPolicy::Adopt); + const std::size_t released = input.ReleaseDesktopModifiers(); + furn::json::Value result = furn::json::Value::Object(); + result.Set("releasedKeys", furn::json::Value::Integer(static_cast(released))); + result.Set("releasedButtons", furn::json::Value::Integer(0)); + result.Set("sweep", furn::json::Value::Bool(true)); + std::cout << result.Serialize() << '\n'; + return 0; + } catch (const furn::HelperError& error) { + std::cerr << "furn-desktop-driver-host: " << error.code() << ": " << error.what() << '\n'; + return error.code() == furn::kErrorInputUnavailable ? 3 : 1; + } +} + +int PrintUsage() { + std::cerr << "Usage: furn-desktop-driver-host --handshake --json | --self-test | --stdio\n" + " furn-desktop-driver-host --release-input " + "(reads {\"keys\":[],\"buttons\":[]} from stdin)\n" + " furn-desktop-driver-host --release-input --sweep " + "(diagnostics: release everything the desktop holds down)\n"; + return 2; +} + +} // namespace + +int wmain(int argc, wchar_t** argv) { + try { + winrt::init_apartment(winrt::apartment_type::multi_threaded); + SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); + if (argc < 2) { + return PrintUsage(); + } + const std::wstring_view mode(argv[1]); + if (mode == L"--handshake") { + std::cout << furn::CreateHello().Serialize() << '\n'; + return 0; + } + if (mode == L"--self-test") { + return furn::RunSelfTest(); + } + if (mode == L"--release-input") { + if (argc >= 3 && std::wstring_view(argv[2]) == L"--sweep") { + return ReleaseInputSweep(); + } + if (argc >= 3) { + return PrintUsage(); + } + return ReleaseInputFromLedger(); + } + if (mode == L"--stdio") { + furn::Host host; + return host.Run(); + } + return PrintUsage(); + } catch (const winrt::hresult_error& error) { + std::cerr << furn::ToUtf8(error.message().c_str()) << '\n'; + return 1; + } catch (const furn::HelperError& error) { + std::cerr << error.code() << ": " << error.what() << '\n'; + return 1; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/packages/agentic/desktop-driver/native/windows/src/selftest.cpp b/packages/agentic/desktop-driver/native/windows/src/selftest.cpp new file mode 100644 index 00000000000..1248ef23ab8 --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/selftest.cpp @@ -0,0 +1,508 @@ +#include "selftest.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "automation.h" +#include "common.h" +#include "framing.h" +#include "geometry.h" +#include "host.h" +#include "input.h" +#include "inputlock.h" +#include "json.h" + +namespace furn { +namespace { + +int failures = 0; +int checks = 0; + +void Check(bool condition, const std::string& name) { + checks += 1; + if (condition) { + std::cout << "ok - " << name << '\n'; + return; + } + failures += 1; + std::cout << "not ok - " << name << '\n'; +} + +std::vector recordedInputs; + +UINT WINAPI RecordInput(UINT count, LPINPUT inputs, int size) { + if (size != sizeof(INPUT)) { + return 0; + } + for (UINT index = 0; index < count; index += 1) { + recordedInputs.push_back(inputs[index]); + } + return count; +} + +void TestFraming() { + const std::string payload = R"({"type":"response","id":"a"})"; + const std::vector frame = EncodeJsonFrame(payload); + Check(frame.size() == kFrameHeaderBytes + payload.size(), "json frame length"); + std::uint8_t type = 0; + std::uint32_t length = 0; + Check(DecodeFrameHeader(frame.data(), type, length), "json frame header decodes"); + Check(type == kJsonFrameType && length == payload.size(), "json frame header fields"); + Check(std::string(reinterpret_cast(frame.data() + kFrameHeaderBytes), length) == payload, + "json frame payload round trip"); + + const std::vector data{1, 2, 3, 4, 5}; + const std::vector binary = EncodeBinaryFrame("image-1", data); + Check(DecodeFrameHeader(binary.data(), type, length), "binary frame header decodes"); + Check(type == kBinaryFrameType, "binary frame type"); + Check(length == 4 + 7 + data.size(), "binary frame payload length"); + const std::uint8_t* payloadStart = binary.data() + kFrameHeaderBytes; + const std::uint32_t identifierLength = static_cast(payloadStart[0]) | + (static_cast(payloadStart[1]) << 8) | + (static_cast(payloadStart[2]) << 16) | + (static_cast(payloadStart[3]) << 24); + Check(identifierLength == 7, "binary frame identifier length"); + Check(std::string(reinterpret_cast(payloadStart + 4), identifierLength) == "image-1", + "binary frame identifier"); + Check(std::equal(data.begin(), data.end(), payloadStart + 4 + identifierLength), "binary frame data"); + + std::uint8_t corrupt[kFrameHeaderBytes] = {'F', 'D', 'R', '0'}; + Check(!DecodeFrameHeader(corrupt, type, length), "invalid magic is rejected"); + std::uint8_t reserved[kFrameHeaderBytes] = {'F', 'D', 'R', '1', kJsonFrameType, 1}; + Check(!DecodeFrameHeader(reserved, type, length), "nonzero reserved frame bytes are rejected"); + std::uint8_t oversized[kFrameHeaderBytes] = {'F', 'D', 'R', '1', kJsonFrameType}; + const std::uint32_t oversizedLength = kMaximumJsonFramePayload + 1; + oversized[8] = static_cast(oversizedLength & 0xffu); + oversized[9] = static_cast((oversizedLength >> 8) & 0xffu); + oversized[10] = static_cast((oversizedLength >> 16) & 0xffu); + oversized[11] = static_cast((oversizedLength >> 24) & 0xffu); + Check(!DecodeFrameHeader(oversized, type, length), "oversized frame headers are rejected"); +} + +void TestJson() { + const std::string document = + R"({"a":1,"b":-2.5,"c":"x\ny\u00e9\ud83d\ude00","d":[true,false,null],"e":{"f":9007199254740991}})"; + const json::Value parsed = json::Value::Parse(document); + Check(parsed.IsObject(), "json object parses"); + Check(parsed.NumberField("a", 0) == 1.0, "json integer field"); + Check(parsed.NumberField("b", 0) == -2.5, "json fractional field"); + Check(parsed.StringField("c").find("\n") == 1, "json escape decoding"); + Check(parsed.StringField("c").size() == 1 + 1 + 1 + 2 + 4, "json utf-8 encoding of escapes"); + const json::Value* list = parsed.Find("d"); + Check(list != nullptr && list->IsArray() && list->Items().size() == 3, "json array parses"); + Check(list->Items()[0].AsBool() && !list->Items()[1].AsBool() && list->Items()[2].IsNull(), "json literal values"); + + const std::string reserialized = parsed.Serialize(); + const json::Value round = json::Value::Parse(reserialized); + Check(round.StringField("c") == parsed.StringField("c"), "json round trip preserves text"); + Check(round.Serialize() == reserialized, "json serialization is stable"); + Check(reserialized.find("\"a\":1") != std::string::npos, "json integers avoid decimal noise"); + Check(reserialized.find("9007199254740991") != std::string::npos, "json preserves large integers"); + + json::Value ordered = json::Value::Object(); + ordered.Set("z", json::Value::Integer(1)); + ordered.Set("a", json::Value::Integer(2)); + ordered.Set("z", json::Value::Integer(3)); + Check(ordered.Serialize() == R"({"z":3,"a":2})", "json preserves insertion order"); + + bool rejected = false; + try { + json::Value::Parse("{\"a\":}"); + } catch (const HelperError& error) { + rejected = std::string(error.code()) == kErrorInvalidRequest; + } + Check(rejected, "malformed json is rejected"); +} + +void TestGeometry() { + Check(Deadline(0).Expired(), "zero-duration deadlines expire immediately"); + Check(std::fabs(DipsFromPhysical(150.0, 144) - 100.0) < 0.001, "physical to dips at 150 percent"); + Check(std::fabs(PhysicalFromDips(100.0, 192) - 200.0) < 0.001, "dips to physical at 200 percent"); + Check(std::fabs(DipsFromPhysical(PhysicalFromDips(37.5, 144), 144) - 37.5) < 0.001, "dip round trip"); + + WindowMetrics metrics; + metrics.dpi = 144; + metrics.clientOrigin = POINT{300, 200}; + metrics.clientSize = SIZE{1200, 900}; + metrics.windowRect = RECT{292, 160, 1508, 1108}; + + const RECT physical{420, 350, 660, 470}; + const RectD logical = PhysicalRectToClientDips(physical, metrics); + Check(std::fabs(logical.x - 80.0) < 0.001, "element x is client relative"); + Check(std::fabs(logical.y - 100.0) < 0.001, "element y is client relative"); + Check(std::fabs(logical.width - 160.0) < 0.001, "element width is logical"); + Check(std::fabs(logical.height - 80.0) < 0.001, "element height is logical"); + + const POINT center = ClientDipsToPhysicalPoint(logical.x + logical.width / 2, logical.y + logical.height / 2, + metrics); + Check(center.x == (physical.left + physical.right) / 2, "center maps back to physical x"); + Check(center.y == (physical.top + physical.bottom) / 2, "center maps back to physical y"); + + const RectD windowRect = ClientRectInScreenDips(metrics); + Check(std::fabs(windowRect.x - 200.0) < 0.001 && std::fabs(windowRect.width - 800.0) < 0.001, + "window rect uses logical client geometry"); + + const RECT crop = IntersectRects(RECT{0, 0, 100, 100}, RECT{50, 50, 200, 200}); + Check(crop.left == 50 && crop.top == 50 && crop.right == 100 && crop.bottom == 100, "capture crop intersects"); + Check(IsEmptyRect(IntersectRects(RECT{0, 0, 10, 10}, RECT{20, 20, 30, 30})), "disjoint crop is empty"); + + const POINT normalized = NormalizeToVirtualDesktop(POINT{0, 0}, RECT{0, 0, 1921, 1081}); + Check(normalized.x == 0 && normalized.y == 0, "virtual desktop origin normalizes to zero"); + const POINT corner = NormalizeToVirtualDesktop(POINT{1920, 1080}, RECT{0, 0, 1921, 1081}); + Check(corner.x == 65535 && corner.y == 65535, "virtual desktop corner normalizes to full scale"); +} + +void TestInputLedger() { + Check(IsSingleWebDriverKeyValue(L"A"), "ASCII key values contain one code point"); + Check(IsSingleWebDriverKeyValue(L"\U0001F600"), "surrogate-pair key values contain one code point"); + Check(!IsSingleWebDriverKeyValue(L"AB") && !IsSingleWebDriverKeyValue(std::wstring(1, 0xd800)), + "multi-code-point and unpaired-surrogate key values are rejected"); + + recordedInputs.clear(); + InputController input; + input.UseTestSender(&RecordInput); + CancellationToken token; + + input.KeyDown(L"\uE008", token); + input.KeyDown(L"a", token); + input.PointerDown(0, token); + Check(input.DepressedKeyCount() == 2 && input.DepressedButtonCount() == 1, "ledger records depressed input"); + Check(recordedInputs.size() == 3, "ledger injected three down events"); + Check(recordedInputs[0].ki.wVk == VK_SHIFT, "special keys map to virtual keys"); + Check((recordedInputs[1].ki.dwFlags & KEYEVENTF_UNICODE) != 0, "printable keys inject unicode"); + + recordedInputs.clear(); + input.ReleaseAll(); + Check(!input.HasDepressedInput(), "release empties the ledger"); + Check(recordedInputs.size() == 3, "release emits one event per depressed input"); + Check(recordedInputs[0].type == INPUT_KEYBOARD && (recordedInputs[0].ki.dwFlags & KEYEVENTF_UNICODE) != 0, + "release unwinds keys in reverse order"); + Check(recordedInputs[1].ki.wVk == VK_SHIFT && (recordedInputs[1].ki.dwFlags & KEYEVENTF_KEYUP) != 0, + "modifier release carries the key-up flag"); + Check(recordedInputs[2].type == INPUT_MOUSE && (recordedInputs[2].mi.dwFlags & MOUSEEVENTF_LEFTUP) != 0, + "pointer button is released last"); + + recordedInputs.clear(); + input.KeyDown(L"b", token); + input.KeyUp(L"b", token); + Check(!input.HasDepressedInput(), "explicit key release clears the ledger"); + + recordedInputs.clear(); + input.PressChord({VK_CONTROL, 'A'}, token); + Check(!input.HasDepressedInput(), "chords leave no depressed input"); + Check(recordedInputs.size() == 4, "chords press and release every key"); + Check(recordedInputs[0].ki.wVk == VK_CONTROL && (recordedInputs[0].ki.dwFlags & KEYEVENTF_KEYUP) == 0, + "chords press modifiers first"); + Check(recordedInputs[3].ki.wVk == VK_CONTROL && (recordedInputs[3].ki.dwFlags & KEYEVENTF_KEYUP) != 0, + "chords release modifiers last"); + + recordedInputs.clear(); + input.TypeText(L"x", token); + Check(!input.HasDepressedInput(), "typed text leaves no depressed input"); + Check(recordedInputs.size() == 2, "typed text emits one down and one up event"); + Check((recordedInputs[0].ki.dwFlags & KEYEVENTF_KEYUP) == 0 && + (recordedInputs[1].ki.dwFlags & KEYEVENTF_KEYUP) != 0, + "typed text releases its ledgered key"); + + recordedInputs.clear(); + input.TypeText(L"\n", token); + Check(recordedInputs.size() == 2 && recordedInputs[0].ki.wVk == VK_RETURN && + recordedInputs[1].ki.wVk == VK_RETURN, + "typed newlines use the recoverable WebDriver Return key"); + + WORD virtualKey = 0; + bool extended = false; + Check(TryMapWebDriverKey(L"\uE012", virtualKey, extended) && virtualKey == VK_LEFT && extended, + "arrow keys are extended keys"); + Check(!TryMapWebDriverKey(L"a", virtualKey, extended), "printable characters are not special keys"); +} + +void TestSnapshotShape() { + ElementSnapshot snapshot; + snapshot.id = "element-1"; + snapshot.automationId = "button-primary"; + snapshot.name = "Primary"; + snapshot.role = "button"; + snapshot.windowId = "window-1"; + snapshot.parentId = "element-0"; + snapshot.scope = ElementScope::Preview; + snapshot.rect = RectD{10.0, 20.0, 120.0, 40.0}; + snapshot.enabled = SupportedBool{true, true, {}}; + snapshot.focused = SupportedBool{true, false, {}}; + snapshot.visible = SupportedBool{true, true, {}}; + snapshot.selected = SupportedBool{false, false, "no selection pattern"}; + snapshot.expanded = SupportedBool{false, false, "leaf"}; + snapshot.checked = SupportedChecked{true, CheckedValue::Mixed, {}}; + + const std::string serialized = SerializeSnapshot(snapshot).Serialize(); + Check(serialized.find(R"("scope":"preview")") != std::string::npos, "snapshot serializes its scope"); + Check(serialized.find(R"("checked":{"supported":true,"value":"mixed"})") != std::string::npos, + "mixed checked state serializes as a string"); + Check(serialized.find(R"("selected":{"supported":false,"reason":"no selection pattern"})") != std::string::npos, + "unsupported state keeps its reason"); + Check(serialized.find(R"("text")") == std::string::npos, "absent optional text is omitted"); + Check(serialized.find(R"("rect":{"height":40,"width":120,"x":10,"y":20})") != std::string::npos, + "rect serializes logical geometry"); + Check(serialized.find(R"("parentId":"element-0")") != std::string::npos, "parent identifier is exposed"); + Check(serialized.find("element-1") != std::string::npos && serialized.find("0x") == std::string::npos, + "identifiers stay opaque"); + + Check(NormalizeRoleQuery(" Edit ") == "textbox", "role aliases normalize"); + Check(RoleForControlType(UIA_CheckBoxControlTypeId) == "checkbox", "control types map to roles"); + + Selector byTestId{"accessibility id", "button-primary"}; + Selector byRole{"tag name", "Button"}; + Selector byName{"link text", "Primary"}; + Selector byPartialName{"partial link text", "rima"}; + Selector byText{"-furn:text", "Primary"}; + Check(MatchesSelector(snapshot, byTestId), "accessibility id matches the automation id"); + Check(MatchesSelector(snapshot, byRole), "tag name matches the normalized role"); + Check(MatchesSelector(snapshot, byName), "link text matches the accessible name"); + Check(MatchesSelector(snapshot, byPartialName), "partial link text matches a substring"); + Check(MatchesSelector(snapshot, byText), "-furn:text falls back to the accessible name"); + Check(!MatchesSelector(snapshot, Selector{"accessibility id", "other"}), "non-matching selectors are rejected"); + + ElementSnapshot root = snapshot; + root.parentId.clear(); + ElementSnapshot child = root; + child.id = "element-2"; + child.parentId = root.id; + child.automationId = "child&NumberField("major", 0) == 1, "hello reports the wire protocol"); +} + +void TestReleaseLedger() { + const auto parse = [](const std::string& document) { return ParseReleaseLedger(json::Value::Parse(document)); }; + const auto rejects = [&parse](const std::string& document, const std::string& label) { + bool rejected = false; + try { + parse(document); + } catch (const HelperError& error) { + rejected = error.code() == kErrorInvalidParams || error.code() == kErrorInvalidRequest; + } + Check(rejected, label); + }; + + const ReleaseLedger parsed = parse(R"({"keys":["a","\uE008"],"buttons":[0,2]})"); + Check(parsed.keys.size() == 2 && parsed.keys[0] == L"a" && parsed.keys[1] == L"\uE008", + "ledger parses webdriver key values in order"); + Check(parsed.buttons.size() == 2 && parsed.buttons[0] == 0 && parsed.buttons[1] == 2, + "ledger parses pointer buttons in order"); + Check(parse("{}").empty() && parse(R"({"keys":[],"buttons":[]})").empty(), "an empty ledger parses"); + + rejects("[]", "a non-object ledger is rejected"); + rejects(R"({"keys":"a"})", "a non-array key list is rejected"); + rejects(R"({"keys":[1]})", "a non-string key is rejected"); + rejects(R"({"keys":["ab"]})", "a multi-character key value is rejected"); + rejects(R"({"keys":[""]})", "an empty key value is rejected"); + rejects(R"({"buttons":2})", "a non-array button list is rejected"); + rejects(R"({"buttons":["0"]})", "a non-numeric button is rejected"); + rejects(R"({"buttons":[1.5]})", "a fractional button is rejected"); + rejects(R"({"buttons":[9]})", "an out-of-range button is rejected"); + rejects(R"({"buttons":[-1]})", "a negative button is rejected"); + rejects(R"({"keys":[],"extra":1})", "an unknown ledger field is rejected"); + rejects(R"({"keys":[)", "malformed ledger json is rejected"); + + InputController empty; + const ReleaseCounts nothing = empty.ReleaseExact(parse("{}")); + Check(nothing.keys == 0 && nothing.buttons == 0, "an empty ledger releases nothing"); + + recordedInputs.clear(); + InputController input; + input.UseTestSender(&RecordInput); + const ReleaseCounts released = input.ReleaseExact(parse(R"({"keys":["\uE008","a"],"buttons":[0,1,2]})")); + Check(released.keys == 2 && released.buttons == 3, "ledger release reports its counts"); + Check(recordedInputs.size() == 5, "ledger release emits one event per entry"); + Check(recordedInputs[0].type == INPUT_KEYBOARD && (recordedInputs[0].ki.dwFlags & KEYEVENTF_UNICODE) != 0 && + recordedInputs[0].ki.wScan == L'a' && (recordedInputs[0].ki.dwFlags & KEYEVENTF_KEYUP) != 0, + "ledger releases keys in reverse order"); + Check(recordedInputs[1].ki.wVk == VK_SHIFT && (recordedInputs[1].ki.dwFlags & KEYEVENTF_KEYUP) != 0, + "ledger releases special keys with their virtual key"); + Check((recordedInputs[2].mi.dwFlags & MOUSEEVENTF_RIGHTUP) != 0, "ledger releases buttons in reverse order"); + Check((recordedInputs[3].mi.dwFlags & MOUSEEVENTF_MIDDLEUP) != 0, "ledger releases the middle button"); + Check((recordedInputs[4].mi.dwFlags & MOUSEEVENTF_LEFTUP) != 0, "ledger releases the left button last"); + + recordedInputs.clear(); + input.ReleaseExact(parse(R"({"keys":["\uE012"]})")); + Check(recordedInputs.size() == 1 && recordedInputs[0].ki.wVk == VK_LEFT && + (recordedInputs[0].ki.dwFlags & KEYEVENTF_EXTENDEDKEY) != 0 && + (recordedInputs[0].ki.dwFlags & KEYEVENTF_KEYUP) != 0, + "ledger releases extended keys with the extended flag"); + + recordedInputs.clear(); + input.ReleaseExact(parse(R"({"keys":["\u00e9"]})")); + Check(recordedInputs.size() == 1 && (recordedInputs[0].ki.dwFlags & KEYEVENTF_UNICODE) != 0 && + recordedInputs[0].ki.wScan == 0x00e9, + "ledger releases printable unicode as a unicode scan code"); + + recordedInputs.clear(); + input.ReleaseExact(parse(R"({"keys":["\ud83d\ude00"]})")); + Check(recordedInputs.size() == 2, "a supplementary code point releases both code units"); + Check(recordedInputs[0].ki.wScan == 0xde00 && recordedInputs[1].ki.wScan == 0xd83d, + "surrogate pairs release in reverse code unit order"); + + recordedInputs.clear(); + const ReleaseCounts buttonsOnly = input.ReleaseExact(parse(R"({"buttons":[3,4]})")); + Check(buttonsOnly.buttons == 2 && recordedInputs.size() == 2, "extended buttons release exactly"); + Check(recordedInputs[0].mi.mouseData == XBUTTON2 && recordedInputs[1].mi.mouseData == XBUTTON1, + "extended buttons carry their button data"); + Check(!input.HasDepressedInput(), "ledger release never adds to the local ledger"); +} + +void TestPhysicalInputLock() { + SetPhysicalInputMutexNameForTesting(L"Local\\FurnDesktopDriverPhysicalInputSelfTest-" + + std::to_wstring(GetCurrentProcessId())); + Check(std::wstring(PhysicalInputMutexName()).find(std::to_wstring(GetCurrentProcessId())) != std::wstring::npos, + "self-test uses an isolated physical input mutex"); + // Every acquisition attempt runs on its own thread because the mutex is + // owned per thread, so this exercises real cross-owner exclusion without + // injecting input. + const auto acquireElsewhere = [](unsigned int timeoutMs, AbandonPolicy policy = AbandonPolicy::Fail) { + std::string outcome = "unknown"; + std::thread worker([&outcome, timeoutMs, policy]() { + const CancellationToken token; + try { + const PhysicalInputScope scope(token, timeoutMs, policy); + outcome = scope.owns() ? "acquired" : "unowned"; + } catch (const HelperError& error) { + outcome = error.code(); + } catch (const CancelledError&) { + outcome = "cancelled"; + } + }); + worker.join(); + return outcome; + }; + + const CancellationToken token; + Check(acquireElsewhere(500) == "acquired", "an idle physical input mutex is available"); + + { + const PhysicalInputScope outer(token, 1000); + Check(outer.owns() && !outer.nested(), "the first scope owns the physical input mutex"); + Check(acquireElsewhere(200) == kErrorInputBusy, "a busy physical input mutex reports input-busy"); + { + const PhysicalInputScope inner(token, 1000); + Check(inner.owns() && inner.nested(), "a nested scope reuses the owned mutex"); + Check(acquireElsewhere(200) == kErrorInputBusy, "nesting keeps other owners out"); + } + Check(acquireElsewhere(200) == kErrorInputBusy, "releasing a nested scope keeps the mutex held"); + + std::string cancelledOutcome = "unknown"; + std::thread waiter([&cancelledOutcome]() { + CancellationToken cancelled; + cancelled.Cancel(); + try { + const PhysicalInputScope scope(cancelled, 30000); + cancelledOutcome = "acquired"; + } catch (const CancelledError&) { + cancelledOutcome = "cancelled"; + } catch (const HelperError& error) { + cancelledOutcome = error.code(); + } + }); + waiter.join(); + Check(cancelledOutcome == "cancelled", "a cancelled wait stops instead of hanging"); + } + Check(acquireElsewhere(1000) == "acquired", "scope destruction releases the mutex"); + + { + PhysicalInputScope moved(token, 500); + Check(moved.owns(), "a scope can be acquired for a move"); + const PhysicalInputScope adopted(std::move(moved)); + Check(adopted.owns() && !moved.owns(), "moving a scope transfers ownership exactly once"); + Check(acquireElsewhere(200) == kErrorInputBusy, "a moved scope still holds the mutex"); + } + Check(acquireElsewhere(500) == "acquired", "a moved scope releases once"); + + // A thread that exits while owning the mutex abandons it, which is exactly + // what a helper crash looks like to the next owner. + std::thread abandoner([]() { + const HANDLE handle = CreateMutexW(nullptr, FALSE, PhysicalInputMutexName()); + if (handle != nullptr) { + WaitForSingleObject(handle, 1000); + CloseHandle(handle); + } + }); + abandoner.join(); + Check(acquireElsewhere(500) == kErrorInputBusy, "an abandoned mutex reports input-busy"); + Check(acquireElsewhere(500) == "acquired", "the reported abandonment leaves the mutex usable"); + + std::thread secondAbandoner([]() { + const HANDLE handle = CreateMutexW(nullptr, FALSE, PhysicalInputMutexName()); + if (handle != nullptr) { + WaitForSingleObject(handle, 1000); + CloseHandle(handle); + } + }); + secondAbandoner.join(); + Check(acquireElsewhere(500, AbandonPolicy::Adopt) == "acquired", "recovery adopts an abandoned mutex"); + Check(acquireElsewhere(500) == "acquired", "adoption releases the mutex when recovery finishes"); +} + +} // namespace + +int RunSelfTest() { + failures = 0; + checks = 0; + TestFraming(); + TestJson(); + TestGeometry(); + TestInputLedger(); + TestReleaseLedger(); + TestPhysicalInputLock(); + TestSnapshotShape(); + TestLeaseIdentity(); + TestHello(); + std::cout << (failures == 0 ? "self-test passed: " : "self-test failed: ") << (checks - failures) << '/' << checks + << " checks\n"; + return failures == 0 ? 0 : 1; +} + +} // namespace furn diff --git a/packages/agentic/desktop-driver/native/windows/src/selftest.h b/packages/agentic/desktop-driver/native/windows/src/selftest.h new file mode 100644 index 00000000000..208f903ce9d --- /dev/null +++ b/packages/agentic/desktop-driver/native/windows/src/selftest.h @@ -0,0 +1,10 @@ +#pragma once + +namespace furn { + +// Runs the helper-internal checks that do not require a live application: +// framing, JSON, coordinate conversion, selector normalization, snapshot shape, +// lease timestamps, and the physical-input release ledger. +int RunSelfTest(); + +} // namespace furn diff --git a/packages/agentic/desktop-driver/references/architecture.md b/packages/agentic/desktop-driver/references/architecture.md new file mode 100644 index 00000000000..03adaf3b44b --- /dev/null +++ b/packages/agentic/desktop-driver/references/architecture.md @@ -0,0 +1,134 @@ +# Architecture + +Desktop Driver separates portable WebDriver behavior from operating-system +automation. Node owns protocol, policy, sessions, tests, and evidence. A +verified native helper owns only window, accessibility, input, and capture +operations for its operating system. + +## Layer model + +```text +WebdriverIO / typed client / bounded agent / JSON CLI + | + W3C remote end + target, session, element state + | + DesktopHost + / \ + deterministic fake host NativeDesktopHost + | + framed FDR1 stdio + / \ + Swift helper C++ helper + macOS Windows/Win32 +``` + +Storybook is an optional consumer: + +```text +component story -- type-only --> desktop-driver/authoring + +storybook-desktop + |- generates static story manifests and plan digests + |- owns the channel/MCP server, Metro, and app lifecycle + |- implements StoryOrchestrator + `- embeds the Desktop Driver listener and registers a native target + +storybook-desktop-runtime + `- reports authenticated story readiness over the Storybook channel +``` + +The package never imports Storybook, React, or React Native. Its +`./storybook` subpath contains only serializable manifest types and the +`StoryOrchestrator` interface. + +## Process boundaries + +A generic native integration normally has: + +1. one trusted Node process hosting the loopback W3C service; +2. one long-lived native helper child per `NativeDesktopHost`; +3. one launched or externally owned application process; +4. one or more WebDriver clients. + +The helper has no listener. Node spawns it with inherited stdin/stdout, verifies +its startup hello against the resolved artifact, and exchanges correlated FDR1 +frames. Binary frames carry PNG data; bounded human-readable diagnostics use +stderr. + +A Storybook integration keeps the Storybook HTTP/WebSocket/MCP listener and W3C +listener on separate loopback ports because they are distinct protocols, but +runs both listeners in the same supervisor process. Metro remains a separate +child process. + +## Request lifecycle + +1. The service matches W3C capabilities against a server-registered + `DesktopTarget`. +2. It reserves the target before asynchronous launch or attach work. +3. The selected `DesktopHost` probes truthful features and creates an + application lease. +4. The session manager exposes opaque window and element IDs; native handles + never cross the WebDriver boundary. +5. Commands run through a per-session queue. Physical input also uses a process + mutex and an operating-system-level cross-process lock. +6. Each host call receives an `AbortSignal` and a bounded deadline. +7. On timeout or cancellation, the helper stops the operation and acknowledges + cancellation before cleanup or later commands proceed. +8. Session deletion releases input and closes only resources owned by that + session. Attached applications survive. +9. Service shutdown rejects new work, drains session creation, deletes + sessions, and disposes each host. + +## State and identity + +The service maintains: + +- registered target identity and provider policy; +- one active session reservation per target; +- application ownership (`launched` or `attached`); +- current window and opaque window IDs; +- session-generated W3C element UUIDs; +- native liveness and Storybook preview generation; +- depressed keyboard and pointer state; +- command and cleanup deadlines. + +A retained element becomes stale when its native object disappears or when a +Storybook preview reset advances its generation. The service does not +reconstruct a replacement element from an index or path. + +## Module ownership + +| Module | Responsibility | +| ------------------ | ----------------------------------------------------------------- | +| `src/protocol` | W3C parsing, actions, capabilities, timeouts, and errors | +| `src/server` | HTTP routing, target registry, sessions, windows, and elements | +| `src/host` | Platform-neutral `DesktopHost` and target contracts | +| `src/hosts/fake` | Deterministic in-memory provider | +| `src/hosts/native` | Long-lived helper process and `DesktopHost` adapter | +| `src/native` | Build, cache, resolution, verification, manifests, and wire types | +| `native/macos` | Swift operating-system implementation | +| `native/windows` | Shared C++ Windows and Win32 implementation | +| `src/client` | Low-level typed W3C client | +| `src/wdio` | Sanctioned WebdriverIO integration and custom commands | +| `src/authoring` | Serializable story plans, selectors, capabilities, and results | +| `src/runner` | Filtering, sharding, execution, and result classification | +| `src/artifacts` | Confined atomic evidence | +| `src/agent` | Bounded JSON-safe operations | +| `src/testing` | Fake harnesses and Storybook test doubles | +| `src/cli` | Commander parsing, JSON output, and process exit adaptation | + +## Deliberate boundaries + +- The service is client-neutral; WebdriverIO is not used by routing or hosts. +- Native helpers receive validated commands and controlled descriptors, not + WebDriver capabilities or arbitrary environment data. +- Native artifacts are source-built or operator-provided. The package never + downloads, mutates, or silently substitutes them. +- Queries are authoritative. Native events are hints for invalidation and + failure classification. +- Screenshots are evidence, not a visual-baseline comparison system. +- The fake host is a protocol and runner test double, not a native fallback. + +See [Service integration](service.md), [Native helpers](native-helpers.md), and +[Security](security.md) for the operational contracts behind these boundaries. diff --git a/packages/agentic/desktop-driver/references/ci-integration.md b/packages/agentic/desktop-driver/references/ci-integration.md new file mode 100644 index 00000000000..0a2e0000c8b --- /dev/null +++ b/packages/agentic/desktop-driver/references/ci-integration.md @@ -0,0 +1,188 @@ +# CI integration + +CI should separate portable package checks, target-OS native contracts, and +real-application automation. A fake-host pass is not a substitute for native +authority, input, accessibility, or capture qualification. + +## Gate 1: portable package checks + +Run on a normal Node runner: + +```sh +yarn +yarn workspace @fluentui-react-native/desktop-driver format +yarn workspace @fluentui-react-native/desktop-driver lint +yarn workspace @fluentui-react-native/desktop-driver build +yarn workspace @fluentui-react-native/desktop-driver test +``` + +When package contents or publishing metadata change, also verify that the +packed artifact: + +- contains built JavaScript, config, documentation, and all `native/**` source; +- contains no `.exe`, `.app`, object files, Swift `.build`, Visual Studio + output, cache selections, or signed artifacts; +- has no install/postinstall native build or downloader; +- can be extracted read-only while a separate external cache remains writable. + +Repository changes to public types, exports, manifests, dependencies, or +project references also require the root build and publishing checks. + +## Gate 2: native build contract + +Run on the target OS with its supported toolchain: + +```sh +FURN_NATIVE_DRIVER_TEST=1 \ + yarn workspace @fluentui-react-native/desktop-driver test --runInBand +``` + +PowerShell: + +```powershell +$env:FURN_NATIVE_DRIVER_TEST = '1' +yarn workspace @fluentui-react-native/desktop-driver test --runInBand +``` + +This gate builds source, verifies cache publication and reuse, starts the exact +long-lived artifact, runs the native self-test, exercises FDR1 framing, and +checks failure behavior. It does not by itself prove automation against a real +React Native app. + +To ensure CI never compiles after a separately provisioned or restored helper, +set: + +```text +FURN_DESKTOP_DRIVER_BUILD_POLICY=never +FURN_DESKTOP_DRIVER_CACHE_ROOT= +``` + +Then run `desktop-driver doctor --platform ` before app startup. +Restore only cache content produced for the same OS, architecture, source, +toolchain, configuration, and macOS signer. Never cache runtime leases or +borrow another logged-in user's native store. + +## Gate 3: real application + +A consuming app must: + +1. build or restore a verified helper; +2. build and launch the exact app under test; +3. start the loopback service with a registered target; +4. provide exact launch or nonce-bound attach identity; +5. run normal WebdriverIO tests or authored plans; +6. upload `run.json`, `host.json`, failure evidence, and bounded native logs; +7. close the session, release input, and stop only owned processes. + +Storybook consumers should use +`storybook-desktop smoke -- --mode stories-and-tests`. The `stories` +mode is intentionally render-only and does not require a helper. + +This repository's current PR workflow provides examples in +`.github/workflows/pr.yml`: + +- `macos-storybook` builds packages, bundles, prepares pods, and runs macOS + `stories-and-tests`. The shared `.github/actions/setup-desktop-driver` action + creates a job-local cache, explicitly builds the helper, runs noninteractive + permission diagnostics without prompting or modifying TCC, and pins later + resolution to that verified artifact. GitHub-hosted runners use an ad hoc + signature because changing trust settings requires interactive + authorization. A managed self-hosted runner can instead pass a + pre-provisioned stable identity through the action's + `macos-signing-identity` input. The hosted job also disables physical clicks + because its desktop can report input authority while another native element + intercepts the target center; managed interactive qualification leaves + physical input enabled; +- `windows-storybook` prepares the generated app, installs the required Windows + App Runtime for that app, runs the shared Windows native build contract, + and asks the same setup action to verify the `windows` endpoint before + running Windows `stories-and-tests`; +- `win32-storybook` asks the setup action to build the same Windows provider + into its own job-local cache, verify the `win32` endpoint, and pin later + resolution before running the authored plans through the prebuilt Paper + endpoint. + +Both Windows jobs upload their doctor report, successful native build log, and +preserved failed-build diagnostics with the Storybook artifacts. Neither +`prep` nor `smoke` may compile a helper after the explicit build step. +Because GitHub-hosted Windows does not provide authoritative global input, +both jobs disable keyboard, physical-click, and wheel capabilities through the +registered host policy. The authored input plans therefore report explicit +capability skips; interactive self-hosted qualification must leave those +features enabled. + +Those jobs are consumer examples, not generic package requirements. The native +helper itself does not require CocoaPods, the Windows App Runtime, or a +Storybook process. + +## macOS runners and signing + +Authoritative AX and input qualification needs: + +- an Apple Silicon Mac in a logged-in Aqua session; +- the helper running as the intended user, not through `sudo`, a daemon, or an + SSH-only bootstrap session; +- a stable Apple Development or organization-managed identity; +- Accessibility and PostEvent authorization for the actual responsible + process; +- an MDM PPPC profile when organization policy supports preauthorization. + +Screen Recording may still require user approval. Tests that explicitly +require screenshots can skip when capture is unavailable; semantic or input +tests must fail clearly when their required authority is missing. + +GitHub-hosted macOS smoke is useful regression signal but is not proof that a +fresh machine can acquire and retain TCC authorization. Use a managed +self-hosted Mac for release qualification. See +[the macOS provider guide](../native/macos/README.md) for signing, TCC, +Developer ID, notarization, and restart behavior. + +## Windows runners + +Physical input and live UI Automation need: + +- a logged-in, unlocked interactive desktop; +- the helper and target app in the same user session; +- compatible integrity levels; +- no concurrent job sending physical input to that desktop. + +Hosted jobs may legitimately report capability skips when they cannot access +the active input session. Treat that as coverage information, not a physical +input pass. Use an interactive self-hosted runner for release qualification of +click, keyboard, wheel, and DPI behavior. + +Windows Graphics Capture and UI Automation results should be retained with the +run metadata. App-specific prerequisites such as Windows App Runtime are the +consumer's responsibility, not the driver's. + +## Sharding and parallelism + +Authored plans can be sharded deterministically: + +```sh +desktop-driver stories run \ + --url "$DESKTOP_DRIVER_URL" \ + --target "$DESKTOP_DRIVER_TARGET" \ + --tag desktop-e2e \ + --shard-index "$SHARD_INDEX" \ + --shard-count "$SHARD_COUNT" \ + --artifacts "artifacts/$PLATFORM/shard-$SHARD_INDEX" +``` + +Do not run shards that send physical input concurrently on one desktop. +Separate them by machine/session or serialize them. The service enforces one +session per target and the helper enforces cross-process input ownership, but +CI should not intentionally create contention. + +## Promotion criteria + +Before making a real-platform job required: + +- failures distinguish assertion, app, helper, authority, and infrastructure; +- required capabilities are available rather than silently skipped; +- helper and app cleanup are ownership-safe; +- artifact collection runs even after failure; +- no helper or app processes leak; +- no input remains depressed after cancellation, crash, or timeout; +- the platform's remaining qualification items in [PLAN.md](../PLAN.md) are + complete. diff --git a/packages/agentic/desktop-driver/references/contributing.md b/packages/agentic/desktop-driver/references/contributing.md new file mode 100644 index 00000000000..41e0f227e08 --- /dev/null +++ b/packages/agentic/desktop-driver/references/contributing.md @@ -0,0 +1,122 @@ +# Contributor reference + +This page expands the repository instructions in [AGENTS.md](../AGENTS.md). +Keep AGENTS.md as the instruction entry point and this file as detailed, +change-oriented guidance. + +## Put changes in the owning module + +| Area | Owns | +| ------------------ | ----------------------------------------------------------------------- | +| `src/protocol` | W3C parsing, actions, capabilities, deadlines, and error mapping | +| `src/server` | HTTP routing, targets, sessions, windows, elements, and shutdown | +| `src/host` | Platform-neutral host and application contracts | +| `src/hosts/fake` | Deterministic contract-test behavior | +| `src/hosts/native` | Process lifecycle, FDR1 framing, cancellation, and native adaptation | +| `src/native` | Source build, cache, selection, verification, and native artifact types | +| `native/macos` | Swift implementation and self-test | +| `native/windows` | Shared C++ Windows/Win32 implementation and self-test | +| `src/authoring` | Static JSON plan and result schemas | +| `src/runner` | Plan selection, capability checks, execution, and classification | +| `src/wdio` | Sanctioned WebdriverIO connection and commands | +| `src/client` | Low-level typed W3C client | +| `src/agent` | Bounded agent-facing operations | +| `src/artifacts` | Evidence naming, confinement, and atomic writes | +| `src/testing` | Reusable fake harnesses, never production fallback | +| `src/cli` | CLI parsing and structured-result adaptation | + +Do not duplicate native build, cache, or verification logic in Storybook +consumers. Do not move Storybook manifest generation or app lifecycle into this +package. + +## Behavioral rules + +- Reserve a target before asynchronous launch or attach. +- Keep session commands and physical input serialized. +- Propagate `AbortSignal` to every host operation. +- Do not release queues or input locks until a timed-out native command settles. +- Preserve attached apps and clean only exact owned resources. +- Keep native events advisory and queries authoritative. +- Use opaque public window/element IDs and native liveness checks. +- A Storybook preview reset invalidates preview elements, not app chrome or + still-live secondary windows. +- Reject unknown capabilities, plan fields, native frames, and unsupported + state explicitly. + +## Native changes + +- Update `native/protocol.json` and [native/PROTOCOL.md](../native/PROTOCOL.md) + together when the FDR1 contract changes. +- Keep frame parsing bounded and recoverable where the protocol permits. +- Build only into external staging/cache roots. +- Verify the actual long-lived process before registering a target. +- Preserve release-only crash recovery and the cross-process physical-input + lock. +- Add native self-tests for provider-local parsing, state, cancellation, + identity, capture geometry, or recovery changes. +- Add or update the opt-in Node native contract when build, cache, selection, + signing, or handshake behavior changes. + +Never patch generated SwiftPM, Xcode, MSBuild output, or cached artifacts. + +## Authored plans + +- Keep plans inline and statically extractable under + `parameters.desktopDriver`. +- Prefer stable `testID` selectors for actions and role/name assertions for + semantic validation. +- Use `requires` and `platforms` for divergence. +- Preserve serializability through manifests, runner results, CLI JSON, and + agent output. +- Do not add an imperative sidecar until a demonstrated scenario cannot be + represented safely by the static contract. + +## Documentation ownership + +- [README.md](../README.md) is the user entry and documentation index. +- `references/` holds long-form user and contributor guidance. +- `native//README.md` documents provider implementation and + operations next to its source. +- [PLAN.md](../PLAN.md) contains only unfinished or deferred work. +- [references/protocol.md](protocol.md) is the public wire/behavior contract. +- [native/PROTOCOL.md](../native/PROTOCOL.md) is the private helper protocol. + +When completing planned work, document the resulting behavior first, then +remove the completed task from PLAN.md. Do not preserve implementation history +as active instructions. + +## Validation order + +During iteration, run the smallest affected test. Before completing a package +change: + +```sh +yarn format +yarn lint +yarn build +yarn test +``` + +For native changes on the target OS: + +```sh +FURN_NATIVE_DRIVER_TEST=1 yarn test --runInBand +``` + +For stable macOS signing: + +```sh +FURN_NATIVE_DRIVER_TEST=1 \ +FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY="" \ + yarn test --runInBand +``` + +After fixing a failed step, rerun that step and downstream checks. Changes to +public types, exports, package contents, manifests, project references, or +dependencies also require relevant root build/publishing checks. Changes to +Storybook orchestration require its package tests and the affected real +endpoint smoke lifecycle. + +Do not treat a fake-host pass as native qualification. Native authority, +physical input, capture, DPI/scale, multi-window, signing, and ownership claims +require the corresponding real-machine evidence. diff --git a/packages/agentic/desktop-driver/references/getting-started.md b/packages/agentic/desktop-driver/references/getting-started.md new file mode 100644 index 00000000000..dbef908ae72 --- /dev/null +++ b/packages/agentic/desktop-driver/references/getting-started.md @@ -0,0 +1,157 @@ +# Getting started + +This guide covers setup for package contributors and consumers who embed the +Desktop Driver service. Native application projects remain responsible for +their own build, installation, identity, and lifecycle policy. + +## Requirements + +All environments need: + +- Node.js 22.12 or later; +- a package manager capable of installing the package and its WebdriverIO + dependency; +- a real, logged-in desktop session for native input and window automation. + +Repository contributors use the checked-in Node and Yarn versions: + +```sh +# from the repository root +yarn +yarn workspace @fluentui-react-native/desktop-driver build +``` + +Do not install dependencies from the package directory. Run `yarn` at the +repository root only after a manifest changes or when a declared command +reports missing dependencies. + +External consumers install the package as a development dependency: + +```sh +yarn add --dev @fluentui-react-native/desktop-driver +``` + +The package is not published from this development branch yet. Until its first +release, repository consumers declare it with `workspace:*`. From the +monorepo, run its binary with: + +```sh +yarn workspace @fluentui-react-native/desktop-driver exec desktop-driver --help +``` + +The shorter `desktop-driver` commands in this documentation assume a consumer +where the package binary is already available. + +The package contains JavaScript output, documentation, and `native/**` source. +It contains no native binary and has no install or postinstall build. + +## Native toolchains + +### Windows and Win32 + +Install: + +- Windows 11 x64; +- Visual Studio 2022 with **Desktop development with C++**; +- the MSVC x64 toolset and a Windows 10 or 11 SDK. + +The helper uses its checked-in MSBuild project. It does not require CMake, +NuGet restore, .NET, or the Windows App SDK. + +### macOS + +Install: + +- macOS 14 or later on Apple Silicon; +- Xcode with its Swift toolchain and command-line tools; +- a reusable Apple Development or local code-signing identity when privacy + authorization must survive rebuilds. + +See [the macOS provider guide](../native/macos/README.md) before choosing a +signing or CI model. + +## Build the package and helper + +Build the TypeScript package first: + +```sh +yarn workspace @fluentui-react-native/desktop-driver build +``` + +Build the native helper on its target operating system: + +```sh +desktop-driver build-driver --platform windows +desktop-driver build-driver --platform win32 +desktop-driver build-driver --platform macos +``` + +`windows` and `win32` select the same C++ helper. The default configurations +are Windows/Win32 x64 release and macOS arm64 release. Unsupported hosts, +cross-builds, and architectures fail before compilation. + +The command prints a JSON `NativeDriverArtifact`, including the verified +executable path, artifact and source identities, provider, endpoints, features, +origin, and signing metadata. + +## Resolve and diagnose + +Normal service startup can resolve a compatible cache artifact or build one if +missing: + +```sh +desktop-driver resolve-driver --platform windows +desktop-driver resolve-driver --platform macos +``` + +CI can prohibit an unexpected source build: + +```sh +desktop-driver resolve-driver --platform windows --build-policy never +desktop-driver doctor --platform windows +``` + +`doctor` never builds. It resolves and verifies an existing helper and returns +JSON with `ready: true`, or a structured readiness error and a nonzero exit +code. + +On macOS, inspect permissions from the exact verified helper: + +```sh +desktop-driver doctor --platform macos --permissions +desktop-driver doctor --platform macos --permissions --prompt +``` + +The first command is noninteractive. `--prompt` is intentionally separate +because it may display Accessibility and Screen Recording authorization UI. + +## First native service + +Follow [Service integration](service.md) to: + +1. resolve a helper; +2. construct a `NativeDesktopHost` with a controlled application descriptor; +3. register a target with `createDesktopDriverServer`; +4. connect using WebdriverIO or the typed client; +5. delete the session and close the service. + +For a Storybook application, use +`@fluentui-react-native/storybook-desktop` as the owner of manifests, app +leases, Metro, the Storybook channel, and the embedded driver listener. The +generic driver package deliberately does not discover or launch Storybook +itself. + +## Common setup failures + +| Failure | Action | +| ----------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `toolchain-missing` on Windows | Install the Visual Studio C++ workload and a Windows SDK; rerun from a normal user desktop | +| `unsupported-host` or `unsupported-platform` | Build on the target OS using Windows x64 or macOS arm64 | +| `no-verified-prebuilt` | Build once, restore the configured cache/install root, or change `buildPolicy` from `never` | +| `integrity-mismatch` | Do not edit cached output; rebuild and let the resolver quarantine the invalid selection | +| macOS permission denial | Run noninteractive `doctor --permissions`, then follow the TCC guidance in the macOS provider guide | +| Session creation cannot identify one app/window | Correct the registered descriptor or owner-provided lease; never loosen it to “first match” | +| Physical input is skipped or unavailable | Use an unlocked interactive desktop with matching user/integrity authority | + +For cache, helper overrides, and all supported environment variables, see +[Native helpers](native-helpers.md). diff --git a/packages/agentic/desktop-driver/references/native-helpers.md b/packages/agentic/desktop-driver/references/native-helpers.md new file mode 100644 index 00000000000..0a4917e5270 --- /dev/null +++ b/packages/agentic/desktop-driver/references/native-helpers.md @@ -0,0 +1,179 @@ +# Native helpers + +The package ships checked-in native source and builds one small helper on the +target operating system. Native binaries are not published in npm, compiled +during installation, or downloaded automatically. + +Windows and Win32 share the C++ x64 provider. macOS uses the Swift arm64 +provider. Both implement the private +[FDR1 framed stdio protocol](../native/PROTOCOL.md) behind the same +`DesktopHost` contract. + +## Build commands + +```sh +desktop-driver build-driver --platform windows +desktop-driver build-driver --platform win32 +desktop-driver build-driver --platform macos +``` + +Common options: + +```text +--architecture arm64|x64 +--configuration debug|release +--cache-root +--force +--macos-signing-identity +``` + +V1 validates the platform combination: Windows/Win32 require x64 and macOS +requires arm64. `build-driver` is an isolated source-build operation. It may +reuse an exactly compatible verified artifact unless `--force` is set, but it +never selects an explicit helper or managed install root. + +See [native/windows/README.md](../native/windows/README.md) and +[native/macos/README.md](../native/macos/README.md) for provider toolchains and +behavior. + +## Resolution order + +`resolve-driver` and `resolveNativeDesktopDriver` select in this order: + +1. exact `helperPath`; +2. compatible artifact beneath `installRoot`; +3. compatible artifact in the shared cache; +4. source build when `buildPolicy` is `if-missing`. + +An invalid explicit helper or install root fails immediately. It never falls +through to a different artifact. `buildPolicy: 'never'` turns a cache miss into +`no-verified-prebuilt`. + +```sh +desktop-driver resolve-driver \ + --platform windows \ + --helper-path D:\tools\furn-desktop-driver-host.exe + +desktop-driver resolve-driver \ + --platform macos \ + --install-root /Library/Application\ Support/Contoso/desktop-driver \ + --build-policy never +``` + +The macOS helper path may name either +`FurnDesktopDriverHost.app` or its nested executable. + +## Configuration + +| CLI/API option | Environment variable | Default | +| ----------------------- | --------------------------------------------- | ------------------------- | +| `buildPolicy` | `FURN_DESKTOP_DRIVER_BUILD_POLICY` | `if-missing` | +| `cacheRoot` | `FURN_DESKTOP_DRIVER_CACHE_ROOT` | User-level platform cache | +| `helperPath` | `FURN_DESKTOP_DRIVER_HELPER_PATH` | None | +| `installRoot` | `FURN_DESKTOP_DRIVER_INSTALL_ROOT` | None | +| `macosSigningIdentity` | `FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY` | Ad hoc signing | +| `configuration` | `FURN_DESKTOP_DRIVER_CONFIGURATION` | `release` | +| Disabled input features | `FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES` | None | + +An explicit API or CLI option takes precedence over its environment variable. +WebDriver capabilities cannot set any of these values. + +Default stores: + +```text +Windows: %LOCALAPPDATA%\Microsoft\FluentUIReactNative\desktop-driver\native +macOS: ~/Library/Caches/com.microsoft.fluentui-react-native.desktop-driver/native +``` + +CI may use a job-local cache root, but should restore only artifacts built for +the matching provider, architecture, source, toolchain, configuration, and +signing identity. + +GitHub Actions examples: + +```text +Windows: $RUNNER_TEMP\furn-desktop-driver-native +macOS: $RUNNER_TEMP/furn-desktop-driver-native +``` + +Build the helper explicitly, run `doctor` against that cache, then set +`FURN_DESKTOP_DRIVER_BUILD_POLICY=never` before application preparation or +smoke so later phases cannot compile implicitly. + +Hosted runners that cannot provide authoritative global input may set +`FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES=keyboard,physicalClick,wheel`. +The Node host masks those capabilities and rejects direct use while leaving +the verified native helper unchanged. Interactive self-hosted runners should +not set this policy. + +## Immutable cache model + +```text +/v1/ + artifacts/-//// + selections// + locks/ + staging/ + diagnostics/ + trash/ +``` + +- The compatibility key covers provider, architecture, configuration, source, + coordinator, protocol, and macOS signing identity. +- The build fingerprint adds the actual toolchain fingerprint. +- The artifact ID hashes the built executable. +- Builds publish through same-volume staging and atomic rename. +- Cross-process directory locks prevent duplicate publication. +- Failed build logs are copied to immutable diagnostics before staging cleanup. +- Corrupt cache selections and artifact collisions are quarantined under + `trash`; in-use artifacts are never replaced. + +Never edit an artifact or selection in place. Rebuild and allow the resolver to +publish a new immutable selection. + +## Verification + +Every artifact is verified before use: + +- supported provider and architecture; +- real path and managed-path confinement; +- source, compatibility, build, and executable identities; +- macOS signature and recorded signing policy; +- FDR1 major/minor compatibility and helper identity; +- one-shot handshake before selection; +- the same handshake from the actual long-lived process before it receives + native commands. + +Explicit paths are trusted only as an operator selection; they still undergo +hash, platform, signature-policy, and handshake verification. An optional +organization-managed prebuilt must use the same artifact layout and be +provisioned out of band. The package does not define a downloader. + +## Runtime + +`NativeHostProcess` starts the verified executable with `--stdio`. Requests, +responses, binary payloads, events, cancellation, and cancellation +acknowledgements are correlated. Stderr is bounded diagnostic output. + +If a command deadline expires, Node requests cancellation and waits for the +helper to settle before the session queue advances. If the helper dies while +input may be depressed, Node invokes the same verified binary in restricted +`--release-input` mode with only the recorded keys and buttons. Further +physical input must not proceed if release cannot be proven. + +## Diagnostics + +`doctor` uses prebuilt-only resolution: + +```sh +desktop-driver doctor --platform windows +desktop-driver doctor --platform win32 +desktop-driver doctor --platform macos +``` + +It prints JSON and exits nonzero for actionable readiness failures. macOS +permission probes are documented in the +[macOS provider guide](../native/macos/README.md). + +For service target setup, see [Service integration](service.md). For CI cache +and native gating, see [CI integration](ci-integration.md). diff --git a/packages/agentic/desktop-driver/references/protocol.md b/packages/agentic/desktop-driver/references/protocol.md new file mode 100644 index 00000000000..dd7edef9ac1 --- /dev/null +++ b/packages/agentic/desktop-driver/references/protocol.md @@ -0,0 +1,152 @@ +# WebDriver contract + +Desktop Driver is a W3C WebDriver Classic-compatible native desktop remote end. +It implements native application semantics rather than claiming browser +conformance. + +V1 platform names are `macos` and `windows`. Endpoint names are `macos`, +`windows`, and `win32`. + +## Sessions and targets + +- Targets are registered by the server. +- Capabilities select one target with `furn:target`. +- Capabilities never carry executable paths, environment variables, helper + selection, leases, or output directories. +- Exactly one active session may own a physical target. +- Commands are serialized per session and input operations share a global + mutex. +- A target records whether the application was launched or attached. +- Deleting an attached session preserves the application. +- Cleanup uses exact process identity, never process-name matching. + +Supported extension capabilities: + +- `furn:target`; +- `furn:endpoint`; +- `furn:renderer`; +- `furn:launchMode`: `launch` or `attach`; +- `furn:clickMode`: `physical`, `accessibility`, or `auto`; +- returned `furn:features`. + +Capability matching implements W3C `alwaysMatch` and ordered `firstMatch`, +rejects duplicate merge keys, and returns `session not created` when no +registered target can satisfy a candidate. + +## W3C routes + +Implemented standard behavior: + +- status and new/delete session; +- get/set timeouts; +- current window, handles, switching, closing, and rectangles; +- element lookup from a window or element; +- active element; +- element name, text, attribute, property, rectangle, enabled, selected, and + displayed queries; +- click, clear, and send keys; +- perform and release actions; +- window and element screenshots; +- accessibility source. + +Implemented `furn` extensions: + +- `GET /session/{id}/furn/manifest`; +- `GET|POST /session/{id}/furn/story`; +- `POST /session/{id}/furn/story/reset`; +- `POST /session/{id}/furn/story/args`; +- `GET /session/{id}/furn/tree`. + +Navigation, history, cookies, frames, shadow roots, prompts, printing, +arbitrary JavaScript execution, browser CSS values, and new-window creation +return `unsupported operation`. + +## Timeouts + +Standard implicit, page-load, and script timeout values are accepted by the +session contract. Native and Storybook operations use driver-owned bounded +deadlines rather than repurposing page-load semantics. + +Every host operation receives an `AbortSignal`. On timeout, the server aborts +and drains native work before input release, teardown, or a later command. A +provider may never apply late input after timeout cleanup. + +## Elements and selectors + +Public references use only the standard +`element-6066-11e4-a52e-4f735466cecf` key with session UUIDs. Native handles, +UIA runtime IDs, AX references, React tags, and HWNDs remain private. + +Every element command validates native liveness. Story reset increments the +preview generation and invalidates preview elements while preserving live app +and Storybook chrome references. + +Pointer and wheel actions may use WebDriver element origins. The server resolves +the public UUID, validates generation and liveness, and passes only the native +element ID to the host. + +Portable lookup strategies: + +| WebDriver strategy | Native meaning | +| ------------------- | -------------------------------------------------- | +| `accessibility id` | Stable React Native `testID`/automation identifier | +| `tag name` | Normalized accessibility role | +| `link text` | Exact accessible name | +| `partial link text` | Partial accessible name | +| `-furn:text` | Normalized visible/value text | + +CSS and XPath are not redefined for non-DOM trees. + +## Errors + +| Condition | WebDriver error | +| ------------------------------------------------- | --------------------------- | +| Target cannot launch or attach | `session not created` | +| Missing or closed session | `invalid session id` | +| Missing or closed window | `no such window` | +| Lookup does not resolve | `no such element` | +| Retained native node is detached or replaced | `stale element reference` | +| Malformed locator | `invalid selector` | +| Disabled, unfocusable, or empty-bounds target | `element not interactable` | +| Another node owns the hit-tested point | `element click intercepted` | +| Capture backend fails | `unable to capture screen` | +| Deadline expires | `timeout` | +| Capability, property, or operation is unavailable | `unsupported operation` | + +Error data may include bounded operation names, native diagnostic codes, and +artifact IDs. It must not expose unrestricted environment values, native +handles, or arbitrary physical paths. + +## Storybook readiness + +When a target provides a `StoryOrchestrator`, the consumer validates: + +- instance, target, endpoint, and catalog identity; +- a private bridge nonce; +- request and run IDs; +- portable-plan digest; +- preview generation; +- the native story-root marker. + +Only the authenticated runtime bridge can complete or fail selection. Every +test receives a new run ID and keyed preview remount. + +## Results and capabilities + +Run status is `passed` or `failed`. Individual tests distinguish: + +- `passed`; +- `failed`; +- `skipped`; +- `timed-out`; +- `cancelled`; +- `infrastructure-error`. + +Steps record status, duration, error, and artifact references. Unsupported +required capabilities skip explicitly. An unavailable native property never +becomes a false-shaped passing assertion. + +See [Test integration](test-integration.md) for plan and result usage, +[Service integration](service.md) for target registration, and +[native/PROTOCOL.md](../native/PROTOCOL.md) for the private Node/helper +transport. diff --git a/packages/agentic/desktop-driver/references/security.md b/packages/agentic/desktop-driver/references/security.md new file mode 100644 index 00000000000..e795b5cd5a5 --- /dev/null +++ b/packages/agentic/desktop-driver/references/security.md @@ -0,0 +1,103 @@ +# Security model + +Desktop Driver controls a real interactive desktop. Its V1 security boundary is +a trusted local supervisor, a loopback-only service, pre-registered targets, +verified native helpers, and confined evidence. It is not a remote automation +service. + +## Service boundary + +- `createDesktopDriverServer` accepts only loopback hosts. +- Browser-origin requests are rejected. +- The service has no TLS, user authentication, or remote authorization. +- Clients can select only server-registered targets. +- Capabilities cannot provide executables, arguments, environment variables, + helper paths, lease paths, cache roots, or artifact destinations. +- One active session may own a target. + +Do not proxy, tunnel, container-port-forward, or expose the listener to another +machine. A remote deployment needs a separately designed authenticated +transport and target authorization model. + +## Native-helper trust + +- Native source is checked in and compilation is explicit. +- Installation never downloads or builds a helper. +- Explicit paths fail verification rather than falling back. +- Managed artifacts use real-path and symlink/junction confinement. +- Immutable cache records bind source, configuration, toolchain, artifact hash, + protocol, and signing identity. +- Invalid cache entries are quarantined. +- The selected executable completes a one-shot handshake and the actual + long-lived process repeats the identity/protocol handshake. +- macOS signatures are checked against their recorded certificate hash, + authority, team identifier, designated requirement, Hardened Runtime, and + timestamp policy. + +An optional organization prebuilt must be provisioned through an external +trusted channel and pinned to organization publisher policy. A hash stored next +to an artifact is integrity metadata, not publisher authentication. + +## Application identity and ownership + +Launch and attach configuration is server-owned. Exact application identities, +window titles, PIDs, process start times, executables, bundle IDs/AUMIDs, and +nonce-bound leases prevent attaching to a merely similar process. + +Ambiguous matches fail. Cleanup preserves attached applications and terminates +only exact resources recorded as owned. Never add process-name or “first +window” fallback cleanup. + +## Native protocol + +The helper has no listener and receives only validated commands over inherited +stdio. FDR1 frames are bounded and correlated. Native handles remain private; +WebDriver uses session-generated UUIDs. + +Every operation is cancellable. Session and global input queues remain held +until native side effects settle. Node mirrors depressed input and uses a +restricted release-only helper mode after an unexpected native exit. + +## Artifacts and diagnostics + +Artifact paths are sanitized and confined beneath the configured run root. +Agent tree responses are bounded by depth and node count. Public errors and +capabilities do not include unrestricted environment data or native handles. + +Screenshots, accessibility trees, source, app titles, and values may contain +sensitive information. CI and local workflows must: + +- collect only evidence required for the test; +- store it in access-controlled artifact systems; +- define retention and deletion; +- avoid publishing it in logs or public issues; +- redact secrets and user data before sharing. + +Native build logs and private generated manifests can contain machine-local +paths. Keep them out of package output and public diagnostics. + +## Platform authority + +Physical input is global to the interactive desktop. Run automation only in a +dedicated logged-in session and serialize it across processes. + +On Windows, use matching user sessions and integrity levels. On macOS, grant +privacy permissions only to the intended responsible process and stable +identity. Never edit `TCC.db`, disable SIP, sandbox the helper to bypass policy, +or use undocumented responsibility APIs. + +## Review triggers + +Require a focused security review before: + +- allowing a non-loopback bind; +- adding authentication, TLS, or remote target discovery; +- accepting executable or artifact configuration from a client; +- adding automatic downloads or updates; +- publishing organization-built native artifacts; +- adding a persistent broker or per-user socket; +- exposing new arbitrary agent operations; +- changing path confinement, signature policy, ownership, or input recovery. + +See [Architecture](architecture.md), [Service integration](service.md), and +[Native helpers](native-helpers.md) for implementation details. diff --git a/packages/agentic/desktop-driver/references/service.md b/packages/agentic/desktop-driver/references/service.md new file mode 100644 index 00000000000..2c59226735e --- /dev/null +++ b/packages/agentic/desktop-driver/references/service.md @@ -0,0 +1,211 @@ +# Service integration + +The Desktop Driver service is an embeddable, loopback-only W3C remote end. +Integrators register controlled targets before listening; clients can select +only those targets. + +## Embed a native target + +```ts +import { createDesktopDriverServer, NativeDesktopHost, resolveNativeDesktopDriver } from '@fluentui-react-native/desktop-driver'; + +const artifact = await resolveNativeDesktopDriver({ + buildPolicy: 'if-missing', + platform: 'windows', +}); + +const nativeHost = new NativeDesktopHost({ + application: { + executablePath: 'C:\\apps\\Contoso.exe', + windowTitle: 'Contoso', + }, + artifact, + endpoint: 'windows', + onStderr: (message) => process.stderr.write(message), +}); + +const service = await createDesktopDriverServer({ + host: '127.0.0.1', + port: 0, + targets: [ + { + endpoint: 'windows', + host: nativeHost, + id: 'contoso-windows', + platformName: 'windows', + renderer: 'fabric', + }, + ], +}); + +console.log(service.url); +``` + +`port: 0` asks the operating system for an available port. Record the returned +URL and target ID in the trusted test supervisor rather than exposing a +discovery listener. + +Always close the service: + +```ts +try { + // Run clients. +} finally { + await service.close(); +} +``` + +Shutdown stops accepting requests, drains session creation, deletes sessions, +releases input, disposes hosts, and reports cleanup failures. + +## Register targets + +Each `DesktopTarget` declares: + +| Field | Meaning | +| ------------------- | ------------------------------------------------------- | +| `id` | Stable service-local identity selected by `furn:target` | +| `platformName` | `macos` or `windows` | +| `endpoint` | `macos`, `windows`, or `win32` | +| `renderer` | `fabric` or `paper` | +| `host` | A fake or native `DesktopHost` implementation | +| `storyOrchestrator` | Optional Storybook selection/reset adapter | +| `storyRootTestId` | Optional native marker verified after story readiness | + +Do not derive a target from incoming capabilities. Executable paths, app +arguments, environment variables, helper paths, lease paths, and artifact roots +are server-owned configuration. + +## Describe the application + +`NativeDesktopHost` receives one immutable +`NativeDesktopApplicationDescriptor`. + +| Field | Use | +| ---------------------------- | --------------------------------------------------------------- | +| `bundleIdentifier` | macOS application identity | +| `aumid` | Windows packaged-app launch identity | +| `executablePath` | Controlled unpackaged launch path | +| `arguments` | Fixed server-owned launch arguments | +| `windowTitle` | Exact expected window title, required by the Windows helper | +| `leasePath` and `leaseNonce` | Exact owner-generated attach lease; do not accept from a client | + +Windows launch needs either `aumid` or `executablePath` plus an exact +`windowTitle`. Attach without an owner-generated lease also requires a title +that resolves to exactly one live window. Ambiguity fails instead of selecting +the first result. + +For macOS descriptor and launch behavior, see +[the macOS native provider](../native/macos/README.md). Storybook supplies a +nonce-bound lease containing the exact PID, process start, executable, endpoint +identity, and app identity; the native helper independently verifies it. + +## Launch and attach + +Clients request one of two modes: + +- `launch` asks the registered host to start the controlled application and + records it as owned; +- `attach` resolves an already running application from the controlled + descriptor or owner-generated lease and records it as external. + +The default is `launch`. Deleting an attached session preserves the app. +Deleting a launched session may close the exact owned process. Cleanup never +kills by process name. + +## Connect with WebdriverIO + +```ts +import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; + +const desktop = await connectDesktopWebdriver({ + clickMode: 'auto', + launchMode: 'launch', + platformName: 'windows', + targetId: 'contoso-windows', + url: service.url, +}); + +try { + const save = await desktop.browser.$('~save-button'); + await save.click(); + await desktop.browser.keys(['Control', 's']); +} finally { + await desktop.delete(); +} +``` + +The returned browser supports standard WebdriverIO APIs backed by the +implemented W3C subset plus these commands when a Storybook orchestrator is +registered: + +- `desktopListStories()`; +- `desktopOpenStory(storyId, runId?)`; +- `desktopResetStory(storyId, runId?)`; +- `desktopExpect(expectation)`; +- `desktopRunStoryTests(options?)`. + +`clickMode` can be `physical`, `accessibility`, or `auto`. The server returns +the negotiated features; it never claims an unsupported input mode. + +## Connect with the typed client + +Use `./client` for protocol integrations and conformance tests that do not want +WebdriverIO: + +```ts +import { createDesktopDriverClient } from '@fluentui-react-native/desktop-driver/client'; + +const client = createDesktopDriverClient({ url: service.url }); +const session = await client.newSession({ + alwaysMatch: { + browserName: 'furn-native-desktop', + platformName: 'windows', + 'furn:target': 'contoso-windows', + }, +}); + +try { + const save = await session.findElement('accessibility id', 'save-button'); + await save.click(); +} finally { + await session.delete(); +} +``` + +Use `./agent` for bounded inspection and evidence operations. It deliberately +does not expose native handles, arbitrary commands, or unrestricted paths. + +## Fake service CLI + +`desktop-driver serve` starts a deterministic fake target: + +```sh +desktop-driver serve \ + --platform windows \ + --endpoint windows \ + --renderer fabric \ + --target fake-windows \ + --manifest story-manifest.windows.json +``` + +The command prints JSON containing the service URL and target ID, stays alive +until a termination signal, then closes cleanly. It is intended for package, +protocol, client, and runner integration tests. Native production integrations +must resolve a verified helper and embed the service. + +## Operations + +- `GET /status` probes every registered target and returns truthful endpoint + features. +- The service accepts a maximum 1 MiB JSON request body by default; override + `maxBodyBytes` only for a measured local integration need. +- Each target has at most one active session. +- Commands are serialized per session. Input is serialized globally. +- Browser-origin requests are rejected. +- Non-loopback hosts are rejected even when passed explicitly. +- There is no network authentication or TLS. Do not proxy or expose the service + beyond the local trusted process boundary. + +See [the WebDriver contract](protocol.md) for routes and errors and +[the security model](security.md) before adding a new service surface. diff --git a/packages/agentic/desktop-driver/references/test-integration.md b/packages/agentic/desktop-driver/references/test-integration.md new file mode 100644 index 00000000000..48d209578d9 --- /dev/null +++ b/packages/agentic/desktop-driver/references/test-integration.md @@ -0,0 +1,236 @@ +# Test integration + +Desktop Driver supports three complementary test levels: + +1. normal WebdriverIO automation against a registered native target; +2. serializable plans colocated with component stories; +3. deterministic package and protocol tests against the fake host. + +The same W3C service and `DesktopHost` contract underpin all three. + +## WebdriverIO tests + +Use `connectDesktopWebdriver` when a test owns service connection and session +lifecycle: + +```ts +import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; + +const desktop = await connectDesktopWebdriver({ + clickMode: 'auto', + launchMode: 'attach', + platformName: 'macos', + targetId: 'component-storybook-macos', + url: process.env.DESKTOP_DRIVER_URL!, +}); + +try { + const input = await desktop.browser.$('~search-input'); + await input.setValue('Fluent'); + await expect(input).toHaveValue('Fluent'); +} finally { + await desktop.delete(); +} +``` + +Use standard WebdriverIO APIs only where the +[implemented protocol](protocol.md) supports them. Browser navigation, cookies, +frames, JavaScript execution, prompts, printing, CSS values, and shadow roots +are intentionally unavailable. + +For Storybook targets, the connection registers: + +- `desktopListStories()`; +- `desktopOpenStory(storyId, runId?)`; +- `desktopResetStory(storyId, runId?)`; +- `desktopExpect(expectation)`; +- `desktopRunStoryTests(options?)`. + +The runner resets the preview for each test, filters and shards a stable +`storyId/testId` ordering, checks required capabilities, and classifies +assertion, timeout, cancellation, skip, and infrastructure outcomes. + +## Author portable story plans + +Plans are static JSON under `parameters.desktopDriver`. Import only their types +from `./authoring`: + +```tsx +import type { DesktopStoryTests } from '@fluentui-react-native/desktop-driver/authoring'; + +export const Default: Story = { + tags: ['desktop-e2e'], + parameters: { + desktopDriver: { + version: 1, + tests: [ + { + id: 'activate', + title: 'Activates from the desktop', + requires: ['physical-click'], + steps: [ + { action: 'wait', target: { testId: 'save-button' } }, + { + expect: { + state: 'role', + target: { testId: 'save-button' }, + value: 'button', + }, + }, + { action: 'click', target: { testId: 'save-button' } }, + { + expect: { + state: 'focused', + target: { testId: 'save-button' }, + value: true, + }, + }, + ], + }, + ], + } satisfies DesktopStoryTests, + }, +}; +``` + +Storybook extraction accepts JSON literals wrapped by TypeScript `satisfies` or +`as`. Do not hide a plan behind variables, functions, spreads, computed +properties, or runtime platform branches. Dynamic or invalid plans fail +generation with source context. + +Portable selectors: + +- `{ testId }` for deterministic interaction; +- `{ role, name? }` for semantic lookup; +- `{ accessibleName }`; +- `{ text }`. + +Actions: + +- click, double-click, clear, type, key sequences, and raw W3C actions; +- scroll and wait; +- Storybook argument updates; +- screenshot, source, and notes. + +Assertions: + +- existence and count; +- role, accessible name, text, and value; +- displayed, enabled, focused, selected, checked/mixed, and expanded state. + +Requirements can declare `accessibility-click`, `element-screenshot`, `focus`, +`keyboard`, `physical-click`, `screenshot`, or `wheel`. A missing requirement +produces an explicit skip. Platform differences belong in `platforms` and +`requires`, not imperative branches. + +## Run plans + +```sh +desktop-driver stories list \ + --url http://127.0.0.1:39859 \ + --target component-storybook-windows + +desktop-driver stories explain components-button--default \ + --url http://127.0.0.1:39859 \ + --target component-storybook-windows + +desktop-driver stories run \ + --url http://127.0.0.1:39859 \ + --target component-storybook-windows \ + --tag desktop-e2e \ + --shard-index 0 \ + --shard-count 2 \ + --artifacts artifacts/windows/desktop-driver +``` + +Selectors can filter by story glob, test glob, tag, and deterministic shard. +The CLI emits JSON and exits nonzero when the run status is not `passed`. + +## Evidence + +Passing `artifactsRoot` writes atomically beneath that root: + +```text +artifactsRoot/ + host.json + run.json + tests/ + -/ + + failure.png + failure-source.xml + failure-tree.json +``` + +Names are sanitized and confined. Failure evidence is best effort and never +replaces the original test error. Trees and screenshots may contain sensitive +application information; apply the retention and access policy described in +[Security](security.md). + +## Fake-host contract tests + +Use `./testing` for deterministic tests that do not need a desktop: + +```ts +import { createDesktopDriverTestHarness } from '@fluentui-react-native/desktop-driver/testing'; +import { createDesktopDriverClient } from '@fluentui-react-native/desktop-driver/client'; + +const harness = await createDesktopDriverTestHarness({ + endpoint: 'windows', +}); +const client = createDesktopDriverClient({ url: harness.server.url }); + +try { + const session = await client.newSession({ + alwaysMatch: { + browserName: 'furn-native-desktop', + platformName: 'windows', + 'furn:target': harness.target.id, + }, + }); + await session.delete(); +} finally { + await harness.close(); +} +``` + +`createDesktopDriverStoryHarness` adds a fake Storybook orchestrator and +manifest-derived elements. The fake provider is appropriate for protocol, +queueing, cancellation, runner, CLI, and consumer contract tests. It is not +evidence that native accessibility, input, window, or capture behavior works. + +## Package tests + +From this package: + +```sh +yarn test +``` + +The default suite covers raw W3C routes, typed clients, WebdriverIO custom +commands, authored plans, fake Storybook orchestration, bounded agents, +artifacts, target concurrency, timeouts, shutdown races, framing, helper +verification, and CLI adaptation. + +Run the opt-in native contract on the matching target OS: + +```sh +FURN_NATIVE_DRIVER_TEST=1 yarn test --runInBand +``` + +PowerShell: + +```powershell +$env:FURN_NATIVE_DRIVER_TEST = '1' +yarn test --runInBand +``` + +The native contract builds into a temporary or configured external cache, +verifies the artifact and long-lived handshake, runs the helper self-test, +checks cache reuse and resolution, and exercises malformed-frame recovery. +Stable-signed macOS coverage also requires +`FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY`. + +Real-app behavior belongs in a consuming application's integration or smoke +suite. This repository uses the Storybook `stories-and-tests` lifecycle; see +[CI integration](ci-integration.md). diff --git a/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts b/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts index 2d291c62424..9ca0e8f5d9d 100644 --- a/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts +++ b/packages/agentic/desktop-driver/src/cli/DesktopDriverCli.test.ts @@ -4,11 +4,257 @@ import { createServer } from 'node:net'; import os from 'node:os'; import path from 'node:path'; +import { NativeDriverError } from '../native/NativeDriverError'; +import type { NativeDriverArtifact } from '../native/types'; import type { DesktopStoryManifest } from '../storybook.js'; +import { createDesktopDriverCommand } from './createDesktopDriverCommand'; jest.setTimeout(30_000); describe('desktop-driver CLI', () => { + afterEach(() => { + process.exitCode = undefined; + }); + + test('exposes isolated native build and resolution commands', async () => { + const artifact = createNativeArtifact(); + const buildDriver = jest.fn(async () => artifact); + const resolveDriver = jest.fn(async () => ({ ...artifact, origin: 'cache' as const })); + const output: string[] = []; + const createProgram = () => + createDesktopDriverCommand({ + buildDriver, + resolveDriver, + stdout: { + write(value) { + output.push(String(value)); + return true; + }, + }, + }); + + await createProgram().parseAsync([ + 'node', + 'test', + 'build-driver', + '--platform', + 'macos', + '--configuration', + 'release', + '--cache-root', + 'native-cache', + '--macos-signing-identity', + 'Apple Development: Example', + ]); + await createProgram().parseAsync([ + 'node', + 'test', + 'resolve-driver', + '--platform', + 'win32', + '--build-policy', + 'never', + '--helper-path', + 'driver.exe', + ]); + + expect(buildDriver).toHaveBeenCalledWith({ + architecture: undefined, + cacheRoot: 'native-cache', + configuration: 'release', + force: undefined, + macosSigningIdentity: 'Apple Development: Example', + platform: 'macos', + }); + expect(resolveDriver).toHaveBeenCalledWith({ + architecture: undefined, + buildPolicy: 'never', + cacheRoot: undefined, + configuration: 'release', + force: undefined, + helperPath: 'driver.exe', + installRoot: undefined, + macosSigningIdentity: undefined, + platform: 'win32', + }); + expect(JSON.parse(output[0])).toMatchObject({ artifactId: 'artifact-id', origin: 'built' }); + expect(JSON.parse(output[1])).toMatchObject({ artifactId: 'artifact-id', origin: 'cache' }); + }); + + test('runs noninteractive permission diagnostics through the resolved macOS helper', async () => { + const artifact = createNativeArtifact({ + architecture: 'arm64', + endpoints: ['macos'], + executablePath: '/verified/FurnDesktopDriverHost.app/Contents/MacOS/furn-desktop-driver-host', + provider: 'macos', + signing: { mode: 'adhoc' }, + }); + const resolveDriver = jest.fn(async () => artifact); + const permissionProbe = jest.fn(async () => ({ + permissions: { + accessibility: { preflight: false }, + postEvent: { preflight: true }, + screenCapture: { preflight: false }, + }, + promptRequested: false, + schemaVersion: 1, + type: 'permissions', + })); + const output: string[] = []; + const program = createDesktopDriverCommand({ + permissionProbe, + resolveDriver, + stdout: { + write(value) { + output.push(String(value)); + return true; + }, + }, + }); + + await program.parseAsync(['node', 'test', 'doctor', '--platform', 'macos', '--permissions']); + + expect(resolveDriver).toHaveBeenCalledWith({ + architecture: undefined, + buildPolicy: 'never', + cacheRoot: undefined, + configuration: 'release', + force: undefined, + helperPath: undefined, + installRoot: undefined, + macosSigningIdentity: undefined, + platform: 'macos', + }); + expect(permissionProbe).toHaveBeenCalledWith(artifact, { prompt: false }); + expect(JSON.parse(output.join(''))).toMatchObject({ + permissions: { + promptRequested: false, + schemaVersion: 1, + type: 'permissions', + }, + ready: true, + result: { executablePath: artifact.executablePath }, + }); + }); + + test('passes the explicit doctor prompt flag only to macOS permission diagnostics', async () => { + const artifact = createNativeArtifact({ + architecture: 'arm64', + endpoints: ['macos'], + provider: 'macos', + signing: { mode: 'signed' }, + }); + const permissionProbe = jest.fn(async () => ({ + promptRequested: true, + schemaVersion: 1, + type: 'permissions', + })); + const output: string[] = []; + const program = createDesktopDriverCommand({ + permissionProbe, + resolveDriver: jest.fn(async () => artifact), + stdout: { + write(value) { + output.push(String(value)); + return true; + }, + }, + }); + + await program.parseAsync(['node', 'test', 'doctor', '--platform', 'macos', '--permissions', '--prompt']); + + expect(permissionProbe).toHaveBeenCalledWith(artifact, { prompt: true }); + expect(JSON.parse(output.join(''))).toMatchObject({ + permissions: { promptRequested: true }, + ready: true, + }); + }); + + test('rejects prompt mode without permissions before resolving a helper', async () => { + const resolveDriver = jest.fn(async () => createNativeArtifact()); + const output: string[] = []; + const program = createDesktopDriverCommand({ + resolveDriver, + stdout: { + write(value) { + output.push(String(value)); + return true; + }, + }, + }); + + await program.parseAsync(['node', 'test', 'doctor', '--platform', 'macos', '--prompt']); + + expect(resolveDriver).not.toHaveBeenCalled(); + expect(JSON.parse(output.join(''))).toEqual({ + error: { + code: 'invalid-params', + message: '--prompt requires --permissions.', + }, + ready: false, + }); + }); + + test('preserves Windows doctor behavior by rejecting macOS-only permission diagnostics', async () => { + const resolveDriver = jest.fn(async () => createNativeArtifact()); + const output: string[] = []; + const program = createDesktopDriverCommand({ + resolveDriver, + stdout: { + write(value) { + output.push(String(value)); + return true; + }, + }, + }); + + await program.parseAsync(['node', 'test', 'doctor', '--platform', 'windows', '--permissions']); + + expect(resolveDriver).not.toHaveBeenCalled(); + expect(JSON.parse(output.join(''))).toMatchObject({ + error: { code: 'unsupported-operation' }, + ready: false, + }); + }); + + test('runs Win32 doctor with prebuilt-only resolution and reports cache misses', async () => { + const resolveDriver = jest.fn(async () => { + throw new NativeDriverError('no-verified-prebuilt', 'No verified Win32 helper is available.'); + }); + const output: string[] = []; + const program = createDesktopDriverCommand({ + resolveDriver, + stdout: { + write(value) { + output.push(String(value)); + return true; + }, + }, + }); + + await program.parseAsync(['node', 'test', 'doctor', '--platform', 'win32', '--cache-root', 'native-cache']); + + expect(resolveDriver).toHaveBeenCalledWith({ + architecture: undefined, + buildPolicy: 'never', + cacheRoot: 'native-cache', + configuration: 'release', + force: undefined, + helperPath: undefined, + installRoot: undefined, + macosSigningIdentity: undefined, + platform: 'win32', + }); + expect(JSON.parse(output.join(''))).toEqual({ + error: { + code: 'no-verified-prebuilt', + message: 'No verified Win32 helper is available.', + }, + ready: false, + }); + expect(process.exitCode).toBe(1); + }); + test('serves, lists, runs, and describes a fake authored plan as JSON', async () => { const port = await getAvailablePort(); const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'desktop-driver-cli-')); @@ -80,6 +326,28 @@ describe('desktop-driver CLI', () => { fs.rmSync(temporaryDirectory, { force: true, recursive: true }); } }); + + function createNativeArtifact(overrides: Partial = {}): NativeDriverArtifact { + return { + architecture: 'x64', + artifactId: 'artifact-id', + artifactRoot: 'artifact-root', + buildFingerprint: 'build-fingerprint', + buildId: 'build-id', + compatibilityKey: 'compatibility-key', + configuration: 'release', + endpoints: ['windows', 'win32'], + executablePath: 'driver.exe', + features: ['probe'], + origin: 'built', + provider: 'windows', + schemaVersion: 1, + signing: { mode: 'none' }, + sourceDigest: 'source-digest', + wireProtocol: { major: 1, minor: 0 }, + ...overrides, + }; + } }); function spawnCli(args: readonly string[]) { diff --git a/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts b/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts index 664fc4ccfc7..3cabf315183 100644 --- a/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts +++ b/packages/agentic/desktop-driver/src/cli/createDesktopDriverCommand.ts @@ -1,13 +1,23 @@ +import { spawn } from 'node:child_process'; import fs from 'node:fs'; import { Command, Option } from 'commander'; import { connectDesktopAgent } from '../agent/DesktopAgent.js'; import { validateDesktopStoryTests } from '../authoring/storyTests.js'; -import type { DesktopStoryManifest } from '../storybook.js'; import { FakeDesktopHost } from '../hosts/fake/FakeDesktopHost.js'; +import { buildNativeDesktopDriver, resolveNativeDesktopDriver } from '../native/nativeDriver.js'; +import type { + NativeDriverArchitecture, + NativeDriverBuildOptions, + NativeDriverBuildPolicy, + NativeDriverConfiguration, + NativeDriverResolveOptions, + NativeDriverArtifact, +} from '../native/types.js'; import type { DesktopEndpoint, DesktopPlatformName, DesktopRenderer } from '../protocol/types.js'; import { createDesktopDriverServer } from '../server/createDesktopDriverServer.js'; +import type { DesktopStoryManifest } from '../storybook.js'; import { FakeStoryOrchestrator } from '../testing/FakeStoryOrchestrator.js'; import { createFakeStoryWindows } from '../testing/fakeStoryElements.js'; import { connectDesktopWebdriver } from '../wdio/DesktopWebdriver.js'; @@ -28,11 +38,19 @@ type SelectionFlags = ConnectionFlags & { }; export type CreateDesktopDriverCommandOptions = { + buildDriver?: typeof buildNativeDesktopDriver; + permissionProbe?: NativePermissionProbe; + resolveDriver?: typeof resolveNativeDesktopDriver; stderr?: Pick; stdout?: Pick; }; +type NativePermissionProbe = (artifact: NativeDriverArtifact, options: { prompt: boolean }) => Promise>; + export function createDesktopDriverCommand(options: CreateDesktopDriverCommandOptions = {}): Command { + const buildDriver = options.buildDriver ?? buildNativeDesktopDriver; + const permissionProbe = options.permissionProbe ?? runNativePermissionProbe; + const resolveDriver = options.resolveDriver ?? resolveNativeDesktopDriver; const stdout = options.stdout ?? process.stdout; const stderr = options.stderr ?? process.stderr; const program = new Command() @@ -43,6 +61,10 @@ export function createDesktopDriverCommand(options: CreateDesktopDriverCommandOp writeOut: (value) => stdout.write(value), }); + addNativeBuildCommand(program, buildDriver, stdout); + addNativeResolveCommand(program, resolveDriver, stdout); + addNativeDoctorCommand(program, resolveDriver, permissionProbe, stdout); + program .command('serve') .description('Start a deterministic fake Desktop Driver target.') @@ -196,6 +218,84 @@ export function createDesktopDriverCommand(options: CreateDesktopDriverCommandOp return program; } +function addNativeBuildCommand( + program: Command, + buildDriver: typeof buildNativeDesktopDriver, + stdout: Pick, +): void { + const command = program.command('build-driver').description('Build the native desktop helper for the selected platform.'); + addNativePlatformOptions(command) + .addOption(new Option('--architecture ', 'native architecture').choices(['arm64', 'x64'])) + .addOption(new Option('--configuration ', 'native build configuration').choices(['debug', 'release']).default('release')) + .option('--cache-root ', 'native helper cache root') + .option('--force', 'build and publish a new immutable selection even when a compatible artifact exists') + .option('--macos-signing-identity ', 'macOS code-signing certificate name or SHA-1 hash') + .action(async (flags: NativeBuildFlags) => { + const result = await buildDriver(toBuildOptions(flags)); + writeJson(stdout, result); + }); +} + +function addNativeResolveCommand( + program: Command, + resolveDriver: typeof resolveNativeDesktopDriver, + stdout: Pick, +): void { + const command = program.command('resolve-driver').description('Resolve and verify a native desktop helper.'); + addNativePlatformOptions(command) + .addOption(new Option('--architecture ', 'native architecture').choices(['arm64', 'x64'])) + .addOption(new Option('--configuration ', 'native build configuration').choices(['debug', 'release']).default('release')) + .addOption(new Option('--build-policy ', 'source build policy').choices(['if-missing', 'never']).default('if-missing')) + .option('--cache-root ', 'native helper cache root') + .option('--helper-path ', 'exact prebuilt helper executable') + .option('--install-root ', 'managed native helper install root') + .option('--macos-signing-identity ', 'macOS code-signing certificate name or SHA-1 hash') + .action(async (flags: NativeResolveFlags) => { + const result = await resolveDriver(toResolveOptions(flags)); + writeJson(stdout, result); + }); +} + +function addNativeDoctorCommand( + program: Command, + resolveDriver: typeof resolveNativeDesktopDriver, + permissionProbe: NativePermissionProbe, + stdout: Pick, +): void { + const command = program.command('doctor').description('Report the selected native helper or an actionable readiness failure.'); + addNativePlatformOptions(command) + .addOption(new Option('--architecture ', 'native architecture').choices(['arm64', 'x64'])) + .addOption(new Option('--configuration ', 'native build configuration').choices(['debug', 'release']).default('release')) + .option('--cache-root ', 'native helper cache root') + .option('--helper-path ', 'exact prebuilt helper executable') + .option('--install-root ', 'managed native helper install root') + .option('--macos-signing-identity ', 'expected macOS code-signing certificate name or SHA-1 hash') + .option('--permissions', 'run the verified macOS helper in noninteractive permission-diagnostic mode') + .option('--prompt', 'allow the permission diagnostic to request Accessibility and Screen Recording access') + .action(async (flags: NativeDoctorFlags) => { + try { + if (flags.prompt && !flags.permissions) { + throw cliError('invalid-params', '--prompt requires --permissions.'); + } + if (flags.permissions && flags.platform !== 'macos') { + throw cliError('unsupported-operation', 'Permission diagnostics are available only for the macOS helper.'); + } + const result = await resolveDriver({ ...toResolveOptions(flags), buildPolicy: 'never' }); + const permissions = flags.permissions ? await permissionProbe(result, { prompt: flags.prompt === true }) : undefined; + writeJson(stdout, { ...(permissions ? { permissions } : {}), ready: true, result }); + } catch (error) { + writeJson(stdout, { + error: { + code: error instanceof Error && 'code' in error ? String(error.code) : 'unknown', + message: error instanceof Error ? error.message : String(error), + }, + ready: false, + }); + process.exitCode = 1; + } + }); +} + export async function runDesktopDriverCli(argv: readonly string[] = process.argv): Promise { await createDesktopDriverCommand().parseAsync([...argv]); } @@ -207,6 +307,62 @@ function addConnectionOptions(command: T): T { .addOption(new Option('--platform ', 'WebDriver platform name').choices(['macos', 'windows']).default('windows')) as T; } +type NativePlatformFlags = { + platform: DesktopEndpoint; +}; + +type NativeBuildFlags = NativePlatformFlags & { + architecture?: NativeDriverArchitecture; + cacheRoot?: string; + configuration: NativeDriverConfiguration; + force?: boolean; + macosSigningIdentity?: string; +}; + +type NativeResolveFlags = NativeBuildFlags & { + buildPolicy?: NativeDriverBuildPolicy; + helperPath?: string; + installRoot?: string; +}; + +type NativeDoctorFlags = NativeResolveFlags & { + permissions?: boolean; + prompt?: boolean; +}; + +function addNativePlatformOptions(command: T): T { + return command.requiredOption('--platform ', 'native endpoint', (value: string) => + parseChoice(value, ['macos', 'windows', 'win32'] as const, 'platform'), + ) as T; +} + +function toBuildOptions(flags: NativeBuildFlags): NativeDriverBuildOptions { + return { + architecture: flags.architecture, + cacheRoot: flags.cacheRoot, + configuration: flags.configuration, + force: flags.force, + macosSigningIdentity: flags.macosSigningIdentity, + platform: flags.platform, + }; +} + +function toResolveOptions(flags: NativeResolveFlags): NativeDriverResolveOptions { + return { + ...toBuildOptions(flags), + buildPolicy: flags.buildPolicy, + helperPath: flags.helperPath, + installRoot: flags.installRoot, + }; +} + +function parseChoice(value: string, choices: readonly T[], name: string): T { + if (!choices.includes(value as T)) { + throw new TypeError(`${name} must be one of ${choices.join(', ')}. Received "${value}".`); + } + return value as T; +} + function addSelectionOptions(command: T): T { return command .option('--story ', 'story id glob') @@ -288,3 +444,85 @@ function parsePositiveInteger(value: string): number { } return parsed; } + +function cliError(code: string, message: string): Error & { code: string } { + return Object.assign(new Error(message), { code }); +} + +async function runNativePermissionProbe(artifact: NativeDriverArtifact, options: { prompt: boolean }): Promise> { + const maximumOutputBytes = 1024 * 1024; + return new Promise((resolve, reject) => { + let settled = false; + let outputBytes = 0; + let errorBytes = 0; + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + const args = options.prompt ? ['--permissions', '--prompt'] : ['--permissions']; + const child = spawn(artifact.executablePath, args, { + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + const timeout = setTimeout(() => { + child.kill(); + settleReject(cliError('permission-probe-timeout', 'The native helper did not complete its permission diagnostic within 30 seconds.')); + }, 30_000); + + const settleReject = (error: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + reject(error); + }; + child.stdout.on('data', (chunk: Buffer) => { + outputBytes += chunk.length; + if (outputBytes > maximumOutputBytes) { + child.kill(); + settleReject(cliError('permission-probe-failed', 'The native helper permission diagnostic exceeded the 1 MiB output limit.')); + return; + } + stdout.push(chunk); + }); + child.stderr.on('data', (chunk: Buffer) => { + errorBytes += chunk.length; + if (errorBytes > maximumOutputBytes) { + child.kill(); + settleReject(cliError('permission-probe-failed', 'The native helper permission diagnostic exceeded the 1 MiB error limit.')); + return; + } + stderr.push(chunk); + }); + child.once('error', (error) => settleReject(error)); + child.once('close', (code) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + if (code !== 0) { + reject( + cliError( + 'permission-probe-failed', + `The native helper exited with code ${String(code)} during permission diagnostics: ${Buffer.concat(stderr).toString('utf8')}`, + ), + ); + return; + } + try { + const value = JSON.parse(Buffer.concat(stdout).toString('utf8')) as Record; + if (value.schemaVersion !== 1 || value.type !== 'permissions') { + throw new Error('The native helper returned an unsupported permission-diagnostic document.'); + } + resolve(value); + } catch (error) { + reject( + cliError( + 'permission-probe-failed', + error instanceof Error ? error.message : 'The native helper returned invalid permission-diagnostic JSON.', + ), + ); + } + }); + }); +} diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts new file mode 100644 index 00000000000..baaa69bfcf2 --- /dev/null +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts @@ -0,0 +1,111 @@ +import { NativeDriverError } from '../../native/NativeDriverError'; +import type { NativeDriverArtifact } from '../../native/types'; +import { toWebDriverError } from '../../protocol/errors'; +import type { WebDriverErrorCode } from '../../protocol/errors'; +import { + applyDisabledInputFeaturePolicy, + FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES, + NativeDesktopHost, + translateNativeError, +} from './NativeDesktopHost'; + +describe('NativeDesktopHost error translation', () => { + test.each<[nativeCode: string, webdriverCode: WebDriverErrorCode]>([ + ['capture-failed', 'unable to capture screen'], + ['element-not-interactable', 'element not interactable'], + ['invalid-params', 'invalid argument'], + ['invalid-request', 'invalid argument'], + ['no-such-element', 'no such element'], + ['no-such-window', 'no such window'], + ])('maps %s to %s', (nativeCode, webdriverCode) => { + const error = toWebDriverError(translateNativeError(new NativeDriverError(nativeCode, 'native failure', { operation: 'test' }))); + + expect(error).toMatchObject({ + code: webdriverCode, + data: { operation: 'test' }, + message: 'native failure', + }); + }); + + describe('NativeDesktopHost input policy', () => { + const artifact: NativeDriverArtifact = { + architecture: 'x64', + artifactId: 'artifact', + artifactRoot: 'artifact-root', + buildFingerprint: 'build', + buildId: 'build', + compatibilityKey: 'compatibility', + configuration: 'release', + endpoints: ['windows', 'win32'], + executablePath: 'driver.exe', + features: [], + origin: 'cache', + provider: 'windows', + schemaVersion: 1, + signing: { mode: 'none' }, + sourceDigest: 'source', + wireProtocol: { major: 1, minor: 0 }, + }; + + afterEach(() => { + delete process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES]; + }); + + test('masks configured input capabilities', () => { + const hostInfo = applyDisabledInputFeaturePolicy( + { + endpoint: 'win32', + features: { + accessibilityClick: true, + elementScreenshot: true, + focus: true, + keyboard: true, + physicalClick: true, + screenshot: true, + setWindowRect: true, + wheel: true, + }, + platformName: 'windows', + protocolVersion: 1, + }, + new Set(['keyboard', 'physicalClick', 'wheel']), + ); + + expect(hostInfo.features).toMatchObject({ + accessibilityClick: true, + keyboard: false, + physicalClick: false, + wheel: false, + }); + }); + + test('rejects disabled physical commands before starting the helper', async () => { + const host = new NativeDesktopHost({ + application: { windowTitle: 'Test App' }, + artifact, + disabledInputFeatures: ['keyboard', 'physicalClick', 'wheel'], + endpoint: 'windows', + }); + + await expect(host.click('element', 'physical')).rejects.toMatchObject({ name: 'HostUnsupportedError' }); + await expect(host.clear('element')).rejects.toMatchObject({ name: 'HostUnsupportedError' }); + await expect(host.sendKeys('element', 'text')).rejects.toMatchObject({ name: 'HostUnsupportedError' }); + await expect( + host.performActions([{ actions: [{ button: 0, type: 'pointerDown' }], id: 'pointer', type: 'pointer' }]), + ).rejects.toMatchObject({ name: 'HostUnsupportedError' }); + }); + + test('rejects unknown disabled input features from the environment', () => { + process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] = 'physicalClick,unknown'; + + expect( + () => + new NativeDesktopHost({ + application: { windowTitle: 'Test App' }, + artifact, + endpoint: 'windows', + }), + ).toThrow('Unsupported disabled native input feature "unknown"'); + }); + }); +}); diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts new file mode 100644 index 00000000000..69bff06e478 --- /dev/null +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts @@ -0,0 +1,477 @@ +import { spawn } from 'node:child_process'; + +import type { + ApplicationLease, + DesktopHost, + DesktopHostEvent, + DesktopHostFeatures, + DesktopHostInfo, + DesktopTarget, + DesktopWindow, + NativeActionSequence, + NativeElementSnapshot, + NativeImage, + NativeSearchRoot, + NativeSelector, + Rect, +} from '../../host/types.js'; +import type { NativeDesktopApplicationDescriptor, NativeDriverArtifact, NativeHostEventMessage } from '../../native/types.js'; +import { NativeDriverError } from '../../native/NativeDriverError.js'; +import { isSingleWebDriverKeyValue } from '../../protocol/actions.js'; +import { HostStaleError, HostUnsupportedError, HostWebDriverError } from '../../protocol/errors.js'; +import type { WebDriverErrorCode } from '../../protocol/errors.js'; +import type { DesktopEndpoint } from '../../protocol/types.js'; +import { NativeHostProcess } from './NativeHostProcess.js'; + +export type NativeDesktopHostOptions = { + application: NativeDesktopApplicationDescriptor; + artifact: NativeDriverArtifact; + disabledInputFeatures?: readonly NativeDisabledInputFeature[]; + endpoint: DesktopEndpoint; + onStderr?: (message: string) => void; +}; + +export type NativeDisabledInputFeature = 'keyboard' | 'physicalClick' | 'wheel'; + +export const FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES = 'FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES'; + +export class NativeDesktopHost implements DesktopHost { + readonly endpoint: DesktopEndpoint; + + private readonly application: NativeDesktopApplicationDescriptor; + private readonly artifact: NativeDriverArtifact; + private readonly disabledInputFeatures: ReadonlySet; + private readonly onStderr?: (message: string) => void; + private readonly listeners = new Set<(event: DesktopHostEvent) => void>(); + private readonly depressedButtons = new Set(); + private readonly depressedKeys = new Set(); + private potentialButtons = new Set(); + private potentialKeys = new Set(); + private failure?: Error; + private processPromise?: Promise; + + constructor({ application, artifact, disabledInputFeatures, endpoint, onStderr }: NativeDesktopHostOptions) { + if (!artifact.endpoints.includes(endpoint)) { + throw new TypeError(`Native helper ${artifact.provider} does not support endpoint "${endpoint}".`); + } + this.application = Object.freeze({ ...application }); + this.artifact = artifact; + this.disabledInputFeatures = resolveDisabledInputFeatures(disabledInputFeatures); + this.endpoint = endpoint; + this.onStderr = onStderr; + } + + async probe(signal?: AbortSignal): Promise { + const hostInfo = await this.request('probe', { endpoint: this.endpoint }, signal); + return applyDisabledInputFeaturePolicy(hostInfo, this.disabledInputFeatures); + } + + async launch(target: DesktopTarget, signal?: AbortSignal): Promise { + return this.request('launch', this.targetParams(target), signal); + } + + async attach(target: DesktopTarget, signal?: AbortSignal): Promise { + return this.request('attach', this.targetParams(target), signal); + } + + async closeApplication(lease: ApplicationLease, signal?: AbortSignal): Promise { + await this.request('closeApplication', { lease }, signal); + } + + windows(lease: ApplicationLease, signal?: AbortSignal): Promise { + return this.request('windows', { lease }, signal); + } + + async closeWindow(windowId: string, signal?: AbortSignal): Promise { + await this.request('closeWindow', { windowId }, signal); + } + + async activate(windowId: string, signal?: AbortSignal): Promise { + await this.request('activate', { windowId }, signal); + } + + getWindowRect(windowId: string, signal?: AbortSignal): Promise { + return this.request('getWindowRect', { windowId }, signal); + } + + setWindowRect(windowId: string, rect: Partial, signal?: AbortSignal): Promise { + return this.request('setWindowRect', { rect, windowId }, signal); + } + + find(root: NativeSearchRoot, selector: NativeSelector, signal?: AbortSignal): Promise { + return this.request('find', { root, selector }, signal); + } + + snapshot(elementId: string, signal?: AbortSignal): Promise { + return this.request('snapshot', { elementId }, signal); + } + + activeElement(windowId: string, signal?: AbortSignal): Promise { + return this.request('activeElement', { windowId }, signal); + } + + hitTest(windowId: string, x: number, y: number, signal?: AbortSignal): Promise { + return this.request('hitTest', { windowId, x, y }, signal); + } + + async click(elementId: string, mode: 'accessibility' | 'auto' | 'physical', signal?: AbortSignal): Promise { + if (this.disabledInputFeatures.has('physicalClick')) { + if (mode === 'auto') { + await this.request('click', { elementId, mode: 'accessibility' }, signal); + return; + } + if (mode === 'physical') { + throw new HostUnsupportedError('Physical pointer input is disabled by the registered target policy.'); + } + } + const includePointer = mode !== 'accessibility'; + if (includePointer) { + this.potentialButtons = new Set([...this.depressedButtons, 0]); + } + try { + await this.request('click', { elementId, mode }, signal); + } finally { + this.potentialButtons = new Set(this.depressedButtons); + } + } + + async clear(elementId: string, signal?: AbortSignal): Promise { + if (this.disabledInputFeatures.has('keyboard')) { + throw new HostUnsupportedError('Physical keyboard fallback is disabled by the registered target policy.'); + } + await this.request('clear', { elementId }, signal); + } + + async sendKeys(elementId: string, text: string, signal?: AbortSignal): Promise { + if (this.disabledInputFeatures.has('keyboard')) { + throw new HostUnsupportedError('Physical keyboard input is disabled by the registered target policy.'); + } + this.potentialKeys = new Set([...this.depressedKeys, ...typedTextRecoveryKeys(text)]); + try { + await this.request('sendKeys', { elementId, text }, signal); + } finally { + this.potentialKeys = new Set(this.depressedKeys); + } + } + + async performActions(actions: readonly NativeActionSequence[], signal?: AbortSignal): Promise { + for (const sequence of actions) { + if ( + (sequence.type === 'key' && this.disabledInputFeatures.has('keyboard')) || + (sequence.type === 'pointer' && this.disabledInputFeatures.has('physicalClick')) || + (sequence.type === 'wheel' && this.disabledInputFeatures.has('wheel')) + ) { + throw new HostUnsupportedError(`${sequence.type} input is disabled by the registered target policy.`); + } + } + const plan = planInputLedger(actions, this.depressedKeys, this.depressedButtons); + this.potentialKeys = plan.potentialKeys; + this.potentialButtons = plan.potentialButtons; + try { + await this.request('performActions', { actions }, signal); + replaceSet(this.depressedKeys, plan.finalKeys); + replaceSet(this.depressedButtons, plan.finalButtons); + } catch (error) { + this.depressedKeys.clear(); + this.depressedButtons.clear(); + throw error; + } finally { + this.potentialKeys = new Set(this.depressedKeys); + this.potentialButtons = new Set(this.depressedButtons); + } + } + + async releaseActions(signal?: AbortSignal): Promise { + this.potentialKeys = new Set(this.depressedKeys); + this.potentialButtons = new Set(this.depressedButtons); + try { + await this.request('releaseActions', undefined, signal); + } finally { + this.depressedKeys.clear(); + this.depressedButtons.clear(); + this.potentialKeys.clear(); + this.potentialButtons.clear(); + } + } + + captureWindow(windowId: string, signal?: AbortSignal): Promise { + return this.requestImage('captureWindow', { windowId }, signal); + } + + captureElement(elementId: string, signal?: AbortSignal): Promise { + return this.requestImage('captureElement', { elementId }, signal); + } + + source(windowId: string, signal?: AbortSignal): Promise { + return this.request('source', { windowId }, signal); + } + + tree(windowId: string, signal?: AbortSignal): Promise { + return this.request('tree', { windowId }, signal); + } + + subscribe(listener: (event: DesktopHostEvent) => void): () => void { + this.listeners.add(listener); + void this.getProcess().catch(() => undefined); + return () => this.listeners.delete(listener); + } + + async dispose(signal?: AbortSignal): Promise { + const process = await this.processPromise; + await process?.dispose(signal); + } + + private targetParams(target: DesktopTarget): Record { + return { + application: this.application, + endpoint: target.endpoint, + renderer: target.renderer, + storyRootTestId: target.storyRootTestId, + targetId: target.id, + }; + } + + private async request(command: string, params?: unknown, signal?: AbortSignal): Promise { + try { + const process = await this.getProcess(); + const response = await process.request(command, params, signal); + return response.result; + } catch (error) { + throw translateNativeError(error); + } + } + + private async requestImage(command: string, params?: unknown, signal?: AbortSignal): Promise { + try { + const process = await this.getProcess(); + const response = await process.request>(command, params, signal); + if (!response.binary) { + throw new Error(`Native helper did not return binary data for "${command}".`); + } + return { ...response.result, data: response.binary }; + } catch (error) { + throw translateNativeError(error); + } + } + + private getProcess(): Promise { + if (this.failure) { + return Promise.reject(this.failure); + } + return (this.processPromise ??= NativeHostProcess.start({ + artifact: this.artifact, + onStderr: this.onStderr, + recoverInput: () => this.recoverInput(), + }).then((process) => { + process.on('event', (message) => this.onEvent(message)); + process.on('exit', (error) => { + this.failure = error; + this.processPromise = undefined; + }); + return process; + })); + } + + private recoverInput(): Promise { + const keys = [...this.potentialKeys]; + const buttons = [...this.potentialButtons]; + if (keys.length === 0 && buttons.length === 0) { + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + const child = spawn(this.artifact.executablePath, ['--release-input'], { + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let settled = false; + let stdinError: Error | undefined; + let timeoutError: NativeDriverError | undefined; + const timeout = setTimeout(() => { + timeoutError = new NativeDriverError('input-recovery-timeout', 'Native input recovery did not complete within 5 seconds.'); + child.kill(); + }, 5000); + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.once('error', (error) => { + if (!settled) { + settled = true; + clearTimeout(timeout); + reject(error); + } + }); + child.stdin.once('error', (error) => { + stdinError = error; + }); + child.once('close', (code) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + if (timeoutError) { + reject(timeoutError); + return; + } + if (code !== 0) { + reject( + new NativeDriverError( + 'input-recovery-failed', + Buffer.concat(stderr).toString('utf8') || stdinError?.message || `Native input recovery exited with code ${String(code)}.`, + ), + ); + return; + } + try { + JSON.parse(Buffer.concat(stdout).toString('utf8')); + this.depressedKeys.clear(); + this.depressedButtons.clear(); + this.potentialKeys.clear(); + this.potentialButtons.clear(); + resolve(); + } catch (error) { + reject(new NativeDriverError('input-recovery-failed', error instanceof Error ? error.message : String(error))); + } + }); + child.stdin.end(JSON.stringify({ buttons, keys })); + }); + } + + private onEvent(message: NativeHostEventMessage): void { + const payload = message.payload as Record | undefined; + let event: DesktopHostEvent | undefined; + if (message.event === 'application-exited' && typeof payload?.applicationId === 'string') { + event = { applicationId: payload.applicationId, type: 'application-exited' }; + } else if (message.event === 'structure-changed' && typeof payload?.windowId === 'string') { + event = { type: 'structure-changed', windowId: payload.windowId }; + } else if (message.event === 'window-closed' && typeof payload?.windowId === 'string') { + event = { type: 'window-closed', windowId: payload.windowId }; + } else if (message.event === 'window-opened' && typeof payload?.windowId === 'string') { + event = { type: 'window-opened', windowId: payload.windowId }; + } + + if (event) { + for (const listener of this.listeners) { + listener(event); + } + } + } +} + +type InputLedgerPlan = { + finalButtons: Set; + finalKeys: Set; + potentialButtons: Set; + potentialKeys: Set; +}; + +function planInputLedger( + actions: readonly NativeActionSequence[], + currentKeys: ReadonlySet, + currentButtons: ReadonlySet, +): InputLedgerPlan { + const finalKeys = new Set(currentKeys); + const finalButtons = new Set(currentButtons); + const potentialKeys = new Set(currentKeys); + const potentialButtons = new Set(currentButtons); + for (const sequence of actions) { + for (const action of sequence.actions) { + if (action.type === 'keyDown' || action.type === 'keyUp') { + const value = action.value; + if (typeof value !== 'string' || !isSingleWebDriverKeyValue(value)) { + throw new TypeError(`${action.type} values must contain exactly one Unicode code point.`); + } + if (action.type === 'keyDown') { + finalKeys.add(value); + potentialKeys.add(value); + } else { + finalKeys.delete(value); + } + } else if (action.type === 'pointerDown' && typeof action.button === 'number') { + finalButtons.add(action.button); + potentialButtons.add(action.button); + } else if (action.type === 'pointerUp' && typeof action.button === 'number') { + finalButtons.delete(action.button); + } + } + } + return { finalButtons, finalKeys, potentialButtons, potentialKeys }; +} + +function replaceSet(target: Set, source: ReadonlySet): void { + target.clear(); + for (const value of source) { + target.add(value); + } +} + +function typedTextRecoveryKeys(text: string): Set { + const keys = new Set(); + for (const value of text) { + if (value === '\r') { + continue; + } + keys.add(value === '\n' ? '\uE006' : value); + } + return keys; +} + +export function applyDisabledInputFeaturePolicy( + hostInfo: DesktopHostInfo, + disabledFeatures: ReadonlySet, +): DesktopHostInfo { + if (disabledFeatures.size === 0) { + return hostInfo; + } + const features: DesktopHostFeatures = { ...hostInfo.features }; + for (const feature of disabledFeatures) { + features[feature] = false; + } + return { ...hostInfo, features }; +} + +function resolveDisabledInputFeatures(configured?: readonly NativeDisabledInputFeature[]): ReadonlySet { + const values = + configured ?? + process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] + ?.split(',') + .map((value) => value.trim()) + .filter(Boolean) ?? + []; + const allowed = new Set(['keyboard', 'physicalClick', 'wheel']); + const resolved = new Set(); + for (const value of values) { + if (!allowed.has(value as NativeDisabledInputFeature)) { + throw new TypeError(`Unsupported disabled native input feature "${value}".`); + } + resolved.add(value as NativeDisabledInputFeature); + } + return resolved; +} + +const nativeWebDriverErrorCodes: Readonly> = { + 'capture-failed': 'unable to capture screen', + 'element-not-interactable': 'element not interactable', + 'invalid-params': 'invalid argument', + 'invalid-request': 'invalid argument', + 'no-such-element': 'no such element', + 'no-such-window': 'no such window', +}; + +export function translateNativeError(error: unknown): unknown { + if (!(error instanceof NativeDriverError)) { + return error; + } + if (error.code === 'unsupported-operation') { + return new HostUnsupportedError(error.message); + } + if (error.code === 'stale-element') { + return new HostStaleError(error.message); + } + const webdriverCode = nativeWebDriverErrorCodes[error.code]; + if (webdriverCode) { + return new HostWebDriverError(webdriverCode, error.message, error.data); + } + return error; +} diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeHostProcess.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeHostProcess.ts new file mode 100644 index 00000000000..ade1f2f7ade --- /dev/null +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeHostProcess.ts @@ -0,0 +1,435 @@ +import { spawn } from 'node:child_process'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { EventEmitter } from 'node:events'; + +import type { + NativeDriverArtifact, + NativeHostEventMessage, + NativeHostHello, + NativeHostJsonMessage, + NativeHostResponse, +} from '../../native/types.js'; +import { nativeDriverWireProtocol } from '../../native/constants.js'; +import { NativeDriverError } from '../../native/NativeDriverError.js'; +import { encodeJsonFrame, NativeFrameDecoder } from './framing.js'; +import type { NativeBinaryFrame } from './framing.js'; + +export type NativeHostProcessOptions = { + artifact: NativeDriverArtifact; + cancellationTimeoutMs?: number; + onStderr?: (message: string) => void; + recoverInput?: () => Promise; + startupTimeoutMs?: number; +}; + +export type NativeHostRequestResult = { + binary?: Uint8Array; + result: T; +}; + +type PendingRequest = { + aborted: boolean; + abortCleanup: () => void; + binary?: Uint8Array; + binaryId?: string; + reject(error: unknown): void; + resolve(value: NativeHostRequestResult): void; + response?: NativeHostResponse; +}; + +class NativeHostResponseError extends NativeDriverError {} + +export class NativeHostProcess extends EventEmitter<{ + event: [NativeHostEventMessage]; + exit: [Error]; +}> { + readonly artifact: NativeDriverArtifact; + readonly hello: NativeHostHello; + + private readonly child: ChildProcessWithoutNullStreams; + private readonly completed: Promise; + private readonly recoverInput?: () => Promise; + private readonly cancellationTimeoutMs: number; + private readonly pending = new Map(); + private readonly pendingBinaryIds = new Map(); + private closed = false; + private closing = false; + private failurePromise?: Promise; + + private constructor( + child: ChildProcessWithoutNullStreams, + artifact: NativeDriverArtifact, + hello: NativeHostHello, + cancellationTimeoutMs: number, + recoverInput?: () => Promise, + ) { + super(); + this.child = child; + this.completed = new Promise((resolve) => child.once('close', () => resolve())); + this.artifact = artifact; + this.hello = hello; + this.cancellationTimeoutMs = cancellationTimeoutMs; + this.recoverInput = recoverInput; + } + + static async start(options: NativeHostProcessOptions): Promise { + const child = spawn(options.artifact.executablePath, ['--stdio'], { + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + }); + const completed = new Promise((resolve) => child.once('close', () => resolve())); + const decoder = new NativeFrameDecoder(); + child.stdout.on('data', (chunk: Buffer) => decoder.write(chunk)); + child.stderr.on('data', (chunk: Buffer) => options.onStderr?.(chunk.toString('utf8'))); + + let hello: NativeHostHello; + try { + hello = await waitForHello(child, decoder, options.startupTimeoutMs ?? 10_000); + validateHello(hello, options.artifact); + } catch (error) { + if (child.exitCode === null && child.signalCode === null) { + child.kill(); + } + await completed; + throw error; + } + const process = new NativeHostProcess(child, options.artifact, hello, options.cancellationTimeoutMs ?? 2000, options.recoverInput); + decoder.on('json', (message) => process.onJson(message)); + decoder.on('binary', (frame) => process.onBinary(frame)); + decoder.on('error', (error) => process.fail(error)); + child.once('error', (error) => process.fail(error)); + child.stdin.on('error', (error) => process.fail(error)); + child.once('exit', (code, signal) => { + if (!process.closed && !process.closing) { + process.fail( + new NativeDriverError( + 'host-exited', + `Native driver helper exited unexpectedly with ${code === null ? `signal ${String(signal)}` : `code ${code}`}.`, + ), + ); + } + }); + return process; + } + + request(command: string, params?: unknown, signal?: AbortSignal): Promise> { + if (this.closed) { + return Promise.reject(new NativeDriverError('host-closed', 'Native driver helper is closed.')); + } + if (signal?.aborted) { + const error = new Error(`Native driver command "${command}" was cancelled before dispatch.`); + error.name = 'AbortError'; + return Promise.reject(error); + } + const id = randomUUID(); + let requestFrame: Buffer; + try { + requestFrame = encodeJsonFrame({ command, id, params, type: 'request' }); + } catch (error) { + return Promise.reject(asError(error)); + } + return new Promise>((resolve, reject) => { + const pending: PendingRequest = { + aborted: false, + abortCleanup: () => undefined, + reject, + resolve: resolve as (value: NativeHostRequestResult) => void, + }; + this.pending.set(id, pending); + const dispatched = this.writeFrame(requestFrame); + if (signal) { + const onAbort = () => { + pending.aborted = true; + void this.write({ id, type: 'cancel' }).catch((error) => { + this.rejectPending(id, error); + this.fail(asError(error)); + }); + const timeout = setTimeout(() => { + void this.failAfterRecovery( + new NativeDriverError( + 'cancellation-timeout', + `Native driver helper did not settle cancelled command "${command}" within ${this.cancellationTimeoutMs} ms.`, + ), + ); + }, this.cancellationTimeoutMs); + timeout.unref(); + const removeAbortListener = pending.abortCleanup; + pending.abortCleanup = () => { + clearTimeout(timeout); + removeAbortListener(); + }; + }; + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener('abort', onAbort, { once: true }); + pending.abortCleanup = () => signal.removeEventListener('abort', onAbort); + } + } + void dispatched.catch((error) => { + this.rejectPending(id, error); + this.fail(asError(error)); + }); + }); + } + + async dispose(signal?: AbortSignal): Promise { + if (this.closed || this.closing) { + return; + } + this.closing = true; + try { + const disposeResult = await Promise.race([ + this.request('dispose', undefined, signal).then(() => 'response' as const), + this.completed.then(() => 'closed' as const), + ]); + if (disposeResult === 'closed') { + this.closed = true; + for (const [id] of this.pending) { + this.rejectPending(id, new NativeDriverError('host-closed', 'Native driver helper closed.')); + } + throw new NativeDriverError('host-exited', 'Native driver helper exited before acknowledging disposal.'); + } + const exited = await Promise.race([this.completed.then(() => true), delay(2000).then(() => false)]); + if (!exited && this.child.exitCode === null && this.child.signalCode === null) { + this.child.kill(); + await this.completed; + } + this.closed = true; + for (const [id] of this.pending) { + this.rejectPending(id, new NativeDriverError('host-closed', 'Native driver helper closed.')); + } + } catch (error) { + if ( + (error instanceof NativeHostResponseError || (error instanceof Error && error.name === 'AbortError')) && + !this.closed && + this.child.exitCode === null && + this.child.signalCode === null + ) { + this.closing = false; + } else if (!this.closed) { + this.closing = false; + this.failurePromise = undefined; + await this.failAfterRecovery(asError(error)); + } + throw error; + } finally { + if (this.closed) { + this.closing = false; + } + } + } + + private onJson(message: NativeHostJsonMessage): void { + if (message.type === 'event') { + this.emit('event', message); + return; + } + if (message.type === 'cancelled') { + const error = new Error(`Native driver command "${message.id}" was cancelled.`); + error.name = 'AbortError'; + this.rejectPending(message.id, error); + return; + } + if (message.type !== 'response') { + this.fail(new Error(`Native driver helper emitted unexpected "${message.type}" message after startup.`)); + return; + } + const pending = this.pending.get(message.id); + if (!pending) { + return; + } + pending.response = message; + pending.binaryId = message.binary?.id; + if (pending.binaryId) { + this.pendingBinaryIds.set(pending.binaryId, message.id); + } + this.completePending(message.id); + } + + private onBinary(frame: NativeBinaryFrame): void { + const requestId = this.pendingBinaryIds.get(frame.id); + if (!requestId) { + this.fail(new Error(`Native driver helper emitted binary payload "${frame.id}" before declaring it in a response.`)); + return; + } + const pending = this.pending.get(requestId); + if (!pending) { + return; + } + pending.binary = frame.data; + this.completePending(requestId); + } + + private completePending(id: string): void { + const pending = this.pending.get(id); + const response = pending?.response; + if (!pending || !response || (pending.binaryId && !pending.binary)) { + return; + } + this.pending.delete(id); + pending.abortCleanup(); + if (pending.binaryId) { + this.pendingBinaryIds.delete(pending.binaryId); + } + if (response.error) { + pending.reject(new NativeHostResponseError(response.error.code, response.error.message, response.error.data)); + return; + } + if (pending.aborted) { + const error = new Error(`Native driver command "${id}" was cancelled.`); + error.name = 'AbortError'; + pending.reject(error); + return; + } + pending.resolve({ binary: pending.binary, result: response.result }); + } + + private rejectPending(id: string, error: unknown): void { + const pending = this.pending.get(id); + if (!pending) { + return; + } + this.pending.delete(id); + pending.abortCleanup(); + if (pending.binaryId) { + this.pendingBinaryIds.delete(pending.binaryId); + } + pending.reject(error); + } + + private async write(message: Parameters[0]): Promise { + await this.writeFrame(encodeJsonFrame(message)); + } + + private async writeFrame(frame: Buffer): Promise { + if (this.child.stdin.destroyed || !this.child.stdin.writable) { + throw new NativeDriverError('host-closed', 'Native driver helper input is closed.'); + } + if (this.child.stdin.write(frame)) { + return; + } + await new Promise((resolve, reject) => { + const cleanup = () => { + this.child.stdin.off('drain', onDrain); + this.child.stdin.off('error', onError); + this.child.off('close', onClose); + }; + const onDrain = () => { + cleanup(); + resolve(); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onClose = () => { + cleanup(); + reject(new NativeDriverError('host-closed', 'Native driver helper closed while writing a request.')); + }; + this.child.stdin.once('drain', onDrain); + this.child.stdin.once('error', onError); + this.child.once('close', onClose); + }); + } + + private fail(error: Error): void { + void this.failAfterRecovery(error); + } + + private failAfterRecovery(error: Error): Promise { + return (this.failurePromise ??= this.recoverAndFail(error)); + } + + private async recoverAndFail(error: Error): Promise { + if (this.closed || this.closing) { + return; + } + this.closed = true; + if (this.child.exitCode === null && this.child.signalCode === null) { + this.child.kill(); + } + await this.completed; + let failure = error; + try { + await this.recoverInput?.(); + } catch (recoveryError) { + failure = new AggregateError([error, recoveryError], `${error.message} Native input recovery also failed.`); + } + for (const [id] of this.pending) { + this.rejectPending(id, failure); + } + this.emit('exit', failure); + } +} + +function waitForHello(child: ChildProcessWithoutNullStreams, decoder: NativeFrameDecoder, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + child.kill(); + reject(new NativeDriverError('handshake-timeout', `Native driver helper did not send hello within ${timeoutMs} ms.`)); + }, timeoutMs); + const onJson = (message: NativeHostJsonMessage) => { + if (message.type !== 'hello') { + cleanup(); + child.kill(); + reject(new NativeDriverError('handshake-failed', `Expected native helper hello, received "${message.type}".`)); + return; + } + cleanup(); + resolve(message); + }; + const onError = (error: Error) => { + cleanup(); + child.kill(); + reject(error); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + reject( + new NativeDriverError( + 'handshake-failed', + `Native helper exited before hello with ${code === null ? `signal ${String(signal)}` : `code ${code}`}.`, + ), + ); + }; + const onChildError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timeout); + decoder.off('json', onJson); + decoder.off('error', onError); + child.off('error', onChildError); + child.off('exit', onExit); + }; + decoder.once('json', onJson); + decoder.once('error', onError); + child.once('error', onChildError); + child.once('exit', onExit); + }); +} + +function validateHello(hello: NativeHostHello, artifact: NativeDriverArtifact): void { + if ( + hello.protocol.major !== nativeDriverWireProtocol.major || + hello.protocol.minor < nativeDriverWireProtocol.minor || + hello.provider !== artifact.provider || + hello.architecture !== artifact.architecture || + hello.buildId !== artifact.buildId || + hello.sourceDigest !== artifact.sourceDigest + ) { + throw new NativeDriverError('handshake-failed', 'Native helper hello does not match the selected artifact.'); + } +} + +function delay(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/packages/agentic/desktop-driver/src/hosts/native/framing.test.ts b/packages/agentic/desktop-driver/src/hosts/native/framing.test.ts new file mode 100644 index 00000000000..af4073f744a --- /dev/null +++ b/packages/agentic/desktop-driver/src/hosts/native/framing.test.ts @@ -0,0 +1,62 @@ +import { NativeFrameDecoder, encodeBinaryFrame, encodeJsonFrame } from './framing'; + +describe('native host framing', () => { + test('decodes chunked JSON and binary frames', () => { + const decoder = new NativeFrameDecoder(); + const messages: unknown[] = []; + const binaries: { data: number[]; id: string }[] = []; + decoder.on('json', (message) => messages.push(message)); + decoder.on('binary', ({ data, id }) => binaries.push({ data: [...data], id })); + + const payload = Buffer.concat([ + encodeJsonFrame({ id: 'request-1', result: { ready: true }, type: 'response' }), + encodeBinaryFrame('image-1', Uint8Array.from([1, 2, 3, 4])), + ]); + decoder.write(payload.subarray(0, 7)); + decoder.write(payload.subarray(7, 19)); + decoder.write(payload.subarray(19)); + + expect(messages).toEqual([{ id: 'request-1', result: { ready: true }, type: 'response' }]); + expect(binaries).toEqual([{ data: [1, 2, 3, 4], id: 'image-1' }]); + }); + + test('rejects invalid frame magic', () => { + const decoder = new NativeFrameDecoder(); + const errors: Error[] = []; + decoder.on('error', (error) => errors.push(error)); + + decoder.write(Buffer.alloc(12)); + + expect(errors[0]?.message).toContain('invalid frame magic'); + }); + + test('rejects reserved header bytes and oversized frames before buffering payloads', () => { + const reservedDecoder = new NativeFrameDecoder(); + const reservedErrors: Error[] = []; + reservedDecoder.on('error', (error) => reservedErrors.push(error)); + const reserved = encodeJsonFrame({ id: 'request-1', result: null, type: 'response' }); + reserved[5] = 1; + reservedDecoder.write(reserved); + expect(reservedErrors[0]?.message).toContain('reserved frame bytes'); + + const oversizedDecoder = new NativeFrameDecoder(); + const oversizedErrors: Error[] = []; + oversizedDecoder.on('error', (error) => oversizedErrors.push(error)); + const header = Buffer.alloc(12); + header.write('FDR1'); + header[4] = 1; + header.writeUInt32LE(8 * 1024 * 1024 + 1, 8); + oversizedDecoder.write(header); + expect(oversizedErrors[0]?.message).toContain('exceeds'); + }); + + test('rejects oversized outbound frames before writing a header', () => { + expect(() => + encodeJsonFrame({ + id: 'request-1', + result: 'x'.repeat(8 * 1024 * 1024), + type: 'response', + }), + ).toThrow('exceeds'); + }); +}); diff --git a/packages/agentic/desktop-driver/src/hosts/native/framing.ts b/packages/agentic/desktop-driver/src/hosts/native/framing.ts new file mode 100644 index 00000000000..10897b5f2ea --- /dev/null +++ b/packages/agentic/desktop-driver/src/hosts/native/framing.ts @@ -0,0 +1,104 @@ +import { Buffer } from 'node:buffer'; +import { EventEmitter } from 'node:events'; + +import type { NativeHostJsonMessage } from '../../native/types.js'; + +const frameMagic = Buffer.from('FDR1'); +const frameHeaderBytes = 12; +const jsonFrameType = 1; +const binaryFrameType = 2; +const maximumJsonFrameBytes = 8 * 1024 * 1024; +const maximumBinaryFrameBytes = 64 * 1024 * 1024; +const maximumBinaryIdentifierBytes = 1024; + +export type NativeBinaryFrame = { + data: Uint8Array; + id: string; +}; + +export class NativeFrameDecoder extends EventEmitter<{ + binary: [NativeBinaryFrame]; + error: [Error]; + json: [NativeHostJsonMessage]; +}> { + private buffer = Buffer.alloc(0); + + write(chunk: Uint8Array): void { + this.buffer = Buffer.concat([this.buffer, Buffer.from(chunk)]); + while (this.buffer.length >= frameHeaderBytes) { + if (!this.buffer.subarray(0, 4).equals(frameMagic)) { + this.emit('error', new Error('Native driver helper emitted an invalid frame magic.')); + return; + } + const type = this.buffer[4]; + const length = this.buffer.readUInt32LE(8); + if (this.buffer[5] !== 0 || this.buffer[6] !== 0 || this.buffer[7] !== 0) { + this.emit('error', new Error('Native driver helper emitted nonzero reserved frame bytes.')); + return; + } + const maximumLength = type === jsonFrameType ? maximumJsonFrameBytes : type === binaryFrameType ? maximumBinaryFrameBytes : undefined; + if (maximumLength === undefined) { + this.emit('error', new Error(`Native driver helper emitted unsupported frame type ${String(type)}.`)); + return; + } + if (length > maximumLength) { + this.emit('error', new Error(`Native driver helper frame length ${length} exceeds the ${maximumLength}-byte type limit.`)); + return; + } + if (this.buffer.length < frameHeaderBytes + length) { + return; + } + const payload = this.buffer.subarray(frameHeaderBytes, frameHeaderBytes + length); + this.buffer = this.buffer.subarray(frameHeaderBytes + length); + try { + if (type === jsonFrameType) { + this.emit('json', JSON.parse(payload.toString('utf8')) as NativeHostJsonMessage); + } else if (type === binaryFrameType) { + this.emit('binary', decodeBinaryPayload(payload)); + } + } catch (error) { + this.emit('error', error instanceof Error ? error : new Error(String(error))); + return; + } + } + } +} + +export function encodeJsonFrame(message: NativeHostJsonMessage): Buffer { + return encodeFrame(jsonFrameType, Buffer.from(JSON.stringify(message), 'utf8')); +} + +export function encodeBinaryFrame(id: string, data: Uint8Array): Buffer { + const idBytes = Buffer.from(id, 'utf8'); + const payload = Buffer.alloc(4 + idBytes.length + data.byteLength); + payload.writeUInt32LE(idBytes.length, 0); + idBytes.copy(payload, 4); + Buffer.from(data).copy(payload, 4 + idBytes.length); + return encodeFrame(binaryFrameType, payload); +} + +function encodeFrame(type: number, payload: Buffer): Buffer { + const maximumLength = type === jsonFrameType ? maximumJsonFrameBytes : type === binaryFrameType ? maximumBinaryFrameBytes : undefined; + if (maximumLength === undefined || payload.length > maximumLength) { + throw new RangeError(`Native driver frame length ${payload.length} exceeds the supported type limit.`); + } + const header = Buffer.alloc(frameHeaderBytes); + frameMagic.copy(header, 0); + header[4] = type; + header.writeUInt32LE(payload.length, 8); + return Buffer.concat([header, payload]); +} + +function decodeBinaryPayload(payload: Buffer): NativeBinaryFrame { + if (payload.length < 4) { + throw new Error('Native driver binary frame is missing its identifier length.'); + } + const idLength = payload.readUInt32LE(0); + if (idLength === 0 || idLength > maximumBinaryIdentifierBytes || payload.length < 4 + idLength) { + throw new Error('Native driver binary frame contains a truncated identifier.'); + } + return { + data: payload.subarray(4 + idLength), + id: payload.subarray(4, 4 + idLength).toString('utf8'), + }; +} diff --git a/packages/agentic/desktop-driver/src/hosts/native/index.ts b/packages/agentic/desktop-driver/src/hosts/native/index.ts new file mode 100644 index 00000000000..10e1374b5dd --- /dev/null +++ b/packages/agentic/desktop-driver/src/hosts/native/index.ts @@ -0,0 +1,6 @@ +export { FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES, NativeDesktopHost } from './NativeDesktopHost.js'; +export type { NativeDesktopHostOptions, NativeDisabledInputFeature } from './NativeDesktopHost.js'; +export { NativeHostProcess } from './NativeHostProcess.js'; +export type { NativeHostProcessOptions, NativeHostRequestResult } from './NativeHostProcess.js'; +export { encodeBinaryFrame, encodeJsonFrame, NativeFrameDecoder } from './framing.js'; +export type { NativeBinaryFrame } from './framing.js'; diff --git a/packages/agentic/desktop-driver/src/index.ts b/packages/agentic/desktop-driver/src/index.ts index 3c68b8d5bd8..6cba01532ea 100644 --- a/packages/agentic/desktop-driver/src/index.ts +++ b/packages/agentic/desktop-driver/src/index.ts @@ -61,8 +61,43 @@ export type { Rect, SupportedValue, } from './host/types.js'; +export { FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES, NativeDesktopHost, NativeHostProcess } from './hosts/native/index.js'; +export type { + NativeDesktopHostOptions, + NativeDisabledInputFeature, + NativeHostProcessOptions, + NativeHostRequestResult, +} from './hosts/native/index.js'; +export { + buildNativeDesktopDriver, + FURN_DESKTOP_DRIVER_BUILD_POLICY, + FURN_DESKTOP_DRIVER_CACHE_ROOT, + FURN_DESKTOP_DRIVER_HELPER_PATH, + FURN_DESKTOP_DRIVER_INSTALL_ROOT, + FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY, + nativeDriverWireProtocol, + NativeDriverError, + resolveNativeDesktopDriver, + verifyNativeDriverArtifact, +} from './native/index.js'; +export type { + NativeDesktopApplicationDescriptor, + NativeDriverArchitecture, + NativeDriverArtifact, + NativeDriverArtifactOrigin, + NativeDriverBuildOptions, + NativeDriverBuildPolicy, + NativeDriverConfiguration, + NativeDriverProvider, + NativeDriverResolveOptions, + NativeDriverSigning, + NativeDriverWireProtocol, + NativeHostEventMessage, + NativeHostHello, + NativeHostJsonMessage, +} from './native/index.js'; export { webElementIdentifier } from './protocol/constants.js'; -export { HostStaleError, HostUnsupportedError, WebDriverError } from './protocol/errors.js'; +export { HostStaleError, HostUnsupportedError, HostWebDriverError, WebDriverError } from './protocol/errors.js'; export type { WebDriverErrorCode } from './protocol/errors.js'; export { type DesktopClickMode, diff --git a/packages/agentic/desktop-driver/src/native/NativeDriverError.ts b/packages/agentic/desktop-driver/src/native/NativeDriverError.ts new file mode 100644 index 00000000000..f0f553a1f00 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/NativeDriverError.ts @@ -0,0 +1,11 @@ +export class NativeDriverError extends Error { + readonly code: string; + readonly data?: Record; + + constructor(code: string, message: string, data?: Record) { + super(message); + this.name = 'NativeDriverError'; + this.code = code; + this.data = data; + } +} diff --git a/packages/agentic/desktop-driver/src/native/constants.ts b/packages/agentic/desktop-driver/src/native/constants.ts new file mode 100644 index 00000000000..1c62d7c67f8 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/constants.ts @@ -0,0 +1 @@ +export const nativeDriverWireProtocol = Object.freeze({ major: 1, minor: 0 }); diff --git a/packages/agentic/desktop-driver/src/native/filesystem.ts b/packages/agentic/desktop-driver/src/native/filesystem.ts new file mode 100644 index 00000000000..76771fc8d6e --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/filesystem.ts @@ -0,0 +1,173 @@ +import { createHash, randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +type JsonValue = boolean | null | number | string | JsonValue[] | { [key: string]: JsonValue }; + +export function sha256(value: string | Uint8Array): string { + return createHash('sha256').update(value).digest('hex'); +} + +export function canonicalJson(value: JsonValue): string { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (value && typeof value === 'object') { + return `{${Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +export function listFiles(root: string): string[] { + const results: string[] = []; + const visit = (directory: string) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(fullPath); + } else if (entry.isFile()) { + results.push(fullPath); + } + } + }; + visit(root); + return results.sort((left, right) => left.localeCompare(right)); +} + +export function hashTree(root: string): string { + const entries = listFiles(root).map((filePath) => ({ + path: path.relative(root, filePath).replaceAll(path.sep, '/'), + sha256: sha256(fs.readFileSync(filePath)), + })); + return sha256(canonicalJson(entries)); +} + +export function atomicWriteJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + fs.writeFileSync(temporaryPath, `${JSON.stringify(value, null, 2)}\n`); + fs.renameSync(temporaryPath, filePath); +} + +export function readJsonFile(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, 'utf8')) as T; +} + +export function defaultNativeDriverCacheRoot(platform = process.platform): string { + if (platform === 'win32') { + const localAppData = process.env.LOCALAPPDATA; + if (!localAppData) { + throw new Error('LOCALAPPDATA is required to resolve the default Windows native driver cache.'); + } + return path.join(localAppData, 'Microsoft', 'FluentUIReactNative', 'desktop-driver', 'native'); + } + if (platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Caches', 'com.microsoft.fluentui-react-native.desktop-driver', 'native'); + } + return path.join(os.homedir(), '.cache', 'fluentui-react-native', 'desktop-driver', 'native'); +} + +export type DirectoryLock = { + release(): void; +}; + +export async function acquireDirectoryLock(lockPath: string, signal?: AbortSignal, timeoutMs = 120_000): Promise { + const deadline = Date.now() + timeoutMs; + const ownerPath = path.join(lockPath, 'owner.json'); + const heartbeatPath = path.join(lockPath, 'heartbeat'); + + while (true) { + throwIfAborted(signal); + try { + fs.mkdirSync(lockPath, { recursive: false }); + atomicWriteJson(ownerPath, { + hostname: os.hostname(), + pid: process.pid, + startedAt: new Date().toISOString(), + }); + fs.writeFileSync(heartbeatPath, new Date().toISOString()); + const heartbeat = setInterval(() => { + try { + fs.writeFileSync(heartbeatPath, new Date().toISOString()); + } catch { + // Release or process shutdown owns the final state. + } + }, 5000); + heartbeat.unref(); + let released = false; + return { + release() { + if (released) { + return; + } + released = true; + clearInterval(heartbeat); + fs.rmSync(lockPath, { force: true, recursive: true }); + }, + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') { + throw error; + } + } + + if (canReclaimLock(ownerPath, heartbeatPath)) { + const stalePath = `${lockPath}.stale-${Date.now()}-${randomUUID()}`; + try { + fs.renameSync(lockPath, stalePath); + fs.rmSync(stalePath, { force: true, recursive: true }); + continue; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ENOENT' && code !== 'EACCES' && code !== 'EPERM') { + throw error; + } + } + } + + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for native driver build lock "${lockPath}".`); + } + await delay(100); + } +} + +function canReclaimLock(ownerPath: string, heartbeatPath: string): boolean { + try { + const owner = readJsonFile<{ hostname?: string; pid?: number }>(ownerPath); + const heartbeat = fs.statSync(heartbeatPath); + return ( + owner.hostname === os.hostname() && + typeof owner.pid === 'number' && + Date.now() - heartbeat.mtimeMs > 60_000 && + !isProcessAlive(owner.pid) + ); + } catch { + return false; + } +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ESRCH'; + } +} + +export function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) { + const error = new Error('The native driver operation was aborted.'); + error.name = 'AbortError'; + throw error; + } +} + +export function delay(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} diff --git a/packages/agentic/desktop-driver/src/native/index.ts b/packages/agentic/desktop-driver/src/native/index.ts new file mode 100644 index 00000000000..e5d3ea25f84 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/index.ts @@ -0,0 +1,26 @@ +export { buildNativeDesktopDriver, resolveNativeDesktopDriver, verifyNativeDriverArtifact } from './nativeDriver.js'; +export { + FURN_DESKTOP_DRIVER_BUILD_POLICY, + FURN_DESKTOP_DRIVER_CACHE_ROOT, + FURN_DESKTOP_DRIVER_HELPER_PATH, + FURN_DESKTOP_DRIVER_INSTALL_ROOT, + FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY, +} from './nativeDriver.js'; +export { nativeDriverWireProtocol } from './constants.js'; +export { NativeDriverError } from './NativeDriverError.js'; +export type { + NativeDesktopApplicationDescriptor, + NativeDriverArchitecture, + NativeDriverArtifact, + NativeDriverArtifactOrigin, + NativeDriverBuildOptions, + NativeDriverBuildPolicy, + NativeDriverConfiguration, + NativeDriverProvider, + NativeDriverResolveOptions, + NativeDriverSigning, + NativeDriverWireProtocol, + NativeHostEventMessage, + NativeHostHello, + NativeHostJsonMessage, +} from './types.js'; diff --git a/packages/agentic/desktop-driver/src/native/macos/buildMacOSDriver.ts b/packages/agentic/desktop-driver/src/native/macos/buildMacOSDriver.ts new file mode 100644 index 00000000000..68d422d2e12 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/macos/buildMacOSDriver.ts @@ -0,0 +1,471 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { atomicWriteJson, canonicalJson, listFiles, sha256, throwIfAborted } from '../filesystem.js'; +import { NativeDriverError } from '../NativeDriverError.js'; +import type { NativeDriverConfiguration, NativeDriverSigning } from '../types.js'; + +const bundleIdentifier = 'com.microsoft.fluentui-react-native.desktop-driver'; +const bundleName = 'FurnDesktopDriverHost.app'; +const executableName = 'furn-desktop-driver-host'; + +export type MacOSToolchain = { + codesignIdentity: string; + fingerprint: string; + macosSdkPath: string; + macosSdkVersion: string; + signing: NativeDriverSigning; + swiftPath: string; + swiftVersion: string; + xcodeVersion: string; +}; + +export type MacOSSigningIdentity = { + hash: string; + name: string; +}; + +export type BuildMacOSDriverOptions = { + buildId: string; + configuration: NativeDriverConfiguration; + packageRoot: string; + signal?: AbortSignal; + sourceDigest: string; + stagingRoot: string; + toolchain: MacOSToolchain; +}; + +export type BuildMacOSDriverResult = { + buildLogPath: string; + bundlePath: string; + executablePath: string; + signing: NativeDriverSigning; +}; + +type CommandResult = { + stderr: string; + stdout: string; +}; + +export function macOSDriverSourceFiles(sourceRoot: string): string[] { + const packageManifest = path.join(sourceRoot, 'Package.swift'); + const sourcesRoot = path.join(sourceRoot, 'Sources'); + if (!fs.existsSync(packageManifest) || !fs.statSync(packageManifest).isFile() || !fs.existsSync(sourcesRoot)) { + throw new NativeDriverError('build-source-missing', `macOS native driver sources are incomplete beneath "${sourceRoot}".`); + } + return [ + packageManifest, + ...(fs.existsSync(path.join(sourceRoot, 'Package.resolved')) ? [path.join(sourceRoot, 'Package.resolved')] : []), + ...listFiles(sourcesRoot).filter((filePath) => path.extname(filePath) === '.swift'), + ] + .map((filePath) => path.relative(sourceRoot, filePath)) + .sort((left, right) => left.localeCompare(right)); +} + +export function hashMacOSDriverSources(sourceRoot: string): string { + const entries = macOSDriverSourceFiles(sourceRoot).map((relativePath) => ({ + path: relativePath.replaceAll(path.sep, '/'), + sha256: sha256(fs.readFileSync(path.join(sourceRoot, relativePath))), + })); + return sha256(canonicalJson(entries)); +} + +export function macOSCodeSignArguments(toolchain: Pick, bundlePath: string): string[] { + return [ + '--force', + '--sign', + toolchain.codesignIdentity, + '--identifier', + bundleIdentifier, + ...(toolchain.signing.mode === 'signed' ? ['--options', 'runtime', '--timestamp'] : ['--timestamp=none']), + bundlePath, + ]; +} + +export function macOSCertificateExtractionArguments(certificatePrefix: string, bundlePath: string): string[] { + return ['--display', `--extract-certificates=${certificatePrefix}`, bundlePath]; +} + +export async function probeMacOSToolchain( + signingIdentity: MacOSSigningIdentity | undefined, + signal?: AbortSignal, +): Promise { + if (process.platform !== 'darwin') { + throw new NativeDriverError('unsupported-host', 'The macOS native driver can only be built on macOS.'); + } + if (process.arch !== 'arm64') { + throw new NativeDriverError('unsupported-host', 'Desktop Driver V1 macOS builds require a native Apple Silicon Node.js process.'); + } + throwIfAborted(signal); + + const swiftPath = combinedOutput(await runCommand('xcrun', ['--find', 'swift'], signal)).trim(); + const swiftVersion = combinedOutput(await runCommand(swiftPath, ['--version'], signal)).trim(); + const macosSdkPath = combinedOutput(await runCommand('xcrun', ['--sdk', 'macosx', '--show-sdk-path'], signal)).trim(); + const macosSdkVersion = combinedOutput(await runCommand('xcrun', ['--sdk', 'macosx', '--show-sdk-version'], signal)).trim(); + const xcodeVersion = combinedOutput(await runCommand('xcodebuild', ['-version'], signal)).trim(); + const signing: NativeDriverSigning = signingIdentity + ? { certificateHash: signingIdentity.hash, identity: signingIdentity.name, mode: 'signed' } + : { mode: 'adhoc' }; + + return { + codesignIdentity: signingIdentity?.hash ?? '-', + fingerprint: sha256( + canonicalJson({ + codesignIdentity: signingIdentity?.hash ?? 'adhoc', + macosSdkPath, + macosSdkVersion, + swiftPath, + swiftVersion, + xcodeVersion, + }), + ), + macosSdkPath, + macosSdkVersion, + signing, + swiftPath, + swiftVersion, + xcodeVersion, + }; +} + +export async function buildMacOSDriver({ + buildId, + configuration, + packageRoot, + signal, + sourceDigest, + stagingRoot, + toolchain, +}: BuildMacOSDriverOptions): Promise { + throwIfAborted(signal); + const sourceRoot = path.join(packageRoot, 'native', 'macos'); + const packageManifest = path.join(sourceRoot, 'Package.swift'); + if (!fs.existsSync(packageManifest)) { + throw new NativeDriverError('build-source-missing', `macOS native driver package does not exist at "${packageManifest}".`); + } + + const stagedPackageRoot = path.join(stagingRoot, 'package'); + const scratchRoot = path.join(stagingRoot, 'build'); + const outputRoot = path.join(stagingRoot, 'output'); + for (const relativePath of macOSDriverSourceFiles(sourceRoot)) { + const destination = path.join(stagedPackageRoot, relativePath); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.copyFileSync(path.join(sourceRoot, relativePath), destination); + } + fs.mkdirSync(outputRoot, { recursive: true }); + fs.writeFileSync( + path.join(stagedPackageRoot, 'Sources', 'DesktopDriverHost', 'GeneratedBuildInfo.swift'), + [ + 'enum FurnBuildInfo {', + ` static let buildId = "${escapeSwiftString(buildId)}"`, + ` static let sourceDigest = "${escapeSwiftString(sourceDigest)}"`, + '}', + '', + ].join('\n'), + ); + + const nativeConfiguration = configuration === 'debug' ? 'debug' : 'release'; + const canonicalSourceRoot = '/furn-desktop-driver/native/macos'; + const buildArguments = [ + 'build', + '--package-path', + stagedPackageRoot, + '--scratch-path', + scratchRoot, + '--configuration', + nativeConfiguration, + '--arch', + 'arm64', + ...(configuration === 'release' ? ['-Xlinker', '-no_uuid'] : []), + '-Xswiftc', + '-file-prefix-map', + '-Xswiftc', + `${stagedPackageRoot}=${canonicalSourceRoot}`, + '-Xswiftc', + '-debug-prefix-map', + '-Xswiftc', + `${stagedPackageRoot}=${canonicalSourceRoot}`, + '-Xswiftc', + '-debug-prefix-map', + '-Xswiftc', + `${scratchRoot}=/furn-desktop-driver/build`, + ]; + const buildLogPath = path.join(stagingRoot, 'build.log'); + const output = await runCommand(toolchain.swiftPath, buildArguments, signal).catch((error) => { + atomicWriteJson(path.join(stagingRoot, 'build-error.json'), { + message: error instanceof Error ? error.message : String(error), + toolchain, + }); + throw error; + }); + fs.writeFileSync(buildLogPath, combinedOutput(output)); + + const binPathResult = await runCommand(toolchain.swiftPath, [...buildArguments, '--show-bin-path'], signal); + const binPath = binPathResult.stdout.trim(); + if (!binPath) { + throw new NativeDriverError( + 'build-failed', + `SwiftPM did not report its binary output path.${binPathResult.stderr.trim() ? `\n${binPathResult.stderr.trim()}` : ''}`, + ); + } + const builtExecutable = path.join(binPath, executableName); + if (!fs.existsSync(builtExecutable)) { + throw new NativeDriverError('build-failed', `SwiftPM completed without producing "${builtExecutable}".`); + } + + const bundlePath = path.join(outputRoot, bundleName); + const contentsPath = path.join(bundlePath, 'Contents'); + const executablePath = path.join(contentsPath, 'MacOS', executableName); + fs.mkdirSync(path.dirname(executablePath), { recursive: true }); + fs.copyFileSync(builtExecutable, executablePath); + fs.chmodSync(executablePath, 0o755); + if (configuration === 'release') { + await runCommand('xcrun', ['strip', '-S', '-x', executablePath], signal); + } + fs.writeFileSync(path.join(contentsPath, 'Info.plist'), makeInfoPlist()); + + if (toolchain.signing.mode === 'adhoc') { + process.emitWarning( + 'The macOS native Desktop Driver helper is using an ad hoc signature. Configure a stable signing identity for durable TCC permissions.', + { code: 'FURN_MACOS_ADHOC_SIGNING' }, + ); + } + await runCommand('codesign', macOSCodeSignArguments(toolchain, bundlePath), signal); + const verifiedSigning = await verifyMacOSDriver(executablePath, toolchain.signing, signal); + return { + buildLogPath, + bundlePath, + executablePath, + signing: verifiedSigning, + }; +} + +export async function verifyMacOSDriver( + executablePath: string, + expectedSigning?: NativeDriverSigning, + signal?: AbortSignal, +): Promise { + const bundlePath = macOSBundleForExecutable(executablePath); + await runCommand('codesign', ['--verify', '--strict', '--verbose=2', bundlePath], signal); + const details = combinedOutput(await runCommand('codesign', ['--display', '--verbose=4', bundlePath], signal)); + const identifier = readCodeSignValue(details, 'Identifier'); + if (identifier !== bundleIdentifier) { + throw new NativeDriverError( + 'signature-mismatch', + `macOS native driver bundle identifier "${identifier ?? 'unknown'}" does not match "${bundleIdentifier}".`, + ); + } + const adhoc = readCodeSignValue(details, 'Signature') === 'adhoc'; + const mode: NativeDriverSigning['mode'] = adhoc ? 'adhoc' : 'signed'; + if (mode === 'signed') { + if (!/^CodeDirectory .*\bflags=.*\bruntime\b/m.test(details)) { + throw new NativeDriverError('signature-mismatch', 'The signed macOS native driver does not enable Hardened Runtime.'); + } + if (!readCodeSignValue(details, 'Timestamp')) { + throw new NativeDriverError('signature-mismatch', 'The signed macOS native driver does not contain a secure timestamp.'); + } + } + const identity = details.match(/^Authority=(.+)$/m)?.[1]; + const teamId = readCodeSignValue(details, 'TeamIdentifier'); + const certificateHash = mode === 'signed' ? await extractSigningCertificateHash(bundlePath, signal) : undefined; + const designatedRequirement = await readDesignatedRequirement(bundlePath, signal); + const signing: NativeDriverSigning = { + ...(certificateHash ? { certificateHash } : {}), + designatedRequirement, + ...(identity ? { identity } : {}), + mode, + ...(teamId && teamId !== 'not set' ? { teamId } : {}), + }; + if (expectedSigning) { + if (expectedSigning.mode !== signing.mode) { + throw new NativeDriverError( + 'signature-mismatch', + `macOS native driver signature mode "${signing.mode}" does not match "${expectedSigning.mode}".`, + ); + } + if (expectedSigning.mode === 'signed' && !expectedSigning.certificateHash) { + throw new NativeDriverError('signature-mismatch', 'The expected macOS signing metadata does not identify its leaf certificate.'); + } + for (const field of ['certificateHash', 'designatedRequirement', 'identity', 'teamId'] as const) { + if (expectedSigning[field] !== undefined && expectedSigning[field] !== signing[field]) { + throw new NativeDriverError( + 'signature-mismatch', + `macOS native driver signer ${field} "${signing[field] ?? 'unknown'}" does not match "${expectedSigning[field]}".`, + ); + } + } + } + return signing; +} + +export function resolveMacOSExecutablePath(helperPath: string): string { + const resolvedPath = path.resolve(helperPath); + if (path.extname(resolvedPath).toLowerCase() !== '.app') { + return resolvedPath; + } + return path.join(resolvedPath, 'Contents', 'MacOS', executableName); +} + +export function macOSBundleForExecutable(executablePath: string): string { + const resolvedPath = path.resolve(executablePath); + const macosDirectory = path.dirname(resolvedPath); + const contentsDirectory = path.dirname(macosDirectory); + const bundlePath = path.dirname(contentsDirectory); + if ( + path.basename(macosDirectory) !== 'MacOS' || + path.basename(contentsDirectory) !== 'Contents' || + path.extname(bundlePath).toLowerCase() !== '.app' + ) { + throw new NativeDriverError('signature-mismatch', `macOS native driver executable is not contained in an application bundle.`); + } + return bundlePath; +} + +export async function resolveMacOSSigningIdentity( + configuredIdentity: string | undefined, + signal?: AbortSignal, +): Promise { + const identity = configuredIdentity?.trim(); + if (!identity) { + return undefined; + } + const output = combinedOutput(await runCommand('security', ['find-identity', '-v', '-p', 'codesigning'], signal)); + const identities = parseMacOSSigningIdentities(output); + const selected = identities.find((candidate) => candidate.hash === identity.toUpperCase() || candidate.name === identity); + if (!selected) { + throw new NativeDriverError('signing-identity-missing', `The macOS code-signing identity "${identity}" is not available.`); + } + return selected; +} + +export function parseMacOSSigningIdentities(output: string): MacOSSigningIdentity[] { + return [...output.matchAll(/^\s*\d+\)\s+([0-9A-F]{40})\s+"([^"]+)"(?:\s+\([^)]*\))?$/gm)].map((match) => ({ + hash: match[1], + name: match[2], + })); +} + +async function extractSigningCertificateHash(bundlePath: string, signal?: AbortSignal): Promise { + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-driver-certificate-')); + const certificatePrefix = path.join(temporaryRoot, 'certificate'); + try { + await runCommand('codesign', macOSCertificateExtractionArguments(certificatePrefix, bundlePath), signal); + const certificatePath = `${certificatePrefix}0`; + if (!fs.existsSync(certificatePath)) { + throw new NativeDriverError('signature-mismatch', 'macOS native driver signature does not contain a leaf certificate.'); + } + return createHash('sha1').update(fs.readFileSync(certificatePath)).digest('hex').toUpperCase(); + } finally { + fs.rmSync(temporaryRoot, { force: true, recursive: true }); + } +} + +async function readDesignatedRequirement(bundlePath: string, signal?: AbortSignal): Promise { + const output = combinedOutput(await runCommand('codesign', ['--display', '-r-', bundlePath], signal)); + const requirement = output.match(/^(?:# )?designated => (.+)$/m)?.[1]?.trim(); + if (!requirement) { + throw new NativeDriverError('signature-mismatch', 'macOS native driver signature does not contain a designated requirement.'); + } + return requirement; +} + +function readCodeSignValue(output: string, key: string): string | undefined { + return output.match(new RegExp(`^${key}=(.+)$`, 'm'))?.[1]; +} + +function combinedOutput(result: CommandResult): string { + return `${result.stdout}${result.stderr}`; +} + +async function runCommand(command: string, args: readonly string[], signal?: AbortSignal): Promise { + throwIfAborted(signal); + return new Promise((resolve, reject) => { + let settled = false; + const child = spawn(command, [...args], { + signal, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let timedOut = false; + const timeout = setTimeout( + () => { + timedOut = true; + child.kill(); + }, + 30 * 60 * 1000, + ); + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.once('error', (error) => { + if (!settled) { + settled = true; + clearTimeout(timeout); + reject(error); + } + }); + child.once('close', (code) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + const output = Buffer.concat(stdout).toString('utf8'); + const errorOutput = Buffer.concat(stderr).toString('utf8'); + if (timedOut) { + reject(new NativeDriverError('build-timeout', `${path.basename(command)} exceeded the 30-minute build deadline.`)); + return; + } + if (code !== 0) { + const details = `${output}${errorOutput}`.trim(); + reject( + new NativeDriverError( + command === 'codesign' ? 'signing-failed' : 'build-failed', + `${path.basename(command)} exited with code ${String(code)}.${details ? `\n${details}` : ''}`, + ), + ); + return; + } + resolve({ stderr: errorOutput, stdout: output }); + }); + }); +} + +function makeInfoPlist(): string { + return [ + '', + '', + '', + '', + ' CFBundleDevelopmentRegion', + ' en', + ' CFBundleExecutable', + ` ${executableName}`, + ' CFBundleIdentifier', + ` ${bundleIdentifier}`, + ' CFBundleInfoDictionaryVersion', + ' 6.0', + ' CFBundleName', + ' Furn Desktop Driver Host', + ' CFBundlePackageType', + ' APPL', + ' CFBundleShortVersionString', + ' 1.0', + ' CFBundleVersion', + ' 1', + ' LSMinimumSystemVersion', + ' 14.0', + ' LSUIElement', + ' ', + '', + '', + '', + ].join('\n'); +} + +function escapeSwiftString(value: string): string { + return value.replaceAll('\\', '\\\\').replaceAll('"', '\\"'); +} diff --git a/packages/agentic/desktop-driver/src/native/nativeDriver.test.ts b/packages/agentic/desktop-driver/src/native/nativeDriver.test.ts new file mode 100644 index 00000000000..36696f1b6c0 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/nativeDriver.test.ts @@ -0,0 +1,424 @@ +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { encodeJsonFrame, NativeFrameDecoder } from '../hosts/native/framing'; +import { NativeHostProcess } from '../hosts/native/NativeHostProcess'; +import type { NativeHostJsonMessage } from './types'; +import { + hashMacOSDriverSources, + macOSCertificateExtractionArguments, + macOSCodeSignArguments, + parseMacOSSigningIdentities, +} from './macos/buildMacOSDriver'; +import { buildNativeDesktopDriver, resolveNativeDesktopDriver } from './nativeDriver'; + +jest.setTimeout(240_000); + +const windowsTest = process.platform === 'win32' && process.env.FURN_NATIVE_DRIVER_TEST === '1' ? test : test.skip; +const macosTest = process.platform === 'darwin' && process.env.FURN_NATIVE_DRIVER_TEST === '1' ? test : test.skip; +const macosSigningIdentity = process.env.FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY; +const signedMacosTest = + process.platform === 'darwin' && macosSigningIdentity && process.env.FURN_NATIVE_DRIVER_TEST === '1' ? test : test.skip; + +describe('native driver build and resolution', () => { + macosTest('builds, signs, reuses, and handshakes with the macOS helper', async () => { + const cacheRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-driver-macos-native-')); + const emitWarning = jest.spyOn(process, 'emitWarning').mockImplementation(() => undefined); + try { + const built = await buildNativeDesktopDriver({ cacheRoot, macosSigningIdentity: '', platform: 'macos' }); + expect(built).toMatchObject({ + architecture: 'arm64', + endpoints: ['macos'], + origin: 'built', + provider: 'macos', + signing: { designatedRequirement: expect.stringContaining('cdhash H"'), mode: 'adhoc' }, + wireProtocol: { major: 1, minor: 0 }, + }); + expect(built.executablePath).toContain(`${path.sep}FurnDesktopDriverHost.app${path.sep}Contents${path.sep}MacOS${path.sep}`); + + const rebuilt = await buildNativeDesktopDriver({ cacheRoot, force: true, macosSigningIdentity: '', platform: 'macos' }); + expect(rebuilt).toMatchObject({ + artifactId: built.artifactId, + buildId: built.buildId, + origin: 'built', + }); + fs.writeFileSync(built.executablePath, 'corrupt'); + const healed = await buildNativeDesktopDriver({ cacheRoot, force: true, macosSigningIdentity: '', platform: 'macos' }); + expect(healed).toMatchObject({ + artifactId: built.artifactId, + buildId: built.buildId, + origin: 'built', + }); + expect(fs.readdirSync(path.join(cacheRoot, 'v1', 'trash'))).not.toHaveLength(0); + + const reused = await buildNativeDesktopDriver({ cacheRoot, macosSigningIdentity: '', platform: 'macos' }); + expect(reused).toMatchObject({ + artifactId: built.artifactId, + origin: 'cache', + }); + const resolved = await resolveNativeDesktopDriver({ + buildPolicy: 'never', + cacheRoot, + macosSigningIdentity: '', + platform: 'macos', + }); + expect(resolved.artifactId).toBe(built.artifactId); + const bundlePath = path.dirname(path.dirname(path.dirname(resolved.executablePath))); + const direct = await resolveNativeDesktopDriver({ + cacheRoot, + helperPath: bundlePath, + macosSigningIdentity: '', + platform: 'macos', + }); + expect(direct).toMatchObject({ + artifactId: built.artifactId, + artifactRoot: bundlePath, + origin: 'explicit-path', + }); + + const helper = await NativeHostProcess.start({ artifact: resolved }); + await expect(helper.request('probe', { endpoint: 'macos' })).resolves.toMatchObject({ + result: { + endpoint: 'macos', + platformName: 'macos', + protocolVersion: 1, + }, + }); + await helper.dispose(); + + const selfTest = spawnSync(resolved.executablePath, ['--self-test'], { + encoding: 'utf8', + timeout: 120_000, + }); + expect(selfTest.status).toBe(0); + expect(selfTest.stdout).toContain('ok '); + } finally { + emitWarning.mockRestore(); + fs.rmSync(cacheRoot, { force: true, maxRetries: 10, recursive: true, retryDelay: 100 }); + } + }); + + signedMacosTest('builds and verifies the macOS helper with a configured signing identity', async () => { + const cacheRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-driver-macos-signed-')); + try { + const built = await buildNativeDesktopDriver({ cacheRoot, macosSigningIdentity, platform: 'macos' }); + expect(built.signing).toMatchObject({ + certificateHash: expect.stringMatching(/^[0-9A-F]{40}$/), + designatedRequirement: expect.any(String), + mode: 'signed', + }); + const rebuilt = await buildNativeDesktopDriver({ cacheRoot, force: true, macosSigningIdentity, platform: 'macos' }); + expect(rebuilt).toMatchObject({ + buildId: built.buildId, + signing: { + certificateHash: built.signing.certificateHash, + designatedRequirement: built.signing.designatedRequirement, + mode: 'signed', + }, + }); + await expect( + resolveNativeDesktopDriver({ buildPolicy: 'never', cacheRoot, macosSigningIdentity, platform: 'macos' }), + ).resolves.toMatchObject({ + artifactId: rebuilt.artifactId, + origin: 'cache', + }); + } finally { + fs.rmSync(cacheRoot, { force: true, maxRetries: 10, recursive: true, retryDelay: 100 }); + } + }); + + test('hashes and stages only declared macOS Swift package sources', () => { + const sourceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-driver-macos-sources-')); + try { + fs.mkdirSync(path.join(sourceRoot, 'Sources', 'DesktopDriverHost'), { recursive: true }); + fs.writeFileSync(path.join(sourceRoot, 'Package.swift'), '// package'); + fs.writeFileSync(path.join(sourceRoot, 'Sources', 'DesktopDriverHost', 'main.swift'), 'print("hello")'); + const digest = hashMacOSDriverSources(sourceRoot); + + fs.mkdirSync(path.join(sourceRoot, '.build', 'arm64-apple-macosx', 'release'), { recursive: true }); + fs.writeFileSync(path.join(sourceRoot, '.build', 'arm64-apple-macosx', 'release', 'helper'), 'generated'); + fs.writeFileSync(path.join(sourceRoot, 'Sources', '.DS_Store'), 'editor'); + + expect(hashMacOSDriverSources(sourceRoot)).toBe(digest); + } finally { + fs.rmSync(sourceRoot, { force: true, recursive: true }); + } + }); + + test('forms unambiguous macOS code-signing arguments', () => { + expect(macOSCertificateExtractionArguments('/tmp/certificate', '/tmp/Helper.app')).toEqual([ + '--display', + '--extract-certificates=/tmp/certificate', + '/tmp/Helper.app', + ]); + expect( + macOSCodeSignArguments( + { + codesignIdentity: 'ABCDEF', + signing: { certificateHash: 'ABCDEF', identity: 'Example', mode: 'signed' }, + }, + '/tmp/Helper.app', + ), + ).toEqual([ + '--force', + '--sign', + 'ABCDEF', + '--identifier', + 'com.microsoft.fluentui-react-native.desktop-driver', + '--options', + 'runtime', + '--timestamp', + '/tmp/Helper.app', + ]); + expect(macOSCodeSignArguments({ codesignIdentity: '-', signing: { mode: 'adhoc' } }, '/tmp/Helper.app')).toContain('--timestamp=none'); + expect( + parseMacOSSigningIdentities( + ' 1) AB592605F8619FCAA952B4A2525AE6D41296FDD7 "FURN Desktop Driver Development" (CSSMERR_TP_NOT_TRUSTED)\n', + ), + ).toEqual([ + { + hash: 'AB592605F8619FCAA952B4A2525AE6D41296FDD7', + name: 'FURN Desktop Driver Development', + }, + ]); + }); + + windowsTest('builds, reuses, resolves, and handshakes with the Windows helper', async () => { + const cacheRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-driver-native-')); + const externalRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-driver-external-')); + try { + const built = await buildNativeDesktopDriver({ cacheRoot, platform: 'windows' }); + expect(built).toMatchObject({ + architecture: 'x64', + endpoints: ['windows', 'win32'], + origin: 'built', + provider: 'windows', + wireProtocol: { major: 1, minor: 0 }, + }); + expect(fs.existsSync(built.executablePath)).toBe(true); + + const reused = await buildNativeDesktopDriver({ cacheRoot, platform: 'win32' }); + expect(reused).toMatchObject({ + artifactId: built.artifactId, + origin: 'cache', + }); + + const resolved = await resolveNativeDesktopDriver({ buildPolicy: 'never', cacheRoot, platform: 'windows' }); + expect(resolved.artifactId).toBe(built.artifactId); + + const emitWarning = jest.spyOn(process, 'emitWarning').mockImplementation(() => undefined); + try { + const compatibilityRoot = path.dirname(path.dirname(resolved.artifactRoot)); + const incompatibleRoot = path.join(compatibilityRoot, 'incompatible-build', 'incompatible-artifact'); + fs.cpSync(resolved.artifactRoot, incompatibleRoot, { recursive: true }); + const incompatibleManifestPath = path.join(incompatibleRoot, 'artifact.json'); + const incompatibleManifest = JSON.parse(fs.readFileSync(incompatibleManifestPath, 'utf8')) as Record; + incompatibleManifest.compatibilityKey = 'incompatible-context'; + fs.writeFileSync(incompatibleManifestPath, `${JSON.stringify(incompatibleManifest, null, 2)}\n`); + const compatibilityDirectory = fs.readdirSync(path.join(cacheRoot, 'v1', 'selections')).at(0); + if (!compatibilityDirectory) { + throw new Error('The native build did not write its compatibility selection.'); + } + const selectionRoot = path.join(cacheRoot, 'v1', 'selections', compatibilityDirectory); + fs.writeFileSync( + path.join(selectionRoot, 'zzzy-incompatible.json'), + `${JSON.stringify({ + artifactPath: path.relative(cacheRoot, incompatibleRoot), + createdAt: new Date().toISOString(), + schemaVersion: 1, + })}\n`, + ); + const unrelatedRoot = path.join(path.dirname(compatibilityRoot), 'unrelated-context', 'build-context', 'artifact-context'); + fs.mkdirSync(unrelatedRoot, { recursive: true }); + fs.writeFileSync(path.join(unrelatedRoot, 'artifact.json'), '{'); + fs.writeFileSync( + path.join(selectionRoot, 'zzzz-unrelated.json'), + `${JSON.stringify({ + artifactPath: path.relative(cacheRoot, unrelatedRoot), + createdAt: new Date().toISOString(), + schemaVersion: 1, + })}\n`, + ); + const externalArtifactRoot = path.join(externalRoot, 'artifact'); + fs.cpSync(resolved.artifactRoot, externalArtifactRoot, { recursive: true }); + const junctionRoot = path.join(compatibilityRoot, 'junction-build', 'junction-artifact'); + fs.mkdirSync(path.dirname(junctionRoot), { recursive: true }); + fs.symlinkSync(externalArtifactRoot, junctionRoot, 'junction'); + fs.writeFileSync( + path.join(selectionRoot, 'zzzx-junction.json'), + `${JSON.stringify({ + artifactPath: path.relative(cacheRoot, junctionRoot), + createdAt: new Date().toISOString(), + schemaVersion: 1, + })}\n`, + ); + await expect(resolveNativeDesktopDriver({ buildPolicy: 'never', cacheRoot, platform: 'windows' })).resolves.toMatchObject({ + artifactId: resolved.artifactId, + }); + expect(fs.readdirSync(path.join(cacheRoot, 'v1', 'trash'))).toHaveLength(3); + expect(fs.existsSync(incompatibleRoot)).toBe(true); + expect(fs.existsSync(unrelatedRoot)).toBe(true); + expect(fs.existsSync(externalArtifactRoot)).toBe(true); + + await expect(NativeHostProcess.start({ artifact: { ...resolved, buildId: 'mismatched-build' } })).rejects.toMatchObject({ + code: 'handshake-failed', + }); + await expectMalformedJsonRecovery(resolved.executablePath); + + const helper = await NativeHostProcess.start({ artifact: resolved }); + await expect(helper.request('probe', { padding: 'x'.repeat(8 * 1024 * 1024) })).rejects.toThrow('exceeds'); + await expect(helper.request('probe', { endpoint: 'windows' })).resolves.toMatchObject({ + result: { + endpoint: 'windows', + platformName: 'windows', + protocolVersion: 1, + }, + }); + await helper.dispose(); + + const selfTest = spawnSync(resolved.executablePath, ['--self-test'], { + encoding: 'utf8', + timeout: 120_000, + }); + expect(selfTest.status).toBe(0); + expect(selfTest.stdout).toContain('self-test passed'); + + fs.writeFileSync(resolved.executablePath, 'corrupt'); + await expect(resolveNativeDesktopDriver({ buildPolicy: 'never', cacheRoot, platform: 'windows' })).rejects.toMatchObject({ + code: 'no-verified-prebuilt', + }); + expect(fs.readdirSync(path.join(cacheRoot, 'v1', 'trash'))).toHaveLength(4); + expect(emitWarning).toHaveBeenCalledWith(expect.stringContaining('Quarantined invalid native driver cache selection'), { + code: 'FURN_NATIVE_DRIVER_CACHE_INVALID', + }); + } finally { + emitWarning.mockRestore(); + } + } finally { + fs.rmSync(cacheRoot, { force: true, maxRetries: 10, recursive: true, retryDelay: 100 }); + fs.rmSync(externalRoot, { force: true, maxRetries: 10, recursive: true, retryDelay: 100 }); + } + }); + + test('rejects unsupported V1 architecture combinations before building', async () => { + await expect(buildNativeDesktopDriver({ architecture: 'arm64', platform: 'windows' })).rejects.toMatchObject({ + code: 'unsupported-platform', + }); + }); + + test('honors the prebuilt-only build policy from the environment', async () => { + const cacheRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-driver-policy-')); + const previousPolicy = process.env.FURN_DESKTOP_DRIVER_BUILD_POLICY; + process.env.FURN_DESKTOP_DRIVER_BUILD_POLICY = 'never'; + try { + await expect(resolveNativeDesktopDriver({ cacheRoot, platform: 'windows' })).rejects.toMatchObject({ + code: 'no-verified-prebuilt', + }); + } finally { + if (previousPolicy === undefined) { + delete process.env.FURN_DESKTOP_DRIVER_BUILD_POLICY; + } else { + process.env.FURN_DESKTOP_DRIVER_BUILD_POLICY = previousPolicy; + } + fs.rmSync(cacheRoot, { force: true, recursive: true }); + } + }); + + async function expectMalformedJsonRecovery(executablePath: string): Promise { + const child = spawn(executablePath, ['--stdio'], { + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + }); + const closed = new Promise((resolve) => child.once('close', () => resolve())); + const decoder = new NativeFrameDecoder(); + child.stdout.on('data', (chunk: Buffer) => decoder.write(chunk)); + child.stderr.resume(); + try { + await waitForMessage(decoder, (message) => message.type === 'hello'); + child.stdin.write(encodeMalformedJsonFrame()); + child.stdin.write( + encodeJsonFrame({ + command: 'probe', + id: 'x'.repeat(1024), + params: { endpoint: 'windows' }, + type: 'request', + }), + ); + child.stdin.write(encodeJsonFrame({ id: 'x'.repeat(1024), type: 'cancel' })); + + const probe = waitForMessage(decoder, (message) => message.type === 'response' && message.id === 'probe-after-malformed'); + child.stdin.write( + encodeJsonFrame({ + command: 'probe', + id: 'probe-after-malformed', + params: { endpoint: 'windows' }, + type: 'request', + }), + ); + await expect(probe).resolves.toMatchObject({ + result: { + endpoint: 'windows', + platformName: 'windows', + protocolVersion: 1, + }, + }); + + const disposed = waitForMessage(decoder, (message) => message.type === 'response' && message.id === 'dispose-after-malformed'); + child.stdin.write( + encodeJsonFrame({ + command: 'dispose', + id: 'dispose-after-malformed', + params: {}, + type: 'request', + }), + ); + await disposed; + await closed; + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill(); + await closed; + } + } + } + + function encodeMalformedJsonFrame(): Buffer { + const payload = Buffer.from('{', 'utf8'); + const header = Buffer.alloc(12); + header.write('FDR1', 0, 'ascii'); + header[4] = 1; + header.writeUInt32LE(payload.length, 8); + return Buffer.concat([header, payload]); + } + + function waitForMessage( + decoder: NativeFrameDecoder, + predicate: (message: NativeHostJsonMessage) => boolean, + ): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error('Timed out waiting for the native helper response.')); + }, 5000); + const onJson = (message: NativeHostJsonMessage) => { + if (predicate(message)) { + cleanup(); + resolve(message); + } + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timeout); + decoder.off('json', onJson); + decoder.off('error', onError); + }; + decoder.on('json', onJson); + decoder.once('error', onError); + }); + } +}); diff --git a/packages/agentic/desktop-driver/src/native/nativeDriver.ts b/packages/agentic/desktop-driver/src/native/nativeDriver.ts new file mode 100644 index 00000000000..3f6c1c6b918 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/nativeDriver.ts @@ -0,0 +1,743 @@ +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; + +import { + acquireDirectoryLock, + atomicWriteJson, + canonicalJson, + defaultNativeDriverCacheRoot, + hashTree, + listFiles, + readJsonFile, + sha256, + throwIfAborted, +} from './filesystem.js'; +import { nativeDriverWireProtocol } from './constants.js'; +import { NativeDriverError } from './NativeDriverError.js'; +import type { + NativeDriverArchitecture, + NativeDriverArtifact, + NativeDriverArtifactOrigin, + NativeDriverBuildOptions, + NativeDriverBuildPolicy, + NativeDriverConfiguration, + NativeDriverProvider, + NativeDriverResolveOptions, + NativeHostHello, +} from './types.js'; +import { + buildMacOSDriver, + hashMacOSDriverSources, + macOSBundleForExecutable, + probeMacOSToolchain, + resolveMacOSExecutablePath, + resolveMacOSSigningIdentity, + verifyMacOSDriver, +} from './macos/buildMacOSDriver.js'; +import type { BuildMacOSDriverResult, MacOSSigningIdentity } from './macos/buildMacOSDriver.js'; +import { buildWindowsDriver, probeWindowsToolchain } from './windows/buildWindowsDriver.js'; +import type { BuildWindowsDriverResult } from './windows/buildWindowsDriver.js'; + +export const FURN_DESKTOP_DRIVER_BUILD_POLICY = 'FURN_DESKTOP_DRIVER_BUILD_POLICY'; +export const FURN_DESKTOP_DRIVER_CACHE_ROOT = 'FURN_DESKTOP_DRIVER_CACHE_ROOT'; +export const FURN_DESKTOP_DRIVER_HELPER_PATH = 'FURN_DESKTOP_DRIVER_HELPER_PATH'; +export const FURN_DESKTOP_DRIVER_INSTALL_ROOT = 'FURN_DESKTOP_DRIVER_INSTALL_ROOT'; +export const FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY = 'FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY'; + +const buildCoordinatorVersion = 2; + +type NativeArtifactManifest = Omit & { + executable: string; +}; + +type NativeArtifactSelection = { + artifactPath: string; + createdAt: string; + schemaVersion: 1; +}; + +type NativeBuildContext = { + architecture: NativeDriverArchitecture; + cacheRoot: string; + compatibilityKey: string; + configuration: NativeDriverConfiguration; + endpoints: readonly ('macos' | 'windows' | 'win32')[]; + packageRoot: string; + provider: NativeDriverProvider; + signingIdentity?: MacOSSigningIdentity; + sourceDigest: string; +}; + +type NativeBuildResult = BuildMacOSDriverResult | BuildWindowsDriverResult; + +export async function buildNativeDesktopDriver(options: NativeDriverBuildOptions): Promise { + const context = await resolveBuildContext(options); + let build: (buildId: string, stagingRoot: string) => Promise; + let toolchainFingerprint: string; + if (context.provider === 'windows') { + const toolchain = await probeWindowsToolchain(options.signal); + toolchainFingerprint = toolchain.fingerprint; + build = (buildId, stagingRoot) => + buildWindowsDriver({ + buildId, + configuration: context.configuration, + packageRoot: context.packageRoot, + signal: options.signal, + sourceDigest: context.sourceDigest, + stagingRoot, + toolchain, + }); + } else { + const toolchain = await probeMacOSToolchain(context.signingIdentity, options.signal); + toolchainFingerprint = toolchain.fingerprint; + build = (buildId, stagingRoot) => + buildMacOSDriver({ + buildId, + configuration: context.configuration, + packageRoot: context.packageRoot, + signal: options.signal, + sourceDigest: context.sourceDigest, + stagingRoot, + toolchain, + }); + } + const buildFingerprint = sha256( + canonicalJson({ + compatibilityKey: context.compatibilityKey, + toolchain: toolchainFingerprint, + }), + ); + const lockRoot = path.join(context.cacheRoot, 'v1', 'locks'); + fs.mkdirSync(lockRoot, { recursive: true }); + const lock = await acquireDirectoryLock(path.join(lockRoot, shortKey(buildFingerprint)), options.signal); + try { + if (!options.force) { + const cached = await resolveCachedArtifact(context, 'cache'); + if (cached?.buildFingerprint === buildFingerprint) { + return cached; + } + } + throwIfAborted(options.signal); + const stagingRoot = path.join(context.cacheRoot, 'v1', 'staging', `${buildFingerprint}-${process.pid}-${randomUUID()}`); + fs.mkdirSync(stagingRoot, { recursive: true }); + try { + const buildId = buildFingerprint; + const built = await build(buildId, stagingRoot); + const handshake = await readOneShotHandshake(built.executablePath, options.signal); + validateHandshake(handshake, { + architecture: context.architecture, + buildId, + provider: context.provider, + sourceDigest: context.sourceDigest, + }); + const artifactId = sha256(fs.readFileSync(built.executablePath)); + const artifactRoot = path.join( + context.cacheRoot, + 'v1', + 'artifacts', + `${context.provider}-${context.architecture}`, + shortKey(context.compatibilityKey), + shortKey(buildFingerprint), + shortKey(artifactId), + ); + const publicationRoot = path.join(context.cacheRoot, 'v1', 'staging', `publish-${artifactId}-${randomUUID()}`); + const publicationBin = path.join(publicationRoot, 'bin'); + fs.mkdirSync(publicationBin, { recursive: true }); + let publishedExecutable: string; + if ('bundlePath' in built) { + const publishedBundle = path.join(publicationBin, path.basename(built.bundlePath)); + fs.cpSync(built.bundlePath, publishedBundle, { recursive: true }); + publishedExecutable = path.join(publishedBundle, path.relative(built.bundlePath, built.executablePath)); + } else { + publishedExecutable = path.join(publicationBin, path.basename(built.executablePath)); + fs.copyFileSync(built.executablePath, publishedExecutable); + } + fs.copyFileSync(built.buildLogPath, path.join(publicationRoot, 'build.log')); + const manifest: NativeArtifactManifest = { + architecture: context.architecture, + artifactId, + buildFingerprint, + buildId, + compatibilityKey: context.compatibilityKey, + configuration: context.configuration, + endpoints: context.endpoints, + executable: path.relative(publicationRoot, publishedExecutable), + features: handshake.features, + provider: context.provider, + schemaVersion: 1, + signing: 'signing' in built ? built.signing : { mode: 'none' }, + sourceDigest: context.sourceDigest, + wireProtocol: handshake.protocol, + }; + atomicWriteJson(path.join(publicationRoot, 'artifact.json'), manifest); + fs.mkdirSync(path.dirname(artifactRoot), { recursive: true }); + try { + fs.renameSync(publicationRoot, artifactRoot); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'EEXIST' && code !== 'ENOTEMPTY') { + throw error; + } + try { + const existing = readArtifact(artifactRoot, 'built'); + validateCachedArtifactContext(existing, context); + await verifyNativeDriverArtifact(existing, options.signal); + fs.rmSync(publicationRoot, { force: true, recursive: true }); + } catch (collisionError) { + quarantinePublishedArtifact(context.cacheRoot, artifactRoot, collisionError); + fs.renameSync(publicationRoot, artifactRoot); + } + } + writeSelection(context.cacheRoot, context.compatibilityKey, artifactRoot); + assertManagedPath(artifactCompatibilityRoot(context), artifactRoot, 'directory'); + return verifyNativeDriverArtifact(readArtifact(artifactRoot, 'built'), options.signal); + } catch (error) { + try { + preserveBuildDiagnostics(context, stagingRoot, error); + } catch (diagnosticError) { + process.emitWarning( + `Could not preserve native build diagnostics: ${ + diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError) + }`, + { code: 'FURN_NATIVE_DRIVER_DIAGNOSTICS_FAILED' }, + ); + } + throw error; + } finally { + try { + fs.rmSync(stagingRoot, { force: true, recursive: true }); + } catch (cleanupError) { + process.emitWarning( + `Could not clean native build staging directory "${stagingRoot}": ${ + cleanupError instanceof Error ? cleanupError.message : String(cleanupError) + }`, + { code: 'FURN_NATIVE_DRIVER_STAGING_CLEANUP_FAILED' }, + ); + } + } + } finally { + lock.release(); + } +} + +function preserveBuildDiagnostics(context: NativeBuildContext, stagingRoot: string, error: unknown): void { + const diagnosticFiles = ['build.log', 'build-error.json'].filter((fileName) => fs.existsSync(path.join(stagingRoot, fileName))); + if (diagnosticFiles.length === 0) { + return; + } + const diagnosticsRoot = path.join( + context.cacheRoot, + 'v1', + 'diagnostics', + `${context.provider}-${context.architecture}`, + shortKey(context.compatibilityKey), + `${Date.now()}-${randomUUID()}`, + ); + fs.mkdirSync(diagnosticsRoot, { recursive: true }); + assertManagedPath(context.cacheRoot, diagnosticsRoot, 'directory'); + for (const fileName of diagnosticFiles) { + fs.copyFileSync(path.join(stagingRoot, fileName), path.join(diagnosticsRoot, fileName)); + } + atomicWriteJson(path.join(diagnosticsRoot, 'failure.json'), { + code: error instanceof NativeDriverError ? error.code : 'unknown', + message: (error instanceof Error ? error.message : String(error)).slice(0, 4096), + }); +} + +function quarantinePublishedArtifact(cacheRoot: string, artifactRoot: string, error: unknown): void { + const trashRoot = path.join(cacheRoot, 'v1', 'trash'); + fs.mkdirSync(trashRoot, { recursive: true }); + assertManagedPath(cacheRoot, trashRoot, 'directory'); + const quarantinePath = path.join(trashRoot, `${Date.now()}-${randomUUID()}-${path.basename(artifactRoot)}`); + fs.renameSync(artifactRoot, quarantinePath); + atomicWriteJson(`${quarantinePath}.error.json`, { + artifactRoot, + message: error instanceof Error ? error.message : String(error), + }); + process.emitWarning(`Quarantined invalid published native driver artifact "${artifactRoot}".`, { + code: 'FURN_NATIVE_DRIVER_CACHE_INVALID', + }); +} + +export async function resolveNativeDesktopDriver(options: NativeDriverResolveOptions): Promise { + const context = await resolveBuildContext(options); + const helperPath = options.helperPath ?? process.env[FURN_DESKTOP_DRIVER_HELPER_PATH]; + if (helperPath) { + return verifyDirectArtifact(helperPath, context, 'explicit-path', options.signal); + } + const installRoot = options.installRoot ?? process.env[FURN_DESKTOP_DRIVER_INSTALL_ROOT]; + if (installRoot) { + const installed = await resolveCachedArtifact({ ...context, cacheRoot: path.resolve(installRoot) }, 'install-root'); + if (!installed) { + throw new NativeDriverError( + 'no-verified-prebuilt', + `No compatible ${context.provider}-${context.architecture} helper was found beneath "${installRoot}".`, + ); + } + return installed; + } + const cached = await resolveCachedArtifact(context, 'cache'); + if (cached) { + return cached; + } + const buildPolicy = resolveBuildPolicy(options.buildPolicy); + if (buildPolicy === 'never') { + throw new NativeDriverError( + 'no-verified-prebuilt', + `No verified ${context.provider}-${context.architecture} helper is available and source builds are disabled.`, + { cacheRoot: context.cacheRoot, compatibilityKey: context.compatibilityKey }, + ); + } + return buildNativeDesktopDriver(options); +} + +export async function verifyNativeDriverArtifact(artifact: NativeDriverArtifact, signal?: AbortSignal): Promise { + throwIfAborted(signal); + const executablePath = fs.realpathSync.native(artifact.executablePath); + if (!fs.statSync(executablePath).isFile()) { + throw new NativeDriverError('integrity-mismatch', `Native driver helper is not a file: "${executablePath}".`); + } + const executableHash = sha256(fs.readFileSync(executablePath)); + if (artifact.artifactId !== executableHash) { + throw new NativeDriverError('integrity-mismatch', `Native driver helper hash does not match "${artifact.artifactId}".`); + } + if (artifact.provider === 'macos') { + await verifyMacOSDriver(executablePath, artifact.signing, signal); + } + const handshake = await readOneShotHandshake(executablePath, signal); + validateHandshake(handshake, artifact); + return Object.freeze({ ...artifact, executablePath, features: Object.freeze([...handshake.features]) }); +} + +async function resolveBuildContext(options: NativeDriverBuildOptions): Promise { + const provider = normalizeProvider(options.platform); + const architecture = options.architecture ?? defaultArchitecture(provider); + validateArchitecture(provider, architecture); + const configuration = options.configuration ?? resolveConfiguration(); + const packageRoot = resolvePackageRoot(); + const nativeRoot = path.join(packageRoot, 'native'); + const providerRoot = path.join(nativeRoot, provider); + if (!fs.existsSync(providerRoot)) { + throw new NativeDriverError('build-source-missing', `Native driver source does not exist at "${providerRoot}".`); + } + const sourceDigest = sha256( + canonicalJson({ + coordinatorVersion: buildCoordinatorVersion, + protocol: sha256(fs.readFileSync(path.join(nativeRoot, 'protocol.json'))), + provider: provider === 'macos' ? hashMacOSDriverSources(providerRoot) : hashTree(providerRoot), + }), + ); + const configuredSigningIdentity = + provider === 'macos' + ? normalizeSigningIdentity(options.macosSigningIdentity ?? process.env[FURN_DESKTOP_DRIVER_MACOS_SIGNING_IDENTITY]) + : undefined; + const signingIdentity = await resolveMacOSSigningIdentity(configuredSigningIdentity, options.signal); + const compatibilityKey = sha256( + canonicalJson({ + architecture, + configuration, + coordinatorVersion: buildCoordinatorVersion, + protocol: nativeDriverWireProtocol, + provider, + signingMode: provider === 'macos' ? (signingIdentity ? 'signed' : 'adhoc') : 'none', + signingIdentity: signingIdentity?.hash, + sourceDigest, + }), + ); + const cacheRoot = path.resolve(options.cacheRoot ?? process.env[FURN_DESKTOP_DRIVER_CACHE_ROOT] ?? defaultNativeDriverCacheRoot()); + return { + architecture, + cacheRoot, + compatibilityKey, + configuration, + endpoints: provider === 'windows' ? ['windows', 'win32'] : ['macos'], + packageRoot, + provider, + signingIdentity, + sourceDigest, + }; +} + +async function resolveCachedArtifact( + context: NativeBuildContext, + origin: Extract, +): Promise { + const selectionRoot = path.join(context.cacheRoot, 'v1', 'selections', shortKey(context.compatibilityKey)); + if (!fs.existsSync(selectionRoot)) { + return undefined; + } + assertManagedPath(context.cacheRoot, selectionRoot, 'directory'); + const selections = listFiles(selectionRoot) + .filter((filePath) => filePath.endsWith('.json')) + .sort((left, right) => right.localeCompare(left)); + for (const selectionPath of selections) { + try { + assertManagedPath(selectionRoot, selectionPath, 'file'); + const selection = readJsonFile(selectionPath); + if (selection.schemaVersion !== 1 || typeof selection.artifactPath !== 'string') { + throw new NativeDriverError('integrity-mismatch', `Invalid native driver selection at "${selectionPath}".`); + } + const artifactRoot = path.resolve(context.cacheRoot, selection.artifactPath); + const compatibilityRoot = artifactCompatibilityRoot(context); + if (!isArtifactRoot(compatibilityRoot, artifactRoot) || !fs.existsSync(path.join(artifactRoot, 'artifact.json'))) { + throw new NativeDriverError('integrity-mismatch', `Native driver selection points to an invalid artifact.`); + } + assertManagedPath(context.cacheRoot, compatibilityRoot, 'directory'); + assertManagedPath(compatibilityRoot, artifactRoot, 'directory'); + const artifact = readArtifact(artifactRoot, origin); + validateCachedArtifactContext(artifact, context); + return await verifyNativeDriverArtifact(artifact); + } catch (error) { + if (origin === 'install-root') { + throw error; + } + quarantineCachedSelection(context.cacheRoot, selectionRoot, selectionPath, error); + } + } + return undefined; +} + +function quarantineCachedSelection(cacheRoot: string, selectionRoot: string, selectionPath: string, error: unknown): void { + try { + assertManagedPath(selectionRoot, selectionPath, 'file'); + } catch (pathError) { + if ((pathError as NodeJS.ErrnoException).code === 'ENOENT') { + return; + } + throw pathError; + } + const trashRoot = path.join(cacheRoot, 'v1', 'trash'); + fs.mkdirSync(trashRoot, { recursive: true }); + assertManagedPath(cacheRoot, trashRoot, 'directory'); + const quarantineRoot = path.join(trashRoot, `${Date.now()}-${randomUUID()}`); + fs.mkdirSync(quarantineRoot, { recursive: true }); + assertManagedPath(trashRoot, quarantineRoot, 'directory'); + try { + fs.renameSync(selectionPath, path.join(quarantineRoot, path.basename(selectionPath))); + } catch (renameError) { + if ((renameError as NodeJS.ErrnoException).code !== 'ENOENT') { + throw renameError; + } + } + atomicWriteJson(path.join(quarantineRoot, 'error.json'), { + message: error instanceof Error ? error.message : String(error), + selectionPath, + }); + process.emitWarning(`Quarantined invalid native driver cache selection "${selectionPath}".`, { + code: 'FURN_NATIVE_DRIVER_CACHE_INVALID', + }); +} + +function readArtifact(artifactRoot: string, origin: NativeDriverArtifactOrigin): NativeDriverArtifact { + const manifestPath = path.join(artifactRoot, 'artifact.json'); + assertManagedPath(artifactRoot, manifestPath, 'file'); + const manifest = readJsonFile(manifestPath); + if (manifest.schemaVersion !== 1 || typeof manifest.executable !== 'string') { + throw new NativeDriverError('integrity-mismatch', `Invalid native driver artifact manifest beneath "${artifactRoot}".`); + } + const executablePath = path.resolve(artifactRoot, manifest.executable); + if (!isWithin(artifactRoot, executablePath)) { + throw new NativeDriverError('integrity-mismatch', 'Native driver artifact executable escapes its artifact root.'); + } + assertManagedPath(artifactRoot, executablePath, 'file'); + const { executable: _executable, ...artifact } = manifest; + return { + ...artifact, + artifactRoot, + executablePath, + origin, + }; +} + +function validateCachedArtifactContext(artifact: NativeDriverArtifact, context: NativeBuildContext): void { + const endpointsMatch = + artifact.endpoints.length === context.endpoints.length && + context.endpoints.every((endpoint, index) => artifact.endpoints[index] === endpoint); + if ( + artifact.architecture !== context.architecture || + artifact.compatibilityKey !== context.compatibilityKey || + artifact.configuration !== context.configuration || + !endpointsMatch || + artifact.provider !== context.provider || + (context.provider === 'macos' && + (artifact.signing.mode !== (context.signingIdentity ? 'signed' : 'adhoc') || + (context.signingIdentity !== undefined && + (artifact.signing.certificateHash !== context.signingIdentity.hash || + artifact.signing.identity !== context.signingIdentity.name)))) || + artifact.sourceDigest !== context.sourceDigest + ) { + throw new NativeDriverError( + 'integrity-mismatch', + `Native driver artifact "${artifact.artifactId}" does not match the requested build context.`, + ); + } +} + +async function verifyDirectArtifact( + helperPath: string, + context: NativeBuildContext, + origin: NativeDriverArtifactOrigin, + signal?: AbortSignal, +): Promise { + const executablePath = fs.realpathSync.native( + context.provider === 'macos' ? resolveMacOSExecutablePath(helperPath) : path.resolve(helperPath), + ); + const signing = + context.provider === 'macos' + ? await verifyMacOSDriver( + executablePath, + context.signingIdentity + ? { + certificateHash: context.signingIdentity.hash, + identity: context.signingIdentity.name, + mode: 'signed', + } + : undefined, + signal, + ) + : ({ mode: 'none' } as const); + const handshake = await readOneShotHandshake(executablePath, signal); + if (handshake.provider !== context.provider || handshake.architecture !== context.architecture) { + throw new NativeDriverError( + 'platform-mismatch', + `Native driver helper reports ${handshake.provider}-${handshake.architecture}, expected ${context.provider}-${context.architecture}.`, + ); + } + if (handshake.protocol.major !== nativeDriverWireProtocol.major || handshake.protocol.minor < nativeDriverWireProtocol.minor) { + throw new NativeDriverError( + 'protocol-mismatch', + `Native helper protocol ${handshake.protocol.major}.${handshake.protocol.minor} is incompatible with ${nativeDriverWireProtocol.major}.${nativeDriverWireProtocol.minor}.`, + ); + } + const artifactId = sha256(fs.readFileSync(executablePath)); + return { + architecture: context.architecture, + artifactId, + artifactRoot: context.provider === 'macos' ? macOSBundleForExecutable(executablePath) : path.dirname(executablePath), + buildFingerprint: handshake.buildId, + buildId: handshake.buildId, + compatibilityKey: context.compatibilityKey, + configuration: context.configuration, + endpoints: context.endpoints, + executablePath, + features: Object.freeze([...handshake.features]), + origin, + provider: context.provider, + schemaVersion: 1, + signing, + sourceDigest: handshake.sourceDigest, + wireProtocol: handshake.protocol, + }; +} + +function writeSelection(cacheRoot: string, compatibilityKey: string, artifactRoot: string): void { + const selectionRoot = path.join(cacheRoot, 'v1', 'selections', shortKey(compatibilityKey)); + const generation = `${new Date().toISOString().replaceAll(/[-:.TZ]/g, '')}-${randomUUID()}`; + atomicWriteJson(path.join(selectionRoot, `${generation}.json`), { + artifactPath: path.relative(cacheRoot, artifactRoot), + createdAt: new Date().toISOString(), + schemaVersion: 1, + } satisfies NativeArtifactSelection); +} + +async function readOneShotHandshake(executablePath: string, signal?: AbortSignal): Promise { + throwIfAborted(signal); + return new Promise((resolve, reject) => { + let settled = false; + const child = spawn(executablePath, ['--handshake', '--json'], { + signal, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill(); + }, 10_000); + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.once('error', (error) => { + if (!settled) { + settled = true; + clearTimeout(timeout); + reject(error); + } + }); + child.once('close', (code) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + if (timedOut) { + reject(new NativeDriverError('handshake-timeout', 'Native driver helper did not complete its handshake within 10 seconds.')); + return; + } + if (code !== 0) { + reject( + new NativeDriverError( + 'handshake-failed', + `Native driver helper exited with code ${String(code)} during handshake: ${Buffer.concat(stderr).toString('utf8')}`, + ), + ); + return; + } + try { + const hello = JSON.parse(Buffer.concat(stdout).toString('utf8')) as NativeHostHello; + if (hello.type !== 'hello') { + throw new Error('Handshake response is not a hello message.'); + } + resolve(hello); + } catch (error) { + reject(new NativeDriverError('handshake-failed', error instanceof Error ? error.message : String(error))); + } + }); + }); +} + +function validateHandshake( + handshake: NativeHostHello, + expected: Pick, +): void { + if (handshake.protocol.major !== nativeDriverWireProtocol.major || handshake.protocol.minor < nativeDriverWireProtocol.minor) { + throw new NativeDriverError( + 'protocol-mismatch', + `Native helper protocol ${handshake.protocol.major}.${handshake.protocol.minor} is incompatible with ${nativeDriverWireProtocol.major}.${nativeDriverWireProtocol.minor}.`, + ); + } + for (const field of ['architecture', 'buildId', 'provider', 'sourceDigest'] as const) { + if (handshake[field] !== expected[field]) { + throw new NativeDriverError( + 'integrity-mismatch', + `Native helper ${field} "${handshake[field]}" does not match "${expected[field]}".`, + ); + } + } +} + +function normalizeProvider(endpoint: 'macos' | 'win32' | 'windows'): NativeDriverProvider { + return endpoint === 'macos' ? 'macos' : 'windows'; +} + +function defaultArchitecture(provider: NativeDriverProvider): NativeDriverArchitecture { + return provider === 'macos' ? 'arm64' : 'x64'; +} + +function validateArchitecture(provider: NativeDriverProvider, architecture: NativeDriverArchitecture): void { + if ((provider === 'windows' && architecture !== 'x64') || (provider === 'macos' && architecture !== 'arm64')) { + throw new NativeDriverError('unsupported-platform', `Desktop Driver V1 does not support ${provider}-${architecture}.`); + } +} + +function resolveConfiguration(): NativeDriverConfiguration { + const value = process.env.FURN_DESKTOP_DRIVER_CONFIGURATION; + if (!value) { + return 'release'; + } + if (value !== 'debug' && value !== 'release') { + throw new NativeDriverError('invalid-configuration', `Unsupported native driver configuration "${value}".`); + } + return value; +} + +function resolveBuildPolicy(value?: NativeDriverBuildPolicy): NativeDriverBuildPolicy { + const configured = value ?? process.env[FURN_DESKTOP_DRIVER_BUILD_POLICY] ?? 'if-missing'; + if (configured !== 'if-missing' && configured !== 'never') { + throw new NativeDriverError('invalid-configuration', `Unsupported native driver build policy "${configured}".`); + } + + return configured; +} + +function normalizeSigningIdentity(value?: string): string | undefined { + const identity = value?.trim(); + return identity || undefined; +} + +function isWithin(root: string, candidate: string): boolean { + const relative = path.relative(path.resolve(root), path.resolve(candidate)); + return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); +} + +function assertManagedPath(root: string, candidate: string, expectedType: 'directory' | 'file'): void { + const absoluteRoot = path.resolve(root); + const absoluteCandidate = path.resolve(candidate); + if (!isWithin(absoluteRoot, absoluteCandidate)) { + throw new NativeDriverError('integrity-mismatch', `Managed native driver path escapes "${absoluteRoot}".`); + } + const relative = path.relative(absoluteRoot, absoluteCandidate); + let current = absoluteRoot; + for (const component of relative.split(path.sep).filter(Boolean)) { + current = path.join(current, component); + if (fs.lstatSync(current).isSymbolicLink()) { + throw new NativeDriverError('integrity-mismatch', `Managed native driver path contains a symbolic link or junction.`); + } + } + const stat = fs.lstatSync(absoluteCandidate); + if ((expectedType === 'file' && !stat.isFile()) || (expectedType === 'directory' && !stat.isDirectory())) { + throw new NativeDriverError('integrity-mismatch', `Managed native driver path is not a ${expectedType}.`); + } + const realRoot = fs.realpathSync.native(absoluteRoot); + const realCandidate = fs.realpathSync.native(absoluteCandidate); + if (!isWithin(realRoot, realCandidate)) { + throw new NativeDriverError('integrity-mismatch', `Managed native driver path resolves outside "${realRoot}".`); + } +} + +function artifactCompatibilityRoot(context: NativeBuildContext): string { + return path.join(context.cacheRoot, 'v1', 'artifacts', `${context.provider}-${context.architecture}`, shortKey(context.compatibilityKey)); +} + +function isArtifactRoot(compatibilityRoot: string, candidate: string): boolean { + if (!isWithin(compatibilityRoot, candidate)) { + return false; + } + const relative = path.relative(path.resolve(compatibilityRoot), path.resolve(candidate)); + return relative !== '' && relative.split(path.sep).length === 2; +} + +function shortKey(value: string): string { + return value.slice(0, 20); +} + +function resolvePackageRoot(): string { + const cliRoot = findPackageRoot(process.argv[1] ? path.dirname(path.resolve(process.argv[1])) : undefined); + if (cliRoot) { + return cliRoot; + } + try { + const requireFromCwd = createRequire(path.join(process.cwd(), 'package.json')); + return path.dirname(requireFromCwd.resolve('@fluentui-react-native/desktop-driver/package.json')); + } catch (error) { + throw new NativeDriverError( + 'build-source-missing', + `Could not resolve @fluentui-react-native/desktop-driver from "${process.cwd()}": ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function findPackageRoot(start?: string): string | undefined { + let current = start; + while (current) { + const manifestPath = path.join(current, 'package.json'); + if (fs.existsSync(manifestPath)) { + const manifest = readJsonFile<{ name?: string }>(manifestPath); + if (manifest.name === '@fluentui-react-native/desktop-driver') { + return current; + } + } + const parent = path.dirname(current); + if (parent === current) { + return undefined; + } + current = parent; + } + return undefined; +} diff --git a/packages/agentic/desktop-driver/src/native/types.ts b/packages/agentic/desktop-driver/src/native/types.ts new file mode 100644 index 00000000000..429095f3722 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/types.ts @@ -0,0 +1,126 @@ +import type { DesktopEndpoint } from '../protocol/types.js'; + +export type NativeDriverProvider = 'macos' | 'windows'; +export type NativeDriverArchitecture = 'arm64' | 'x64'; +export type NativeDriverConfiguration = 'debug' | 'release'; +export type NativeDriverBuildPolicy = 'if-missing' | 'never'; +export type NativeDriverArtifactOrigin = 'built' | 'cache' | 'explicit-path' | 'install-root'; + +export type NativeDriverWireProtocol = { + major: number; + minor: number; +}; + +export type NativeDriverSigning = { + certificateHash?: string; + designatedRequirement?: string; + identity?: string; + mode: 'adhoc' | 'none' | 'signed'; + teamId?: string; +}; + +export type NativeDriverArtifact = { + architecture: NativeDriverArchitecture; + artifactId: string; + artifactRoot: string; + buildFingerprint: string; + buildId: string; + compatibilityKey: string; + configuration: NativeDriverConfiguration; + endpoints: readonly DesktopEndpoint[]; + executablePath: string; + features: readonly string[]; + origin: NativeDriverArtifactOrigin; + provider: NativeDriverProvider; + schemaVersion: 1; + signing: NativeDriverSigning; + sourceDigest: string; + wireProtocol: NativeDriverWireProtocol; +}; + +export type NativeDriverBuildOptions = { + architecture?: NativeDriverArchitecture; + cacheRoot?: string; + configuration?: NativeDriverConfiguration; + force?: boolean; + macosSigningIdentity?: string; + platform: DesktopEndpoint; + signal?: AbortSignal; +}; + +export type NativeDriverResolveOptions = NativeDriverBuildOptions & { + buildPolicy?: NativeDriverBuildPolicy; + helperPath?: string; + installRoot?: string; +}; + +export type NativeDesktopApplicationDescriptor = { + arguments?: readonly string[]; + aumid?: string; + bundleIdentifier?: string; + executablePath?: string; + leaseNonce?: string; + leasePath?: string; + windowTitle?: string; +}; + +export type NativeHostHello = { + architecture: NativeDriverArchitecture; + buildId: string; + features: string[]; + minimumOs?: string; + protocol: NativeDriverWireProtocol; + provider: NativeDriverProvider; + sourceDigest: string; + type: 'hello'; +}; + +export type NativeHostRequest = { + command: string; + id: string; + params?: unknown; + type: 'request'; +}; + +export type NativeHostResponse = { + binary?: { + height?: number; + id: string; + mimeType?: string; + scaleFactor?: number; + width?: number; + }; + error?: { + code: string; + data?: Record; + message: string; + }; + id: string; + result?: unknown; + type: 'response'; +}; + +export type NativeHostEventMessage = { + event: string; + payload?: unknown; + sequence?: number; + type: 'event'; +}; + +export type NativeHostCancel = { + id: string; + type: 'cancel'; +}; + +export type NativeHostCancelled = { + id: string; + type: 'cancelled'; +}; + +export type NativeHostJsonMessage = + | NativeHostCancel + | NativeHostCancelled + | NativeHostEventMessage + | NativeHostHello + | NativeHostRequest + | NativeHostResponse; diff --git a/packages/agentic/desktop-driver/src/native/windows/buildWindowsDriver.ts b/packages/agentic/desktop-driver/src/native/windows/buildWindowsDriver.ts new file mode 100644 index 00000000000..56bf89bc405 --- /dev/null +++ b/packages/agentic/desktop-driver/src/native/windows/buildWindowsDriver.ts @@ -0,0 +1,233 @@ +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { atomicWriteJson, sha256, throwIfAborted } from '../filesystem.js'; +import { NativeDriverError } from '../NativeDriverError.js'; +import type { NativeDriverConfiguration } from '../types.js'; + +export type WindowsToolchain = { + fingerprint: string; + msbuildPath: string; + msbuildVersion: string; + platformToolset: string; + sdkVersion: string; + visualStudioPath: string; +}; + +export type BuildWindowsDriverOptions = { + buildId: string; + configuration: NativeDriverConfiguration; + packageRoot: string; + signal?: AbortSignal; + sourceDigest: string; + stagingRoot: string; + toolchain: WindowsToolchain; +}; + +export type BuildWindowsDriverResult = { + buildLogPath: string; + executablePath: string; +}; + +export async function probeWindowsToolchain(signal?: AbortSignal): Promise { + if (process.platform !== 'win32') { + throw new NativeDriverError('unsupported-host', 'The Windows native driver can only be built on Windows.'); + } + throwIfAborted(signal); + const programFilesX86 = process.env['ProgramFiles(x86)']; + if (!programFilesX86) { + throw new NativeDriverError('toolchain-missing', 'ProgramFiles(x86) is not available.'); + } + const vswherePath = path.join(programFilesX86, 'Microsoft Visual Studio', 'Installer', 'vswhere.exe'); + if (!fs.existsSync(vswherePath)) { + throw new NativeDriverError('toolchain-missing', `Visual Studio Installer was not found at "${vswherePath}".`); + } + const visualStudioPath = ( + await runCommand( + vswherePath, + ['-latest', '-products', '*', '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', '-property', 'installationPath'], + signal, + ) + ).trim(); + if (!visualStudioPath) { + throw new NativeDriverError('toolchain-missing', 'Visual Studio with the Desktop development with C++ workload was not found.'); + } + const msbuildCandidates = [ + path.join(visualStudioPath, 'MSBuild', 'Current', 'Bin', 'MSBuild.exe'), + path.join(visualStudioPath, 'MSBuild', '15.0', 'Bin', 'MSBuild.exe'), + ]; + const msbuildPath = msbuildCandidates.find((candidate) => fs.existsSync(candidate)); + if (!msbuildPath) { + throw new NativeDriverError('toolchain-missing', `MSBuild was not found beneath "${visualStudioPath}".`); + } + const msbuildVersion = (await runCommand(msbuildPath, ['-version', '-nologo'], signal)).trim().split(/\r?\n/).at(-1) ?? ''; + const platformToolset = fs + .readdirSync(path.join(visualStudioPath, 'VC', 'Auxiliary', 'Build'), { withFileTypes: true }) + .filter((entry) => entry.isFile() && /^Microsoft\.VCToolsVersion\.v\d+\.default\.txt$/.test(entry.name)) + .map((entry) => entry.name.match(/\.v(\d+)\./)?.[1]) + .filter((value): value is string => value !== undefined) + .map((value) => `v${value}`) + .sort((left, right) => Number(left.slice(1)) - Number(right.slice(1))) + .at(-1); + if (!platformToolset) { + throw new NativeDriverError('toolchain-missing', `No MSVC platform toolset was found beneath "${visualStudioPath}".`); + } + const sdkRoot = path.join(programFilesX86, 'Windows Kits', '10', 'Include'); + const sdkVersion = fs + .readdirSync(sdkRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && /^\d+\.\d+\.\d+\.\d+$/.test(entry.name)) + .map((entry) => entry.name) + .sort(compareVersions) + .at(-1); + if (!sdkVersion) { + throw new NativeDriverError('toolchain-missing', `No Windows SDK was found beneath "${sdkRoot}".`); + } + return { + fingerprint: sha256(JSON.stringify({ msbuildPath, msbuildVersion, platformToolset, sdkVersion, visualStudioPath })), + msbuildPath, + msbuildVersion, + platformToolset, + sdkVersion, + visualStudioPath, + }; +} + +export async function buildWindowsDriver({ + buildId, + configuration, + packageRoot, + signal, + sourceDigest, + stagingRoot, + toolchain, +}: BuildWindowsDriverOptions): Promise { + throwIfAborted(signal); + const projectPath = path.join(packageRoot, 'native', 'windows', 'DesktopDriverHost.vcxproj'); + if (!fs.existsSync(projectPath)) { + throw new NativeDriverError('build-source-missing', `Windows native driver project does not exist at "${projectPath}".`); + } + + const buildRoot = path.join(stagingRoot, 'build'); + const outputRoot = path.join(stagingRoot, 'output'); + const generatedRoot = path.join(buildRoot, 'generated'); + fs.mkdirSync(generatedRoot, { recursive: true }); + fs.mkdirSync(outputRoot, { recursive: true }); + fs.writeFileSync( + path.join(generatedRoot, 'build_info.h'), + [ + '#pragma once', + `inline constexpr wchar_t FurnDesktopDriverBuildId[] = L"${escapeCppString(buildId)}";`, + `inline constexpr wchar_t FurnDesktopDriverSourceDigest[] = L"${escapeCppString(sourceDigest)}";`, + '', + ].join('\n'), + ); + + const buildLogPath = path.join(stagingRoot, 'build.log'); + const nativeConfiguration = configuration === 'debug' ? 'Debug' : 'Release'; + const output = await runCommand( + toolchain.msbuildPath, + [ + projectPath, + '-nologo', + '-m', + '-restore:false', + `/p:Configuration=${nativeConfiguration}`, + '/p:Platform=x64', + `/p:PlatformToolset=${toolchain.platformToolset}`, + `/p:OutDir=${ensureTrailingSeparator(outputRoot)}`, + `/p:IntDir=${ensureTrailingSeparator(path.join(buildRoot, 'obj'))}`, + `/p:FurnGeneratedDir=${ensureTrailingSeparator(generatedRoot)}`, + `/p:WindowsTargetPlatformVersion=${toolchain.sdkVersion}`, + ], + signal, + ).catch((error) => { + atomicWriteJson(path.join(stagingRoot, 'build-error.json'), { + message: error instanceof Error ? error.message : String(error), + toolchain, + }); + throw error; + }); + fs.writeFileSync(buildLogPath, output); + + const executablePath = path.join(outputRoot, 'furn-desktop-driver-host.exe'); + if (!fs.existsSync(executablePath)) { + throw new NativeDriverError('build-failed', `MSBuild completed without producing "${executablePath}".`); + } + return { buildLogPath, executablePath }; +} + +async function runCommand(command: string, args: readonly string[], signal?: AbortSignal): Promise { + throwIfAborted(signal); + return new Promise((resolve, reject) => { + let settled = false; + const child = spawn(command, [...args], { + signal, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let timedOut = false; + const timeout = setTimeout( + () => { + timedOut = true; + child.kill(); + }, + 30 * 60 * 1000, + ); + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.once('error', (error) => { + if (!settled) { + settled = true; + clearTimeout(timeout); + reject(error); + } + }); + child.once('close', (code) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + const output = Buffer.concat(stdout).toString('utf8'); + const errorOutput = Buffer.concat(stderr).toString('utf8'); + if (timedOut) { + reject(new NativeDriverError('build-timeout', `${path.basename(command)} exceeded the 30-minute build deadline.`)); + return; + } + if (code !== 0) { + const details = `${output}${errorOutput}`.trim(); + reject( + new NativeDriverError( + 'build-failed', + `${path.basename(command)} exited with code ${String(code)}.${details ? `\n${details}` : ''}`, + ), + ); + return; + } + resolve(`${output}${errorOutput}`); + }); + }); +} + +function compareVersions(left: string, right: string): number { + const leftParts = left.split('.').map(Number); + const rightParts = right.split('.').map(Number); + for (let index = 0; index < Math.max(leftParts.length, rightParts.length); index += 1) { + const difference = (leftParts[index] ?? 0) - (rightParts[index] ?? 0); + if (difference !== 0) { + return difference; + } + } + return 0; +} + +function ensureTrailingSeparator(value: string): string { + return value.endsWith(path.sep) ? value : `${value}${path.sep}`; +} + +function escapeCppString(value: string): string { + return value.replaceAll('\\', '\\\\').replaceAll('"', '\\"'); +} diff --git a/packages/agentic/desktop-driver/src/protocol/actions.test.ts b/packages/agentic/desktop-driver/src/protocol/actions.test.ts new file mode 100644 index 00000000000..1dced169fbb --- /dev/null +++ b/packages/agentic/desktop-driver/src/protocol/actions.test.ts @@ -0,0 +1,27 @@ +import { createInputState, parseActionSequences } from './actions'; + +describe('WebDriver actions', () => { + test('requires key values to contain exactly one Unicode code point', () => { + expect(() => + parseActionSequences([{ actions: [{ type: 'keyDown', value: 'ab' }], id: 'keyboard', type: 'key' }], createInputState()), + ).toThrow('exactly one Unicode code point'); + expect(() => + parseActionSequences([{ actions: [{ type: 'keyDown', value: '\ud800' }], id: 'keyboard', type: 'key' }], createInputState()), + ).toThrow('exactly one Unicode code point'); + + const parsed = parseActionSequences( + [ + { + actions: [ + { type: 'keyDown', value: '\uE008' }, + { type: 'keyDown', value: '\u{1F600}' }, + ], + id: 'keyboard', + type: 'key', + }, + ], + createInputState(), + ); + expect([...parsed.nextState.pressedKeys]).toEqual(['\uE008', '\u{1F600}']); + }); +}); diff --git a/packages/agentic/desktop-driver/src/protocol/actions.ts b/packages/agentic/desktop-driver/src/protocol/actions.ts index 667baf6426f..3d2d635cbc8 100644 --- a/packages/agentic/desktop-driver/src/protocol/actions.ts +++ b/packages/agentic/desktop-driver/src/protocol/actions.ts @@ -90,11 +90,15 @@ function parseAction( if (typeof action.value !== 'string' || action.value.length === 0) { throw invalidArgument(`${path}.value must be a non-empty string.`); } + if (!isSingleWebDriverKeyValue(action.value)) { + throw invalidArgument(`${path}.value must contain exactly one Unicode code point.`); + } if (action.type === 'keyDown') { state.pressedKeys.add(action.value); } else { state.pressedKeys.delete(action.value); } + return action as WebDriverAction; } if (sourceType === 'pointer') { @@ -133,6 +137,11 @@ function parseAction( return action as WebDriverAction; } +export function isSingleWebDriverKeyValue(value: string): boolean { + const codePoint = value.codePointAt(0); + return codePoint !== undefined && [...value].length === 1 && (codePoint < 0xd800 || codePoint > 0xdfff); +} + function validateCoordinates(action: Record, path: string): void { if (typeof action.x !== 'number' || typeof action.y !== 'number') { throw invalidArgument(`${path}.x and y must be numbers.`); diff --git a/packages/agentic/desktop-driver/src/protocol/capabilities.ts b/packages/agentic/desktop-driver/src/protocol/capabilities.ts index 46aa3c1e7f6..74a74cfadc4 100644 --- a/packages/agentic/desktop-driver/src/protocol/capabilities.ts +++ b/packages/agentic/desktop-driver/src/protocol/capabilities.ts @@ -1,5 +1,5 @@ import type { DesktopHostInfo, DesktopTarget } from '../host/types.js'; -import { invalidArgument, WebDriverError } from './errors.js'; +import { invalidArgument, toWebDriverError, WebDriverError } from './errors.js'; import { withCommandTimeout } from './timeouts.js'; import type { DesktopClickMode, NewSessionCapabilities } from './types.js'; @@ -19,6 +19,8 @@ const standardCapabilities = new Set([ export type MatchedCapabilities = { clickMode: DesktopClickMode; + hostInfo: DesktopHostInfo; + launchMode: 'attach' | 'launch'; requested: Record; target: DesktopTarget; }; @@ -51,6 +53,8 @@ export async function matchCapabilities( const candidates = getCapabilityCandidates(capabilities); for (const requested of candidates) { validateCapabilityNames(requested); + const requestedClickMode = validateClickMode(requested['furn:clickMode']); + const launchMode = validateLaunchMode(requested['furn:launchMode']); const targetId = requested['furn:target']; const candidatesForTarget = typeof targetId === 'string' ? targets.filter((target) => target.id === targetId) : targets.length === 1 ? targets : []; @@ -69,9 +73,15 @@ export async function matchCapabilities( continue; } - const host = await withCommandTimeout((signal) => target.host.probe(signal), 10_000, `Probing target "${target.id}"`); - const clickMode = resolveClickMode(requested['furn:clickMode'], host); - return { clickMode, requested, target }; + let host: DesktopHostInfo; + try { + host = await withCommandTimeout((signal) => target.host.probe(signal), 10_000, `Probing target "${target.id}"`); + } catch (error) { + const webdriverError = toWebDriverError(error); + throw new WebDriverError('session not created', webdriverError.message, webdriverError.data); + } + const clickMode = resolveClickMode(requestedClickMode, host); + return { clickMode, hostInfo: host, launchMode, requested, target }; } } @@ -111,11 +121,23 @@ function validateCapabilityNames(capabilities: Record): void { } } -function resolveClickMode(requested: unknown, host: DesktopHostInfo): DesktopClickMode { +function validateClickMode(requested: unknown): DesktopClickMode | 'auto' { const clickMode = requested ?? 'auto'; if (clickMode !== 'auto' && clickMode !== 'physical' && clickMode !== 'accessibility') { throw invalidArgument('"furn:clickMode" must be "auto", "physical", or "accessibility".'); } + return clickMode; +} + +function validateLaunchMode(requested: unknown): 'attach' | 'launch' { + const launchMode = requested ?? 'launch'; + if (launchMode !== 'attach' && launchMode !== 'launch') { + throw invalidArgument('"furn:launchMode" must be "attach" or "launch".'); + } + return launchMode; +} + +function resolveClickMode(clickMode: DesktopClickMode | 'auto', host: DesktopHostInfo): DesktopClickMode { if (clickMode === 'physical' && !host.features.physicalClick) { throw new WebDriverError('session not created', 'The target does not support physical click input.'); } diff --git a/packages/agentic/desktop-driver/src/protocol/errors.ts b/packages/agentic/desktop-driver/src/protocol/errors.ts index b3edd35e28b..9d1ba3b9d9c 100644 --- a/packages/agentic/desktop-driver/src/protocol/errors.ts +++ b/packages/agentic/desktop-driver/src/protocol/errors.ts @@ -52,6 +52,9 @@ export function toWebDriverError(error: unknown): WebDriverError { if (error instanceof WebDriverError) { return error; } + if (error instanceof HostWebDriverError) { + return new WebDriverError(error.code, error.message, error.data); + } if (error instanceof HostStaleError) { return new WebDriverError('stale element reference', error.message); } @@ -61,6 +64,18 @@ export function toWebDriverError(error: unknown): WebDriverError { return new WebDriverError('unknown error', error instanceof Error ? error.message : String(error)); } +export class HostWebDriverError extends Error { + readonly code: WebDriverErrorCode; + readonly data?: Record; + + constructor(code: WebDriverErrorCode, message: string, data?: Record) { + super(message); + this.name = 'HostWebDriverError'; + this.code = code; + this.data = data; + } +} + export class HostStaleError extends Error { constructor(message = 'The native element is no longer available.') { super(message); diff --git a/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts b/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts index 8dae3394e03..dcb886591bc 100644 --- a/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts +++ b/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts @@ -44,6 +44,8 @@ export async function createDesktopDriverServer(options: DesktopDriverServerOpti const targets = new TargetRegistry(options.targets); const sessions = new SessionManager(); let closing = false; + let listenerClosePromise: Promise | undefined; + let cleanupQueue: Promise = Promise.resolve(); const httpServer = createServer((request, response) => { if (closing) { const error = new WebDriverError('unknown error', 'Desktop Driver is shutting down.'); @@ -61,29 +63,39 @@ export async function createDesktopDriverServer(options: DesktopDriverServerOpti targets, // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- WebDriver is an intentionally loopback-only local protocol. url: `http://${formatHost(host)}:${port}`, - async close() { + close() { closing = true; sessions.beginClose(); - let sessionError: unknown; - await closeServer(httpServer); - await sessions.waitForCreates(); - try { - await sessions.deleteAll(); - } catch (error) { - sessionError = error; - } - const hosts = new Set(targets.list().map(({ host: targetHost }) => targetHost)); - const results = await Promise.allSettled( - [...hosts].map((targetHost) => withCommandTimeout((signal) => targetHost.dispose(signal), 10_000, 'Disposing a desktop host')), + const attempt = cleanupQueue.then(runCleanup, runCleanup); + cleanupQueue = attempt.then( + () => undefined, + () => undefined, ); - const hostErrors = results - .filter((result): result is PromiseRejectedResult => result.status === 'rejected') - .map(({ reason }) => reason); - if (sessionError || hostErrors.length > 0) { - throw new AggregateError([...(sessionError ? [sessionError] : []), ...hostErrors], 'Desktop Driver cleanup failed.'); - } + return attempt; }, }; + + async function runCleanup(): Promise { + let sessionError: unknown; + listenerClosePromise ??= httpServer.listening ? closeServer(httpServer) : Promise.resolve(); + await listenerClosePromise; + await sessions.waitForCreates(); + try { + await sessions.deleteAll(); + } catch (error) { + sessionError = error; + } + const hosts = new Set(targets.list().map(({ host: targetHost }) => targetHost)); + const results = await Promise.allSettled( + [...hosts].map((targetHost) => withCommandTimeout((signal) => targetHost.dispose(signal), 10_000, 'Disposing a desktop host')), + ); + const hostErrors = results + .filter((result): result is PromiseRejectedResult => result.status === 'rejected') + .map(({ reason }) => reason); + if (sessionError || hostErrors.length > 0) { + throw new AggregateError([...(sessionError ? [sessionError] : []), ...hostErrors], 'Desktop Driver cleanup failed.'); + } + } } async function handleRequest( @@ -133,20 +145,19 @@ async function route(context: RouteContext, sessions: SessionManager, targets: T throw invalidArgument('New Session requires "capabilities".'); } const matched = await matchCapabilities(request.capabilities, targets.list()); - const hostInfo = await withCommandTimeout( - (signal) => matched.target.host.probe(signal), - 10_000, - `Probing target "${matched.target.id}"`, - ); - const launchMode = matched.requested['furn:launchMode'] ?? 'launch'; - if (launchMode !== 'attach' && launchMode !== 'launch') { - throw invalidArgument('"furn:launchMode" must be "attach" or "launch".'); + try { + const session = await sessions.create(matched.target, matched.hostInfo, matched.clickMode, matched.launchMode); + return { + sessionId: session.id, + capabilities: createReturnedCapabilities(matched, matched.hostInfo, session.timeouts), + }; + } catch (error) { + const webdriverError = toWebDriverError(error); + if (webdriverError.code === 'session not created') { + throw webdriverError; + } + throw new WebDriverError('session not created', webdriverError.message, webdriverError.data); } - const session = await sessions.create(matched.target, hostInfo, matched.clickMode, launchMode); - return { - sessionId: session.id, - capabilities: createReturnedCapabilities(matched, hostInfo, session.timeouts), - }; } if (segments[0] !== 'session' || !segments[1]) { @@ -525,10 +536,10 @@ async function handleElementCommand( throw new WebDriverError('element not interactable', `Element "${record.id}" cannot be clicked.`); } await sessions.runInputCommand(async () => { - await hostCommand(session, `Activating window "${snapshot.windowId}"`, (signal) => - session.target.host.activate(snapshot.windowId, signal), - ); if (session.clickMode !== 'accessibility') { + await hostCommand(session, `Activating window "${snapshot.windowId}"`, (signal) => + session.target.host.activate(snapshot.windowId, signal), + ); const point = { x: snapshot.rect.x + snapshot.rect.width / 2, y: snapshot.rect.y + snapshot.rect.height / 2, @@ -536,7 +547,7 @@ async function handleElementCommand( const hit = await hostCommand(session, `Hit testing element "${record.id}"`, (signal) => session.target.host.hitTest(snapshot.windowId, point.x, point.y, signal), ); - if (hit && hit.id !== snapshot.id) { + if (hit && !(await isElementOrDescendant(session, snapshot.id, hit))) { throw new WebDriverError('element click intercepted', `Element "${record.id}" is obscured by another native element.`); } } @@ -580,6 +591,21 @@ async function handleElementCommand( throw new WebDriverError('unknown command', `Unknown element command "${operation.join('/')}".`); } +async function isElementOrDescendant(session: DesktopSession, targetNativeId: string, hit: NativeElementSnapshot): Promise { + let current: NativeElementSnapshot | undefined = hit; + for (let depth = 0; current && depth < 100; depth += 1) { + if (current.id === targetNativeId) { + return true; + } + current = current.parentId + ? await hostCommand(session, 'Resolving the hit-test ancestor', (signal) => + session.target.host.snapshot(current!.parentId!, signal), + ).catch(() => undefined) + : undefined; + } + return false; +} + async function findElements( session: DesktopSession, sessions: SessionManager, diff --git a/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts b/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts index 56c3014945a..f7589c7c962 100644 --- a/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts +++ b/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts @@ -3,6 +3,7 @@ import { Buffer } from 'node:buffer'; import { createDesktopDriverClient } from '../client/DesktopDriverClient.js'; import type { FakeDesktopHost } from '../hosts/fake/FakeDesktopHost.js'; import { webElementIdentifier } from '../protocol/constants.js'; +import { HostWebDriverError } from '../protocol/errors.js'; import { runDesktopStoryTests } from '../runner/StoryTestRunner.js'; import { createDesktopDriverTestHarness } from '../testing/protocolHarness.js'; import type { DesktopStoryManifest, StoryOrchestrator, StoryReadyResult } from '../storybook.js'; @@ -133,6 +134,75 @@ describe('Desktop Driver W3C remote end', () => { } }); + test('reports native launch failures as session not created', async () => { + const harness = await createDesktopDriverTestHarness(); + jest.spyOn(harness.host, 'launch').mockRejectedValue( + new HostWebDriverError('invalid argument', 'The native launch descriptor was rejected.', { + operation: 'launch', + }), + ); + try { + const client = createDesktopDriverClient({ url: harness.server.url }); + await expect( + client.newSession({ + alwaysMatch: { platformName: 'windows', 'furn:target': harness.target.id }, + }), + ).rejects.toMatchObject({ + code: 'session not created', + data: { operation: 'launch' }, + message: 'The native launch descriptor was rejected.', + }); + } finally { + await harness.close(); + } + }); + + test('reports native probe failures as session not created', async () => { + const harness = await createDesktopDriverTestHarness(); + jest.spyOn(harness.host, 'probe').mockRejectedValue( + new HostWebDriverError('unable to capture screen', 'The native helper could not initialize.', { + operation: 'probe', + }), + ); + try { + const client = createDesktopDriverClient({ url: harness.server.url }); + await expect( + client.newSession({ + alwaysMatch: { platformName: 'windows', 'furn:target': harness.target.id }, + }), + ).rejects.toMatchObject({ + code: 'session not created', + data: { operation: 'probe' }, + message: 'The native helper could not initialize.', + }); + } finally { + await harness.close(); + } + }); + + test.each([ + ['furn:clickMode', 'not-a-click-mode'], + ['furn:launchMode', 'not-a-launch-mode'], + ])('rejects invalid %s before probing the target', async (capability, value) => { + const harness = await createDesktopDriverTestHarness(); + const probe = jest.spyOn(harness.host, 'probe'); + try { + const client = createDesktopDriverClient({ url: harness.server.url }); + await expect( + client.newSession({ + alwaysMatch: { + platformName: 'windows', + 'furn:target': harness.target.id, + [capability]: value, + }, + }), + ).rejects.toMatchObject({ code: 'invalid argument' }); + expect(probe).not.toHaveBeenCalled(); + } finally { + await harness.close(); + } + }); + test('waits for in-flight session creation before server cleanup', async () => { const harness = await createDesktopDriverTestHarness({ launchDelayMs: 80 }); const client = createDesktopDriverClient({ url: harness.server.url }); @@ -149,6 +219,18 @@ describe('Desktop Driver W3C remote end', () => { expect(harness.host.actions).toContainEqual(expect.objectContaining({ type: 'close-application' })); }); + test('retries host cleanup after the listener has already closed', async () => { + const harness = await createDesktopDriverTestHarness(); + const dispose = jest.spyOn(harness.host, 'dispose').mockRejectedValueOnce(new Error('first cleanup failed')).mockResolvedValueOnce(); + try { + await expect(harness.server.close()).rejects.toThrow('Desktop Driver cleanup failed.'); + await expect(harness.server.close()).resolves.toBeUndefined(); + expect(dispose).toHaveBeenCalledTimes(2); + } finally { + await harness.server.close(); + } + }); + test('serializes release behind a timed-out native action and drains the host operation', async () => { const harness = await createDesktopDriverTestHarness({ actionDelayMs: 50 }); try { diff --git a/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts b/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts index 76f796ae486..c845bf69650 100644 --- a/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts +++ b/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts @@ -15,6 +15,7 @@ declare global { interface Capabilities { 'furn:clickMode'?: 'accessibility' | 'auto' | 'physical'; 'furn:endpoint'?: 'macos' | 'win32' | 'windows'; + 'furn:launchMode'?: 'attach' | 'launch'; 'furn:target'?: string; } @@ -30,6 +31,7 @@ declare global { export type DesktopWebdriverOptions = { clickMode?: DesktopClickMode; + launchMode?: 'attach' | 'launch'; logLevel?: 'debug' | 'error' | 'info' | 'silent' | 'trace' | 'warn'; platformName: DesktopPlatformName; targetId: string; @@ -105,6 +107,7 @@ export async function connectDesktopWebdriver(options: DesktopWebdriverOptions): }, { 'furn:clickMode': options.clickMode ?? 'auto', + 'furn:launchMode': options.launchMode ?? 'launch', 'furn:target': options.targetId, }, ); diff --git a/packages/agentic/storybook-desktop-runtime/config/metro.cjs b/packages/agentic/storybook-desktop-runtime/config/metro.cjs index 0eaa13ed9e2..b19ce00ad7c 100644 --- a/packages/agentic/storybook-desktop-runtime/config/metro.cjs +++ b/packages/agentic/storybook-desktop-runtime/config/metro.cjs @@ -106,7 +106,12 @@ function readDriverManifest(manifestPath) { throw new Error(`Desktop Driver manifest does not exist at ${manifestPath}.`); } const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); - if (manifest.schemaVersion !== 1) { + if ( + manifest.schemaVersion !== 2 || + !manifest.nativeDriver || + !manifest.application || + typeof manifest.application.leasePath !== 'string' + ) { throw new Error(`Unsupported Desktop Driver manifest schema "${manifest.schemaVersion}".`); } return manifest; diff --git a/packages/agentic/storybook-desktop-runtime/src/DesktopStoryRoot.tsx b/packages/agentic/storybook-desktop-runtime/src/DesktopStoryRoot.tsx index 6bd6ece9796..2cd1acc8fa6 100644 --- a/packages/agentic/storybook-desktop-runtime/src/DesktopStoryRoot.tsx +++ b/packages/agentic/storybook-desktop-runtime/src/DesktopStoryRoot.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { StyleSheet, View } from 'react-native'; +import { Platform, StyleSheet, View } from 'react-native'; import { addons } from 'storybook/preview-api'; import { desktopStoryErrorEvent, desktopStoryReadyEvent } from './DesktopDriverBridge'; @@ -13,6 +13,7 @@ type DesktopStoryRootProps = { export function DesktopStoryRoot({ children, storyId }: DesktopStoryRootProps) { const { runtimeInstance, selection } = useDesktopStorybookConfig(); const testID = useDesktopStorybookTestID('story-root'); + const exposeNativeMarker = Platform.OS === 'macos' || Platform.OS === 'windows' || Platform.OS === ('win32' as any); const activeSelection = selection?.storyId === storyId ? selection : undefined; React.useEffect(() => { @@ -49,7 +50,7 @@ export function DesktopStoryRoot({ children, storyId }: DesktopStoryRootProps) { } }} > - + {children} diff --git a/packages/agentic/storybook-desktop/README.md b/packages/agentic/storybook-desktop/README.md index 9c61c7f002a..7442d85b3f5 100644 --- a/packages/agentic/storybook-desktop/README.md +++ b/packages/agentic/storybook-desktop/README.md @@ -101,6 +101,8 @@ platform option, or omit it to use `FURN_STORYBOOK_PLATFORM` and then the host d ```sh storybook-desktop server --win32 +storybook-desktop build-driver --windows +storybook-desktop build-driver --macos storybook-desktop driver --windows storybook-desktop manifest --windows storybook-desktop instance --windows @@ -112,13 +114,22 @@ storybook-desktop smoke --win32 --mode stories storybook-desktop smoke --windows --mode stories-and-tests ``` -Use `--config ` for a differently named configuration file. `prep` installs CocoaPods on macOS, generates the -React Native Test App solution on Windows, and is a no-op for the prebuilt Win32 host. `bundle` generates the selected +Use `--config ` for a differently named configuration file. +`build-driver` builds only the source-shipped native helper. `prep` first +ensures that helper, then installs CocoaPods on macOS or generates the React +Native Test App solution on Windows; Win32 prep now ensures the shared Windows +helper. `bundle` generates the selected story catalog and routes to `rnx-cli bundle`. `build` and `run` route to `rnx-cli` by default using native project names derived from the app manifest. The config supplies default macOS workspace/scheme and Windows solution arguments from the app key, and reads the macOS bundle identifier directly from `app.json`. Win32 has no native project to build, so its default build and run operations are unsupported until the consumer provides a prebuilt-host launch command. +Set `platformOptions.macos.nativeDriver.macosSigningIdentity` to use a stable +macOS code-signing identity. When omitted, the source build uses an ad hoc +signature and TCC permissions may need to be granted again after a rebuild. +The shared standalone macOS `run` command applies the enlistment-specific +xcconfig, matching the bundle identity used by `driver` and `smoke`. + `server` loads the same config, selects the matching platform catalog, and derives the app-owned Storybook config directory automatically. It accepts `--host` and `--port`; the separate `storybook-server` binary is a convenience alias for this subcommand. Consumer package scripts should forward arguments rather than define one server alias per @@ -129,8 +140,9 @@ and agent workflows. `parameters.desktopDriver` plans, then writes exact-platform and portable-plan digests. `instance` prints the enlistment-specific channel, Metro, and driver identity. `driver` starts the Storybook channel/MCP server and the W3C Desktop -Driver listener on separate loopback ports in one Node process. The initial -target uses the deterministic fake host; native providers are a later stage. +Driver listener on separate loopback ports in one Node process. It resolves the +verified native helper before starting Metro and registers a process-backed +target. The deterministic fake host remains test-only. Component authors tag portable plans with `desktop-e2e`. Button, Checkbox, and Input provide the initial examples. Plan extraction evaluates only the inline @@ -161,12 +173,23 @@ the processes it recorded. `createWin32SmokeCommand` bundles and launches the co desktop chrome, resize handles, and addon surface through the configured test-ID prefix, traverses every story, and performs the same ownership-safe cleanup. `--mode stories` is the default renderability gate; `--mode stories-and-tests` performs the same complete traversal and then runs every `desktop-e2e` authored plan through -the Stage 1 manifest-derived fake target. Native plan execution replaces that target when the Stage 2 providers land. +the native provider. Storybook owns app launch and supplies an exact +nonce-bound process lease; WebDriver attaches and preserves the app until the +Storybook lifecycle performs final cleanup. +The reusable macOS lifecycle resolves the launched app by its isolated bundle +identifier and atomically records its PID, start time, executable, and nonce +before creating the attached WebDriver session. Consumers provide only native identity, title, test-ID prefix, and optional required story IDs. The Windows helper also records React Native Test App's Debug Metro port (`8081` by default), while Storybook and Desktop Driver ports remain enlistment-specific. Artifacts are written beneath the consuming app's `artifacts/windows` or `artifacts/win32` directory. +The Windows Fabric lifecycle restarts only its exact owned app after the full +catalog traversal, then rewrites the nonce-bound application lease and runs +authored tests against the warm Metro bundle. This avoids carrying accumulated +Fabric story state into the native test phase while preserving the same server, +ports, and process ownership. + `smoke` can also use a complete consumer command or the generic reusable lifecycle. The generic lifecycle starts the shared channel server and Metro, builds and launches the app, selects every indexed story, runs the configured app stop command, and terminates only the server processes it started. macOS uses the package's bundle-ID-based stop command by diff --git a/packages/agentic/storybook-desktop/config/server.cjs b/packages/agentic/storybook-desktop/config/server.cjs index c8e20128365..8563be5e404 100644 --- a/packages/agentic/storybook-desktop/config/server.cjs +++ b/packages/agentic/storybook-desktop/config/server.cjs @@ -35,34 +35,48 @@ async function startDesktopStorybookServer({ throw new Error(`Desktop Driver manifest does not exist at ${driverManifestPath}.`); } const driverManifest = JSON.parse(fs.readFileSync(driverManifestPath, 'utf8')); - const [ - { createDesktopDriverServer }, - { createFakeStoryWindows, FakeDesktopHost, FakeStoryOrchestrator }, - { StorybookChannelOrchestrator }, - ] = await Promise.all([ - import('@fluentui-react-native/desktop-driver/server'), - import('@fluentui-react-native/desktop-driver/testing'), + if ( + driverManifest.schemaVersion !== 2 || + !driverManifest.nativeDriver || + !driverManifest.application || + typeof driverManifest.application.leasePath !== 'string' || + typeof driverManifest.application.leaseNonce !== 'string' + ) { + throw new Error(`Invalid native Desktop Driver manifest at ${driverManifestPath}.`); + } + const [{ NativeDesktopHost, createDesktopDriverServer }, { StorybookChannelOrchestrator }] = await Promise.all([ + import('@fluentui-react-native/desktop-driver'), import('../lib/driver/index.js'), ]); if (!server) { throw new Error('Desktop Driver Storybook orchestration requires WebSockets.'); } - const targetHost = new FakeDesktopHost({ - endpoint: driverManifest.endpoint, - platformName: driverManifest.endpoint === 'macos' ? 'macos' : 'windows', - storyRootTestId: `${driverManifest.testIDPrefix}-story-root`, - windows: createFakeStoryWindows(driverManifest.storyManifest, `${driverManifest.testIDPrefix}-story-root`), - }); - const orchestrator = - process.env.STORYBOOK_SMOKE_MODE === 'stories-and-tests' - ? new FakeStoryOrchestrator(driverManifest.storyManifest, targetHost) - : createChannelOrchestrator({ - channelServer: server, - driverManifest, - serverUrl: loopbackUrl(host, port), - targetHost, - StorybookChannelOrchestrator, - }); + let targetHost; + let orchestrator; + if (process.env.STORYBOOK_NATIVE_DRIVER_FAKE === '1') { + const { createFakeStoryWindows, FakeDesktopHost, FakeStoryOrchestrator } = + await import('@fluentui-react-native/desktop-driver/testing'); + targetHost = new FakeDesktopHost({ + endpoint: driverManifest.endpoint, + platformName: driverManifest.endpoint === 'macos' ? 'macos' : 'windows', + storyRootTestId: `${driverManifest.testIDPrefix}-story-root`, + windows: createFakeStoryWindows(driverManifest.storyManifest, `${driverManifest.testIDPrefix}-story-root`), + }); + orchestrator = new FakeStoryOrchestrator(driverManifest.storyManifest, targetHost); + } else { + targetHost = new NativeDesktopHost({ + application: driverManifest.application, + artifact: driverManifest.nativeDriver, + endpoint: driverManifest.endpoint, + onStderr: (message) => process.stderr.write(`[desktop-driver-native] ${message}`), + }); + orchestrator = createChannelOrchestrator({ + channelServer: server, + driverManifest, + serverUrl: loopbackUrl(host, port), + StorybookChannelOrchestrator, + }); + } driver = await createDesktopDriverServer({ host, port: driverManifest.driverPort, @@ -91,24 +105,12 @@ async function startDesktopStorybookServer({ return { channelServer: server, driver }; } -function createChannelOrchestrator({ channelServer, driverManifest, serverUrl, targetHost, StorybookChannelOrchestrator }) { - const channelOrchestrator = new StorybookChannelOrchestrator({ +function createChannelOrchestrator({ channelServer, driverManifest, serverUrl, StorybookChannelOrchestrator }) { + return new StorybookChannelOrchestrator({ channelServer, driverManifest, serverUrl, }); - const setStoryMarker = (result) => { - targetHost.resetPreview(); - targetHost.setElementName('story-root', JSON.stringify(result)); - return result; - }; - return { - getManifest: () => channelOrchestrator.getManifest(), - getCurrentStory: () => channelOrchestrator.getCurrentStory(), - selectStory: async (request) => setStoryMarker(await channelOrchestrator.selectStory(request)), - resetStory: async (request) => setStoryMarker(await channelOrchestrator.resetStory(request)), - updateArgs: (storyId, args) => channelOrchestrator.updateArgs(storyId, args), - }; } function loopbackUrl(host, port) { diff --git a/packages/agentic/storybook-desktop/config/smoke-win32.ps1 b/packages/agentic/storybook-desktop/config/smoke-win32.ps1 index 1b6547f6d9d..9097934822d 100644 --- a/packages/agentic/storybook-desktop/config/smoke-win32.ps1 +++ b/packages/agentic/storybook-desktop/config/smoke-win32.ps1 @@ -21,6 +21,7 @@ $cliPath = Join-Path $PSScriptRoot 'cli.cjs' $controlPath = Join-Path $PSScriptRoot 'storybook-control.cjs' $hostPath = Join-Path $PSScriptRoot 'run-win32.cjs' $requiredStories = @($RequiredStoryIds -split ',' | Where-Object { $_ }) +$applicationLeasePath = $null if ($SmokeMode -eq 'stories-and-tests' -and (-not $driverPort -or -not $env:STORYBOOK_DRIVER_MANIFEST)) { throw 'STORYBOOK_DRIVER_PORT and STORYBOOK_DRIVER_MANIFEST are required for stories-and-tests smoke mode.' @@ -52,6 +53,39 @@ function Start-OwnedCommand { -RedirectStandardError (Join-Path $logRoot "$LogName.err.log") -PassThru -WindowStyle Hidden } +function Write-ApplicationLease { + param( + [Parameter(Mandatory)] + [System.Diagnostics.Process]$Process, + [Parameter(Mandatory)] + [Microsoft.Management.Infrastructure.CimInstance]$ProcessInfo + ) + + $driverManifest = Get-Content -LiteralPath $env:STORYBOOK_DRIVER_MANIFEST -Raw | ConvertFrom-Json + if ( + $driverManifest.schemaVersion -ne 2 -or + -not $driverManifest.application.leasePath -or + -not $driverManifest.application.leaseNonce + ) { + throw "Invalid native Desktop Driver manifest at '$($env:STORYBOOK_DRIVER_MANIFEST)'." + } + $leasePath = [string]$driverManifest.application.leasePath + $leaseDirectory = Split-Path -Parent $leasePath + New-Item -ItemType Directory -Path $leaseDirectory -Force | Out-Null + $temporaryPath = "$leasePath.$PID.tmp" + @{ + schemaVersion = 1 + endpoint = 'win32' + nonce = [string]$driverManifest.application.leaseNonce + processId = $Process.Id + processStartedAt = $Process.StartTime.ToUniversalTime().ToString('o') + windowTitle = $WindowTitle + executablePath = [string]$ProcessInfo.ExecutablePath + } | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $temporaryPath -Encoding utf8NoBOM + Move-Item -LiteralPath $temporaryPath -Destination $leasePath -Force + return $leasePath +} + function Wait-ForTcpPort { param( [Parameter(Mandatory)] @@ -192,6 +226,9 @@ try { $appProcessInfo = Get-CimInstance Win32_Process -Filter "ProcessId=$($appProcess.Id)" Add-OwnedProcess -Id $appProcessInfo.ParentProcessId Add-OwnedProcess -Id $appProcess.Id + if ($SmokeMode -eq 'stories-and-tests') { + $applicationLeasePath = Write-ApplicationLease -Process $appProcess -ProcessInfo $appProcessInfo + } Wait-ForAutomationId -Process $appProcess -AutomationId (StorybookId 'sidebar-header') Wait-ForAutomationId -Process $appProcess -AutomationId (StorybookId 'sidebar-resize') @@ -211,6 +248,9 @@ try { throw 'The Win32 Storybook host exited during the smoke test.' } } finally { + if ($applicationLeasePath) { + Remove-Item -LiteralPath $applicationLeasePath -Force -ErrorAction SilentlyContinue + } for ($index = $ownedProcessIds.Count - 1; $index -ge 0; $index -= 1) { $ownedProcessId = $ownedProcessIds[$index] if ($ownedProcessId -and $ownedProcessId -ne $PID) { diff --git a/packages/agentic/storybook-desktop/config/smoke-windows.ps1 b/packages/agentic/storybook-desktop/config/smoke-windows.ps1 index 7651fb3f2e9..1b09523935e 100644 --- a/packages/agentic/storybook-desktop/config/smoke-windows.ps1 +++ b/packages/agentic/storybook-desktop/config/smoke-windows.ps1 @@ -21,6 +21,7 @@ $metroPort = if ($env:RCT_METRO_PORT) { [int]$env:RCT_METRO_PORT } else { 8081 } $driverPort = if ($env:STORYBOOK_DRIVER_PORT) { [int]$env:STORYBOOK_DRIVER_PORT } else { 0 } $cliPath = Join-Path $PSScriptRoot 'cli.cjs' $controlPath = Join-Path $PSScriptRoot 'storybook-control.cjs' +$applicationLeasePath = $null if ($SmokeMode -eq 'stories-and-tests' -and (-not $driverPort -or -not $env:STORYBOOK_DRIVER_MANIFEST)) { throw 'STORYBOOK_DRIVER_PORT and STORYBOOK_DRIVER_MANIFEST are required for stories-and-tests smoke mode.' @@ -107,6 +108,40 @@ function Invoke-DesktopCli { } } +function Write-ApplicationLease { + param( + [Parameter(Mandatory)] + [System.Diagnostics.Process]$Process, + [Parameter(Mandatory)] + [string]$LaunchTarget + ) + + $driverManifest = Get-Content -LiteralPath $env:STORYBOOK_DRIVER_MANIFEST -Raw | ConvertFrom-Json + if ( + $driverManifest.schemaVersion -ne 2 -or + -not $driverManifest.application.leasePath -or + -not $driverManifest.application.leaseNonce + ) { + throw "Invalid native Desktop Driver manifest at '$($env:STORYBOOK_DRIVER_MANIFEST)'." + } + $leasePath = [string]$driverManifest.application.leasePath + $leaseDirectory = Split-Path -Parent $leasePath + New-Item -ItemType Directory -Path $leaseDirectory -Force | Out-Null + $temporaryPath = "$leasePath.$PID.tmp" + $aumid = $LaunchTarget -replace '^shell:AppsFolder\\', '' + @{ + schemaVersion = 1 + endpoint = 'windows' + nonce = [string]$driverManifest.application.leaseNonce + processId = $Process.Id + processStartedAt = $Process.StartTime.ToUniversalTime().ToString('o') + windowTitle = $WindowTitle + aumid = $aumid + } | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $temporaryPath -Encoding utf8NoBOM + Move-Item -LiteralPath $temporaryPath -Destination $leasePath -Force + return $leasePath +} + function Install-WindowsFrameworkPackage { param( [Parameter(Mandatory)] @@ -185,6 +220,48 @@ function Add-OwnedProcess { } } +function Start-StorybookApp { + $existingAppIds = @(Get-Process -Name ReactApp -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Id) + Start-Process explorer.exe $launchTarget + $process = Wait-ForApp -ExcludedProcessIds $existingAppIds + Add-OwnedProcess -Id $process.Id + return $process +} + +function Stop-StorybookApp { + param([Parameter(Mandatory)][System.Diagnostics.Process]$Process) + + if (Get-Process -Id $Process.Id -ErrorAction SilentlyContinue) { + Stop-Process -Id $Process.Id -ErrorAction Stop + } + $deadline = (Get-Date).AddSeconds(30) + while ((Get-Date) -lt $deadline) { + if (-not (Get-Process -Id $Process.Id -ErrorAction SilentlyContinue)) { + return + } + Start-Sleep -Milliseconds 250 + } + throw "Timed out stopping owned Windows Storybook process $($Process.Id)." +} + +function Invoke-StorybookControl { + param( + [Parameter(Mandatory)] + [ValidateSet('stories', 'tests')] + [string]$Phase, + [Parameter(Mandatory)] + [System.Diagnostics.Process]$Process + ) + + & node $controlPath --phase $Phase + if ($LASTEXITCODE -ne 0) { + if (-not (Get-Process -Id $Process.Id -ErrorAction SilentlyContinue)) { + throw "Windows Storybook host terminated during the '$Phase' smoke phase." + } + throw "Windows Storybook '$Phase' smoke phase failed with exit code $LASTEXITCODE." + } +} + try { if (Get-NetTCPConnection -State Listen -LocalPort $storybookPort -ErrorAction SilentlyContinue) { throw "Storybook port $storybookPort is already in use." @@ -194,7 +271,11 @@ try { } Invoke-DesktopCli -Arguments @('bundle', '--windows') - Invoke-DesktopCli -Arguments @('prep', '--windows') + $prepArguments = @('prep', '--windows') + if ($SmokeMode -eq 'stories') { + $prepArguments += '--no-driver' + } + Invoke-DesktopCli -Arguments $prepArguments $serverLauncher = Start-OwnedCommand -FilePath 'node' ` -ArgumentList @($cliPath, 'server', '--windows', '--port', [string]$storybookPort) -LogName 'storybook-server' @@ -215,21 +296,21 @@ try { Wait-ForTcpPort -Port $metroPort Add-OwnedProcess -Id (Get-PortOwner -Port $metroPort) - $existingAppIds = @(Get-Process -Name ReactApp -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Id) - Start-Process explorer.exe $launchTarget - $appProcess = Wait-ForApp -ExcludedProcessIds $existingAppIds - Add-OwnedProcess -Id $appProcess.Id - $env:STORYBOOK_WS_PORT = [string]$storybookPort $env:STORYBOOK_SMOKE_FAIL_FAST = '1' - & node $controlPath - if ($LASTEXITCODE -ne 0) { - if (-not (Get-Process -Id $appProcess.Id -ErrorAction SilentlyContinue)) { - throw 'Windows Storybook host terminated during smoke validation.' - } - throw "Windows Storybook smoke validation failed with exit code $LASTEXITCODE." + + $appProcess = Start-StorybookApp + Invoke-StorybookControl -Phase 'stories' -Process $appProcess + if ($SmokeMode -eq 'stories-and-tests') { + Stop-StorybookApp -Process $appProcess + $appProcess = Start-StorybookApp + $applicationLeasePath = Write-ApplicationLease -Process $appProcess -LaunchTarget $launchTarget + Invoke-StorybookControl -Phase 'tests' -Process $appProcess } } finally { + if ($applicationLeasePath) { + Remove-Item -LiteralPath $applicationLeasePath -Force -ErrorAction SilentlyContinue + } for ($index = $ownedProcessIds.Count - 1; $index -ge 0; $index -= 1) { $ownedProcessId = $ownedProcessIds[$index] if ($ownedProcessId -and $ownedProcessId -ne $PID) { diff --git a/packages/agentic/storybook-desktop/config/storybook-control.cjs b/packages/agentic/storybook-desktop/config/storybook-control.cjs index 87f6203d42b..0ecc27c7ee5 100644 --- a/packages/agentic/storybook-desktop/config/storybook-control.cjs +++ b/packages/agentic/storybook-desktop/config/storybook-control.cjs @@ -5,6 +5,17 @@ const { pathToFileURL } = require('node:url'); const host = process.env.STORYBOOK_WS_HOST || '127.0.0.1'; const port = Number(process.env.STORYBOOK_WS_PORT) || 7007; const smokeMode = process.env.STORYBOOK_SMOKE_MODE || 'stories'; +const smokePhase = parseSmokePhase(process.argv.slice(2)); + +function parseSmokePhase(args) { + if (args.length === 0) { + return 'all'; + } + if (args.length === 2 && args[0] === '--phase' && ['stories', 'tests'].includes(args[1])) { + return args[1]; + } + throw new Error(`Unsupported Storybook control arguments: ${args.join(' ')}`); +} function baseUrl() { // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- native Storybook services are loopback-only @@ -39,39 +50,40 @@ async function smoke() { if (smokeMode !== 'stories' && smokeMode !== 'stories-and-tests') { throw new Error(`Unsupported Storybook smoke mode "${smokeMode}".`); } + if (smokePhase !== 'tests') { + const index = await request('/index.json'); + const entries = Object.values(index.entries || {}).filter(({ type }) => type === 'story'); + if (entries.length === 0) { + throw new Error('The Storybook index did not contain any stories.'); + } - const index = await request('/index.json'); - const entries = Object.values(index.entries || {}).filter(({ type }) => type === 'story'); - if (entries.length === 0) { - throw new Error('The Storybook index did not contain any stories.'); - } - - const settleMilliseconds = Number(process.env.STORYBOOK_SMOKE_SETTLE_MS) || 0; - const failFast = process.env.STORYBOOK_SMOKE_FAIL_FAST === '1'; - const failures = []; + const settleMilliseconds = Number(process.env.STORYBOOK_SMOKE_SETTLE_MS) || 0; + const failFast = process.env.STORYBOOK_SMOKE_FAIL_FAST === '1'; + const failures = []; - for (const { id } of entries) { - try { - await selectStory(id); - if (settleMilliseconds > 0) { - await new Promise((resolve) => setTimeout(resolve, settleMilliseconds)); - } - process.stdout.write(`rendered ${id}\n`); - } catch (error) { - failures.push({ id, error: error.message }); - process.stderr.write(`failed ${id}: ${error.message}\n`); - if (failFast) { - break; + for (const { id } of entries) { + try { + await selectStory(id); + if (settleMilliseconds > 0) { + await new Promise((resolve) => setTimeout(resolve, settleMilliseconds)); + } + process.stdout.write(`rendered ${id}\n`); + } catch (error) { + failures.push({ id, error: error.message }); + process.stderr.write(`failed ${id}: ${error.message}\n`); + if (failFast) { + break; + } } } - } - if (failures.length > 0) { - throw new Error(`${failures.length} of ${entries.length} stories failed to render`); + if (failures.length > 0) { + throw new Error(`${failures.length} of ${entries.length} stories failed to render`); + } + process.stdout.write(`Rendered ${entries.length} stories.\n`); } - process.stdout.write(`Rendered ${entries.length} stories.\n`); - if (smokeMode === 'stories-and-tests') { + if (smokeMode === 'stories-and-tests' && smokePhase !== 'stories') { await runAuthoredTests(); } } @@ -83,7 +95,7 @@ async function runAuthoredTests() { } const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); if ( - manifest.schemaVersion !== 1 || + manifest.schemaVersion !== 2 || !['macos', 'win32', 'windows'].includes(manifest.endpoint) || !Number.isInteger(manifest.driverPort) || typeof manifest.targetId !== 'string' diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts index db2f1c54023..ae4d0203fd5 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts @@ -1,15 +1,41 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; +import type { NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; + import { STORYBOOK_SMOKE_MODE } from '../config/commands'; import { makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig'; import { FURN_STORYBOOK_BUNDLE_IDENTIFIER, FURN_STORYBOOK_INSTANCE_ID } from '../config/instance'; import { FURN_STORYBOOK_PLATFORM } from '../config/platforms'; import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commandRunner'; import { createDesktopStorybookCommand } from './createDesktopStorybookCommand'; -import { DesktopStorybookCli } from './DesktopStorybookCli'; +import { DesktopStorybookCli, parseMacOSRunningApplications, writeMacOSApplicationLeaseFile } from './DesktopStorybookCli'; const storybookRoot = path.resolve(__dirname, '../../../../../apps/storybook'); +const nativeDriverArtifact: NativeDriverArtifact = { + architecture: 'x64', + artifactId: 'artifact', + artifactRoot: 'artifact-root', + buildFingerprint: 'build', + buildId: 'build-id', + compatibilityKey: 'compatibility', + configuration: 'release', + endpoints: ['windows', 'win32', 'macos'], + executablePath: 'driver.exe', + features: ['probe'], + origin: 'cache', + provider: 'windows', + schemaVersion: 1, + signing: { mode: 'none' }, + sourceDigest: 'source', + wireProtocol: { major: 1, minor: 0 }, +}; +const nativeDriverTestOptions = { + buildNativeDriver: async () => nativeDriverArtifact, + resolveNativeDriver: async () => nativeDriverArtifact, + writeMacOSApplicationLease: async () => undefined, +}; const createEmptyStoryManifest = async (_config: unknown, platform: 'macos' | 'windows' | 'win32') => ({ endpoint: platform, entries: [], @@ -18,6 +44,62 @@ const createEmptyStoryManifest = async (_config: unknown, platform: 'macos' | 'w schemaVersion: 1 as const, }); +describe('macOS application leases', () => { + test('validates exact running application records and rejects ambiguity', () => { + expect( + parseMacOSRunningApplications( + JSON.stringify([{ executablePath: '/Applications/Storybook.app/Contents/MacOS/Storybook', processId: 42, processStartedAt: 1 }]), + 'com.example.storybook', + ), + ).toEqual([{ executablePath: '/Applications/Storybook.app/Contents/MacOS/Storybook', processId: 42, processStartedAt: 1 }]); + expect(() => + parseMacOSRunningApplications( + JSON.stringify([ + { executablePath: '/Applications/First.app/Contents/MacOS/First', processId: 1, processStartedAt: 1 }, + { executablePath: '/Applications/Second.app/Contents/MacOS/Second', processId: 2, processStartedAt: 2 }, + ]), + 'com.example.storybook', + ), + ).toThrow('More than one running application'); + expect(() => + parseMacOSRunningApplications( + JSON.stringify([{ executablePath: null, processId: 42, processStartedAt: null }]), + 'com.example.storybook', + ), + ).toThrow('does not expose an executable path'); + }); + + test('writes an atomic owner-only lease', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-storybook-macos-lease-')); + const leasePath = path.join(root, 'state', 'application.json'); + try { + writeMacOSApplicationLeaseFile( + { + bundleIdentifier: 'com.example.storybook', + leaseNonce: 'nonce', + leasePath, + }, + { + executablePath: '/Applications/Storybook.app/Contents/MacOS/Storybook', + processId: 42, + processStartedAt: 1, + }, + ); + expect(JSON.parse(fs.readFileSync(leasePath, 'utf8'))).toMatchObject({ + bundleIdentifier: 'com.example.storybook', + nonce: 'nonce', + processId: 42, + schemaVersion: 1, + }); + if (process.platform !== 'win32') { + expect(fs.statSync(leasePath).mode & 0o777).toBe(0o600); + } + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } + }); +}); + class RecordingRunner implements DesktopCommandRunner { readonly foreground: PreparedDesktopCommand[] = []; readonly background: PreparedDesktopCommand[] = []; @@ -53,7 +135,7 @@ function makeConfig(platformOptions = {}) { describe('DesktopStorybookCli', () => { test('starts the config-owned server with platform and connection options', async () => { const runner = new RecordingRunner(); - const cli = new DesktopStorybookCli(makeConfig(), { runner }); + const cli = new DesktopStorybookCli(makeConfig(), { ...nativeDriverTestOptions, runner }); await cli.server('win32', { host: '0.0.0.0', port: 7100 }); @@ -72,7 +154,7 @@ describe('DesktopStorybookCli', () => { test('uses shared preparation and rnx-cli bundle defaults', async () => { const runner = new RecordingRunner(); - const cli = new DesktopStorybookCli(makeConfig(), { runner }); + const cli = new DesktopStorybookCli(makeConfig(), { ...nativeDriverTestOptions, runner }); await cli.prep('macos'); await cli.bundle('win32'); @@ -99,7 +181,7 @@ describe('DesktopStorybookCli', () => { test('uses native project defaults and rejects an unconfigured Win32 build', async () => { const runner = new RecordingRunner(); - const cli = new DesktopStorybookCli(makeConfig(), { runner }); + const cli = new DesktopStorybookCli(makeConfig(), { ...nativeDriverTestOptions, runner }); await cli.run('windows'); @@ -110,6 +192,25 @@ describe('DesktopStorybookCli', () => { await expect(cli.build('win32')).rejects.toThrow('build is not configured for win32'); }); + test('runs the standalone macOS app with the enlistment-specific identity', async () => { + const runner = new RecordingRunner(); + const cli = new DesktopStorybookCli(makeConfig({ macos: { run: { command: 'launch-storybook' } } }), { + ...nativeDriverTestOptions, + runner, + }); + + await cli.run('macos'); + + expect(runner.foreground[0]).toMatchObject({ + command: 'launch-storybook', + env: { + [FURN_STORYBOOK_BUNDLE_IDENTIFIER]: cli.instance.bundleIdentifier, + [FURN_STORYBOOK_INSTANCE_ID]: cli.instance.id, + XCODE_XCCONFIG_FILE: path.join(storybookRoot, 'macos', '.storybook-desktop', `${cli.instance.id}.xcconfig`), + }, + }); + }); + test('always runs app and process cleanup when the reusable smoke lifecycle fails', async () => { const runner = new RecordingRunner(); runner.failCommand = 'launch-storybook'; @@ -123,6 +224,7 @@ describe('DesktopStorybookCli', () => { }, }), { + ...nativeDriverTestOptions, createStoryManifest: createEmptyStoryManifest, runner, fetch: jest.fn(async () => new Response('{}')), @@ -145,6 +247,7 @@ describe('DesktopStorybookCli', () => { test('renders every indexed story before cleanup', async () => { const runner = new RecordingRunner(); + const resolveNativeDriver = jest.fn(async () => nativeDriverArtifact); const output: string[] = []; const fetch = jest.fn(async (input: Parameters[0]) => { const url = input.toString(); @@ -172,6 +275,7 @@ describe('DesktopStorybookCli', () => { }, }), { + ...nativeDriverTestOptions, createStoryManifest: createEmptyStoryManifest, runner, fetch, @@ -182,6 +286,7 @@ describe('DesktopStorybookCli', () => { return true; }, }, + resolveNativeDriver, }, ); blockedPorts.add(cli.instance.storybookPort); @@ -201,6 +306,7 @@ describe('DesktopStorybookCli', () => { expect(runner.background[1].args).toEqual(['start', '--no-interactive', '--port', String(cli.instance.metroPort + 1)]); expect(runner.foreground.at(-1)?.command).toBe('stop-storybook'); expect(runner.stopped).toBe(2); + expect(resolveNativeDriver).not.toHaveBeenCalled(); }); test('keeps retrying the first story while the initial Metro bundle is compiling', async () => { @@ -238,6 +344,7 @@ describe('DesktopStorybookCli', () => { }, }), { + ...nativeDriverTestOptions, createStoryManifest: createEmptyStoryManifest, fetch, isPortAvailable: async () => true, @@ -257,7 +364,11 @@ describe('DesktopStorybookCli', () => { test('runs authored tests after traversing the complete story index', async () => { const runner = new RecordingRunner(); + const resolveNativeDriver = jest.fn(async () => nativeDriverArtifact); const events: string[] = []; + const writeMacOSApplicationLease = jest.fn(async () => { + events.push('lease'); + }); const fetch = jest.fn(async (input: Parameters[0]) => { const url = input.toString(); if (url.endsWith('/index.json')) { @@ -303,17 +414,24 @@ describe('DesktopStorybookCli', () => { }, }), { + ...nativeDriverTestOptions, createStoryManifest: createEmptyStoryManifest, fetch, isPortAvailable: async () => true, runSmokeTests, runner, + resolveNativeDriver, + writeMacOSApplicationLease, }, ); await cli.smoke('macos', { mode: 'stories-and-tests' }); - expect(events).toEqual(['select:first--story', 'select:second--story', 'tests']); + expect(events).toEqual(['lease', 'select:first--story', 'select:second--story', 'tests']); + expect(writeMacOSApplicationLease).toHaveBeenCalledWith( + path.join(storybookRoot, 'storybook-desktop.generated', 'driver-manifest.macos.json'), + 120_000, + ); expect(runSmokeTests).toHaveBeenCalledWith({ // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- the test exercises the loopback Desktop Driver driverUrl: `http://127.0.0.1:${cli.instance.driverPort}`, @@ -328,13 +446,47 @@ describe('DesktopStorybookCli', () => { expect(runner.background[0].env).toMatchObject({ [STORYBOOK_SMOKE_MODE]: 'stories-and-tests', }); + expect(resolveNativeDriver).toHaveBeenCalledWith({ + buildPolicy: undefined, + cacheRoot: undefined, + configuration: 'release', + helperPath: undefined, + installRoot: undefined, + macosSigningIdentity: undefined, + platform: 'macos', + }); }); }); describe('createDesktopStorybookCommand', () => { + test.each(['windows', 'win32'] as const)('builds the %s native helper without running app preparation', async (platform) => { + const runner = new RecordingRunner(); + const buildNativeDriver = jest.fn(async () => nativeDriverArtifact); + const program = createDesktopStorybookCommand({ + ...nativeDriverTestOptions, + buildNativeDriver, + config: makeConfig(), + output: { write: () => true }, + runner, + }); + + await program.parseAsync(['node', 'test', 'build-driver', `--${platform}`, '--force']); + + expect(buildNativeDriver).toHaveBeenCalledWith({ + cacheRoot: undefined, + configuration: 'release', + force: true, + macosSigningIdentity: undefined, + platform, + }); + expect(runner.foreground).toEqual([]); + expect(runner.background).toEqual([]); + }); + test('forwards the selected smoke mode and isolated instance to a package-owned lifecycle', async () => { const runner = new RecordingRunner(); const program = createDesktopStorybookCommand({ + ...nativeDriverTestOptions, config: makeConfig({ windows: { smoke: { @@ -370,6 +522,7 @@ describe('createDesktopStorybookCommand', () => { test('starts the channel and embedded driver from one server command', async () => { const runner = new RecordingRunner(); const program = createDesktopStorybookCommand({ + ...nativeDriverTestOptions, config: makeConfig(), createStoryManifest: createEmptyStoryManifest, isPortAvailable: async () => true, @@ -404,6 +557,7 @@ describe('createDesktopStorybookCommand', () => { const secondRunner = new RecordingRunner(); const secondProgram = createDesktopStorybookCommand({ + ...nativeDriverTestOptions, config: makeConfig(), createStoryManifest: createEmptyStoryManifest, isPortAvailable: async () => true, @@ -419,7 +573,7 @@ describe('createDesktopStorybookCommand', () => { test('forwards server platform and connection options', async () => { const runner = new RecordingRunner(); - const program = createDesktopStorybookCommand({ config: makeConfig(), runner }); + const program = createDesktopStorybookCommand({ ...nativeDriverTestOptions, config: makeConfig(), runner }); await program.parseAsync(['node', 'test', 'server', '--win32', '--host', 'localhost', '--port', '7101']); @@ -435,7 +589,7 @@ describe('createDesktopStorybookCommand', () => { test('honors an explicit platform flag', async () => { const runner = new RecordingRunner(); - const program = createDesktopStorybookCommand({ config: makeConfig(), runner }); + const program = createDesktopStorybookCommand({ ...nativeDriverTestOptions, config: makeConfig(), runner }); await program.parseAsync(['node', 'test', 'bundle', '--windows']); @@ -449,7 +603,7 @@ describe('createDesktopStorybookCommand', () => { const previousPlatform = process.env[FURN_STORYBOOK_PLATFORM]; process.env[FURN_STORYBOOK_PLATFORM] = 'win32'; const runner = new RecordingRunner(); - const program = createDesktopStorybookCommand({ config: makeConfig(), runner }); + const program = createDesktopStorybookCommand({ ...nativeDriverTestOptions, config: makeConfig(), runner }); try { await program.parseAsync(['node', 'test', 'bundle']); @@ -465,7 +619,11 @@ describe('createDesktopStorybookCommand', () => { }); test('rejects multiple platform flags', async () => { - const program = createDesktopStorybookCommand({ config: makeConfig(), runner: new RecordingRunner() }); + const program = createDesktopStorybookCommand({ + ...nativeDriverTestOptions, + config: makeConfig(), + runner: new RecordingRunner(), + }); program.exitOverride(); program.configureOutput({ writeErr: () => {} }); program.commands.forEach((command) => { diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts index 52ae62ca62a..2143e5a59f0 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts @@ -1,7 +1,15 @@ +import { spawn } from 'node:child_process'; import fs from 'node:fs'; import { createServer } from 'node:net'; import path from 'node:path'; +import { + buildNativeDesktopDriver, + resolveNativeDesktopDriver, + type NativeDriverArtifact, + type NativeDriverBuildOptions, +} from '@fluentui-react-native/desktop-driver'; + import { desktopSmokeModes, STORYBOOK_SMOKE_MODE, @@ -32,13 +40,24 @@ type ResolvedDesktopStorybookInstance = DesktopStorybookInstance & { macosXcconfigPath?: string; }; +export type DesktopStorybookBuildDriverOptions = { + force?: boolean; +}; + +export type DesktopStorybookPrepOptions = { + driver?: boolean; +}; + export type DesktopStorybookCliOptions = { + buildNativeDriver?: typeof buildNativeDesktopDriver; createStoryManifest?: typeof createDesktopStoryManifest; runner?: DesktopCommandRunner; fetch?: typeof globalThis.fetch; output?: Pick; isPortAvailable?: (port: number) => Promise; runSmokeTests?: typeof runDesktopStorybookSmokeTests; + resolveNativeDriver?: typeof resolveNativeDesktopDriver; + writeMacOSApplicationLease?: typeof writeMacOSApplicationLease; }; export type DesktopStorybookServerOptions = { @@ -51,11 +70,14 @@ export class DesktopStorybookCli { readonly instance: DesktopStorybookInstance; private readonly runner: DesktopCommandRunner; + private readonly buildNativeDriver: typeof buildNativeDesktopDriver; private readonly createStoryManifest: typeof createDesktopStoryManifest; private readonly fetch: typeof globalThis.fetch; private readonly output: Pick; private readonly isPortAvailable: (port: number) => Promise; private readonly runSmokeTests: typeof runDesktopStorybookSmokeTests; + private readonly resolveNativeDriver: typeof resolveNativeDesktopDriver; + private readonly writeMacOSApplicationLease: typeof writeMacOSApplicationLease; constructor(config: DesktopStorybookConfig, options: DesktopStorybookCliOptions = {}) { this.config = config; @@ -64,11 +86,14 @@ export class DesktopStorybookCli { bundleIdentifierPrefix: config.macosBundleIdentifier, }); this.runner = options.runner ?? new NodeDesktopCommandRunner(); + this.buildNativeDriver = options.buildNativeDriver ?? buildNativeDesktopDriver; this.createStoryManifest = options.createStoryManifest ?? createDesktopStoryManifest; this.fetch = options.fetch ?? globalThis.fetch; this.output = options.output ?? process.stdout; this.isPortAvailable = options.isPortAvailable ?? isLoopbackPortAvailable; this.runSmokeTests = options.runSmokeTests ?? runDesktopStorybookSmokeTests; + this.resolveNativeDriver = options.resolveNativeDriver ?? resolveNativeDesktopDriver; + this.writeMacOSApplicationLease = options.writeMacOSApplicationLease ?? writeMacOSApplicationLease; } async server(platform: Platforms, options: DesktopStorybookServerOptions = {}): Promise { @@ -97,6 +122,7 @@ export class DesktopStorybookCli { if (command === false || command === undefined) { throw unsupportedAction('server', platform); } + const nativeDriver = await this.resolveDriver(platform); const storybookPort = options.port ?? (await findAvailablePort(this.instance.storybookPort, this.isPortAvailable)); const metroPort = await findAvailablePort(this.instance.metroPort, this.isPortAvailable, new Set([storybookPort])); const driverPort = await findAvailablePort(this.instance.driverPort, this.isPortAvailable, new Set([storybookPort, metroPort])); @@ -106,7 +132,7 @@ export class DesktopStorybookCli { metroPort, storybookPort, }; - resolvedInstance.driverManifestPath = await this.writeDriverManifest(platform, resolvedInstance); + resolvedInstance.driverManifestPath = await this.writeDriverManifest(platform, resolvedInstance, nativeDriver); this.output.write(`Storybook instance ${resolvedInstance.id}: channel=${storybookPort}, metro=${metroPort}, driver=${driverPort}\n`); const metro = this.runner.start(this.prepareCommand(defaultMetroCommand(resolvedInstance), platform, resolvedInstance)); try { @@ -152,8 +178,21 @@ export class DesktopStorybookCli { ); } - prep(platform: Platforms): Promise { - return this.executeAction('prep', platform); + async buildDriver(platform: Platforms, options: DesktopStorybookBuildDriverOptions = {}): Promise { + const nativeOptions = this.requireNativeDriverOptions(platform); + const artifact = await this.buildNativeDriver({ + ...this.toBuildOptions(platform, nativeOptions), + force: options.force, + }); + this.output.write(`${JSON.stringify(artifact, null, 2)}\n`); + return artifact; + } + + async prep(platform: Platforms, options: DesktopStorybookPrepOptions = {}): Promise { + if (options.driver !== false) { + await this.resolveDriver(platform); + } + await this.executeAction('prep', platform); } bundle(platform: Platforms): Promise { @@ -161,7 +200,14 @@ export class DesktopStorybookCli { } run(platform: Platforms): Promise { - return this.executeAction('run', platform); + if (platform !== 'macos') { + return this.executeAction('run', platform); + } + const instance: ResolvedDesktopStorybookInstance = { + ...this.instance, + macosXcconfigPath: writeMacOSInstanceConfig(this.config.projectRoot, this.instance), + }; + return this.executeAction('run', platform, instance); } build(platform: Platforms): Promise { @@ -175,7 +221,7 @@ export class DesktopStorybookCli { throw unsupportedAction('smoke', platform); } if (smoke?.command) { - const instance = await this.resolveSmokeInstance(platform, smoke); + const instance = await this.resolveSmokeInstance(platform, smoke, mode); await this.executePlan(withEnvironment(smoke.command, { [STORYBOOK_SMOKE_MODE]: mode }), platform, instance); return; } @@ -185,12 +231,12 @@ export class DesktopStorybookCli { ); } - const instance = await this.resolveSmokeInstance(platform, smoke); + const instance = await this.resolveSmokeInstance(platform, smoke, mode); await this.runSmokeLifecycle(platform, smoke, instance, mode); } private async executeAction( - action: Exclude, + action: Exclude, platform: Platforms, instance?: ResolvedDesktopStorybookInstance, ): Promise { @@ -249,6 +295,12 @@ export class DesktopStorybookCli { await Promise.all(readiness); await this.executeAction('run', platform, instance); + if (mode === 'stories-and-tests' && platform === 'macos') { + if (!instance.driverManifestPath) { + throw new Error('The macOS authored-test lifecycle did not create a Desktop Driver manifest.'); + } + await this.writeMacOSApplicationLease(instance.driverManifestPath, smoke.startupTimeoutMs ?? 120_000); + } await this.renderEveryStory(serverUrl, smoke.settleMs ?? 0, smoke.startupTimeoutMs); if (mode === 'stories-and-tests') { const result = await this.runSmokeTests({ @@ -287,7 +339,11 @@ export class DesktopStorybookCli { } } - private async resolveSmokeInstance(platform: Platforms, smoke: DesktopSmokeOptions): Promise { + private async resolveSmokeInstance( + platform: Platforms, + smoke: DesktopSmokeOptions, + mode: DesktopSmokeMode, + ): Promise { const storybookPort = smoke.serverUrl ? portFromUrl(smoke.serverUrl) : await findAvailablePort(this.instance.storybookPort, this.isPortAvailable); @@ -302,7 +358,10 @@ export class DesktopStorybookCli { metroPort, }; - resolvedInstance.driverManifestPath = await this.writeDriverManifest(platform, resolvedInstance); + if (mode === 'stories-and-tests') { + const nativeDriver = await this.resolveDriver(platform); + resolvedInstance.driverManifestPath = await this.writeDriverManifest(platform, resolvedInstance, nativeDriver); + } if (platform === 'macos') { resolvedInstance.macosXcconfigPath = writeMacOSInstanceConfig(this.config.projectRoot, resolvedInstance); } @@ -314,13 +373,18 @@ export class DesktopStorybookCli { return Object.freeze(resolvedInstance); } - private async writeDriverManifest(platform: Platforms, instance: DesktopStorybookInstance): Promise { + private async writeDriverManifest( + platform: Platforms, + instance: DesktopStorybookInstance, + nativeDriver: NativeDriverArtifact, + ): Promise { const storyManifest = await this.createStoryManifest(this.config, platform); const outputPath = path.join(this.config.projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); const driverManifest = createDesktopStorybookDriverManifest({ bridgeNonce: readReusableBridgeNonce(outputPath, instance, storyManifest.platformManifestDigest), config: this.config, instance, + nativeDriver, platform, storyManifest, }); @@ -328,6 +392,37 @@ export class DesktopStorybookCli { return outputPath; } + private async resolveDriver(platform: Platforms): Promise { + const options = this.requireNativeDriverOptions(platform); + return this.resolveNativeDriver({ + ...this.toBuildOptions(platform, options), + buildPolicy: options.buildPolicy, + helperPath: options.helperPath, + installRoot: options.installRoot, + macosSigningIdentity: options.macosSigningIdentity, + }); + } + + private requireNativeDriverOptions(platform: Platforms): Exclude, false> { + const options = this.config.getNativeDriverOptions(platform); + if (options === false) { + throw new Error(`Native Desktop Driver is disabled for ${platform}.`); + } + return options; + } + + private toBuildOptions( + platform: Platforms, + options: Exclude, false>, + ): NativeDriverBuildOptions { + return { + cacheRoot: options.cacheRoot, + configuration: options.configuration, + macosSigningIdentity: options.macosSigningIdentity, + platform, + }; + } + private async renderEveryStory(serverUrl: string, settleMs: number, startupTimeoutMs = 120_000): Promise { const storyIndex = await this.getJson(new URL('/index.json', serverUrl)); const entries = Object.values((storyIndex.entries ?? {}) as Record).filter( @@ -511,6 +606,166 @@ function writeMacOSInstanceConfig(projectRoot: string, instance: DesktopStoryboo return xcconfigPath; } +type MacOSRunningApplication = { + executablePath: string; + processId: number; + processStartedAt: number; +}; + +export async function writeMacOSApplicationLease(manifestPath: string, timeoutMs = 120_000): Promise { + if (process.platform !== 'darwin') { + throw new Error('A macOS application lease can only be written on macOS.'); + } + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as { + application?: { + bundleIdentifier?: string; + leaseNonce?: string; + leasePath?: string; + }; + }; + const application = manifest.application; + if (!application?.bundleIdentifier || !application.leaseNonce || !application.leasePath) { + throw new Error(`The Desktop Driver manifest at "${manifestPath}" does not contain a complete macOS application descriptor.`); + } + + const runningApplication = await waitForMacOSApplication(application.bundleIdentifier, timeoutMs); + writeMacOSApplicationLeaseFile( + { + bundleIdentifier: application.bundleIdentifier, + leaseNonce: application.leaseNonce, + leasePath: application.leasePath, + }, + runningApplication, + ); +} + +export function writeMacOSApplicationLeaseFile( + application: { bundleIdentifier: string; leaseNonce: string; leasePath: string }, + runningApplication: MacOSRunningApplication, +): void { + validateMacOSRunningApplication(runningApplication); + const processStartedAt = new Date(runningApplication.processStartedAt * 1000); + const temporaryPath = `${application.leasePath}.${process.pid}.tmp`; + fs.mkdirSync(path.dirname(application.leasePath), { mode: 0o700, recursive: true }); + try { + fs.writeFileSync( + temporaryPath, + `${JSON.stringify( + { + bundleIdentifier: application.bundleIdentifier, + endpoint: 'macos', + executablePath: runningApplication.executablePath, + nonce: application.leaseNonce, + processId: runningApplication.processId, + processStartedAt: processStartedAt.toISOString(), + schemaVersion: 1, + }, + null, + 2, + )}\n`, + { mode: 0o600 }, + ); + fs.renameSync(temporaryPath, application.leasePath); + fs.chmodSync(application.leasePath, 0o600); + } finally { + fs.rmSync(temporaryPath, { force: true }); + } +} + +async function waitForMacOSApplication(bundleIdentifier: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + let lastError: unknown; + do { + let output: string; + try { + output = await runProcess('/usr/bin/osascript', [ + '-l', + 'JavaScript', + '-e', + [ + "ObjC.import('AppKit');", + 'function run(argv) {', + ' const applications = $.NSRunningApplication.runningApplicationsWithBundleIdentifier(argv[0]);', + ' const result = [];', + ' for (let index = 0; index < applications.count; index += 1) {', + ' const application = applications.objectAtIndex(index);', + ' const executableURL = application.executableURL;', + ' const launchDate = application.launchDate;', + ' result.push({', + ' executablePath: executableURL ? ObjC.unwrap(executableURL.path) : null,', + ' processId: Number(application.processIdentifier),', + ' processStartedAt: launchDate ? Number(launchDate.timeIntervalSince1970) : null,', + ' });', + ' }', + ' return JSON.stringify(result);', + '}', + ].join('\n'), + bundleIdentifier, + ]); + } catch (error) { + lastError = error; + await delay(250); + continue; + } + const applications = parseMacOSRunningApplications(output, bundleIdentifier); + if (applications.length === 1) { + return applications[0]; + } + await delay(250); + } while (Date.now() < deadline); + throw new Error(`Timed out waiting for the macOS application "${bundleIdentifier}"${lastError ? `: ${errorMessage(lastError)}` : '.'}`); +} + +export function parseMacOSRunningApplications(output: string, bundleIdentifier: string): MacOSRunningApplication[] { + const value: unknown = JSON.parse(output); + if (!Array.isArray(value)) { + throw new Error(`The macOS application query for "${bundleIdentifier}" returned a non-array result.`); + } + if (value.length > 1) { + throw new Error(`More than one running application has bundle identifier "${bundleIdentifier}".`); + } + return value.map((entry) => { + if (!entry || typeof entry !== 'object') { + throw new Error(`The macOS application query for "${bundleIdentifier}" returned an invalid application record.`); + } + const application = entry as Partial; + validateMacOSRunningApplication(application); + return application as MacOSRunningApplication; + }); +} + +function validateMacOSRunningApplication(application: Partial): void { + if (typeof application.executablePath !== 'string' || !application.executablePath) { + throw new Error('The running macOS application does not expose an executable path.'); + } + if (!Number.isInteger(application.processId) || (application.processId ?? 0) <= 0) { + throw new Error('The running macOS application does not expose a valid process identifier.'); + } + if (!Number.isFinite(application.processStartedAt) || (application.processStartedAt ?? 0) <= 0) { + throw new Error(`Could not determine the start time for macOS process ${String(application.processId ?? 'unknown')}.`); + } +} + +function runProcess(command: string, args: readonly string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, [...args], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.once('error', reject); + child.once('close', (code) => { + if (code === 0) { + resolve(Buffer.concat(stdout).toString('utf8')); + } else { + reject(new Error(Buffer.concat(stderr).toString('utf8') || `${command} exited with code ${String(code)}.`)); + } + }); + }); +} + async function findAvailablePort( preferredPort: number, isPortAvailable: (port: number) => Promise, diff --git a/packages/agentic/storybook-desktop/src/cli/README.md b/packages/agentic/storybook-desktop/src/cli/README.md index 21968f026cc..671b237e51b 100644 --- a/packages/agentic/storybook-desktop/src/cli/README.md +++ b/packages/agentic/storybook-desktop/src/cli/README.md @@ -60,14 +60,15 @@ root filename. ## Command responsibilities -| Command | Responsibility | -| -------- | ------------------------------------------------------------------------------------------------- | -| `server` | Start the Storybook channel, REST control, and MCP server for the selected catalog. | -| `prep` | Install or generate native prerequisites. Run after checkout and when native dependencies change. | -| `bundle` | Generate the selected story catalog and produce its release JavaScript bundle through `rnx-cli`. | -| `build` | Build the selected native project without launching it. | -| `run` | Build and launch the selected native app or configured prebuilt host. | -| `smoke` | Own the server, Metro, app launch, all-story traversal, optional authored tests, and cleanup. | +| Command | Responsibility | +| -------------- | ------------------------------------------------------------------------------------------------ | +| `server` | Start the Storybook channel, REST control, and MCP server for the selected catalog. | +| `build-driver` | Build only the source-shipped native Desktop Driver helper. | +| `prep` | Ensure the helper, then install or generate native app prerequisites. | +| `bundle` | Generate the selected story catalog and produce its release JavaScript bundle through `rnx-cli`. | +| `build` | Build the selected native project without launching it. | +| `run` | Build and launch the selected native app or configured prebuilt host. | +| `smoke` | Own the server, Metro, app launch, all-story traversal, optional authored tests, and cleanup. | The TypeScript API exposes the same operations through `DesktopStorybookCli`. Use it when a test coordinator needs injected process @@ -85,7 +86,8 @@ yarn start yarn storybook run ``` -Run `yarn storybook prep` before the first native launch and after changes that +Run `yarn storybook build-driver` for an isolated helper build. Run +`yarn storybook prep` before the first native launch and after changes that invalidate generated native projects or dependencies. The `run` command includes the native build; use `build` when launch or deployment is not desired. @@ -125,8 +127,9 @@ yarn storybook smoke --win32 --mode stories-and-tests `stories` is the default and traverses the complete indexed catalog. `stories-and-tests` performs the same traversal and then runs the -component-authored `desktop-e2e` plans against the Stage 1 manifest-derived -fake target; native plan execution begins with the Stage 2 providers. Both modes are preferable to a shell +component-authored `desktop-e2e` plans against the native provider. The +render-only `stories` mode neither resolves a helper nor starts WebDriver. +Both modes are preferable to a shell chain because they own the exact server, Metro, app identity, traversal, test session, and cleanup. Consumer configuration should provide any platform-specific app stop command. For Windows Fabric and the prebuilt REX Win32 host, configure the package-owned commands returned by diff --git a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts index 52fe80ae633..21a9ff54afa 100644 --- a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts +++ b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts @@ -3,7 +3,12 @@ import { Command, Option } from 'commander'; import { desktopSmokeModes, type DesktopSmokeMode } from '../config/commands.js'; import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { Platforms } from '../config/platforms.js'; -import type { DesktopStorybookCliOptions, DesktopStorybookServerOptions } from './DesktopStorybookCli.js'; +import type { + DesktopStorybookBuildDriverOptions, + DesktopStorybookCliOptions, + DesktopStorybookPrepOptions, + DesktopStorybookServerOptions, +} from './DesktopStorybookCli.js'; import { DesktopStorybookCli } from './DesktopStorybookCli.js'; import { loadDesktopStorybookConfig } from './loadConfig.js'; @@ -16,6 +21,8 @@ type PlatformFlags = { type ServerFlags = PlatformFlags & DesktopStorybookServerOptions; type ManifestFlags = PlatformFlags & { out?: string }; type SmokeFlags = PlatformFlags & { mode: DesktopSmokeMode }; +type BuildDriverFlags = PlatformFlags & DesktopStorybookBuildDriverOptions; +type PrepFlags = PlatformFlags & { driver: DesktopStorybookPrepOptions['driver'] }; export type CreateDesktopStorybookCommandOptions = DesktopStorybookCliOptions & { config?: DesktopStorybookConfig; @@ -35,20 +42,23 @@ export function createDesktopStorybookCommand(options: CreateDesktopStorybookCom ).then( (config) => new DesktopStorybookCli(config, { + buildNativeDriver: options.buildNativeDriver, runner: options.runner, createStoryManifest: options.createStoryManifest, fetch: options.fetch, output: options.output, isPortAvailable: options.isPortAvailable, runSmokeTests: options.runSmokeTests, + resolveNativeDriver: options.resolveNativeDriver, }), )); addServerCommand(program, getApi); + addBuildDriverCommand(program, getApi); addDriverCommand(program, getApi); addManifestCommand(program, getApi); addInstanceCommand(program, getApi); - addActionCommand(program, 'prep', 'Prepare native dependencies and generated projects.', getApi); + addPrepCommand(program, getApi); addActionCommand(program, 'bundle', 'Generate stories and create the platform JavaScript bundle.', getApi); addActionCommand(program, 'run', 'Build and launch the native Storybook app.', getApi); addActionCommand(program, 'build', 'Build the native Storybook app without launching it.', getApi); @@ -57,6 +67,28 @@ export function createDesktopStorybookCommand(options: CreateDesktopStorybookCom return program; } +function addPrepCommand(program: Command, getApi: () => Promise): void { + const command = program + .command('prep') + .description('Prepare the native helper, dependencies, and generated projects.') + .option('--no-driver', 'prepare only the native app project'); + addPlatformOptions(command); + command.action(async (flags: PrepFlags) => { + const api = await getApi(); + await api.prep(resolvePlatform(flags, api), { driver: flags.driver }); + }); +} + +function addBuildDriverCommand(program: Command, getApi: () => Promise): void { + const command = program.command('build-driver').description('Build the native Desktop Driver helper without preparing the app.'); + command.option('--force', 'publish a new immutable helper selection'); + addPlatformOptions(command); + command.action(async (flags: BuildDriverFlags) => { + const api = await getApi(); + await api.buildDriver(resolvePlatform(flags, api), { force: flags.force }); + }); +} + function addDriverCommand(program: Command, getApi: () => Promise): void { const command = program .command('driver') @@ -94,7 +126,7 @@ export async function runDesktopStorybookCli(argv: readonly string[] = process.a function addActionCommand( program: Command, - action: 'prep' | 'bundle' | 'run' | 'build', + action: 'bundle' | 'run' | 'build', description: string, getApi: () => Promise, ): void { diff --git a/packages/agentic/storybook-desktop/src/cli/index.ts b/packages/agentic/storybook-desktop/src/cli/index.ts index 5f2c71962f8..bd9b0beca07 100644 --- a/packages/agentic/storybook-desktop/src/cli/index.ts +++ b/packages/agentic/storybook-desktop/src/cli/index.ts @@ -3,7 +3,13 @@ export { runDesktopStorybookCli, type CreateDesktopStorybookCommandOptions, } from './createDesktopStorybookCommand.js'; -export { DesktopStorybookCli, type DesktopStorybookCliOptions, type DesktopStorybookServerOptions } from './DesktopStorybookCli.js'; +export { + DesktopStorybookCli, + type DesktopStorybookBuildDriverOptions, + type DesktopStorybookCliOptions, + type DesktopStorybookPrepOptions, + type DesktopStorybookServerOptions, +} from './DesktopStorybookCli.js'; export { NodeDesktopCommandRunner, type DesktopCommandRunner, diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts index 2913997b47f..b6ac58f0fcd 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts @@ -61,6 +61,7 @@ describe('runDesktopStorybookSmokeTests', () => { ); expect(connect).toHaveBeenCalledWith({ + launchMode: 'attach', platformName: 'windows', targetId: 'storybook-windows', // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- test-only loopback service diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index 2381b028e98..34fb63ad5ae 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -23,6 +23,7 @@ export async function runDesktopStorybookSmokeTests( connect: DesktopStorybookSmokeConnector = connectDesktopWebdriver, ): Promise { const desktop = await connect({ + launchMode: 'attach', platformName: options.platform === 'macos' ? 'macos' : 'windows', targetId: options.targetId, url: options.driverUrl, diff --git a/packages/agentic/storybook-desktop/src/config/commands.ts b/packages/agentic/storybook-desktop/src/config/commands.ts index 09550fda9ce..0acc9411970 100644 --- a/packages/agentic/storybook-desktop/src/config/commands.ts +++ b/packages/agentic/storybook-desktop/src/config/commands.ts @@ -1,9 +1,15 @@ import path from 'node:path'; import { createRequire } from 'node:module'; +import type { + NativeDesktopApplicationDescriptor, + NativeDriverBuildPolicy, + NativeDriverConfiguration, +} from '@fluentui-react-native/desktop-driver'; + import type { Platforms } from './platforms.ts'; -export const desktopStorybookActions = ['server', 'prep', 'bundle', 'run', 'build', 'smoke'] as const; +export const desktopStorybookActions = ['server', 'build-driver', 'prep', 'bundle', 'run', 'build', 'smoke'] as const; export type DesktopStorybookAction = (typeof desktopStorybookActions)[number]; @@ -58,6 +64,16 @@ export type DesktopNativeProjectOptions = { device?: string; }; +export type DesktopNativeDriverOptions = { + application?: Omit; + buildPolicy?: NativeDriverBuildPolicy; + cacheRoot?: string; + configuration?: NativeDriverConfiguration; + helperPath?: string; + installRoot?: string; + macosSigningIdentity?: string; +}; + export type DesktopSmokeOptions = { /** * A complete app-owned smoke command. When set, the reusable server, Metro, traversal, and cleanup lifecycle is skipped. @@ -107,6 +123,11 @@ export type DesktopSmokeOptions = { }; export type DesktopPlatformOptions = { + /** + * Native desktop helper selection and registered application identity. + */ + nativeDriver?: DesktopNativeDriverOptions | false; + nativeProject?: DesktopNativeProjectOptions; /** diff --git a/packages/agentic/storybook-desktop/src/config/index.ts b/packages/agentic/storybook-desktop/src/config/index.ts index 89360a4c967..2727544faf4 100644 --- a/packages/agentic/storybook-desktop/src/config/index.ts +++ b/packages/agentic/storybook-desktop/src/config/index.ts @@ -8,6 +8,7 @@ export { STORYBOOK_SMOKE_MODE, type DesktopCommand, type DesktopCommandPlan, + type DesktopNativeDriverOptions, type DesktopNativeProjectOptions, type DesktopPlatformOptions, type DesktopPlatformOptionsMap, diff --git a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.test.ts b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.test.ts index 7c734ad6746..dcf4c56e28b 100644 --- a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.test.ts +++ b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.test.ts @@ -135,6 +135,51 @@ describe('DesktopStorybookConfig', () => { }); }); + test('resolves native driver paths and application identity from the consuming project', () => { + const config = makeDesktopStorybookConfig({ + projectRoot: storybookRoot, + platformOptions: { + macos: { + nativeDriver: { + macosSigningIdentity: 'Apple Development: Example', + }, + }, + win32: { + nativeDriver: { + application: { + executablePath: 'tools/ReactTest.exe', + windowTitle: 'Consumer Storybook (Win32)', + }, + buildPolicy: 'never', + cacheRoot: '.cache/native-driver', + helperPath: 'tools/driver.exe', + installRoot: 'tools/native-driver', + }, + }, + }, + }); + + expect(config.getNativeDriverOptions('win32')).toEqual({ + application: { + executablePath: path.join(storybookRoot, 'tools/ReactTest.exe'), + windowTitle: 'Consumer Storybook (Win32)', + }, + buildPolicy: 'never', + cacheRoot: path.join(storybookRoot, '.cache/native-driver'), + configuration: 'release', + helperPath: path.join(storybookRoot, 'tools/driver.exe'), + installRoot: path.join(storybookRoot, 'tools/native-driver'), + }); + expect(config.getNativeDriverOptions('macos')).toMatchObject({ + application: { + bundleIdentifier: 'com.microsoft.fluentui.agenticstorybook', + windowTitle: 'Agentic Components Storybook', + }, + configuration: 'release', + macosSigningIdentity: 'Apple Development: Example', + }); + }); + test('creates package-owned Windows and Win32 smoke commands', () => { const configDirectory = path.resolve(__dirname, '../../config'); diff --git a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts index 5bca34f8b01..504f168ac0f 100644 --- a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts +++ b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts @@ -2,9 +2,11 @@ import fs from 'node:fs'; import { createRequire } from 'node:module'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import type { NativeDesktopApplicationDescriptor } from '@fluentui-react-native/desktop-driver'; import type { DesktopCommand, DesktopCommandPlan, + DesktopNativeDriverOptions, DesktopNativeProjectOptions, DesktopPlatformOptions, DesktopPlatformOptionsMap, @@ -254,8 +256,24 @@ export class DesktopStorybookConfig { ...defaultNativeProjectOptions(this, platform), ...configured.nativeProject, }; + const configuredNativeDriver = configured.nativeDriver; return Object.freeze({ + nativeDriver: + configuredNativeDriver === false + ? false + : Object.freeze({ + ...defaultNativeDriverOptions(this, platform), + ...configuredNativeDriver, + cacheRoot: resolveOptionalProjectPath(this.projectRoot, configuredNativeDriver?.cacheRoot), + helperPath: resolveOptionalProjectPath(this.projectRoot, configuredNativeDriver?.helperPath), + installRoot: resolveOptionalProjectPath(this.projectRoot, configuredNativeDriver?.installRoot), + application: Object.freeze({ + ...defaultNativeApplication(this, platform), + ...configuredNativeDriver?.application, + executablePath: resolveOptionalProjectPath(this.projectRoot, configuredNativeDriver?.application?.executablePath), + }), + }), nativeProject: Object.freeze(nativeProject), server: configured.server !== undefined ? configured.server : defaultServerCommand(this), prep: configured.prep !== undefined ? configured.prep : defaultPrepPlan(platform), @@ -267,7 +285,7 @@ export class DesktopStorybookConfig { } getCommandPlan( - action: Exclude, + action: Exclude, platformSetting: Platforms | string | undefined = this.platform, ): DesktopCommandPlan | false { const platform = requirePlatform(platformSetting); @@ -283,6 +301,14 @@ export class DesktopStorybookConfig { return this.getPlatformOptions(platform).smoke; } + getNativeDriverOptions(platformSetting: Platforms | string | undefined = this.platform): Readonly | false { + const options = this.getPlatformOptions(platformSetting).nativeDriver; + if (options === undefined) { + throw new Error('Native driver options were not resolved.'); + } + return options; + } + private resolveStoryPackage(spec: StoryPackageSpec, platform?: Platforms): ResolvedStoryPackage | undefined { const [packageName, packageSettings] = typeof spec === 'string' ? [spec, {}] : spec; const enabledPlatforms = packageSettings.platforms ?? this.config.platforms ?? getAllPlatforms(this.projectRoot); @@ -349,6 +375,10 @@ function resolveFromProject(projectRoot: string, setting: string): string { return path.isAbsolute(setting) ? path.normalize(setting) : path.resolve(projectRoot, setting); } +function resolveOptionalProjectPath(projectRoot: string, setting: string | undefined): string | undefined { + return setting === undefined ? undefined : resolveFromProject(projectRoot, setting); +} + function requirePlatform(platformSetting: Platforms | string | undefined): Platforms { const platform = getPlatform(platformSetting); if (!platform) { @@ -447,6 +477,19 @@ function defaultNativeProjectOptions(config: DesktopStorybookConfig, platform: P } } +function defaultNativeDriverOptions(_config: DesktopStorybookConfig, _platform: Platforms): DesktopNativeDriverOptions { + return { + configuration: 'release', + }; +} + +function defaultNativeApplication(config: DesktopStorybookConfig, platform: Platforms): NativeDesktopApplicationDescriptor { + return { + ...(platform === 'macos' ? { bundleIdentifier: config.macosBundleIdentifier } : {}), + windowTitle: config.displayName, + }; +} + function defaultSmokeOptions(config: DesktopStorybookConfig, platform: Platforms): DesktopSmokeOptions | undefined { if (platform !== 'macos') { return undefined; @@ -514,6 +557,15 @@ function normalizeDesktopPlatformOptions(platformOptions: DesktopPlatformOptions platform, { ...options, + nativeDriver: + options.nativeDriver === false + ? false + : options.nativeDriver + ? { + ...options.nativeDriver, + application: options.nativeDriver.application ? { ...options.nativeDriver.application } : undefined, + } + : undefined, nativeProject: options.nativeProject ? { ...options.nativeProject } : undefined, server: normalizeCommand(options.server), prep: normalizeCommandPlan(options.prep), @@ -599,6 +651,7 @@ function validateDesktopPlatformOptions(platformOptions: DesktopPlatformOptionsM throw new RangeError(`config contains unsupported desktop platform options "${platform}".`); } validateCommand(options.server, `${platform}.server`); + validateNativeDriverOptions(options.nativeDriver, `${platform}.nativeDriver`); validateCommandPlan(options.prep, `${platform}.prep`); validateCommandPlan(options.bundle, `${platform}.bundle`); validateCommandPlan(options.run, `${platform}.run`); @@ -609,6 +662,35 @@ function validateDesktopPlatformOptions(platformOptions: DesktopPlatformOptionsM validateCommand(options.smoke?.metro, `${platform}.smoke.metro`); validateCommandPlan(options.smoke?.stop, `${platform}.smoke.stop`); } + + function validateNativeDriverOptions(options: DesktopNativeDriverOptions | false | undefined, source: string): void { + if (options === false || options === undefined) { + return; + } + if (options.buildPolicy !== undefined && options.buildPolicy !== 'if-missing' && options.buildPolicy !== 'never') { + throw new RangeError(`${source}.buildPolicy must be "if-missing" or "never".`); + } + if (options.configuration !== undefined && options.configuration !== 'debug' && options.configuration !== 'release') { + throw new RangeError(`${source}.configuration must be "debug" or "release".`); + } + for (const [name, value] of Object.entries({ + cacheRoot: options.cacheRoot, + helperPath: options.helperPath, + installRoot: options.installRoot, + macosSigningIdentity: options.macosSigningIdentity, + })) { + if (value !== undefined && !value.trim()) { + throw new TypeError(`${source}.${name} cannot be empty.`); + } + } + if (options.application) { + for (const [name, value] of Object.entries(options.application)) { + if (typeof value === 'string' && !value.trim()) { + throw new TypeError(`${source}.application.${name} cannot be empty.`); + } + } + } + } } } diff --git a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts index 7eadef78b3f..19d61ab5623 100644 --- a/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/StorybookChannelOrchestrator.test.ts @@ -72,6 +72,11 @@ const storyManifest: DesktopStoryManifest = { }; const driverManifest: DesktopStorybookDriverManifest = { + application: { + leaseNonce: 'nonce', + leasePath: 'application-lease.json', + windowTitle: 'Agentic Components Storybook', + }, appName: 'AgenticStorybook', bridgeNonce: 'nonce', displayName: 'Agentic Components Storybook', @@ -79,10 +84,28 @@ const driverManifest: DesktopStorybookDriverManifest = { endpoint: 'windows', instanceId: 'instance', metroPort: 8081, + nativeDriver: { + architecture: 'x64', + artifactId: 'artifact', + artifactRoot: 'artifact-root', + buildFingerprint: 'build', + buildId: 'build-id', + compatibilityKey: 'compatibility', + configuration: 'release', + endpoints: ['windows', 'win32'], + executablePath: 'driver.exe', + features: ['probe'], + origin: 'cache', + provider: 'windows', + schemaVersion: 1, + signing: { mode: 'none' }, + sourceDigest: 'source', + wireProtocol: { major: 1, minor: 0 }, + }, platformManifestDigest: storyManifest.platformManifestDigest, portablePlanDigest: storyManifest.portablePlanDigest, renderer: 'fabric', - schemaVersion: 1, + schemaVersion: 2, storyManifest, storybookPort: 7007, targetId: 'agenticstorybook-windows', diff --git a/packages/agentic/storybook-desktop/src/driver/driverManifest.test.ts b/packages/agentic/storybook-desktop/src/driver/driverManifest.test.ts new file mode 100644 index 00000000000..ba4eb8735ed --- /dev/null +++ b/packages/agentic/storybook-desktop/src/driver/driverManifest.test.ts @@ -0,0 +1,114 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import type { DesktopStoryManifest, NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; + +import { createDesktopStorybookInstance } from '../config/instance'; +import { makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig'; +import { createDesktopStorybookDriverManifest, writeDesktopStorybookDriverManifest } from './driverManifest'; + +const storybookRoot = path.resolve(__dirname, '../../../../../apps/storybook'); +const storyManifest: DesktopStoryManifest = { + endpoint: 'windows', + entries: [], + platformManifestDigest: 'platform-digest', + portablePlanDigest: 'portable-digest', + schemaVersion: 1, +}; +const nativeDriver: NativeDriverArtifact = { + architecture: 'x64', + artifactId: 'artifact', + artifactRoot: 'artifact-root', + buildFingerprint: 'build', + buildId: 'build-id', + compatibilityKey: 'compatibility', + configuration: 'release', + endpoints: ['windows', 'win32'], + executablePath: 'driver.exe', + features: ['probe'], + origin: 'cache', + provider: 'windows', + schemaVersion: 1, + signing: { mode: 'none' }, + sourceDigest: 'source', + wireProtocol: { major: 1, minor: 0 }, +}; + +describe('Desktop Storybook driver manifest', () => { + test('records native helper and nonce-bound application identity', () => { + const config = makeDesktopStorybookConfig({ projectRoot: storybookRoot }); + const instance = createDesktopStorybookInstance({ + bundleIdentifierPrefix: config.macosBundleIdentifier, + projectRoot: config.projectRoot, + }); + const manifest = createDesktopStorybookDriverManifest({ + bridgeNonce: 'nonce', + config, + instance, + nativeDriver, + platform: 'windows', + storyManifest, + }); + + expect(manifest).toMatchObject({ + application: { + leaseNonce: 'nonce', + leasePath: path.join(storybookRoot, 'storybook-desktop.generated', 'application-lease.windows.json'), + windowTitle: 'Agentic Components Storybook', + }, + nativeDriver, + schemaVersion: 2, + }); + + const macosManifest = createDesktopStorybookDriverManifest({ + bridgeNonce: 'nonce', + config, + instance, + nativeDriver: { + ...nativeDriver, + architecture: 'arm64', + endpoints: ['macos'], + provider: 'macos', + signing: { mode: 'adhoc' }, + }, + platform: 'macos', + storyManifest: { ...storyManifest, endpoint: 'macos' }, + }); + expect(macosManifest.application).toMatchObject({ + bundleIdentifier: instance.bundleIdentifier, + leasePath: path.join(storybookRoot, 'storybook-desktop.generated', 'application-lease.macos.json'), + }); + }); + + test('removes a stale application lease before writing a new manifest', () => { + const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'storybook-driver-manifest-')); + try { + const config = makeDesktopStorybookConfig({ + projectRoot: storybookRoot, + }); + const instance = createDesktopStorybookInstance({ + bundleIdentifierPrefix: config.macosBundleIdentifier, + projectRoot: config.projectRoot, + }); + const manifest = createDesktopStorybookDriverManifest({ + bridgeNonce: 'nonce', + config, + instance, + nativeDriver, + platform: 'windows', + storyManifest, + }); + fs.mkdirSync(path.dirname(manifest.application.leasePath), { recursive: true }); + fs.writeFileSync(manifest.application.leasePath, '{}'); + const outputPath = path.join(temporaryDirectory, 'driver-manifest.json'); + + writeDesktopStorybookDriverManifest(manifest, outputPath); + + expect(fs.existsSync(manifest.application.leasePath)).toBe(false); + expect(JSON.parse(fs.readFileSync(outputPath, 'utf8'))).toMatchObject({ schemaVersion: 2 }); + } finally { + fs.rmSync(temporaryDirectory, { force: true, recursive: true }); + } + }); +}); diff --git a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts index 870b482319b..ae1aae07716 100644 --- a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts +++ b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts @@ -2,13 +2,17 @@ import { randomBytes } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; +import type { DesktopStoryManifest, NativeDesktopApplicationDescriptor, NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { DesktopStorybookInstance } from '../config/instance.js'; import type { Platforms } from '../config/platforms.js'; export type DesktopStorybookDriverManifest = { + application: NativeDesktopApplicationDescriptor & { + leaseNonce: string; + leasePath: string; + }; appName: string; bridgeNonce: string; displayName: string; @@ -16,10 +20,11 @@ export type DesktopStorybookDriverManifest = { endpoint: Platforms; instanceId: string; metroPort: number; + nativeDriver: NativeDriverArtifact; platformManifestDigest: string; portablePlanDigest: string; renderer: 'fabric' | 'paper'; - schemaVersion: 1; + schemaVersion: 2; storyManifest: DesktopStoryManifest; storybookPort: number; targetId: string; @@ -30,6 +35,7 @@ export type CreateDesktopStorybookDriverManifestOptions = { bridgeNonce?: string; config: DesktopStorybookConfig; instance: DesktopStorybookInstance; + nativeDriver: NativeDriverArtifact; platform: Platforms; storyManifest: DesktopStoryManifest; }; @@ -38,10 +44,22 @@ export function createDesktopStorybookDriverManifest({ bridgeNonce = randomBytes(24).toString('base64url'), config, instance, + nativeDriver, platform, storyManifest, }: CreateDesktopStorybookDriverManifestOptions): DesktopStorybookDriverManifest { + const nativeOptions = config.getNativeDriverOptions(platform); + if (nativeOptions === false) { + throw new Error(`Native Desktop Driver is disabled for ${platform}.`); + } + const leasePath = path.join(config.projectRoot, 'storybook-desktop.generated', `application-lease.${platform}.json`); return Object.freeze({ + application: Object.freeze({ + ...nativeOptions.application, + ...(platform === 'macos' ? { bundleIdentifier: instance.bundleIdentifier } : {}), + leaseNonce: bridgeNonce, + leasePath, + }), appName: config.appName, bridgeNonce, displayName: config.displayName, @@ -49,10 +67,11 @@ export function createDesktopStorybookDriverManifest({ endpoint: platform, instanceId: instance.id, metroPort: instance.metroPort, + nativeDriver, platformManifestDigest: storyManifest.platformManifestDigest, portablePlanDigest: storyManifest.portablePlanDigest, renderer: platform === 'win32' ? 'paper' : 'fabric', - schemaVersion: 1, + schemaVersion: 2, storyManifest, storybookPort: instance.storybookPort, targetId: `${config.appName}-${platform}`.toLowerCase(), @@ -62,6 +81,7 @@ export function createDesktopStorybookDriverManifest({ export function writeDesktopStorybookDriverManifest(manifest: DesktopStorybookDriverManifest, outputPath: string): void { fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.rmSync(manifest.application.leasePath, { force: true }); const content = `${JSON.stringify(manifest, null, 2)}\n`; if (!fs.existsSync(outputPath) || fs.readFileSync(outputPath, 'utf8') !== content) { fs.writeFileSync(outputPath, content); diff --git a/packages/agentic/storybook-desktop/src/driver/representativePlans.test.ts b/packages/agentic/storybook-desktop/src/driver/representativePlans.test.ts index a6ea286e20e..e04c4bddb92 100644 --- a/packages/agentic/storybook-desktop/src/driver/representativePlans.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/representativePlans.test.ts @@ -6,13 +6,13 @@ import path from 'node:path'; jest.setTimeout(30_000); describe('representative desktop story plans', () => { - test('extracts and runs Button, Checkbox, and Input plans unchanged through WebdriverIO', async () => { + test('extracts and runs the applicable Button, Checkbox, and Input plans through WebdriverIO', async () => { const artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'representative-story-plans-')); try { const response = await runContract(artifactsRoot); expect(response).toMatchObject({ planned: [ - { id: 'components-button--default', tests: ['pointer-focus'] }, + { id: 'components-button--default', tests: ['pointer-focus', 'pointer-activation'] }, { id: 'components-checkbox--default', tests: ['toggles-checked-state'] }, { id: 'components-input--default', tests: ['types-and-clears'] }, ], diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index 2931d706311..ffc2d9c8328 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -17,6 +17,11 @@ describe('desktop Storybook server integration', () => { const manifestPath = path.join(temporaryDirectory, 'driver-manifest.json'); const projectRoot = path.resolve(__dirname, '../../../../../apps/storybook'); const driverManifest: DesktopStorybookDriverManifest = { + application: { + leaseNonce: 'integration-nonce', + leasePath: path.join(temporaryDirectory, 'application-lease.json'), + windowTitle: 'Agentic Components Storybook', + }, appName: 'AgenticStorybook', bridgeNonce: 'integration-nonce', displayName: 'Agentic Components Storybook', @@ -24,10 +29,28 @@ describe('desktop Storybook server integration', () => { endpoint: 'windows', instanceId: 'integration', metroPort: 8081, + nativeDriver: { + architecture: 'x64', + artifactId: 'artifact', + artifactRoot: temporaryDirectory, + buildFingerprint: 'build', + buildId: 'build-id', + compatibilityKey: 'compatibility', + configuration: 'release', + endpoints: ['windows', 'win32'], + executablePath: path.join(temporaryDirectory, 'driver.exe'), + features: ['probe'], + origin: 'cache', + provider: 'windows', + schemaVersion: 1, + signing: { mode: 'none' }, + sourceDigest: 'source', + wireProtocol: { major: 1, minor: 0 }, + }, platformManifestDigest: 'platform-digest', portablePlanDigest: 'portable-digest', renderer: 'fabric', - schemaVersion: 1, + schemaVersion: 2, storyManifest: { endpoint: 'windows', entries: [ @@ -55,6 +78,7 @@ describe('desktop Storybook server integration', () => { ...process.env, STORYBOOK_CONFIG_PATH: path.join(projectRoot, 'src'), STORYBOOK_DRIVER_MANIFEST: manifestPath, + STORYBOOK_NATIVE_DRIVER_FAKE: '1', STORYBOOK_PROJECT_ROOT: projectRoot, STORYBOOK_SMOKE_MODE: 'stories-and-tests', STORYBOOK_WS_PORT: String(storybookPort), diff --git a/packages/agentic/storybook-desktop/src/index.ts b/packages/agentic/storybook-desktop/src/index.ts index 8cbaea15464..f3f3400ef2e 100644 --- a/packages/agentic/storybook-desktop/src/index.ts +++ b/packages/agentic/storybook-desktop/src/index.ts @@ -8,7 +8,9 @@ export { export type { CreateDesktopStorybookCommandOptions, DesktopCommandRunner, + DesktopStorybookBuildDriverOptions, DesktopStorybookCliOptions, + DesktopStorybookPrepOptions, DesktopStorybookServerOptions, PreparedDesktopCommand, RunningDesktopCommand, diff --git a/packages/components/FocusZone/windows/FRNFocusZone/FRNFocusZone.vcxproj b/packages/components/FocusZone/windows/FRNFocusZone/FRNFocusZone.vcxproj index 229df020a76..3d23cae5051 100644 --- a/packages/components/FocusZone/windows/FRNFocusZone/FRNFocusZone.vcxproj +++ b/packages/components/FocusZone/windows/FRNFocusZone/FRNFocusZone.vcxproj @@ -9,8 +9,8 @@ FRNFocusZone Win32Proj FRNFocusZone - 10.0.26100.0 - 10.0 + + 10.0.22621.0 en-US 17.0 false