diff --git a/.github/workflows/channel-gateway.yml b/.github/workflows/channel-gateway.yml index 031d6fddf19..f6c4e08c9aa 100644 --- a/.github/workflows/channel-gateway.yml +++ b/.github/workflows/channel-gateway.yml @@ -8,6 +8,8 @@ on: - 'package.json' - 'package-lock.json' - 'scripts/typecheck.mts' + - '.dockerignore' + - 'docker-compose.channels.yml' - '.github/workflows/channel-gateway.yml' push: branches: [dev] @@ -17,6 +19,8 @@ on: - 'package.json' - 'package-lock.json' - 'scripts/typecheck.mts' + - '.dockerignore' + - 'docker-compose.channels.yml' - '.github/workflows/channel-gateway.yml' workflow_dispatch: @@ -50,3 +54,71 @@ jobs: run: npm run test:ci --workspace @librechat/channel-gateway - name: Build gateway run: npm run build --workspace @librechat/channel-gateway + - name: Build standalone image + run: docker build -f packages/channel-gateway/Dockerfile -t channel-gateway:ci . + - name: Verify standalone image and fail-closed startup + shell: bash + run: | + docker run --rm --entrypoint node channel-gateway:ci -e "const fs=require('node:fs');if(process.getuid()===0||!fs.existsSync('/app/main.cjs')||fs.existsSync('/app/node_modules'))process.exit(1)" + set +e + output=$(docker run --rm channel-gateway:ci 2>&1) + status=$? + set -e + if [ "$status" -eq 0 ]; then + echo 'Gateway unexpectedly started without a channel binding.' + exit 1 + fi + if ! grep -q 'channel_gateway_startup_failed' <<< "$output"; then + echo 'Gateway did not report its expected safe startup failure.' + exit 1 + fi + - name: Smoke-test isolated webhook process + run: | + docker run --rm -i --read-only --entrypoint node channel-gateway:ci - <<'NODE' + const http = require('node:http'); + const { spawn } = require('node:child_process'); + const { createHmac } = require('node:crypto'); + const { setTimeout: delay } = require('node:timers/promises'); + const bundle = { + channel: { channelId: 'fixture', tenantId: 'fixture', agentId: 'agent_fixture', createdBy: 'owner', type: 'slack', status: 'active', revision: 'fixture', slack: { teamId: 'T1', appId: 'A1', botUserId: 'U1' } }, + gateway: { version: 1, transport: 'webhook', nativeSessions: false, lifecycleReactions: false, signatureToleranceSeconds: 300 }, + credentials: { botToken: 'fixture', signingSecret: 'fixture-signing' }, agent: { id: 'agent_fixture' }, + }; + const backend = http.createServer((req, res) => { + if (req.url.endsWith('/runs/claim')) { req.resume(); res.writeHead(204); res.end(); return; } + res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify(bundle)); + }); + backend.listen(0, '127.0.0.1', async () => { + const child = spawn(process.execPath, ['/app/main.cjs'], { + env: { ...process.env, LIBRECHAT_INTERNAL_URL: `http://127.0.0.1:${backend.address().port}`, CHANNEL_SERVICE_TOKEN: 'fixture', CHANNEL_TENANT_ID: 'fixture', CHANNEL_ID: 'fixture' }, + stdio: 'ignore', + }); + const exited = new Promise(resolve => child.once('exit', resolve)); + const deadline = setTimeout(() => child.kill('SIGKILL'), 10000); + try { + let ready = false; + for (let attempt = 0; attempt < 50; attempt++) { + try { ready = (await fetch('http://127.0.0.1:8080/healthz', { signal: AbortSignal.timeout(500) })).ok; } catch {} + if (ready) break; + await delay(100); + } + if (!ready) throw new Error('Gateway did not become healthy'); + const body = JSON.stringify({ type: 'url_verification', challenge: 'verified' }); + const url = 'http://127.0.0.1:8080/slack/events/fixture/fixture'; + const unsigned = await fetch(url, { method: 'POST', body }); + if (unsigned.status !== 401) throw new Error('Unsigned challenge accepted'); + const timestamp = String(Math.floor(Date.now() / 1000)); + const signature = 'v0=' + createHmac('sha256', 'fixture-signing').update(`v0:${timestamp}:${body}`).digest('hex'); + const signed = await fetch(url, { method: 'POST', body, headers: { 'x-slack-request-timestamp': timestamp, 'x-slack-signature': signature } }); + if (signed.status !== 200 || (await signed.json()).challenge !== 'verified') throw new Error('Signed challenge rejected'); + + child.kill('SIGTERM'); + if (await exited !== 0) throw new Error('Gateway did not exit cleanly'); + console.log('Standalone gateway became healthy and stopped cleanly.'); + } catch { + child.kill('SIGKILL'); process.exitCode = 1; + } finally { + clearTimeout(deadline); backend.closeAllConnections(); backend.close(); + } + }); + NODE diff --git a/docker-compose.channels.yml b/docker-compose.channels.yml new file mode 100644 index 00000000000..141aa3c023a --- /dev/null +++ b/docker-compose.channels.yml @@ -0,0 +1,20 @@ +services: + channel-gateway: + profiles: [channels] + build: + context: . + dockerfile: packages/channel-gateway/Dockerfile + init: true + restart: unless-stopped + read_only: true + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + environment: + LIBRECHAT_INTERNAL_URL: http://api:${PORT:-3080} + CHANNEL_SERVICE_TOKEN: ${CHANNEL_SERVICE_TOKEN:-} + CHANNEL_TENANT_ID: ${CHANNEL_TENANT_ID:-} + CHANNEL_ID: ${CHANNEL_ID:-} + APP_TITLE: ${APP_TITLE:-LibreChat} + SLACK_APP_TOKEN: ${SLACK_APP_TOKEN:-} + stop_grace_period: 10s diff --git a/packages/api/src/channels/service.spec.ts b/packages/api/src/channels/service.spec.ts index 7205b1fac3b..a73ab169186 100644 --- a/packages/api/src/channels/service.spec.ts +++ b/packages/api/src/channels/service.spec.ts @@ -349,6 +349,7 @@ it('uses one policy snapshot and supplies the verified webhook credentials only lifecycleReactions: true, signatureToleranceSeconds: 60, maxDeliveryParts: 1, + socketReconnectDelayMs: 100, })); deps.config = config; const id = await createChannel(); @@ -378,6 +379,7 @@ it('uses one policy snapshot and supplies the verified webhook credentials only nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }); expect(result.body.delivery).toEqual({ maxDeliveryParts: 1 }); expect(result.body.gateway).not.toHaveProperty('enabled'); diff --git a/packages/api/src/endpoints/config/endpoints.spec.ts b/packages/api/src/endpoints/config/endpoints.spec.ts index cd4e525859e..d5c07701d06 100644 --- a/packages/api/src/endpoints/config/endpoints.spec.ts +++ b/packages/api/src/endpoints/config/endpoints.spec.ts @@ -244,6 +244,7 @@ describe('createEndpointsConfigService', () => { nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }; const deps = createMockDeps({ loadDefaultEndpointsConfig: jest.fn().mockResolvedValue({ diff --git a/packages/channel-gateway/Dockerfile b/packages/channel-gateway/Dockerfile new file mode 100644 index 00000000000..ef76bf2f963 --- /dev/null +++ b/packages/channel-gateway/Dockerfile @@ -0,0 +1,35 @@ +FROM node:24.22.1-alpine AS build + +ENV SCARF_ANALYTICS=false HUSKY=0 +WORKDIR /build + +COPY package.json package-lock.json ./ +COPY api/package.json ./api/package.json +COPY client/package.json ./client/package.json +COPY packages/api/package.json ./packages/api/package.json +COPY packages/client/package.json ./packages/client/package.json +COPY packages/chat/package.json ./packages/chat/package.json +COPY packages/data-schemas/package.json ./packages/data-schemas/package.json +COPY packages/data-provider/package.json ./packages/data-provider/package.json +COPY packages/channel-gateway/package.json ./packages/channel-gateway/package.json +COPY vendor/code ./vendor/code +RUN npm ci --ignore-scripts --no-audit --workspace @librechat/channel-gateway --workspace librechat-data-provider --include-workspace-root=false + +COPY packages/data-provider ./packages/data-provider +COPY packages/channel-gateway ./packages/channel-gateway +COPY scripts/typecheck.mts ./scripts/typecheck.mts +RUN npm run build --workspace librechat-data-provider \ + && npm run typecheck --workspace @librechat/channel-gateway \ + && npm run build --workspace @librechat/channel-gateway + +FROM node:24.22.1-alpine AS runtime + +ENV NODE_ENV=production PORT=8080 LISTEN_HOST=0.0.0.0 +WORKDIR /app +COPY --from=build --chown=node:node /build/packages/channel-gateway/dist/main.cjs ./main.cjs +COPY --chown=node:node LICENSE ./LICENSE +USER node +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+process.env.PORT+'/healthz',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})" +CMD ["node", "main.cjs"] diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index 1e5fb6e3e0d..7cd21eaea66 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -1,8 +1,75 @@ -# Slack transport adapters +# Slack Channel gateway -This private workspace contains transport, native presentation and private interaction adapters for Agent Channels. The executable consumer is the [standalone gateway runtime in #17618](https://github.com/LibreChat-AI/LibreChat/pull/17618), which composes these adapters with native presentation, HTTP and Socket Mode ingress, durable workers, process lifecycle and container packaging. +## Configure one server-owned policy -The adapter build produces seven internal modules: +Configure the application in `librechat.yaml`: + +```yaml +endpoints: + agents: + channels: + enabled: true + transport: socket_mode + nativeSessions: false + lifecycleReactions: false + signatureToleranceSeconds: 300 + socketReconnectDelayMs: 5000 +``` + +Socket Mode, disabled optional native/lifecycle emission and a 300-second signature window are the defaults. The signature window accepts 1–300 seconds. Socket reconnect delay accepts 100–60,000 milliseconds and defaults to 5,000; consecutive failed attempts use linear backoff. For webhooks, set `transport: webhook` and `gatewayUrl` to the externally reachable HTTPS gateway base URL. Keep the private application service URL internal. + +The gateway loads this versioned policy with its private channel bundle before opening its listener or selecting ingress. Deploy matching application and gateway versions: policy version 1 is required, and absent or unsupported versions fail closed. The four ingress/security/feature fields remain required; an older version-1 bundle may omit only the additive reconnect delay and receives its default. Configure `CHANNEL_SERVICE_TOKEN`, `CHANNEL_TENANT_ID`, `CHANNEL_ID`, `LIBRECHAT_INTERNAL_URL` and, for Socket Mode, `SLACK_APP_TOKEN` in the gateway environment. These are process bindings and secrets. `APP_TITLE`, listening settings, provider/service addresses and process request timeouts also remain deployment settings. + +Remove legacy `SLACK_NATIVE_SESSIONS`, `SLACK_LIFECYCLE_REACTIONS` and `SLACK_SIGNATURE_TOLERANCE_SECS` environment variables; their presence now fails startup instead of overriding the server. Use the YAML fields above. `SLACK_TRANSPORT` is optional and only asserts the server-selected mode; a mismatch fails startup. Restart gateway replicas after changing ingress mode or the Slack app identity used by Socket Mode. A replica which observes a changed mode or Socket app binding rejects that delivery and becomes unhealthy. Workers read optional feature policy from the current private bundle, while already-frozen delivery plans retain their original operations. + +## Install a webhook app + +`/healthz` and `/readyz` share the actual readiness state: startup must complete, Socket Mode must be connected to its selected authenticated app when selected, and an observed ingress-policy mismatch makes the replica unready. `/livez` reports only that the HTTP process is alive. The container healthcheck uses `/healthz`. + +1. Select webhook mode and set the gateway base URL in the application configuration. Create the Slack app and apply the generated bot capabilities; install it to obtain its bot token. Keep its signing secret private. +2. Activate the Agent Channel with the app/workspace/bot identity, bot token and signing secret, and configure the matching gateway service/tenant/channel bindings. The private bundle requires an active channel with usable signing credentials before webhook verification can succeed. +3. Start the gateway, then apply the generated Events API and interaction URLs to the Slack app. Installation challenges authenticate the original request bytes with that channel's signing secret before echoing the challenge. Missing, stale or tampered signatures cannot pass URL verification. Socket Mode instead requires the app-level token and matching authenticated app identity. + +Native Stop remains available for an authenticated existing session even when optional native emission is disabled. The application accepts bounded future clock skew and durably pins each click's first observed target, lease epoch and cutoff, including no target. A retry cannot advance its cutoff or stop a newer run. Past occurrence times remain unchanged; clock skew does not establish a globally exact event order. + +## Delivery recovery + +The gateway preserves each frozen delivery operation and its receipt. A required operation whose provider result is unknown holds its actor lane: retrying blindly could post the same answer twice. Stopping a worker or requesting cancellation does not prove whether Slack accepted an earlier request. + +### Reconcile a verified provider result + +1. Pause gateway processing for the affected channel while investigating. Keep the channel enabled and preserve its actor link and source authority; disabling these is not a recovery bypass. +2. Inspect the frozen run plan and retained receipt. Verify in the provider that the exact operation reached the intended channel or thread. Record its message key or file ID, matching the retained attempt and any upload checkpoint. If the result cannot be verified, leave the receipt unknown and the lane held. +3. After the old lease expires, use `POST /api/channels/service/:channelId/runs/claim` with `{ "owner": "" }`, normal private service authentication and tenant binding. Verify that the returned run ID is the run being investigated; claims select eligible work and do not take a target run ID. Use the returned current fence, renewing it through `POST /renew` as needed. An expired or replaced worker's fence cannot reconcile the run. +4. Submit the confirmed receipt through the private channel API. These endpoints confirm existing attempts; they do not send messages, replace the frozen plan, reset attempts or accept a retryable outcome. + +All paths below are relative to `/api/channels/service/:channelId/runs/:runId`. Send the channel service bearer token and the configured tenant header, using the same internal service URL as the gateway. Preserve the complete fence returned by the claim. + +For a delivery operation, `POST /delivery/reconcile`: + +```json +{ + "fence": { "runId": "", "owner": "", "epoch": 1 }, + "receipt": { + "operationId": "", + "outcome": "confirmed", + "attempts": 1, + "externalMessageKey": ":" + } +} +``` + +Replace every illustrative value with the actual current fence and retained attempt count; do not assume the example epoch is current. Supply `externalFileId` for a verified file operation as required by its frozen operation and checkpoint. Existing provider identities are immutable; reconciliation cannot replace them with a different message or file. + +For a progress update, `POST /progress/reconcile` with the current `fence`, the retained progress `revision`, and a `receipt` containing `outcome: "confirmed"`, the retained `attempts` and verified `externalMessageKey`. The message identity remains fixed across progress revisions. + +Once every required delivery operation is confirmed, `POST /commit` with the current `fence` to complete delivery and release the actor lane. Resume normal gateway processing afterward. If any required result remains uncertain, keep the run held rather than resending or recreating it. Optional exhausted effects follow the gateway's bounded recovery policy and do not justify replaying a required answer. + +## Transport adapter boundary + +The standalone process composes private transport adapters with native presentation, HTTP and Socket Mode ingress, durable workers and process lifecycle. + +The production bundle incorporates these seven internal source modules: | Module | Responsibility | | ---------- | -------------------------------------------------------------------------------------------- | @@ -14,9 +81,9 @@ The adapter build produces seven internal modules: | `identity` | Actor-private account-link tickets and exact-account confirmation | | `review` | Current actor/action validation, native forms and immutable saved-decision retry | -The application defaults `endpoints.agents.channels.enabled` to `false`. Enable the capability in a deployment that includes the matching executable consumer. These adapter artifacts are internal dependencies; deploying the runnable gateway also requires its entry point, presentation layer and durable worker. +The application defaults `endpoints.agents.channels.enabled` to `false`. Enable the capability in a deployment that includes the matching executable consumer. The deployed `main.cjs` bundle includes its entry point, presentation layer, durable worker and these internal modules. -## Consumer contract +### Consumer contract Before opening ingress, fetch the private channel bundle, validate its versioned gateway policy and call `resolveGatewayConfig`. The server selects Socket Mode or webhook transport; an optional process transport assertion must match. Missing or unsupported policy fails closed. Application and gateway must both support policy version 1. @@ -25,3 +92,7 @@ Webhook consumers pass the original bounded bytes and signature headers to `pars Forward native Stop with its original occurrence time and complete actor/channel/thread scope. The application owns bounded clock-skew normalization and the durable first-observation target, epoch and cutoff. Retained observation identity follows the external actor/thread across account relinks and channel/agent revisions; it is separate from the versioned execution principal. A retry cannot select a later lane or run. The observation joins account-erasure admission and channel serialization fences; cancellation still requires current authority and the exact lease fence. Optional native-session emission does not revoke cancellation for an existing session. The runnable consumer preserves canonical execution identities and frozen delivery receipts across restarts. An uncertain required provider write requires reconciliation rather than a blind repeat. The standalone runtime's HTTP/Socket, worker, process and container checks validate that composition in addition to the adapter tests here. + +Socket connections use one gateway-owned supervision task. Connection loss immediately fails readiness; retries own every SDK startup result and require a fresh matching-app hello before ingress resumes. Shutdown aborts API requests and backoff, closes the SDK, lets the SDK close its WebSocket transport and drains supervision alongside the worker. + +The private bundle also supplies the server’s `maxDeliveryParts` storage limit. New plans reserve required answer output first, group attachment attempts with their failure notice, and explicitly report shortening or omitted optional effects. Already persisted plans retain their frozen operations. diff --git a/packages/channel-gateway/package.json b/packages/channel-gateway/package.json index 63c2015ee57..106b3b500e2 100644 --- a/packages/channel-gateway/package.json +++ b/packages/channel-gateway/package.json @@ -2,11 +2,12 @@ "name": "@librechat/channel-gateway", "version": "0.1.0", "private": true, - "description": "Slack transport adapters for LibreChat Agent Channels", + "description": "Standalone Slack gateway for LibreChat Agent Channels", "type": "module", "scripts": { "build": "tsdown", "build:watch": "tsdown --watch", + "start": "node dist/main.cjs", "typecheck": "node ../../scripts/typecheck.mts", "test": "jest --watch", "test:ci": "jest --ci --runInBand" diff --git a/packages/channel-gateway/src/app.spec.ts b/packages/channel-gateway/src/app.spec.ts new file mode 100644 index 00000000000..1e4f2651e6b --- /dev/null +++ b/packages/channel-gateway/src/app.spec.ts @@ -0,0 +1,704 @@ +import { createHmac } from 'node:crypto'; +import { SocketModeClient } from '@slack/socket-mode'; +import { createServer, request as httpRequest } from 'node:http'; +import { DEFAULT_CHANNEL_GATEWAY_POLICY } from 'librechat-data-provider'; +import type { AddressInfo } from 'node:net'; +import type { ChannelBundle } from './types'; +import { loadConfig, resolveGatewayConfig } from './config'; +import { ChannelClient } from './client'; +import { connectSocket } from './socket'; +import { SlackClient } from './slack'; +import { GatewayApp } from './app'; + +const bundle: ChannelBundle = { + gateway: { version: 1, ...DEFAULT_CHANNEL_GATEWAY_POLICY, transport: 'webhook' }, + channel: { + channelId: 'binding', + tenantId: 'tenant', + agentId: 'agent', + createdBy: 'owner', + type: 'slack', + status: 'active', + credentialMode: 'per_user', + revision: 'r1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'B1' }, + }, + agent: { id: 'agent' }, + credentials: { botToken: 'test-bot', signingSecret: 'test-signing' }, +}; +const envelope = (event: object = {}) => ({ + type: 'event_callback', + event_id: 'E1', + team_id: 'T1', + api_app_id: 'A1', + event: { + type: 'app_mention', + user: 'U1', + channel: 'C1', + ts: '1793000000.123456', + text: '<@B1> hello', + ...event, + }, +}); +const callback = ( + action = 'channel_run_cancel', + value = JSON.stringify({ runId: 'run', revision: 'r1' }), +) => ({ + type: 'block_actions', + team: { id: 'T1' }, + api_app_id: 'A1', + user: { id: 'U1' }, + channel: { id: 'C1' }, + container: { channel_id: 'C1' }, + actions: [{ action_id: action, value }], +}); +const json = (body: object, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); +function setup(env: NodeJS.ProcessEnv = {}) { + const currentBundle: ChannelBundle = { + ...bundle, + gateway: { + ...bundle.gateway, + transport: env.SLACK_TRANSPORT === 'socket_mode' ? 'socket_mode' : 'webhook', + }, + }; + const config = resolveGatewayConfig( + loadConfig({ + CHANNEL_SERVICE_TOKEN: 'test-service', + SLACK_TRANSPORT: 'webhook', + CHANNEL_TENANT_ID: 'tenant', + CHANNEL_ID: 'binding', + ...env, + }), + currentBundle.gateway, + ); + const backend = jest + .fn, Parameters>() + .mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(currentBundle) + : json({ runId: 'run', duplicate: false }), + ); + const slack = jest + .fn, Parameters>() + .mockImplementation(async () => json({ ok: true })); + const log = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; + const app = new GatewayApp( + config, + new ChannelClient(config, backend), + new SlackClient(config, undefined, slack), + log, + ); + if (config.SLACK_TRANSPORT === 'socket_mode') + app.bindSocketApp(currentBundle.channel.slack.appId); + return { app, backend, slack, log, config, bundle: currentBundle }; +} +async function serve(app: GatewayApp, check: (base: string) => Promise) { + const server = createServer((request, response) => { + void app.http(request, response); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + try { + await check(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +} +function signed(body: string) { + const timestamp = String(Math.floor(Date.now() / 1000)); + return { + 'Content-Type': 'application/json', + 'x-slack-request-timestamp': timestamp, + 'x-slack-signature': + 'v0=' + createHmac('sha256', 'test-signing').update(`v0:${timestamp}:${body}`).digest('hex'), + }; +} + +describe('durable Slack ingress', () => { + test('does not acknowledge until admission has been durably confirmed', async () => { + const { app, backend } = setup(); + let accept: (response: Response) => void = () => {}; + const admission = new Promise((resolve) => { + accept = resolve; + }); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') ? json(bundle) : admission, + ); + let acknowledged = false; + const pending = app.dispatch(envelope(), false).then(() => { + acknowledged = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(acknowledged).toBe(false); + accept(json({ runId: 'run', duplicate: false })); + await pending; + expect(acknowledged).toBe(true); + expect(JSON.parse(String(backend.mock.calls[1][1]?.body))).toMatchObject({ + input: 'hello', + actor: { externalUserId: 'U1', teamId: 'T1', appId: 'A1' }, + }); + }); + + test.each(['team_id', 'api_app_id'])('does not admit a foreign %s', async (field) => { + const { app, backend } = setup(); + await expect(app.dispatch({ ...envelope(), [field]: 'foreign' }, false)).rejects.toThrow(); + expect(backend).toHaveBeenCalledTimes(1); + }); + + test('native Stop is constrained to its actor, thread and occurrence time', async () => { + const { app, backend } = setup(); + await app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.123456', + }), + false, + ); + expect(String(backend.mock.calls[1][0])).toContain('/runs/stop-thread'); + expect(JSON.parse(String(backend.mock.calls[1][1]?.body))).toEqual({ + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + externalThreadKey: 'C1:1792999999.5', + externalChannelId: 'C1', + eventAt: 1793000001123, + }); + }); + + test('native Stop remains available when native emission is disabled', async () => { + const { app, backend } = setup(); + await app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + false, + ); + expect(backend).toHaveBeenCalledTimes(2); + expect(String(backend.mock.calls[1][0])).toContain('/runs/stop-thread'); + }); + + test.each([400, 403])('native Stop reports a safe diagnostic for backend %i', async (status) => { + const { app, backend, log } = setup(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'private stop content' }, status), + ); + const dispatch = app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + false, + ); + if (status === 400) await expect(dispatch).rejects.toThrow(); + else await dispatch; + expect(log.warn.mock.calls).toEqual([ + [status === 400 ? 'native_stop_invalid_request' : 'native_stop_forbidden'], + ]); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('private'); + }); + + test('native Stop malformed envelope and binding errors are distinguishable without payload logging', async () => { + const { app, log } = setup(); + const stop = envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }); + await expect(app.dispatch({ ...stop, team_id: 'foreign' }, false)).rejects.toThrow(); + await expect(app.dispatch({ ...stop, event_id: undefined }, false)).rejects.toThrow(); + await expect( + app.dispatch(envelope({ type: 'agent_session_stopped', event_ts: '1793000001.1' }), false), + ).rejects.toThrow(); + expect(log.warn.mock.calls).toEqual([ + ['native_stop_binding_rejected'], + ['native_stop_envelope_rejected'], + ['native_stop_event_rejected'], + ]); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('foreign'); + }); + + test.each(['1793000000.1', '1793000010.1'])( + 'forwards future Stop %s immediately with its original occurrence time for durable service normalization', + async (event_ts) => { + const { app, backend } = setup(); + jest.spyOn(Date, 'now').mockReturnValue(1793000000000); + await app.dispatch( + envelope({ type: 'agent_session_stopped', thread_ts: '1792999999.5', event_ts }), + false, + ); + expect(JSON.parse(String(backend.mock.calls[1][1]?.body)).eventAt).toBe( + Math.floor(Number(event_ts) * 1000), + ); + }, + ); + + test('already-aborted Stop does not reach the backend', async () => { + const { app, backend } = setup(); + const controller = new AbortController(); + controller.abort(); + await expect( + app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + false, + controller.signal, + ), + ).rejects.toThrow(); + expect(backend).not.toHaveBeenCalled(); + }); + + test('Socket Mode leaves rejected Stop unacknowledged for redelivery', async () => { + const { app, backend, log, config, bundle } = setup({ + SLACK_TRANSPORT: 'socket_mode', + SLACK_APP_TOKEN: 'test-app', + }); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'invalid_run_action' }, 400), + ); + jest.spyOn(SocketModeClient.prototype, 'start').mockResolvedValue({ ok: true }); + jest.spyOn(SocketModeClient.prototype, 'disconnect').mockResolvedValue(undefined); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit( + 'ws_message', + JSON.stringify({ type: 'hello', connection_info: { app_id: 'A1' } }), + false, + ); + await new Promise((resolve) => setImmediate(resolve)); + const ack = jest.fn().mockResolvedValue(undefined); + socket.emit('slack_event', { + type: 'events_api', + envelope_id: 'stop', + body: envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + ack, + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(ack).not.toHaveBeenCalled(); + expect(log.warn.mock.calls).toContainEqual(['native_stop_invalid_request']); + expect(log.warn.mock.calls).toContainEqual(['slack_socket_delivery_not_accepted']); + controller.abort(); + }); + + test('only explicit link_required invokes private linking', async () => { + const { app, backend, slack } = setup(); + backend.mockImplementation(async (url) => { + if (String(url).includes('/service/slack/')) return json(bundle); + if (String(url).endsWith('/runs')) return json({ error: 'channel_link_required' }, 403); + if (String(url).endsWith('/identity/U1')) return json({ ok: true, value: null }); + return json({ + ok: true, + value: { + url: `https://chat.example/channels/link#${'a'.repeat(43)}`, + expiresAtMs: Date.now() + 60000, + }, + }); + }); + await app.dispatch(envelope(), false); + expect(slack).toHaveBeenCalledTimes(1); + expect(String(slack.mock.calls[0][0])).toContain('chat.postEphemeral'); + }); + + test.each([{ error: 'forbidden' }, { error: 'expired' }, { message: 'channel_link_required' }])( + 'does not reinterpret arbitrary 403 as linking %p', + async (failure) => { + const { app, backend, slack } = setup(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') ? json(bundle) : json(failure, 403), + ); + await expect(app.dispatch(envelope(), false)).rejects.toThrow(); + expect(slack).not.toHaveBeenCalled(); + expect(backend).toHaveBeenCalledTimes(2); + }, + ); + + test('rejects foreign interaction workspace/app and mismatched container', async () => { + const { app, backend } = setup(); + for (const body of [ + { ...callback(), team: { id: 'other' } }, + { ...callback(), api_app_id: 'other' }, + { ...callback(), container: { channel_id: 'other' } }, + ]) + await expect(app.dispatch(body, true)).rejects.toThrow(); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); + + test('stale controls cannot cancel work', async () => { + const { app, backend, slack } = setup(); + await app.dispatch( + callback('channel_run_cancel', JSON.stringify({ runId: 'run', revision: 'old' })), + true, + ); + expect(backend).toHaveBeenCalledTimes(1); + expect(slack).toHaveBeenCalledTimes(1); + }); + + test('keeps overload unacknowledged rather than queueing memory', async () => { + const { app, backend } = setup(); + let release: (response: Response) => void = () => {}; + const waiting = new Promise((resolve) => { + release = resolve; + }); + backend.mockImplementation(async () => waiting.then(() => json(bundle))); + const pending = Array.from({ length: 64 }, () => + app.dispatch(envelope({ bot_id: 'bot' }), false), + ); + await expect(app.dispatch(envelope(), false)).rejects.toThrow('capacity'); + expect(backend).toHaveBeenCalledTimes(64); + release(json(bundle)); + await Promise.all(pending); + backend.mockImplementation(async () => json(bundle)); + await expect(app.dispatch(envelope({ bot_id: 'bot' }), false)).resolves.toBeUndefined(); + }); +}); + +describe('signed webhook boundary', () => { + test('a Socket process cannot replace its authenticated app binding in place', () => { + const { app } = setup({ SLACK_TRANSPORT: 'socket_mode', SLACK_APP_TOKEN: 'test-app' }); + app.ready = true; + app.connected = true; + expect(() => app.bindSocketApp('A2')).toThrow('restart required'); + expect(app.ready).toBe(false); + expect(app.connected).toBe(false); + }); + test.each([ + ['webhook', false, false, 503], + ['webhook', true, false, 200], + ['socket_mode', false, true, 503], + ['socket_mode', true, false, 503], + ['socket_mode', true, true, 200], + ] as const)( + 'shares truthful readiness for %s ready=%s connected=%s', + async (transport, ready, connected, status) => { + const { app } = setup({ SLACK_TRANSPORT: transport, SLACK_APP_TOKEN: 'test-app' }); + app.ready = ready; + app.connected = connected; + await serve(app, async (base) => { + for (const path of ['/healthz', '/readyz']) { + const response = await fetch(base + path); + expect(response.status).toBe(status); + expect(await response.json()).toEqual({ status: status === 200 ? 'ready' : 'not_ready' }); + } + expect((await fetch(base + '/livez')).status).toBe(200); + }); + }, + ); + test('does not echo a challenge after the ingress deadline expires during channel lookup', async () => { + const { app, backend } = setup(); + backend.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 2100)); + return json(bundle); + }); + await serve(app, async (base) => { + const body = JSON.stringify({ type: 'url_verification', challenge: 'too-late' }); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(503); + expect(await response.text()).not.toContain('too-late'); + }); + }); + test('echoes only an authenticated installation challenge without admitting a run', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify({ type: 'url_verification', challenge: 'verified-challenge' }); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ challenge: 'verified-challenge' }); + const tampered = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body + 'changed'), + body, + }); + expect(tampered.status).toBe(401); + }); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); + + test('fails closed and becomes unhealthy if the current server policy changes ingress', async () => { + const { app, backend, bundle: current } = setup(); + app.ready = true; + backend.mockImplementation(async () => + json({ ...current, gateway: { ...current.gateway, transport: 'socket_mode' } }), + ); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + expect( + ( + await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }) + ).status, + ).toBe(503); + expect((await fetch(base + '/healthz')).status).toBe(503); + expect((await fetch(base + '/readyz')).status).toBe(503); + }); + expect(backend).toHaveBeenCalledTimes(1); + }); + + test('rejects unsigned installation challenges before echoing their payload', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ type: 'url_verification', challenge: 'must-authenticate-first' }), + }); + expect(response.status).toBe(401); + expect(await response.text()).not.toContain('must-authenticate-first'); + expect(backend).toHaveBeenCalledTimes(1); + }); + }); + + test('HTTP leaves rejected Stop retryable rather than acknowledging it', async () => { + const { app, backend } = setup(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'invalid_run_action' }, 400), + ); + await serve(app, async (base) => { + const body = JSON.stringify( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + ); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ error: 'delivery_not_accepted' }); + }); + }); + test('bounds incomplete incoming bodies by the acknowledgement deadline', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + let request: ReturnType | undefined; + const response = new Promise((resolve) => { + request = httpRequest( + base + '/slack/events/tenant/binding', + { method: 'POST', headers: { 'Content-Type': 'application/json' } }, + (incoming) => { + incoming.resume(); + resolve(incoming.statusCode ?? 0); + }, + ); + request.on('error', () => resolve('closed')); + request.write('{'); + }); + const result = await Promise.race([ + response, + new Promise((resolve) => setTimeout(() => resolve('stalled'), 2600)), + ]); + request?.destroy(); + expect(result).not.toBe('stalled'); + expect(backend).not.toHaveBeenCalled(); + }); + }); + + test('bounds webhook capacity before backend channel lookup', async () => { + const { app, backend } = setup(); + let release: () => void = () => {}; + const waiting = new Promise((resolve) => { + release = resolve; + }); + backend.mockImplementation(async () => { + await waiting; + return json(bundle); + }); + await serve(app, async (base) => { + const body = JSON.stringify(envelope({ bot_id: 'bot' })); + const pending = Array.from({ length: 65 }, () => + fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }), + ); + for (let i = 0; i < 100 && backend.mock.calls.length < 64; i++) + await new Promise((resolve) => setTimeout(resolve, 5)); + await new Promise((resolve) => setTimeout(resolve, 50)); + const lookups = backend.mock.calls.length; + release(); + const results = await Promise.all(pending); + await Promise.all(results.map((result) => result.arrayBuffer())); + expect(lookups).toBeLessThanOrEqual(64); + expect(results.filter((result) => result.status === 503)).toHaveLength(1); + }); + }); + + test('rejects unsigned/tampered requests without admitting work', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body + 'tampered'), + body, + }); + expect(response.status).toBe(401); + expect(await response.json()).toEqual({ error: 'invalid_signature' }); + }); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); + + test('accepts exact signed payload only after durable admission', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({}); + }); + expect(backend.mock.calls.filter(([url]) => String(url).endsWith('/runs'))).toHaveLength(1); + }); + + test('requires exact configured route and rejects webhook use in Socket Mode', async () => { + const { app, backend } = setup({ SLACK_TRANSPORT: 'socket_mode', SLACK_APP_TOKEN: 'test-app' }); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + for (const path of ['/slack/events/tenant/binding', '/slack/events/other/binding']) + expect( + (await fetch(base + path, { method: 'POST', headers: signed(body), body })).status, + ).toBe(404); + }); + expect(backend).not.toHaveBeenCalled(); + }); + + test('failed admission returns retryable HTTP failure, not false success', async () => { + const { app, backend, log } = setup(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'private-secret-details' }, 500), + ); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ error: 'delivery_not_accepted' }); + }); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('private-secret-details'); + }); + + test('interactions require signed form encoding and one payload', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify(callback()); + expect( + ( + await fetch(base + '/slack/interactions/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }) + ).status, + ).toBe(400); + const form = new URLSearchParams({ + payload: JSON.stringify(callback('channel_link_open')), + }).toString(); + const response = await fetch(base + '/slack/interactions/tenant/binding', { + method: 'POST', + headers: { ...signed(form), 'Content-Type': 'application/x-www-form-urlencoded' }, + body: form, + }); + expect(response.status).toBe(200); + }); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); +}); + +test.each([false, true])( + 'pending modal work owns ingress capacity until settlement (failure=%s)', + async (fail) => { + const { app, backend } = setup(); + let release!: () => void; + const waiting = new Promise((resolve) => { + release = resolve; + }); + let reads = 0; + const reference = { + runId: 'run', + revision: 'r1', + actionId: 'action', + channelId: 'C1', + actorId: 'U1', + expiresAt: Date.now() + 60000, + }; + backend.mockImplementation(async () => { + reads++; + await waiting; + if (fail) throw new Error('outage'); + return json({ + actionId: 'action', + pendingAction: { + actionId: 'action', + expiresAt: reference.expiresAt, + payload: { type: 'ask_user_question', question: { question: 'Answer this' } }, + }, + }); + }); + const body = { + type: 'view_submission', + team: { id: 'T1' }, + api_app_id: 'A1', + user: { id: 'U1' }, + view: { + callback_id: 'channel_native_review', + private_metadata: JSON.stringify(reference), + state: { values: {} }, + }, + }; + const pending = Array.from({ length: 64 }, () => app.dispatch(body, true, undefined, bundle)); + for (let attempt = 0; attempt < 20 && reads < 64; attempt++) + await new Promise((resolve) => setTimeout(resolve, 0)); + try { + expect(reads).toBe(64); + await expect(app.dispatch(envelope(), false, undefined, bundle)).rejects.toThrow('capacity'); + } finally { + release(); + await Promise.all(pending); + } + await expect( + app.dispatch(envelope({ bot_id: 'bot' }), false, undefined, bundle), + ).resolves.toBeUndefined(); + }, +); diff --git a/packages/channel-gateway/src/app.ts b/packages/channel-gateway/src/app.ts new file mode 100644 index 00000000000..5a0454b4f64 --- /dev/null +++ b/packages/channel-gateway/src/app.ts @@ -0,0 +1,382 @@ +import { z } from 'zod'; +import { createHash } from 'node:crypto'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import type { ChannelBundle, Logger } from './types'; +import type { GatewayRuntimeConfig } from './config'; +import type { ReviewInteraction } from './review'; +import { + eventTimestampUs, + normalizeEvent, + parseEnvelope, + parseWebhookRequest, + SLACK_REQUEST_MAX_BYTES, +} from './events'; +import { ChannelClient, ServiceError } from './client'; +import { IdentityFlow } from './identity'; +import { ReviewFlow } from './review'; +import { SlackClient } from './slack'; + +const id = z.string().min(1).max(256); +const stateValue = z + .object({ + value: z.string().nullable().optional(), + selected_option: z.object({ value: z.string() }).nullable().optional(), + selected_options: z + .array(z.object({ value: z.string() })) + .nullable() + .optional(), + }) + .transform((value) => ({ + value: value.value ?? undefined, + selected_option: value.selected_option ?? undefined, + selected_options: value.selected_options ?? undefined, + })); +const interactionSchema = z.object({ + type: z.enum(['block_actions', 'view_submission', 'view_closed']), + team: z.object({ id }), + api_app_id: id, + user: z.object({ id }), + channel: z.object({ id }).optional(), + trigger_id: z.string().optional(), + container: z.object({ channel_id: id.optional(), message_ts: z.string().optional() }).optional(), + message: z.object({ ts: z.string().optional() }).optional(), + actions: z + .array( + z.object({ action_id: id, value: z.string().optional(), action_ts: z.string().optional() }), + ) + .max(1) + .optional(), + view: z + .object({ + callback_id: z.string(), + private_metadata: z.string().max(3000), + state: z + .object({ values: z.record(z.string(), z.record(z.string(), stateValue)) }) + .optional(), + }) + .optional(), +}); + +export class GatewayApp { + ready = false; + connected = false; + private active = 0; + private httpActive = 0; + private socketAppId?: string; + private readonly identity: IdentityFlow; + private readonly review: ReviewFlow; + constructor( + private readonly config: GatewayRuntimeConfig, + private readonly client: ChannelClient, + slack: SlackClient, + private readonly log: Logger, + ) { + this.identity = new IdentityFlow(client, slack); + this.review = new ReviewFlow(client, slack); + } + + /** No queue before admission: overload is left unacknowledged for Slack redelivery. */ + async dispatch( + body: unknown, + interactive: boolean, + parent?: AbortSignal, + verifiedBundle?: ChannelBundle, + ): Promise { + if (this.active >= 64) throw new Error('Gateway capacity exhausted'); + this.active++; + const signal = AbortSignal.any([AbortSignal.timeout(2000), ...(parent ? [parent] : [])]); + try { + signal.throwIfAborted(); + const bundle = verifiedBundle ?? (await this.client.channel(signal)); + this.assertPolicy(bundle); + if (interactive) { + const callback = interactionSchema.parse(body); + if ( + callback.team.id !== bundle.channel.slack.teamId || + callback.api_app_id !== bundle.channel.slack.appId + ) + throw new Error('Interaction workspace or app changed'); + const channelId = callback.channel?.id ?? callback.container?.channel_id ?? ''; + if ( + callback.channel?.id && + callback.container?.channel_id && + callback.channel.id !== callback.container.channel_id + ) + throw new Error('Interaction channel mismatch'); + const interaction: ReviewInteraction = { + teamId: callback.team.id, + appId: callback.api_app_id, + actorId: callback.user.id, + channelId, + triggerId: callback.trigger_id, + view: callback.view + ? { + callbackId: callback.view.callback_id, + privateMetadata: callback.view.private_metadata, + values: callback.view.state?.values, + } + : undefined, + }; + if (callback.type === 'view_closed') return; + if (callback.type === 'view_submission') + return await this.review.submit(interaction, bundle, signal); + const action = callback.actions?.[0]; + if (!action) throw new Error('Missing Slack action'); + if (action.action_id === 'channel_feedback') { + const target = callback.container?.message_ts; + if ( + !channelId || + !target || + (callback.message?.ts && callback.message.ts !== target) || + !action.action_ts || + !['+1', '-1'].includes(action.value ?? '') + ) + throw new Error('Invalid answer feedback'); + const eventId = + 'slack-feedback:' + + createHash('sha256') + .update( + [ + callback.team.id, + callback.api_app_id, + channelId, + target, + callback.user.id, + action.action_ts, + action.value, + ].join(':'), + ) + .digest('hex'); + await this.reaction( + { + eventId, + eventTimestampUs: eventTimestampUs(action.action_ts), + externalMessageKey: `${channelId}:${target}`, + externalUserId: callback.user.id, + reaction: action.value!, + action: 'added', + }, + signal, + ); + } else if (action.action_id === 'channel_link_confirm') + await this.identity.confirm( + channelId, + callback.user.id, + action.value ?? '', + bundle, + signal, + ); + else if (action.action_id.startsWith('channel_native_choice_')) + await this.review.choice(action.value ?? '', interaction, bundle, signal); + else if ( + ['channel_run_cancel', 'channel_run_review', 'channel_native_write'].includes( + action.action_id, + ) + ) + await this.review.control( + action.action_id, + action.value ?? '', + interaction, + bundle, + signal, + ); + else if (!['channel_link_open', 'channel_run_open'].includes(action.action_id)) + throw new Error('Unsupported Slack action'); + signal.throwIfAborted(); + return; + } + const nativeStop = z + .object({ event: z.object({ type: z.literal('agent_session_stopped') }) }) + .safeParse(body).success; + let envelope; + try { + envelope = parseEnvelope(body, bundle); + } catch (error) { + if (nativeStop) + this.log.warn( + error instanceof z.ZodError + ? 'native_stop_envelope_rejected' + : 'native_stop_binding_rejected', + ); + throw error; + } + let event; + try { + event = normalizeEvent(envelope, bundle); + } catch (error) { + if (nativeStop) this.log.warn('native_stop_event_rejected'); + throw error; + } + if (envelope.event.type === 'agent_session_stopped' && !event) + this.log.warn('native_stop_actor_ignored'); + if (!event) return; + if (event.kind === 'stop') { + if (event.eventAt > Date.now()) this.log.warn('native_stop_future_timestamp'); + // Emission policy does not revoke cancellation for an already-created native session. + // The service binds clock-skew normalization to one durable target across retries. + signal.throwIfAborted(); + try { + const result = await this.client.stop( + event.actor, + event.externalThreadKey, + event.eventAt, + event.externalChannelId, + signal, + ); + this.log.info( + result.cancellationRequested ? 'native_stop_requested' : 'native_stop_no_active_run', + ); + } catch (error) { + if (!(error instanceof ServiceError && [400, 403].includes(error.status))) throw error; + this.log.warn( + error.status === 400 ? 'native_stop_invalid_request' : 'native_stop_forbidden', + ); + if (error.status === 400) throw error; + } + } else if (event.kind === 'reaction') await this.reaction(event.reaction, signal); + else { + try { + const admitted = await this.client.admit(event.admission, signal); + if (!admitted.runId) throw new Error('Admission not confirmed'); + } catch (error) { + if (!(error instanceof ServiceError && error.code === 'channel_link_required')) + throw error; + await this.identity.prompt( + event.admission.externalChannelId, + event.admission.actor.externalUserId, + bundle, + signal, + ); + } + } + signal.throwIfAborted(); + } finally { + this.active--; + } + } + + bindSocketApp(appId: string): void { + if ( + !appId || + this.config.SLACK_TRANSPORT !== 'socket_mode' || + (this.socketAppId !== undefined && this.socketAppId !== appId) + ) { + this.ready = false; + this.connected = false; + throw new Error('Socket application binding changed; restart required'); + } + this.socketAppId = appId; + } + + private assertPolicy(bundle: ChannelBundle): void { + if ( + bundle.gateway.transport !== this.config.SLACK_TRANSPORT || + (this.config.SLACK_TRANSPORT === 'socket_mode' && + bundle.channel.slack.appId !== this.socketAppId) + ) { + this.ready = false; + this.connected = false; + throw new Error('Gateway ingress policy changed; restart required'); + } + } + + private async reaction(record: object, signal: AbortSignal): Promise { + try { + await this.client.request(this.client.root + '/reactions', record, signal); + } catch (error) { + if (!(error instanceof ServiceError && error.status === 404)) throw error; + } + } + + async http(request: IncomingMessage, response: ServerResponse): Promise { + const path = request.url?.split('?')[0]; + const send = (status: number, body: object) => { + response.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' }); + response.end(JSON.stringify(body)); + }; + if (request.method === 'GET' && path === '/livez') { + send(200, { status: 'ok' }); + return; + } + if (request.method === 'GET' && (path === '/healthz' || path === '/readyz')) { + const ready = + this.ready && + (this.config.SLACK_TRANSPORT !== 'socket_mode' || + (this.connected && this.socketAppId !== undefined)); + send(ready ? 200 : 503, { status: ready ? 'ready' : 'not_ready' }); + return; + } + const route = path?.match(/^\/slack\/(events|interactions)\/([^/]+)\/([^/]+)$/); + if ( + !route || + request.method !== 'POST' || + this.config.SLACK_TRANSPORT !== 'webhook' || + route[2] !== encodeURIComponent(this.config.CHANNEL_TENANT_ID) || + route[3] !== encodeURIComponent(this.config.CHANNEL_ID) + ) { + send(404, { error: 'not_found' }); + return; + } + if (this.httpActive >= 64) { + send(503, { error: 'capacity_exhausted' }); + return; + } + this.httpActive++; + const controller = new AbortController(); + const stopReading = () => { + if (!request.complete) request.destroy(); + }; + controller.signal.addEventListener('abort', stopReading, { once: true }); + const timeout = setTimeout(() => controller.abort(), 2000); + request.on('aborted', () => controller.abort()); + try { + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of request) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += bytes.length; + if (size > SLACK_REQUEST_MAX_BYTES) { + send(413, { error: 'payload_too_large' }); + return; + } + chunks.push(bytes); + controller.signal.throwIfAborted(); + } + const body = Buffer.concat(chunks); + const interactive = route[1] === 'interactions'; + const bundle = await this.client.channel(controller.signal); + controller.signal.throwIfAborted(); + this.assertPolicy(bundle); + const parsed = parseWebhookRequest( + body, + { + signature: request.headers['x-slack-signature'], + timestamp: request.headers['x-slack-request-timestamp'], + contentType: request.headers['content-type'], + }, + bundle, + interactive, + ); + if (!parsed.ok) { + send(parsed.status, { error: parsed.code }); + return; + } + if (parsed.request.kind === 'challenge') { + send(200, { challenge: parsed.request.challenge }); + return; + } + const payload = + parsed.request.kind === 'event' ? parsed.request.envelope : parsed.request.payload; + const ack = await this.dispatch(payload, interactive, controller.signal, bundle); + controller.signal.throwIfAborted(); + send(200, ack ?? {}); + } catch { + this.log.warn('slack_delivery_not_accepted'); + if (!response.headersSent) send(503, { error: 'delivery_not_accepted' }); + } finally { + clearTimeout(timeout); + controller.signal.removeEventListener('abort', stopReading); + this.httpActive--; + } + } +} diff --git a/packages/channel-gateway/src/client.spec.ts b/packages/channel-gateway/src/client.spec.ts index 768506b853e..60bae5e7ab2 100644 --- a/packages/channel-gateway/src/client.spec.ts +++ b/packages/channel-gateway/src/client.spec.ts @@ -565,3 +565,16 @@ test('Stop and cancel retain the complete external-channel scope at the service { actor, revision: run.revision, externalChannelId: 'C1' }, ]); }); + +test('exposes only the approved trace-unavailable code for a409; other diagnostics stay private', async () => { + const transport = jest + .fn, Parameters>() + .mockResolvedValueOnce(json({ error: 'trace_unavailable', secret: 'private' }, 409)) + .mockResolvedValueOnce(json({ error: 'private-database-error', secret: 'private' }, 409)); + const client = new ChannelClient(config, transport); + await expect(client.claim('worker')).rejects.toMatchObject({ + status: 409, + code: 'trace_unavailable', + }); + await expect(client.claim('worker')).rejects.toMatchObject({ status: 409, code: '' }); +}); diff --git a/packages/channel-gateway/src/client.ts b/packages/channel-gateway/src/client.ts index 51348601d33..3c8cd2208f6 100644 --- a/packages/channel-gateway/src/client.ts +++ b/packages/channel-gateway/src/client.ts @@ -122,7 +122,10 @@ export class ChannelClient { let code = ''; try { const failure: { error?: string } = JSON.parse(bytes.toString()); - if (response.status === 403 && failure.error === 'channel_link_required') + if ( + (response.status === 403 && failure.error === 'channel_link_required') || + (response.status === 409 && failure.error === 'trace_unavailable') + ) code = failure.error; } catch { /* Untrusted error bodies are never exposed. */ diff --git a/packages/channel-gateway/src/config.spec.ts b/packages/channel-gateway/src/config.spec.ts index f2fb2eb23dc..d7578ae7994 100644 --- a/packages/channel-gateway/src/config.spec.ts +++ b/packages/channel-gateway/src/config.spec.ts @@ -25,6 +25,21 @@ describe('gateway configuration', () => { ).toBe(channelConfigSchema.parse({}).transport); }); + test('older v1 policies default only reconnect pacing and newer ones honor the override', () => { + const bootstrap = loadConfig({ ...required, SLACK_APP_TOKEN: 'test-app-token' }); + const older = { + transport: 'socket_mode' as const, + nativeSessions: false, + lifecycleReactions: false, + signatureToleranceSeconds: 300, + }; + expect(resolveGatewayConfig(bootstrap, older).socketReconnectDelayMs).toBe(5000); + expect( + resolveGatewayConfig(bootstrap, { ...older, socketReconnectDelayMs: 100 }) + .socketReconnectDelayMs, + ).toBe(100); + }); + test('supports legacy binding aliases and equal canonical/legacy values', () => { const legacy = { CHANNEL_SERVICE_TOKEN: required.CHANNEL_SERVICE_TOKEN, diff --git a/packages/channel-gateway/src/config.ts b/packages/channel-gateway/src/config.ts index 3dd26a35a67..02da3431324 100644 --- a/packages/channel-gateway/src/config.ts +++ b/packages/channel-gateway/src/config.ts @@ -36,6 +36,7 @@ const environment = z.object({ export type GatewayConfig = z.infer; export type GatewayRuntimeConfig = GatewayConfig & { SLACK_TRANSPORT: ChannelGatewayPolicy['transport']; + socketReconnectDelayMs: number; }; export function loadConfig(env: NodeJS.ProcessEnv): GatewayConfig { @@ -85,7 +86,7 @@ export function loadConfig(env: NodeJS.ProcessEnv): GatewayConfig { /** Bootstrap settings cannot override the server's current installation/ingress policy. */ export function resolveGatewayConfig( config: GatewayConfig, - value: ChannelGatewayPolicy, + value: z.input, ): GatewayRuntimeConfig { const parsed = channelGatewayPolicySchema.safeParse(value); if (!parsed.success) throw new Error('Invalid channel gateway policy'); @@ -94,5 +95,9 @@ export function resolveGatewayConfig( throw new Error('Gateway transport does not match channel policy'); if (policy.transport === 'socket_mode' && !config.SLACK_APP_TOKEN) throw new Error('SLACK_APP_TOKEN is required for Socket Mode'); - return { ...config, SLACK_TRANSPORT: policy.transport }; + return { + ...config, + SLACK_TRANSPORT: policy.transport, + socketReconnectDelayMs: policy.socketReconnectDelayMs, + }; } diff --git a/packages/channel-gateway/src/main.ts b/packages/channel-gateway/src/main.ts new file mode 100644 index 00000000000..9fba7c1edab --- /dev/null +++ b/packages/channel-gateway/src/main.ts @@ -0,0 +1,112 @@ +import { createServer } from 'node:http'; +import { Registry, collectDefaultMetrics, Counter } from 'prom-client'; +import type { Server } from 'node:http'; +import type { SocketConnection } from './socket'; +import { loadConfig, resolveGatewayConfig } from './config'; +import { connectSocket } from './socket'; +import { ChannelWorker } from './worker'; +import { ChannelClient } from './client'; +import { SlackClient } from './slack'; +import { GatewayApp } from './app'; +import { logger } from './types'; + +async function main(): Promise { + const bootstrap = loadConfig(process.env); + const controller = new AbortController(); + const client = new ChannelClient(bootstrap); + let app: GatewayApp | undefined; + let server: Server | undefined; + let connection: SocketConnection | undefined; + let worker: Promise | undefined; + let stopping = false; + const shutdown = () => { + if (stopping) return; + stopping = true; + if (app) app.ready = false; + controller.abort(); + if (server) { + server.close(); + const listener = server; + const timer = setTimeout(() => listener.closeAllConnections(), 5000); + timer.unref(); + } + }; + process.once('SIGTERM', shutdown); + process.once('SIGINT', shutdown); + try { + const bundle = await client.channel(controller.signal); + const config = resolveGatewayConfig(bootstrap, bundle.gateway); + const registry = new Registry(); + collectDefaultMetrics({ register: registry, prefix: 'channel_gateway_' }); + const httpRequests = new Counter({ + name: 'channel_gateway_http_requests_total', + help: 'Gateway HTTP requests by bounded outcome', + labelNames: ['status'], + registers: [registry], + }); + const slack = new SlackClient(config, logger); + const application = new GatewayApp(config, client, slack, logger); + app = application; + const listener = createServer((request, response) => { + response.on('finish', () => httpRequests.labels(String(response.statusCode)).inc()); + if (request.method === 'GET' && request.url === '/metrics') { + void registry + .metrics() + .then((metrics) => { + response.writeHead(200, { 'Content-Type': registry.contentType }); + response.end(metrics); + }) + .catch(() => { + response.writeHead(500); + response.end(); + }); + return; + } + void application.http(request, response).catch(() => { + logger.error('gateway_http_failed'); + if (!response.headersSent) response.writeHead(500); + response.end(); + }); + }); + server = listener; + listener.requestTimeout = 5000; + listener.headersTimeout = 5000; + listener.maxConnections = 128; + await new Promise((resolve, reject) => { + listener.once('error', reject); + listener.listen(config.PORT, config.LISTEN_HOST, resolve); + }); + controller.signal.throwIfAborted(); + if (config.SLACK_TRANSPORT === 'socket_mode') + connection = await connectSocket( + config, + application, + bundle.channel.slack.appId, + logger, + controller.signal, + ); + application.ready = true; + logger.info('channel_gateway_started'); + worker = new ChannelWorker(config, client, slack, logger).run(controller.signal); + if (connection) + await Promise.race([ + worker, + connection.closed.then(() => { + if (!controller.signal.aborted) throw new Error('Socket supervision ended'); + }), + ]); + else await worker; + } catch (error) { + if (!controller.signal.aborted) throw error; + } finally { + shutdown(); + await Promise.allSettled([worker, connection?.closed]); + process.removeListener('SIGTERM', shutdown); + process.removeListener('SIGINT', shutdown); + } +} + +void main().catch(() => { + logger.error('channel_gateway_startup_failed'); + process.exitCode = 1; +}); diff --git a/packages/channel-gateway/src/socket.spec.ts b/packages/channel-gateway/src/socket.spec.ts new file mode 100644 index 00000000000..3f82198cc43 --- /dev/null +++ b/packages/channel-gateway/src/socket.spec.ts @@ -0,0 +1,467 @@ +import { SocketModeClient } from '@slack/socket-mode'; +import { DEFAULT_CHANNEL_GATEWAY_POLICY } from 'librechat-data-provider'; +import type { ChannelBundle } from './types'; +import { loadConfig, resolveGatewayConfig } from './config'; +import { BoundSocket, connectSocket } from './socket'; +import { ChannelClient } from './client'; +import { SlackClient } from './slack'; +import { GatewayApp } from './app'; + +const config = resolveGatewayConfig( + loadConfig({ + CHANNEL_SERVICE_TOKEN: 'test-service', + CHANNEL_TENANT_ID: 'tenant', + CHANNEL_ID: 'binding', + SLACK_TRANSPORT: 'socket_mode', + SLACK_APP_TOKEN: 'test-app', + }), + DEFAULT_CHANNEL_GATEWAY_POLICY, +); +const bundle: ChannelBundle = { + gateway: { version: 1, ...DEFAULT_CHANNEL_GATEWAY_POLICY, transport: 'socket_mode' }, + channel: { + channelId: 'binding', + tenantId: 'tenant', + agentId: 'agent', + createdBy: 'owner', + type: 'slack', + status: 'active', + credentialMode: 'per_user', + revision: 'r1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'B1' }, + }, + agent: { id: 'agent' }, + credentials: { botToken: 'test-bot' }, +}; +const json = (body: object) => + new Response(JSON.stringify(body), { headers: { 'Content-Type': 'application/json' } }); +const envelope = () => ({ + type: 'event_callback', + event_id: 'E1', + team_id: 'T1', + api_app_id: 'A1', + event: { + type: 'app_mention', + user: 'U1', + channel: 'C1', + ts: '1793000000.123456', + text: '<@B1> private prompt', + }, +}); +const hello = (appId = 'A1') => + JSON.stringify({ type: 'hello', connection_info: { app_id: appId } }); +const tick = () => new Promise((resolve) => setImmediate(resolve)); +class TestSocket extends BoundSocket { + receive(data: string | ArrayBuffer, binary = false) { + return this.onWebSocketMessage(data, binary); + } +} +function setup() { + const backend = jest + .fn, Parameters>() + .mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ runId: 'run', duplicate: false }), + ); + const log = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; + const slack = new SlackClient(config, log, async () => json({ ok: true })); + return { + backend, + log, + app: new GatewayApp( + resolveGatewayConfig(config, bundle.gateway), + new ChannelClient(config, backend), + slack, + log, + ), + }; +} +function event(ack = jest.fn().mockResolvedValue(undefined)) { + return { type: 'events_api', envelope_id: 'envelope', body: envelope(), ack }; +} + +describe('Socket Mode app binding', () => { + let start: jest.SpyInstance; + let disconnect: jest.SpyInstance; + beforeEach(() => { + start = jest.spyOn(SocketModeClient.prototype, 'start').mockResolvedValue({ ok: true }); + disconnect = jest.spyOn(SocketModeClient.prototype, 'disconnect').mockResolvedValue(undefined); + }); + + test('validates hello app ID before making the connection ready', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + await socket.receive(hello()); + expect(state).toHaveBeenCalledWith(true); + expect(disconnect).not.toHaveBeenCalled(); + }); + + test('SDK never receives application payloads before a verified hello', async () => { + const socket = new TestSocket('A1', jest.fn(), config); + const delivered = jest.fn(); + socket.on('slack_event', delivered); + const wire = JSON.stringify({ type: 'events_api', envelope_id: 'wire', payload: envelope() }); + await socket.receive(wire); + expect(delivered).not.toHaveBeenCalled(); + await socket.receive(hello()); + await socket.receive(wire); + expect(delivered).toHaveBeenCalledTimes(1); + socket.emit('reconnecting'); + await socket.receive(wire); + expect(delivered).toHaveBeenCalledTimes(1); + }); + + test('malformed hello removes an existing validation, not just readiness at startup', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + const delivered = jest.fn(); + socket.on('slack_event', delivered); + await socket.receive(hello()); + state.mockClear(); + await socket.receive(JSON.stringify({ type: 'hello', connection_info: {} })); + expect(state).toHaveBeenCalledWith(false); + await socket.receive( + JSON.stringify({ type: 'events_api', envelope_id: 'wire', payload: envelope() }), + ); + expect(delivered).not.toHaveBeenCalled(); + }); + + test('disconnect failure cannot become an unhandled SDK callback rejection', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + disconnect.mockRejectedValueOnce(new Error('private disconnect payload')); + await expect(socket.receive(hello('wrong'))).resolves.toBeUndefined(); + expect(state).toHaveBeenCalledWith(false); + }); + + test('SDK logging cannot print raw prompts, tokens or malformed payloads', async () => { + const prints = [ + jest.spyOn(console, 'debug'), + jest.spyOn(console, 'log'), + jest.spyOn(console, 'warn'), + jest.spyOn(console, 'error'), + ]; + const socket = new TestSocket('A1', jest.fn(), config); + await socket.receive(hello()); + await socket.receive( + JSON.stringify({ type: 'events_api', envelope_id: 'wire', payload: envelope() }), + ); + await socket.receive('{private token malformed'); + expect(prints.flatMap((print) => print.mock.calls)).toEqual([]); + }); + + test.each(['different', '', undefined])( + 'wrong or missing hello app ID %p disables the socket', + async (appId) => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + await socket.receive(JSON.stringify({ type: 'hello', connection_info: { app_id: appId } })); + expect(state).toHaveBeenCalledWith(false); + expect(state).not.toHaveBeenCalledWith(true); + expect(disconnect).toHaveBeenCalledTimes(1); + }, + ); + + test('malformed/binary/oversized input never authenticates the socket', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + for (const raw of [ + '{', + 'null', + 'true', + JSON.stringify({ type: 'hello' }).padEnd(1024 * 1024 + 1, ' '), + ]) + await socket.receive(raw); + await socket.receive(hello(), true); + expect(state).not.toHaveBeenCalledWith(true); + }); + + test('malformed events are contained rather than rejecting the SDK callback', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + await socket.receive(hello()); + for (const payload of [ + { type: 'events_api' }, + { type: 'events_api', payload: {} }, + { type: 'events_api', payload: { event: null } }, + ]) + await expect(socket.receive(JSON.stringify(payload))).resolves.toBeUndefined(); + }); + + test('events before verified hello cannot reach application dispatch', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(backend).not.toHaveBeenCalled(); + expect(delivery.ack).not.toHaveBeenCalled(); + controller.abort(); + }); + + test('marks the app unready when its current channel changes Socket application', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + try { + socket.emit('ws_message', hello(), false); + await tick(); + app.ready = true; + backend.mockImplementation(async () => + json({ + ...bundle, + channel: { + ...bundle.channel, + revision: 'r2', + slack: { ...bundle.channel.slack, appId: 'A2' }, + }, + }), + ); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(delivery.ack).not.toHaveBeenCalled(); + expect(app.ready).toBe(false); + expect(backend).toHaveBeenCalledTimes(1); + } finally { + controller.abort(); + } + }); + + test('valid hello permits durable admission followed by acknowledgment', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(app.connected).toBe(true); + expect(backend).toHaveBeenCalledTimes(2); + expect(delivery.ack).toHaveBeenCalledWith(undefined); + controller.abort(); + expect(app.connected).toBe(false); + }); + + test('a wrong hello stops delivery and repeated wrong hello never becomes ready', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello('wrong'), false); + await tick(); + socket.emit('slack_event', event()); + await tick(); + socket.emit('ws_message', hello('wrong'), false); + await tick(); + socket.emit('slack_event', event()); + await tick(); + expect(app.connected).toBe(false); + expect(backend).not.toHaveBeenCalled(); + expect(disconnect).toHaveBeenCalledTimes(2); + controller.abort(); + }); + + test('reconnect requires another valid hello before accepting more events', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + socket.emit('reconnecting'); + socket.emit('slack_event', event()); + await tick(); + expect(app.connected).toBe(false); + expect(backend).not.toHaveBeenCalled(); + socket.emit('connecting'); + socket.emit('ws_message', hello(), false); + await tick(); + socket.emit('slack_event', event()); + await tick(); + expect(backend).toHaveBeenCalledTimes(2); + controller.abort(); + }); + + test('transport failure revokes hello authority until the next connection attempt', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket, closed } = await connectSocket(config, app, 'A1', log, controller.signal); + try { + socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + socket.emit('error', new Error('private transport failure')); + expect(app.connected).toBe(false); + socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(false); + socket.emit('slack_event', event()); + await tick(); + expect(backend).not.toHaveBeenCalled(); + socket.emit('connecting'); + socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + } finally { + controller.abort(); + await closed; + } + }); + + test('SDK error and rejected admission never log raw content or acknowledge failure', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : Promise.reject(new Error('private provider credential')), + ); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(delivery.ack).not.toHaveBeenCalled(); + socket.emit('error', new Error('private Slack app token')); + expect(log.warn.mock.calls).toEqual([ + ['slack_socket_delivery_not_accepted'], + ['slack_socket_transport_error'], + ]); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('private'); + controller.abort(); + }); + + test('invalid SDK deliveries are ignored without authority or acknowledgement', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + for (const delivery of [ + { ...event(), envelope_id: '' }, + { ...event(), type: 'other' }, + { ...event(), ack: 'not a function' }, + ]) + socket.emit('slack_event', delivery); + await tick(); + expect(backend).not.toHaveBeenCalled(); + controller.abort(); + }); + + test('initial startup failure is owned and closes the transport', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + start.mockRejectedValueOnce(new Error('private startup failure')); + await expect(connectSocket(config, app, 'A1', log, controller.signal)).rejects.toThrow(); + expect(disconnect).toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); + + test('retries a failed reconnect and requires a fresh authenticated hello', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + const connection = await connectSocket( + { ...config, socketReconnectDelayMs: 100 }, + app, + 'A1', + log, + controller.signal, + ); + try { + connection.socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + start.mockRejectedValueOnce(new Error('private retry failure')); + connection.socket.emit('disconnected'); + await tick(); + expect(app.connected).toBe(false); + const deadline = Date.now() + 3000; + while (start.mock.calls.length < 3 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(start).toHaveBeenCalledTimes(3); + expect(app.connected).toBe(false); + expect(log.warn).toHaveBeenCalledWith('slack_socket_reconnect_failed'); + connection.socket.emit('connecting'); + connection.socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + } finally { + controller.abort(); + await connection.closed; + } + }); + + test('abort during a pending reconnect drains without an unhandled rejection', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + const connection = await connectSocket( + { ...config, socketReconnectDelayMs: 100 }, + app, + 'A1', + log, + controller.signal, + ); + let fail: (error: Error) => void = () => {}; + start.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + fail = reject; + }), + ); + connection.socket.emit('disconnected'); + const deadline = Date.now() + 3000; + while (start.mock.calls.length < 2 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(start).toHaveBeenCalledTimes(2); + controller.abort(); + fail(new Error('private late rejection')); + await expect(connection.closed).resolves.toBeUndefined(); + expect(app.connected).toBe(false); + expect(disconnect).toHaveBeenCalled(); + }); + + test('pre-aborted startup never starts the SDK', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + controller.abort(); + await expect(connectSocket(config, app, 'A1', log, controller.signal)).rejects.toThrow(); + expect(start).not.toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); + + test('abort during startup rejects and cleans up a late successful start', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + let finish: (value: { ok: true }) => void = () => {}; + start.mockImplementation( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const pending = connectSocket(config, app, 'A1', log, controller.signal); + controller.abort(); + finish({ ok: true }); + await expect(pending).rejects.toThrow(); + expect(disconnect).toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); + + test('aborted active transport cannot dispatch or acknowledge later input', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + controller.abort(); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(backend).not.toHaveBeenCalled(); + expect(delivery.ack).not.toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); +}); diff --git a/packages/channel-gateway/src/socket.ts b/packages/channel-gateway/src/socket.ts new file mode 100644 index 00000000000..fe9fcd474c6 --- /dev/null +++ b/packages/channel-gateway/src/socket.ts @@ -0,0 +1,273 @@ +import { z } from 'zod'; +import { LogLevel } from '@slack/web-api'; +import { SocketModeClient } from '@slack/socket-mode'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { GatewayRuntimeConfig } from './config'; +import type { GatewayConfig } from './config'; +import type { GatewayApp } from './app'; +import type { Logger } from './types'; + +const quiet = { + debug() {}, + info() {}, + warn() {}, + error() {}, + setLevel() {}, + setName() {}, + getLevel: () => LogLevel.ERROR, +}; +const event = z.object({ + type: z.enum(['events_api', 'interactive']), + body: z.unknown(), + envelope_id: z.string().min(1), + ack: z.custom<(value?: object) => Promise>((value) => typeof value === 'function'), +}); + +/** The SDK hides hello's connection_info; validate it before permitting its event dispatch. */ +export class BoundSocket extends SocketModeClient { + private phase: 'handshaking' | 'verified' | 'inactive' = 'handshaking'; + constructor( + private readonly appId: string, + private readonly state: (connected: boolean) => void, + config: GatewayConfig, + signal?: AbortSignal, + ) { + super({ + appToken: config.SLACK_APP_TOKEN!, + logger: quiet, + // SDK 3.1's automatic reconnect discards start rejections. The gateway owns that task. + autoReconnectEnabled: false, + clientOptions: { + slackApiUrl: config.SLACK_API_BASE_URL.replace(/\/?$/, '/'), + timeout: config.API_CALL_TIMEOUT_MS, + retryConfig: { retries: 0 }, + rejectRateLimitedCalls: true, + allowAbsoluteUrls: false, + fetch: (url, init) => + fetch(url, { + ...init, + redirect: 'error', + signal: AbortSignal.any([ + AbortSignal.timeout(config.API_CALL_TIMEOUT_MS), + ...(signal ? [signal] : []), + ]), + }), + }, + }); + this.on('connecting', () => { + // An aborted URL fetch may already have settled; close a late-created SDK socket too. + if (signal?.aborted) + queueMicrotask(() => { + void this.disconnect().catch(() => {}); + }); + }); + for (const phase of ['connecting', 'reconnecting', 'disconnecting', 'disconnected', 'error']) + this.on(phase, () => { + this.phase = phase === 'connecting' ? 'handshaking' : 'inactive'; + this.state(false); + }); + } + + protected override async onWebSocketMessage( + data: string | ArrayBuffer, + binary: boolean, + ): Promise { + if (binary) return; + const raw = typeof data === 'string' ? data : new TextDecoder().decode(data); + if (Buffer.byteLength(raw) > 1024 * 1024) return; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return; + } + const envelope = z + .object({ + type: z.string(), + connection_info: z.unknown().optional(), + envelope_id: z.string().optional(), + payload: z.record(z.string(), z.unknown()).optional(), + }) + .safeParse(parsed); + if (!envelope.success) return; + const value = envelope.data; + if (value.type === 'hello') { + const connection = z + .object({ app_id: z.literal(this.appId) }) + .safeParse(value.connection_info); + if (!connection.success) { + this.state(false); + this.phase = 'inactive'; + await this.disconnect().catch(() => {}); + return; + } + if (this.phase !== 'handshaking') return; + this.phase = 'verified'; + this.state(true); + } else if (value.type === 'disconnect') { + this.phase = 'inactive'; + this.state(false); + } else { + if (this.phase !== 'verified' || !value.envelope_id || !value.payload) return; + if (value.type === 'events_api') { + if ( + !z.object({ event: z.object({ type: z.string().min(1) }) }).safeParse(value.payload) + .success + ) + return; + } else if (value.type !== 'interactive') return; + } + try { + await super.onWebSocketMessage(data, binary); + } catch { + // The SDK invokes this async callback without awaiting it. Contain malformed frames here. + this.phase = 'inactive'; + this.state(false); + this.emit('error', new Error('Socket message rejected')); + await this.disconnect().catch(() => {}); + } + } +} + +export interface SocketConnection { + socket: BoundSocket; + /** Owned supervision/cleanup; rejects only an unexpected lifecycle failure. */ + closed: Promise; +} + +async function untilAbort(work: Promise, signal: AbortSignal): Promise { + let stop: () => void = () => {}; + const aborted = new Promise((_resolve, reject) => { + stop = () => reject(new Error('Gateway stopped')); + if (signal.aborted) stop(); + else signal.addEventListener('abort', stop, { once: true }); + }); + try { + return await Promise.race([work, aborted]); + } finally { + signal.removeEventListener('abort', stop); + } +} + +export async function connectSocket( + config: GatewayRuntimeConfig, + app: GatewayApp, + appId: string, + log: Logger, + signal: AbortSignal, +): Promise { + signal.throwIfAborted(); + app.bindSocketApp(appId); + app.connected = false; + const socket = new BoundSocket( + appId, + (connected) => { + app.connected = connected && !signal.aborted; + }, + config, + signal, + ); + for (const phase of ['connecting', 'reconnecting', 'disconnected']) + socket.on(phase, () => { + app.connected = false; + }); + socket.on('error', () => log.warn('slack_socket_transport_error')); + socket.on('slack_event', (input: unknown) => { + if (!app.connected || signal.aborted) return; + const delivery = event.safeParse(input); + if (!delivery.success) return; + if ( + z + .object({ event: z.object({ type: z.literal('agent_session_stopped') }) }) + .safeParse(delivery.data.body).success + ) + log.info('slack_native_stop_received'); + void app + .dispatch(delivery.data.body, delivery.data.type === 'interactive', signal) + .then((payload) => delivery.data.ack(payload)) + .catch(() => log.warn('slack_socket_delivery_not_accepted')); + }); + + let resolveReady: () => void = () => {}; + let rejectReady: (error: unknown) => void = () => {}; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + let cleanup: Promise | undefined; + const close = () => { + if (!cleanup) + cleanup = (async () => { + app.connected = false; + let disconnect: Promise; + try { + disconnect = socket.disconnect(); + } catch (error) { + disconnect = Promise.reject(error); + } + const results = await Promise.allSettled([disconnect]); + if (results.some((result) => result.status === 'rejected')) + log.warn('slack_socket_shutdown_unconfirmed'); + })(); + return cleanup; + }; + const stop = () => { + app.connected = false; + void close(); + }; + signal.addEventListener('abort', stop, { once: true }); + const closed = (async () => { + let hasConnected = false; + let failures = 0; + try { + while (!signal.aborted) { + let disconnected: () => void = () => {}; + const ended = new Promise((resolve) => { + disconnected = resolve; + }); + socket.once('disconnected', disconnected); + try { + await untilAbort(socket.start(), signal); + signal.throwIfAborted(); + failures = 0; + if (!hasConnected) { + hasConnected = true; + resolveReady(); + } + await untilAbort(ended, signal); + } catch (error) { + if (signal.aborted) break; + if (!hasConnected) throw error; + log.warn('slack_socket_reconnect_failed'); + } finally { + socket.removeListener('disconnected', disconnected); + app.connected = false; + } + if (signal.aborted) break; + failures++; + // Node's timer ceiling is a platform constraint, not a retry policy limit. + const wait = Math.min(config.socketReconnectDelayMs * failures, 2_147_483_647); + try { + await delay(wait, undefined, { signal }); + } catch { + break; + } + socket.emit('reconnecting'); + } + } finally { + if (!hasConnected && signal.aborted) rejectReady(new Error('Gateway stopped')); + signal.removeEventListener('abort', stop); + await close(); + } + })(); + // Own failures even before the initial handle can be returned; main also observes completion. + void closed.catch(rejectReady); + try { + await ready; + signal.throwIfAborted(); + } catch (error) { + await closed.catch(() => {}); + throw error; + } + return { socket, closed }; +} diff --git a/packages/channel-gateway/src/worker.spec.ts b/packages/channel-gateway/src/worker.spec.ts new file mode 100644 index 00000000000..ae6b26fa5c3 --- /dev/null +++ b/packages/channel-gateway/src/worker.spec.ts @@ -0,0 +1,824 @@ +import { createHash } from 'node:crypto'; +import { setTimeout as delay } from 'node:timers/promises'; +import { DEFAULT_CHANNEL_GATEWAY_POLICY } from 'librechat-data-provider'; +import type { + ChannelDeliveryPlan, + ChannelDeliveryReceipt, + ChannelRunView, +} from 'librechat-data-provider'; +import type { ChannelBundle, Lease, Logger, ResponseGrant, ResponsePreparation } from './types'; +import { ChannelWorker, progressText } from './worker'; +import { ChannelClient } from './client'; +import { makeSlackPlan } from './render'; +import { loadConfig } from './config'; +import { SlackClient } from './slack'; + +const json = (body: object, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +const slackOK = (body: object = {}) => json({ ok: true, ...body }); + +interface MutationBody { + owner?: string; + fence?: Lease['fence']; + responseId?: string; + revision?: number; + rendererVersion?: string; + text?: string; + plan?: ChannelDeliveryPlan; + operationId?: string; + attempts?: number; + upload?: ChannelDeliveryReceipt['upload']; + receipt?: ChannelDeliveryReceipt; +} + +function fixture() { + const config = loadConfig({ + CHANNEL_SERVICE_TOKEN: 'service-token', + CHANNEL_ID: 'channel-1', + CHANNEL_TENANT_ID: 'tenant-1', + LIBRECHAT_INTERNAL_URL: 'http://backend.example', + SLACK_API_BASE_URL: 'http://127.0.0.1:8080/api/', + API_CALL_TIMEOUT_MS: '2000', + AGENT_INVOKE_TIMEOUT_MS: '2000', + }); + const bundle: ChannelBundle = { + gateway: { version: 1, ...DEFAULT_CHANNEL_GATEWAY_POLICY, transport: 'socket_mode' }, + channel: { + channelId: config.CHANNEL_ID, + tenantId: config.CHANNEL_TENANT_ID, + agentId: 'agent-1', + createdBy: 'user-1', + credentialMode: 'per_user', + type: 'slack', + status: 'active', + revision: 'revision-1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'BOT1' }, + }, + credentials: { botToken: 'bot-token' }, + agent: { id: 'agent-1', name: 'Test' }, + }; + const state: ChannelRunView = { + runId: 'run-1', + channelId: config.CHANNEL_ID, + revision: 'revision-1', + state: 'executing', + sequence: 1, + cancellationRequested: false, + receipts: [], + admission: { + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + eventId: 'event-1', + externalChannelId: 'C1', + externalThreadKey: 'C1:123.456', + externalMessageKey: 'C1:123.457', + input: 'hello', + }, + }; + const grant: ResponseGrant = { + apiKey: 'sk-bound-test', + requestKey: 'channel:run-1', + responseId: 'resp_test', + request: { + model: bundle.agent.id, + input: state.admission.input, + background: true, + store: true, + }, + }; + const lease: Lease = { + run: state, + fence: { runId: state.runId, owner: 'worker-1', epoch: 1 }, + leaseExpiresAt: Date.now() + 60_000, + }; + const requests: { + path: string; + body: MutationBody; + authorization: string | null; + key: string | null; + }[] = []; + const slackRequests: { method: string; body: URLSearchParams }[] = []; + const artifactBytes = Buffer.from('test artifact'); + const controller = new AbortController(); + let claimCount = 0; + const log: Logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; + const hooks: { + advance?: () => void; + renew?: () => Promise; + response?: () => Promise; + slack?: (method: string, body: URLSearchParams) => Promise; + prepare?: () => ResponsePreparation; + checkpoint?: () => Promise; + begin?: (body: MutationBody) => Response | undefined; + } = {}; + const snapshot = () => structuredClone(state); + const backend = jest.fn, Parameters>( + async (input, init) => { + const path = new URL(String(input)).pathname; + const body: MutationBody = init?.body ? JSON.parse(String(init.body)) : {}; + const headers = new Headers(init?.headers); + requests.push({ + path, + body, + authorization: headers.get('authorization'), + key: headers.get('idempotency-key'), + }); + if (path.includes('/service/slack/')) return json(bundle); + if (path.includes('/api/agents/v1/responses')) { + if (hooks.response) return hooks.response(); + return json({ + id: grant.responseId ?? 'resp_test', + object: 'response', + model: bundle.agent.id, + background: true, + store: true, + status: 'completed', + }); + } + if (path.endsWith('/claim')) { + if (claimCount++) return new Response(null, { status: 204 }); + lease.fence.owner = body.owner ?? ''; + return json({ ...lease, run: snapshot() }); + } + if (path.endsWith('/renew')) { + if (hooks.renew) return hooks.renew(); + return json({ fence: lease.fence, leaseExpiresAt: Date.now() + 90 }); + } + if (path.endsWith('/yield')) { + controller.abort(); + return new Response(null, { status: 204 }); + } + if (path.endsWith('/response/prepare')) return json(hooks.prepare?.() ?? grant); + if (path.endsWith('/response/bind')) grant.responseId = body.responseId; + else if (path.endsWith('/advance')) { + if (hooks.advance) hooks.advance(); + else { + state.state = 'delivering'; + state.snapshot = { + status: 'completed', + text: 'Final answer', + responseId: grant.responseId, + elapsedMs: 200, + }; + } + } else if (path.endsWith('/progress/plan')) { + state.progress = { + revision: body.revision ?? 0, + rendererVersion: body.rendererVersion ?? '', + text: body.text ?? '', + outcome: 'planned', + attempts: 0, + externalMessageKey: state.progress?.externalMessageKey, + }; + } else if (path.endsWith('/progress/begin')) { + if (!state.progress) throw new Error('Missing progress fixture'); + state.progress.outcome = 'inflight'; + state.progress.attempts++; + } else if (path.endsWith('/progress/settle')) { + if (!state.progress || !body.receipt) + throw new Error('Missing progress settlement fixture'); + state.progress = { ...state.progress, ...body.receipt }; + } else if (path.endsWith('/plan')) { + state.plan = body.plan; + state.receipts = + body.plan?.parts.map((part) => ({ + operationId: part.operationId, + outcome: 'planned', + attempts: 0, + })) ?? []; + } else if (path.includes('/artifacts/')) { + const artifact = state.snapshot?.artifacts?.[0]; + if (!artifact) throw new Error('Missing artifact fixture'); + return new Response(artifactBytes, { + headers: { + 'Content-Type': 'application/octet-stream', + 'X-Artifact-Filename': encodeURIComponent(artifact.filename), + 'X-Artifact-SHA256': artifact.sha256, + 'X-Artifact-Media-Type': artifact.mediaType, + }, + }); + } else if (path.endsWith('/delivery/begin')) { + const intercepted = hooks.begin?.(body); + if (intercepted) return intercepted; + const receipt = state.receipts.find((item) => item.operationId === body.operationId); + if (!receipt) throw new Error('Missing begin fixture'); + receipt.outcome = 'inflight'; + receipt.attempts++; + } else if (path.endsWith('/delivery/checkpoint')) { + const receipt = state.receipts.find((item) => item.operationId === body.operationId); + if (!receipt) throw new Error('Missing checkpoint fixture'); + receipt.upload = body.upload; + if (hooks.checkpoint) return hooks.checkpoint(); + } else if (path.endsWith('/delivery/settle')) { + const receipt = state.receipts.find( + (item) => item.operationId === body.receipt?.operationId, + ); + if (!receipt || !body.receipt) throw new Error('Missing settle fixture'); + Object.assign(receipt, body.receipt); + } else if (path.endsWith('/commit')) state.state = 'delivered'; + else throw new Error('Unexpected backend fixture path'); + return json(snapshot()); + }, + ); + const slackTransport = jest.fn, Parameters>( + async (input, init) => { + const method = new URL(String(input)).pathname.split('/').at(-1) ?? ''; + const body = new URLSearchParams(String(init?.body)); + slackRequests.push({ method, body }); + if (hooks.slack) return hooks.slack(method, body); + if (method === 'files.getUploadURLExternal') + return slackOK({ file_id: 'F1', upload_url: 'http://127.0.0.1:8080/upload' }); + if (method === 'upload') return new Response('ok'); + if (method === 'files.completeUploadExternal') return slackOK({ files: [{ id: 'F1' }] }); + if (method === 'agents.sessions.setStatus') return slackOK(); + return slackOK({ channel: 'C1', ts: method === 'chat.update' ? body.get('ts') : '123.789' }); + }, + ); + const client = new ChannelClient(config, backend); + const slack = new SlackClient(config, log, slackTransport); + const worker = new ChannelWorker(config, client, slack, log); + const savedPlan = (artifacts = false) => { + state.state = 'delivering'; + state.snapshot = { + status: 'completed', + text: 'Saved final answer', + responseId: grant.responseId, + elapsedMs: 200, + }; + if (artifacts) + state.snapshot.artifacts = [ + { + artifactId: 'artifact-1', + filename: 'report.txt', + bytes: artifactBytes.length, + mediaType: 'text/plain', + sha256: createHash('sha256').update(artifactBytes).digest('hex'), + }, + ]; + state.plan = makeSlackPlan(state, true); + state.receipts = state.plan.parts.map((part) => ({ + operationId: part.operationId, + outcome: 'planned', + attempts: 0, + })); + }; + const execute = () => worker.process({ ...lease, run: snapshot() }, controller.signal); + const paths = () => requests.map((request) => request.path); + return { + config, + bundle, + state, + grant, + lease, + requests, + slackRequests, + artifactBytes, + controller, + log, + hooks, + backend, + slackTransport, + worker, + savedPlan, + execute, + paths, + }; +} + +describe('durable Channels worker', () => { + test('a reduced server delivery budget reaches the persisted plan and completes without optional uploads', async () => { + const f = fixture(); + f.bundle.delivery = { maxDeliveryParts: 1 }; + f.hooks.advance = () => { + f.state.state = 'delivering'; + f.state.snapshot = { + status: 'completed', + text: 'Final answer', + responseId: 'resp_test', + elapsedMs: 200, + artifacts: [ + { + artifactId: 'file-1', + filename: 'report.txt', + mediaType: 'text/plain', + bytes: 1, + sha256: 'a'.repeat(64), + }, + ], + }; + }; + await f.execute(); + expect(f.state.plan?.parts).toHaveLength(1); + expect(f.state.plan?.parts[0].text).toContain('Final answer'); + expect(f.state.plan?.parts[0].text).toContain('omitted'); + expect(f.slackRequests.some((request) => request.method.startsWith('files.'))).toBe(false); + expect(f.state.state).toBe('delivered'); + }); + + test.each([false, true])( + 'uses the current server lifecycle-reaction policy (%s)', + async (enabled) => { + const f = fixture(); + f.bundle.gateway.lifecycleReactions = enabled; + await f.execute(); + const reactions = f.slackRequests.filter((request) => + request.method.startsWith('reactions.'), + ); + expect(reactions.map((request) => [request.method, request.body.get('name')])).toEqual( + enabled + ? [ + ['reactions.add', 'eyes'], + ['reactions.add', 'hourglass_flowing_sand'], + ['reactions.remove', 'hourglass_flowing_sand'], + ['reactions.add', 'white_check_mark'], + ] + : [], + ); + expect(f.state.state).toBe('delivered'); + }, + ); + + test('executes only through a bound Responses grant and persists progress before writes', async () => { + const f = fixture(); + delete f.grant.responseId; + await f.execute(); + const responseRequests = f.requests.filter((request) => + request.path.startsWith('/api/agents/v1/responses'), + ); + expect(responseRequests.map((request) => request.authorization)).toEqual([ + 'Bearer sk-bound-test', + 'Bearer sk-bound-test', + ]); + expect(responseRequests[0].key).toBe('channel:run-1'); + expect(responseRequests[0].body).toMatchObject({ + model: 'agent-1', + input: 'hello', + background: true, + store: true, + stream: false, + }); + expect(f.paths().some((path) => path.endsWith('/response/bind'))).toBe(true); + const writes = f.slackRequests.filter((request) => + ['chat.postMessage', 'chat.update'].includes(request.method), + ); + expect(writes.map((request) => request.method)).toEqual([ + 'chat.postMessage', + 'chat.update', + 'chat.postMessage', + ]); + expect(writes[0].body.get('text')).toBe(':hourglass_flowing_sand: Working…'); + expect(writes[1].body.get('text')).toBe(':white_check_mark: Completed.'); + expect(f.paths().filter((path) => path.endsWith('/progress/begin'))).toHaveLength(2); + expect(f.paths().filter((path) => path.endsWith('/progress/settle'))).toHaveLength(2); + expect(f.state.state).toBe('delivered'); + }); + + test('reuses a saved immutable plan without preparing or reexecuting Responses', async () => { + const f = fixture(); + f.savedPlan(); + f.state.snapshot!.text = 'Mutated snapshot must not render'; + await f.execute(); + expect( + f.slackRequests.find((request) => request.method === 'chat.postMessage')?.body.get('text'), + ).toContain('Saved final answer'); + expect(f.paths().some((path) => /response|advance|\/plan$/.test(path))).toBe(false); + expect(f.state.state).toBe('delivered'); + }); + + test.each(['unknown', 'inflight'] as const)( + 'never replays a required %s delivery', + async (outcome) => { + const f = fixture(); + f.savedPlan(); + f.state.receipts[0].outcome = outcome; + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /delivery\/begin|commit|yield/.test(path))).toBe(false); + }, + ); + + test('records an uncertain create once and holds it on recovery', async () => { + const f = fixture(); + f.savedPlan(); + f.hooks.slack = async () => { + throw new Error('Uncertain connection reset'); + }; + await expect(f.execute()).rejects.toThrow('not confirmed'); + expect(f.state.receipts[0]).toMatchObject({ outcome: 'unknown', attempts: 1 }); + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests).toHaveLength(1); + expect(f.paths().filter((path) => path.endsWith('/delivery/begin'))).toHaveLength(1); + }); + + test.each(['apiKey', 'requestKey', 'model', 'input'] as const)( + 'rejects a changed %s grant before Responses execution', + async (field) => { + const f = fixture(); + f.bundle.gateway.nativeSessions = true; + if (field === 'apiKey' || field === 'requestKey') f.grant[field] = 'wrong'; + else f.grant.request[field] = 'wrong'; + await expect(f.execute()).rejects.toThrow( + field === 'apiKey' ? 'Invalid run-bound response credential' : 'Invalid Responses grant', + ); + expect(f.paths().some((path) => path.startsWith('/api/agents/'))).toBe(false); + }, + ); + + test('renews independently while a Slack create is slow', async () => { + const f = fixture(); + f.savedPlan(); + f.lease.leaseExpiresAt = Date.now() + 60; + f.hooks.slack = async () => { + await delay(120); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + await f.execute(); + expect(f.paths().filter((path) => path.endsWith('/renew')).length).toBeGreaterThan(1); + expect(f.state.state).toBe('delivered'); + expect(f.paths().some((path) => path.endsWith('/yield'))).toBe(false); + }); + + test('uncertain renewal stops settlement and does not blindly yield an inflight create', async () => { + const f = fixture(); + f.savedPlan(); + f.lease.leaseExpiresAt = Date.now() + 60; + f.hooks.renew = async () => json({ error: 'unavailable' }, 503); + f.hooks.slack = async () => { + await delay(100); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + f.log.warn = () => f.controller.abort(); + await f.worker.run(f.controller.signal); + expect(f.paths().some((path) => path.endsWith('/renew'))).toBe(true); + expect(f.state.receipts[0].outcome).toBe('inflight'); + expect(f.paths().some((path) => /delivery\/settle|commit|yield/.test(path))).toBe(false); + expect(f.slackRequests).toHaveLength(1); + }); + + test('unknown progress is not replayed and cannot safely yield a waiting run', async () => { + const f = fixture(); + f.state.progress = { + revision: 1, + rendererVersion: 'slack-progress-v1', + text: 'Working', + outcome: 'unknown', + attempts: 1, + }; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + responseId: 'resp_test', + elapsedMs: 10, + }; + }; + f.log.warn = () => f.controller.abort(); + await f.worker.run(f.controller.signal); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /progress\/(begin|settle)|yield/.test(path))).toBe(false); + }); + + test('cleans terminal progress before persisting a final plan', async () => { + const f = fixture(); + f.state.progress = { + revision: 1, + rendererVersion: 'slack-progress-v1', + text: progressText(f.state), + outcome: 'confirmed', + attempts: 1, + externalMessageKey: 'C1:123.700', + }; + await f.execute(); + expect(f.slackRequests[0].method).toBe('chat.update'); + expect(f.slackRequests[0].body.get('text')).toBe(':white_check_mark: Completed.'); + const cleanup = f.paths().findIndex((path) => path.endsWith('/progress/settle')); + const final = f.paths().findIndex((path) => path.endsWith('/run-1/plan')); + expect(cleanup).toBeLessThan(final); + expect(cleanup).toBeGreaterThan(-1); + }); + + test('checkpoint allocation and upload before completion, without a false failure notice', async () => { + const f = fixture(); + f.savedPlan(true); + await f.execute(); + expect(f.slackRequests.map((request) => request.method)).toEqual([ + 'chat.postMessage', + 'files.getUploadURLExternal', + 'upload', + 'files.completeUploadExternal', + ]); + expect( + f.requests + .filter((request) => request.path.endsWith('/delivery/checkpoint')) + .map((request) => request.body.upload), + ).toEqual([ + { fileId: 'F1', stage: 'allocated' }, + { fileId: 'F1', stage: 'uploaded' }, + ]); + expect( + f.state.receipts.find((receipt) => receipt.operationId.includes(':artifact:')), + ).toMatchObject({ + outcome: 'confirmed', + externalFileId: 'F1', + upload: { fileId: 'F1', stage: 'uploaded' }, + }); + expect(f.state.state).toBe('delivered'); + }); + + test('unknown artifact completion retains its checkpoint and is never allocated/uploaded again', async () => { + const f = fixture(); + f.savedPlan(true); + f.hooks.slack = async (method) => { + if (method === 'files.getUploadURLExternal') + return slackOK({ file_id: 'F1', upload_url: 'http://127.0.0.1:8080/upload' }); + if (method === 'upload') return new Response('ok'); + if (method === 'files.completeUploadExternal') + throw new Error('Connection lost after completion'); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + await f.execute(); + expect( + f.state.receipts.find((receipt) => receipt.operationId.includes(':artifact:')), + ).toMatchObject({ outcome: 'unknown', upload: { fileId: 'F1', stage: 'uploaded' } }); + const before = f.slackRequests.length; + await f.execute(); + expect(f.slackRequests).toHaveLength(before); + expect( + f.slackRequests.filter((request) => request.method === 'files.getUploadURLExternal'), + ).toHaveLength(1); + expect( + f.slackRequests.filter((request) => request.method === 'files.completeUploadExternal'), + ).toHaveLength(1); + }); + + test('an inflight artifact checkpoint is held, never resumed by allocating a second file', async () => { + const f = fixture(); + f.savedPlan(true); + f.state.receipts[0].outcome = 'confirmed'; + const artifact = f.state.receipts.find((receipt) => + receipt.operationId.includes(':artifact:'), + )!; + artifact.outcome = 'inflight'; + artifact.attempts = 1; + artifact.upload = { fileId: 'F-existing', stage: 'allocated' }; + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /delivery\/begin|commit/.test(path))).toBe(false); + }); + + test('an invalid actor binding cannot create Responses or send Slack output', async () => { + const f = fixture(); + f.state.admission.actor.appId = 'FOREIGN'; + await expect(f.execute()).rejects.toThrow('binding'); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /prepare|agents\/v1/.test(path))).toBe(false); + }); + + test('falls back to the persisted text only after an explicit rich-block rejection', async () => { + const f = fixture(); + f.savedPlan(); + f.hooks.slack = async (_method, body) => + body.has('blocks') && body.get('blocks') !== '[]' + ? slackOK({ ok: false, error: 'invalid_blocks' }) + : slackOK({ channel: 'C1', ts: '123.789' }); + await f.execute(); + expect(f.slackRequests).toHaveLength(2); + expect(f.slackRequests[0].body.get('text')).toBe(f.slackRequests[1].body.get('text')); + expect(f.slackRequests[1].body.get('blocks')).toBe('[]'); + expect(f.paths().filter((path) => path.endsWith('/delivery/begin'))).toHaveLength(1); + expect(f.state.receipts[0]).toMatchObject({ outcome: 'confirmed', attempts: 1 }); + }); + + test('exhausted optional artifact retries settle as failed instead of blocking final answer commit', async () => { + const f = fixture(); + f.savedPlan(true); + f.state.receipts[0].outcome = 'confirmed'; + const receipt = f.state.receipts.find((item) => item.operationId.includes(':artifact:'))!; + receipt.outcome = 'retryable'; + receipt.attempts = 5; + await f.execute(); + expect(receipt.outcome).toBe('permanent_failure'); + expect(f.slackRequests.some((request) => request.method === 'files.getUploadURLExternal')).toBe( + false, + ); + expect(f.state.state).toBe('delivered'); + }); + + test('does not yield when a renewal becomes uncertain while the independent renewer drains', async () => { + const f = fixture(); + f.bundle.gateway.nativeSessions = true; + f.lease.leaseExpiresAt = Date.now() + 60; + f.hooks.renew = async () => { + await delay(30); + return json({ error: 'unavailable' }, 503); + }; + f.hooks.response = async () => { + await delay(25); + return json({ + id: 'resp_test', + object: 'response', + model: 'agent-1', + background: true, + store: true, + status: 'requires_action', + }); + }; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + responseId: 'resp_test', + elapsedMs: 10, + }; + }; + f.log.warn = () => f.controller.abort(); + await f.worker.run(f.controller.signal); + expect(f.paths().some((path) => path.endsWith('/renew'))).toBe(true); + expect(f.paths().some((path) => path.endsWith('/yield'))).toBe(false); + }); + + test('a lost allocation checkpoint acknowledgement cannot trigger binary upload or another allocation', async () => { + const f = fixture(); + f.savedPlan(true); + f.hooks.checkpoint = async () => { + throw new Error('Checkpoint acknowledgement lost'); + }; + await expect(f.execute()).rejects.toThrow('Checkpoint acknowledgement lost'); + const receipt = f.state.receipts.find((item) => item.operationId.includes(':artifact:'))!; + expect(receipt).toMatchObject({ + outcome: 'inflight', + upload: { fileId: 'F1', stage: 'allocated' }, + }); + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests.map((request) => request.method)).toEqual([ + 'chat.postMessage', + 'files.getUploadURLExternal', + ]); + }); + + test('verified unstarted cancellation never creates or polls a Response', async () => { + const f = fixture(); + f.bundle.gateway.nativeSessions = true; + f.state.cancellationRequested = true; + f.hooks.prepare = () => ({ + settled: { + ...f.state, + state: 'delivering', + snapshot: { text: '', status: 'cancelled', elapsedMs: 0 }, + }, + }); + await f.execute(); + expect(f.paths().some((path) => /agents\/v1|advance|response\/bind/.test(path))).toBe(false); + expect( + f.slackRequests.find((request) => request.method === 'chat.postMessage')?.body.get('text'), + ).toContain('This request was cancelled'); + }); + + test('unproved cancellation without a Response cursor is held instead of creating new work', async () => { + const f = fixture(); + f.bundle.gateway.nativeSessions = true; + f.state.cancellationRequested = true; + delete f.grant.responseId; + await expect(f.execute()).rejects.toThrow('awaits reconciliation'); + expect(f.paths().some((path) => /agents\/v1|advance|response\/bind/.test(path))).toBe(false); + }); + + test('a canonical cancellation polls only the exact bound Response cancellation route', async () => { + const f = fixture(); + f.bundle.gateway.nativeSessions = true; + f.state.cancellationRequested = true; + f.hooks.advance = () => { + f.state.state = 'delivering'; + f.state.snapshot = { text: '', status: 'cancelled', responseId: 'resp_test', elapsedMs: 0 }; + }; + await f.execute(); + expect( + f.requests + .filter((request) => request.path.startsWith('/api/agents/')) + .map((request) => request.path), + ).toEqual(['/api/agents/v1/responses/resp_test/cancel']); + expect( + f.slackRequests + .filter((request) => request.method === 'agents.sessions.setStatus') + .map((request) => request.body.get('status')), + ).toEqual(['processing', 'active']); + }); + + test('requires-action uses native suspended state and one safe actor-lane yield', async () => { + const f = fixture(); + f.bundle.gateway.nativeSessions = true; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + responseId: 'resp_test', + elapsedMs: 10, + }; + }; + await f.worker.run(f.controller.signal); + expect(f.slackRequests.map((request) => [request.method, request.body.get('status')])).toEqual([ + ['agents.sessions.setStatus', 'processing'], + ['agents.sessions.setStatus', 'suspended'], + ]); + expect(f.paths().filter((path) => path.endsWith('/yield'))).toHaveLength(1); + expect(f.paths().some((path) => path.endsWith('/commit'))).toBe(false); + }); +}); + +describe('optional delivery and uncertain execution boundaries', () => { + function trace(f: ReturnType) { + const part = { + operationId: 'run-1:trace:0', + kind: 'message' as const, + required: false, + text: '', + traceUrl: 'https://chat.example.com/session', + }; + f.state.plan!.parts.push(part); + f.state.receipts.push({ operationId: part.operationId, outcome: 'planned', attempts: 0 }); + return part; + } + test('an exhausted optional message cannot hold a confirmed required answer or cause a repost', async () => { + const f = fixture(); + f.savedPlan(); + f.state.receipts[0].outcome = 'confirmed'; + f.state.plan!.parts.push({ + operationId: 'notice', + kind: 'message', + required: false, + text: 'Attachment unavailable', + }); + f.state.receipts.push({ + operationId: 'notice', + outcome: 'retryable', + attempts: 5, + retryAt: Date.now() + 60000, + }); + await f.execute(); + expect(f.state.state).toBe('delivered'); + expect(f.slackRequests).toHaveLength(0); + }); + test('a trace omitted by the latest view after answer delivery is skipped before receipt lookup', async () => { + const f = fixture(); + f.savedPlan(); + const part = trace(f); + f.hooks.slack = async () => { + f.state.plan!.parts = f.state.plan!.parts.filter((p) => p.operationId !== part.operationId); + f.state.receipts = f.state.receipts.filter((p) => p.operationId !== part.operationId); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + await f.execute(); + expect(f.state.state).toBe('delivered'); + expect(f.slackRequests).toHaveLength(1); + expect(f.slackRequests[0].body.get('text')).not.toContain(part.traceUrl); + }); + test.each(['trace_unavailable', 'stale_fence'])( + 'only the exact trace authority outcome is skippable: %s', + async (code) => { + const f = fixture(); + f.savedPlan(); + f.state.receipts[0].outcome = 'confirmed'; + const part = trace(f); + f.hooks.begin = (body) => + body.operationId === part.operationId ? json({ error: code }, 409) : undefined; + if (code === 'trace_unavailable') { + await f.execute(); + expect(f.state.state).toBe('delivered'); + } else { + await expect(f.execute()).rejects.toThrow('HTTP 409'); + expect(f.state.state).toBe('delivering'); + } + expect(f.slackRequests).toHaveLength(0); + }, + ); + test('native status success still publishes recovery guidance and suppresses stale review controls', async () => { + const f = fixture(); + f.bundle.gateway.nativeSessions = true; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + elapsedMs: 10, + responseId: 'resp_test', + recovery: 'execution_state_unavailable', + pendingAction: { actionId: 'stale', kind: 'tool_approval', expiresAt: Date.now() + 60000 }, + }; + }; + await f.worker.run(f.controller.signal); + const warning = f.slackRequests.find((r) => r.method === 'chat.postMessage')!; + expect(warning.body.get('text')).toContain('do not repeat actions'); + expect(warning.body.get('text')).toContain('reconcile'); + expect(warning.body.get('blocks')).not.toContain('channel_run_review'); + expect(f.slackRequests.some((r) => r.method === 'agents.sessions.setStatus')).toBe(true); + expect(f.paths().some((p) => p.endsWith('/yield'))).toBe(true); + f.state.cancellationRequested = true; + expect(progressText(f.state)).toContain('Cancellation requested.'); + expect(progressText(f.state)).toContain('do not repeat actions'); + }); +}); diff --git a/packages/channel-gateway/src/worker.ts b/packages/channel-gateway/src/worker.ts new file mode 100644 index 00000000000..bfc1f6db66a --- /dev/null +++ b/packages/channel-gateway/src/worker.ts @@ -0,0 +1,711 @@ +import { randomBytes } from 'node:crypto'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { + ChannelDeliveryReceipt, + ChannelDeliveryPart, + ChannelRunView, + ChannelRunFence, +} from 'librechat-data-provider'; +import type { ChannelBundle, Lease, Logger, ResponseGrant } from './types'; +import type { GatewayConfig } from './config'; +import { durableSlackMessage, makeSlackPlan, supportedRenderer } from './render'; +import { SlackClient, SlackWriteError } from './slack'; +import { ChannelClient, ServiceError } from './client'; + +const progressRenderer = 'slack-progress-v1'; +const terminalStates = new Set(['delivered', 'failed', 'cancelled']); +class YieldRun extends Error {} + +export function terminalSnapshot(run: ChannelRunView): boolean { + return !!run.snapshot && ['completed', 'failed', 'cancelled'].includes(run.snapshot.status); +} + +export function pendingReview(run: ChannelRunView): boolean { + const pending = run.snapshot?.pendingAction; + return ( + !run.snapshot?.recovery && + run.snapshot?.status === 'requires_action' && + !!pending?.actionId && + pending.expiresAt > Date.now() && + ['tool_approval', 'ask_user_question', 'oauth'].includes(pending.kind) + ); +} + +export function progressText(run: ChannelRunView): string { + if (terminalSnapshot(run)) { + if (run.snapshot?.status === 'completed') return ':white_check_mark: Completed.'; + if (run.snapshot?.status === 'cancelled') return 'Cancelled.'; + return ':warning: Failed.'; + } + if (run.snapshot?.recovery === 'execution_state_unavailable') + return ( + (run.cancellationRequested ? 'Cancellation requested. ' : '') + + ':warning: This request’s execution status is unavailable, possibly after a restart. Sign-in may be saved, but continuation and completion are not confirmed. Check the result; do not repeat actions that may already have run. If this stays unavailable, ask an administrator to reconcile the request before retrying. Stop requests cancellation, not proof that work has stopped.' + ); + if (run.cancellationRequested) return 'Cancellation requested.'; + if (run.snapshot?.status === 'requires_action') + return pendingReview(run) + ? '⏸ Your input is needed. Open the private review to continue.' + : '⏸ Waiting for input. The review action is unavailable or expired.'; + return ':hourglass_flowing_sand: Working…'; +} + +export function deliveryReceipt(run: ChannelRunView, operationId: string): ChannelDeliveryReceipt { + const receipt = run.receipts.find((item) => item.operationId === operationId); + if (!receipt) throw new Error('Missing persisted delivery receipt'); + return { ...receipt }; +} + +export function writeOutcome( + error: unknown, + attempts: number, +): Omit { + if (error instanceof SlackWriteError && error.outcome === 'permanent') + return { attempts, outcome: 'permanent_failure', errorCode: 'transport_rejected' }; + if (error instanceof SlackWriteError && error.outcome === 'retryable') + return { + attempts, + outcome: 'retryable', + errorCode: 'transport_retryable', + retryAt: Date.now() + Math.max(1000, error.retryAfterMs), + }; + return { attempts, outcome: 'unknown', errorCode: 'transport_uncertain' }; +} + +function address(run: ChannelRunView): { channel: string; thread_ts: string } { + const channel = run.admission.externalChannelId; + const prefix = channel + ':'; + const thread = run.admission.externalThreadKey.slice(prefix.length); + if ( + !channel || + !run.admission.externalThreadKey.startsWith(prefix) || + !thread || + /[:\s]/.test(thread) + ) + throw new Error('Invalid durable thread address'); + return { channel, thread_ts: thread }; +} + +function assertView(run: ChannelRunView, fence: ChannelRunFence, channelId: string): void { + if (run.runId !== fence.runId || run.channelId !== channelId || !run.revision) + throw new Error('Durable identity changed'); + address(run); +} + +function canYield(run: ChannelRunView): void { + if ( + [run.progress, ...run.receipts].some( + (item) => item?.outcome === 'inflight' || item?.outcome === 'unknown', + ) + ) + throw new Error('Delivery awaits reconciliation'); + throw new YieldRun(); +} + +export class ChannelWorker { + private readonly owner = randomBytes(16).toString('hex'); + constructor( + private readonly config: GatewayConfig, + private readonly client: ChannelClient, + private readonly slack: SlackClient, + private readonly log: Logger, + ) {} + + async run(signal: AbortSignal): Promise { + while (!signal.aborted) { + let lease: Lease | null = null; + try { + lease = await this.client.claim(this.owner, signal); + if (lease) { + if (lease.fence.owner !== this.owner) throw new Error('Claim owner changed'); + await this.process(lease, signal); + } + } catch (error) { + if (signal.aborted) break; + if (error instanceof YieldRun && lease) { + try { + await this.client.yield(lease.fence, signal); + } catch { + this.log.warn('channel_yield_unconfirmed'); + } + } else this.log.warn('channel_run_held_for_recovery'); + } + try { + await delay(1000, undefined, { signal }); + } catch { + break; + } + } + } + + async process(lease: Lease, parent: AbortSignal): Promise { + const { fence } = lease; + assertView(lease.run, fence, this.config.CHANNEL_ID); + if (!fence.owner || fence.epoch <= 0 || lease.leaseExpiresAt <= Date.now()) + throw new Error('Invalid durable lease'); + const controller = new AbortController(); + const signal = AbortSignal.any([parent, controller.signal]); + let expiry = lease.leaseExpiresAt; + let renewalUncertain = false; + const check = () => { + if (signal.aborted || Date.now() >= expiry) throw new Error('Durable lease lost'); + }; + const renewer = (async () => { + while (!signal.aborted) { + const remaining = expiry - Date.now(); + if (remaining <= 0) { + renewalUncertain = true; + controller.abort(); + return; + } + try { + await delay(Math.max(1, Math.floor(remaining / 3)), undefined, { signal }); + } catch { + return; + } + try { + const renewed = await this.client.renew(fence, signal); + if ( + renewed.fence.runId !== fence.runId || + renewed.fence.owner !== fence.owner || + renewed.fence.epoch !== fence.epoch || + renewed.leaseExpiresAt <= Date.now() + ) + throw new Error('Renewal identity changed'); + expiry = renewed.leaseExpiresAt; + } catch { + renewalUncertain = true; + controller.abort(); + return; + } + } + })(); + let failure: unknown; + try { + await this.execute(lease.run, fence, signal, check); + } catch (error) { + failure = error; + } finally { + controller.abort(); + await renewer; + } + if (failure instanceof YieldRun && renewalUncertain) throw new Error('Renewal uncertain'); + if (failure) throw failure; + } + + private async execute( + initial: ChannelRunView, + fence: ChannelRunFence, + signal: AbortSignal, + check: () => void, + ): Promise { + const bundle = await this.client.channel(signal); + if ( + initial.revision !== bundle.channel.revision || + initial.admission.actor.teamId !== bundle.channel.slack.teamId || + initial.admission.actor.appId !== bundle.channel.slack.appId + ) + throw new Error('Run binding changed'); + const token = bundle.credentials.botToken; + let sessionStatus = ''; + let sessionUpdated = 0; + let sessionRetry = 0; + let lastProgressWrite = 0; + const syncProgress = async (current: ChannelRunView) => { + check(); + let native = false; + if (bundle.gateway.nativeSessions) { + let status: 'active' | 'suspended' | 'processing' = 'processing'; + if (terminalSnapshot(current) || current.plan) status = 'active'; + else if (current.snapshot?.status === 'requires_action') status = 'suspended'; + if (!sessionRetry && sessionStatus === status && Date.now() - sessionUpdated < 60000) + native = true; + else if (terminalSnapshot(current) || Date.now() >= sessionRetry) { + const target = address(current); + try { + await this.slack.setSessionStatus( + token, + { + channel: target.channel, + threadTs: target.thread_ts, + status, + title: bundle.agent.name + ? `${bundle.agent.name} conversation`.slice(0, 200) + : undefined, + }, + signal, + ); + sessionStatus = status; + sessionUpdated = Date.now(); + sessionRetry = 0; + native = true; + } catch { + sessionRetry = Date.now() + 60000; + this.log.warn('native_status_unavailable'); + } + } + } + if (native && !current.progress && !pendingReview(current) && !current.snapshot?.recovery) + return current; + const updated = await this.progress(current, fence, bundle, signal, check, lastProgressWrite); + if ( + updated.progress?.revision !== current.progress?.revision || + updated.progress?.attempts !== current.progress?.attempts + ) + lastProgressWrite = Date.now(); + return updated; + }; + await this.lifecycle(initial, bundle, 'started', signal); + let run = await syncProgress(initial); + let grant: ResponseGrant | undefined; + let unstartedCancellation = false; + if (!run.plan) { + const prepared = await this.client.prepare(fence, signal); + if (prepared.settled) { + assertView(prepared.settled, fence, this.config.CHANNEL_ID); + if ( + prepared.settled.state !== 'delivering' || + !prepared.settled.cancellationRequested || + prepared.settled.snapshot?.status !== 'cancelled' || + prepared.settled.snapshot.responseId + ) + throw new Error('Invalid unstarted cancellation proof'); + run = prepared.settled; + unstartedCancellation = true; + } else { + if ( + !prepared.apiKey.startsWith('sk-bound-') || + prepared.requestKey !== `channel:${run.runId}` || + prepared.request.model !== bundle.agent.id || + prepared.request.input !== run.admission.input || + !prepared.request.background || + !prepared.request.store + ) + throw new Error('Invalid Responses grant'); + if (!prepared.responseId) { + if (run.cancellationRequested) + throw new Error('Unstarted cancellation awaits reconciliation'); + check(); + const response = await this.client.response(prepared, 'create', signal); + prepared.responseId = response.id; + run = await this.client.mutate( + fence, + 'response/bind', + { responseId: response.id }, + signal, + ); + } + grant = prepared; + } + } + for (let polls = 0; !run.plan; polls++) { + check(); + if (grant) + await this.client.response(grant, run.cancellationRequested ? 'cancel' : 'read', signal); + if (!unstartedCancellation) run = await this.client.mutate(fence, 'advance', {}, signal); + assertView(run, fence, this.config.CHANNEL_ID); + if (grant && run.snapshot?.responseId && run.snapshot.responseId !== grant.responseId) + throw new Error('Response projection changed'); + if (run.plan) break; + run = await syncProgress(run); + if ( + terminalSnapshot(run) && + run.progress?.outcome === 'confirmed' && + run.progress.text !== progressText(run) + ) + run = await syncProgress(run); + if (run.state === 'delivering' && run.snapshot) { + if ( + terminalSnapshot(run) && + run.progress?.externalMessageKey && + ['planned', 'retryable'].includes(run.progress.outcome) + ) + throw new Error('Terminal progress cleanup pending'); + run = await this.client.mutate( + fence, + 'plan', + { + plan: makeSlackPlan( + run, + bundle.gateway.nativeSessions, + bundle.delivery?.maxDeliveryParts, + ), + }, + signal, + ); + break; + } + if (terminalStates.has(run.state)) return; + if (run.state === 'blocked' || run.state === 'requires_action' || polls >= 4) canYield(run); + await delay(1000, undefined, { signal }); + } + if ( + !run.plan || + !supportedRenderer(run.plan.rendererVersion) || + !run.plan.parts.length || + run.plan.parts.length > 100 + ) + throw new Error('Unsupported delivery plan'); + const renderer = run.plan.rendererVersion; + for (const part of run.plan.parts) { + check(); + const traceAvailable = () => + !part.traceUrl || + run.plan?.parts.some( + (current) => + current.operationId === part.operationId && current.traceUrl === part.traceUrl, + ); + if (!traceAvailable()) continue; + if (part.kind === 'artifact') + run = await this.artifact(run, fence, part, bundle, signal, check); + else { + if ( + part.operationId === `${run.runId}:artifact-status` && + !part.required && + !run.plan?.parts.some( + (item) => + item.kind === 'artifact' && + ['unknown', 'inflight', 'permanent_failure'].includes( + deliveryReceipt(run, item.operationId).outcome, + ), + ) + ) + continue; + let receipt = deliveryReceipt(run, part.operationId); + if ( + receipt.outcome === 'confirmed' || + (!part.required && + (['unknown', 'permanent_failure'].includes(receipt.outcome) || receipt.attempts >= 5)) + ) + continue; + this.retryable(receipt); + const message = durableSlackMessage(part, renderer); + try { + run = await this.client.mutate( + fence, + 'delivery/begin', + { operationId: part.operationId }, + signal, + ); + } catch (error) { + if ( + part.traceUrl && + error instanceof ServiceError && + error.status === 409 && + error.code === 'trace_unavailable' + ) + continue; + throw error; + } + if (!traceAvailable()) continue; + receipt = deliveryReceipt(run, part.operationId); + if (receipt.outcome !== 'inflight') throw new Error('Delivery begin not persisted'); + check(); + try { + const result = await this.slack.deliverPart( + token, + { ...address(run), ...message }, + signal, + ); + receipt = { + ...receipt, + outcome: 'confirmed', + externalMessageKey: `${result.channel}:${result.ts}`, + }; + } catch (error) { + receipt = { operationId: part.operationId, ...writeOutcome(error, receipt.attempts) }; + } + check(); + run = await this.settle(run, fence, receipt, signal); + if (receipt.outcome !== 'confirmed' && (part.required || receipt.outcome === 'retryable')) + throw new Error('Delivery not confirmed'); + } + } + check(); + run = await this.client.mutate(fence, 'commit', {}, signal); + if (!terminalStates.has(run.state)) throw new Error('Commit not confirmed'); + await this.lifecycle(run, bundle, run.state === 'delivered' ? 'completed' : 'failed', signal); + } + + private retryable(receipt: ChannelDeliveryReceipt): void { + if ( + !['planned', 'retryable'].includes(receipt.outcome) || + receipt.upload || + receipt.attempts >= 5 || + (receipt.retryAt ?? 0) > Date.now() + ) + throw new Error('Delivery deferred or awaits reconciliation'); + } + + private async settle( + run: ChannelRunView, + fence: ChannelRunFence, + receipt: ChannelDeliveryReceipt, + signal: AbortSignal, + ): Promise { + const { upload: checkpoint, ...wire } = receipt; + const updated = await this.client.mutate(fence, 'delivery/settle', { receipt: wire }, signal); + const stored = deliveryReceipt(updated, receipt.operationId); + if ( + stored.outcome !== receipt.outcome || + stored.externalMessageKey !== receipt.externalMessageKey || + stored.externalFileId !== receipt.externalFileId || + (checkpoint && + (stored.upload?.fileId !== checkpoint.fileId || stored.upload.stage !== checkpoint.stage)) + ) + throw new Error('Delivery settlement not persisted'); + return updated; + } + + private async progress( + run: ChannelRunView, + fence: ChannelRunFence, + bundle: ChannelBundle, + signal: AbortSignal, + check: () => void, + lastWrite: number, + ): Promise { + if (run.plan || terminalStates.has(run.state) || run.state === 'blocked') return run; + let progress = run.progress; + if (terminalSnapshot(run) && !progress?.externalMessageKey) return run; + if ( + progress && + (progress.rendererVersion !== progressRenderer || + ['unknown', 'inflight', 'permanent_failure'].includes(progress.outcome)) + ) + return run; + const text = progressText(run); + if (!progress || progress.outcome === 'confirmed') { + if (progress?.text === text) return run; + const revision = (progress?.revision ?? 0) + 1; + check(); + run = await this.client.mutate( + fence, + 'progress/plan', + { revision, rendererVersion: progressRenderer, text }, + signal, + ); + progress = run.progress; + if (!progress || progress.revision !== revision || progress.text !== text) + throw new Error('Progress plan not persisted'); + } + if (!['planned', 'retryable'].includes(progress.outcome) || progress.text.length > 4000) + throw new Error('Unsupported progress state'); + if ((progress.retryAt ?? 0) > Date.now()) return run; + if (Date.now() < lastWrite + 1000) + await delay(lastWrite + 1000 - Date.now(), undefined, { signal }); + check(); + const revision = progress.revision; + run = await this.client.mutate(fence, 'progress/begin', { revision }, signal); + progress = run.progress; + if ( + !progress || + progress.revision !== revision || + progress.outcome !== 'inflight' || + progress.rendererVersion !== progressRenderer || + !progress.text || + progress.text.length > 4000 + ) + throw new Error('Progress begin not persisted'); + const target = address(run); + const blocks: Parameters[1]['blocks'] = [ + { type: 'section', text: { type: 'mrkdwn', text: progress.text }, expand: true }, + ]; + if (!terminalSnapshot(run) && !run.cancellationRequested) { + const value = JSON.stringify({ runId: run.runId, revision: run.revision }); + const elements = [ + { + type: 'button' as const, + text: { type: 'plain_text' as const, text: 'Stop' }, + action_id: 'channel_run_cancel', + value, + }, + ]; + if (pendingReview(run)) + elements.push({ + type: 'button', + text: { type: 'plain_text', text: 'Review request' }, + action_id: 'channel_run_review', + value, + }); + blocks.push({ type: 'actions', elements }); + } + let receipt: Omit; + check(); + try { + if (progress.attempts > 5) throw new SlackWriteError('permanent'); + let posted: { channel: string; ts: string }; + if (progress.externalMessageKey) { + const prefix = target.channel + ':'; + const ts = progress.externalMessageKey.slice(prefix.length); + if (!progress.externalMessageKey.startsWith(prefix) || !ts || /[:\s]/.test(ts)) + throw new Error('Invalid progress address'); + posted = await this.slack.updateMessageOnce( + bundle.credentials.botToken, + { channel: target.channel, ts, text: progress.text, blocks }, + signal, + ); + } else + posted = await this.slack.postMessage( + bundle.credentials.botToken, + { ...target, text: progress.text, blocks }, + signal, + ); + receipt = { + attempts: progress.attempts, + outcome: 'confirmed', + externalMessageKey: `${posted.channel}:${posted.ts}`, + }; + } catch (error) { + receipt = writeOutcome(error, progress.attempts); + } + check(); + run = await this.client.mutate(fence, 'progress/settle', { revision, receipt }, signal); + if ( + run.progress?.revision !== revision || + run.progress.outcome !== receipt.outcome || + (receipt.outcome === 'confirmed' && + run.progress.externalMessageKey !== receipt.externalMessageKey) + ) + throw new Error('Progress settlement not persisted'); + return run; + } + + private async artifact( + run: ChannelRunView, + fence: ChannelRunFence, + part: ChannelDeliveryPart, + bundle: ChannelBundle, + signal: AbortSignal, + check: () => void, + ): Promise { + const artifacts = run.snapshot?.artifacts ?? []; + const artifact = artifacts.find((item) => item.artifactId === part.artifactId); + if (!artifact || artifacts.length > 20 || part.required || part.presentation) + throw new Error('Unsupported artifact part'); + let receipt = deliveryReceipt(run, part.operationId); + if (['confirmed', 'unknown', 'permanent_failure'].includes(receipt.outcome)) return run; + if ( + !['planned', 'retryable'].includes(receipt.outcome) || + receipt.upload || + (receipt.retryAt ?? 0) > Date.now() + ) + throw new Error('Artifact deferred or awaits reconciliation'); + check(); + run = await this.client.mutate( + fence, + 'delivery/begin', + { operationId: part.operationId }, + signal, + ); + receipt = deliveryReceipt(run, part.operationId); + if (receipt.outcome !== 'inflight' || receipt.upload) + throw new Error('Artifact begin not persisted'); + const settle = async (outcome: ChannelDeliveryReceipt['outcome'], errorCode?: string) => { + check(); + run = await this.settle(run, fence, { ...receipt, outcome, errorCode }, signal); + if (outcome === 'retryable') throw new Error('Artifact retry deferred'); + return run; + }; + if (receipt.attempts > 5) return settle('permanent_failure', 'artifact_retry_exhausted'); + let bytes: Buffer; + try { + bytes = await this.client.artifact(fence, artifact, signal); + } catch (error) { + if (error instanceof ServiceError && [401, 403, 409].includes(error.status)) throw error; + return settle('permanent_failure', 'artifact_unavailable'); + } + check(); + let allocation: { fileId: string; uploadUrl: string }; + try { + allocation = await this.slack.allocateArtifact( + bundle.credentials.botToken, + artifact.filename, + artifact.bytes, + signal, + ); + } catch (error) { + const outcome = writeOutcome(error, receipt.attempts); + receipt.retryAt = outcome.retryAt; + return settle(outcome.outcome, outcome.errorCode); + } + const checkpoint = async (stage: 'allocated' | 'uploaded') => { + check(); + const upload = { fileId: allocation.fileId, stage }; + run = await this.client.mutate( + fence, + 'delivery/checkpoint', + { operationId: part.operationId, attempts: receipt.attempts, upload }, + signal, + ); + const stored = deliveryReceipt(run, part.operationId); + if ( + stored.outcome !== 'inflight' || + stored.attempts !== receipt.attempts || + stored.upload?.fileId !== upload.fileId || + stored.upload.stage !== stage + ) + throw new Error('Upload checkpoint not persisted'); + receipt = stored; + }; + await checkpoint('allocated'); + check(); + try { + await this.slack.uploadArtifact(allocation.uploadUrl, bytes, signal); + } catch { + return settle('unknown', 'artifact_upload_uncertain'); + } + await checkpoint('uploaded'); + check(); + const target = address(run); + try { + await this.slack.completeArtifact( + bundle.credentials.botToken, + { + channel: target.channel, + threadTs: target.thread_ts, + fileId: allocation.fileId, + filename: artifact.filename, + }, + signal, + ); + } catch { + return settle('unknown', 'artifact_completion_uncertain'); + } + receipt.externalFileId = allocation.fileId; + return settle('confirmed'); + } + + private async lifecycle( + run: ChannelRunView, + bundle: ChannelBundle, + phase: 'started' | 'completed' | 'failed', + signal: AbortSignal, + ): Promise { + if (!bundle.gateway.lifecycleReactions) return; + const prefix = run.admission.externalChannelId + ':'; + const ts = run.admission.externalMessageKey.slice(prefix.length); + if (!run.admission.externalMessageKey.startsWith(prefix) || !ts || /[:\s]/.test(ts)) return; + const limited = AbortSignal.any([signal, AbortSignal.timeout(2000)]); + const reaction = (operation: 'add' | 'remove', name: string) => + this.slack.reaction( + bundle.credentials.botToken, + operation, + run.admission.externalChannelId, + ts, + name, + limited, + ); + try { + if (phase === 'started') { + await reaction('add', 'eyes'); + await reaction('add', 'hourglass_flowing_sand'); + } else { + await reaction('remove', 'hourglass_flowing_sand'); + await reaction('add', phase === 'completed' ? 'white_check_mark' : 'warning'); + } + } catch { + this.log.warn('optional_lifecycle_unavailable'); + } + } +} diff --git a/packages/channel-gateway/tsdown.config.mjs b/packages/channel-gateway/tsdown.config.mjs index 5a66c1503a9..1295f517a9f 100644 --- a/packages/channel-gateway/tsdown.config.mjs +++ b/packages/channel-gateway/tsdown.config.mjs @@ -1,15 +1,7 @@ import { defineConfig } from 'tsdown'; export default defineConfig({ - entry: [ - 'src/client.ts', - 'src/config.ts', - 'src/events.ts', - 'src/slack.ts', - 'src/render.ts', - 'src/identity.ts', - 'src/review.ts', - ], + entry: ['src/main.ts'], platform: 'node', format: ['cjs'], dts: false, diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 66d382b8390..92584a1f179 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -3409,6 +3409,7 @@ describe('complete channel operator configuration', () => { nativeSessions: false, lifecycleReactions: false, signatureToleranceSeconds: 300, + socketReconnectDelayMs: 5000, }); expect( configSchema.parse({ @@ -3421,6 +3422,7 @@ describe('complete channel operator configuration', () => { nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }, }, }, @@ -3430,9 +3432,14 @@ describe('complete channel operator configuration', () => { nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }); }); it.each([ + { socketReconnectDelayMs: 99 }, + { socketReconnectDelayMs: 60001 }, + { socketReconnectDelayMs: 100.5 }, + { socketReconnectDelayMs: '5000' }, { nativeSessions: 'true' }, { lifecycleReactions: 'false' }, { signatureToleranceSeconds: 0 }, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 6807cdedc99..6ca4d0f677b 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1234,11 +1234,19 @@ export const CHANNEL_ARTIFACT_FILES_HARD_MAX = 20; export const CHANNEL_ARTIFACT_READ_TIMEOUT_HARD_MAX_MS = 60_000; /** One policy controls installation, ingress and optional Slack effects across gateway replicas. */ +const SOCKET_RECONNECT_DELAY_DEFAULT_MS = 5000; export const channelGatewayPolicySchema = z.object({ transport: z.enum(['socket_mode', 'webhook']), nativeSessions: z.boolean(), lifecycleReactions: z.boolean(), signatureToleranceSeconds: z.number().int().min(1).max(300), + /** Additive v1 pacing setting; omitted older bundles retain the SDK's existing base delay. */ + socketReconnectDelayMs: z + .number() + .int() + .min(100) + .max(60_000) + .default(SOCKET_RECONNECT_DELAY_DEFAULT_MS), }); export type ChannelGatewayPolicy = z.infer; export const DEFAULT_CHANNEL_GATEWAY_POLICY: ChannelGatewayPolicy = { @@ -1246,6 +1254,7 @@ export const DEFAULT_CHANNEL_GATEWAY_POLICY: ChannelGatewayPolicy = { nativeSessions: false, lifecycleReactions: false, signatureToleranceSeconds: 300, + socketReconnectDelayMs: SOCKET_RECONNECT_DELAY_DEFAULT_MS, }; export const CHANNEL_GATEWAY_POLICY_VERSION = 1; export const channelGatewayWirePolicySchema = channelGatewayPolicySchema.extend({ @@ -1285,6 +1294,7 @@ export const channelConfigSchema = channelStorageConfigSchema.extend({ signatureToleranceSeconds: channelGatewayPolicySchema.shape.signatureToleranceSeconds.default( DEFAULT_CHANNEL_GATEWAY_POLICY.signatureToleranceSeconds, ), + socketReconnectDelayMs: channelGatewayPolicySchema.shape.socketReconnectDelayMs, gatewayUrl: z.string().url().optional(), ticketLifetimeMs: z .number()