From d53fdc6648019aad7b0e4030da261a6efb16beab Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:38:21 +0100 Subject: [PATCH 01/12] =?UTF-8?q?=F0=9F=9A=80=20feat:=20Run=20the=20Standa?= =?UTF-8?q?lone=20Slack=20Channel=20Gateway?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/channel-gateway.yml | 61 ++ docker-compose.channels.yml | 23 + packages/channel-gateway/Dockerfile | 35 + packages/channel-gateway/package.json | 1 + packages/channel-gateway/src/app.spec.ts | 547 ++++++++++++++++ packages/channel-gateway/src/app.ts | 369 +++++++++++ packages/channel-gateway/src/main.ts | 77 +++ packages/channel-gateway/src/socket.spec.ts | 329 ++++++++++ packages/channel-gateway/src/socket.ts | 180 ++++++ packages/channel-gateway/src/worker.spec.ts | 673 +++++++++++++++++++ packages/channel-gateway/src/worker.ts | 680 ++++++++++++++++++++ packages/channel-gateway/tsdown.config.mjs | 10 +- 12 files changed, 2976 insertions(+), 9 deletions(-) create mode 100644 docker-compose.channels.yml create mode 100644 packages/channel-gateway/Dockerfile create mode 100644 packages/channel-gateway/src/app.spec.ts create mode 100644 packages/channel-gateway/src/app.ts create mode 100644 packages/channel-gateway/src/main.ts create mode 100644 packages/channel-gateway/src/socket.spec.ts create mode 100644 packages/channel-gateway/src/socket.ts create mode 100644 packages/channel-gateway/src/worker.spec.ts create mode 100644 packages/channel-gateway/src/worker.ts diff --git a/.github/workflows/channel-gateway.yml b/.github/workflows/channel-gateway.yml index 031d6fddf19..bc2bca2fab4 100644 --- a/.github/workflows/channel-gateway.yml +++ b/.github/workflows/channel-gateway.yml @@ -8,6 +8,8 @@ on: - 'package.json' - 'package-lock.json' - 'scripts/typecheck.mts' + - '.dockerignore' + - 'docker-compose.channels.yml' - '.github/workflows/channel-gateway.yml' push: branches: [dev] @@ -17,6 +19,8 @@ on: - 'package.json' - 'package-lock.json' - 'scripts/typecheck.mts' + - '.dockerignore' + - 'docker-compose.channels.yml' - '.github/workflows/channel-gateway.yml' workflow_dispatch: @@ -50,3 +54,60 @@ jobs: run: npm run test:ci --workspace @librechat/channel-gateway - name: Build gateway run: npm run build --workspace @librechat/channel-gateway + - name: Build standalone image + run: docker build -f packages/channel-gateway/Dockerfile -t channel-gateway:ci . + - name: Verify standalone image and fail-closed startup + shell: bash + run: | + docker run --rm --entrypoint node channel-gateway:ci -e "const fs=require('node:fs');if(process.getuid()===0||!fs.existsSync('/app/main.cjs')||fs.existsSync('/app/node_modules'))process.exit(1)" + set +e + output=$(docker run --rm channel-gateway:ci 2>&1) + status=$? + set -e + if [ "$status" -eq 0 ]; then + echo 'Gateway unexpectedly started without a channel binding.' + exit 1 + fi + if ! grep -q 'channel_gateway_startup_failed' <<< "$output"; then + echo 'Gateway did not report its expected safe startup failure.' + exit 1 + fi + - name: Smoke-test isolated webhook process + run: | + docker run --rm -i --read-only --entrypoint node channel-gateway:ci - <<'NODE' + const http = require('node:http'); + const { spawn } = require('node:child_process'); + const { setTimeout: delay } = require('node:timers/promises'); + const bundle = { + channel: { channelId: 'fixture', tenantId: 'fixture', agentId: 'agent_fixture', type: 'slack', status: 'active', revision: 'fixture', slack: { teamId: 'T1', appId: 'A1', botUserId: 'U1' } }, + credentials: { botToken: 'fixture' }, agent: { id: 'agent_fixture' }, + }; + const backend = http.createServer((req, res) => { + if (req.url.endsWith('/runs/claim')) { req.resume(); res.writeHead(204); res.end(); return; } + res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify(bundle)); + }); + backend.listen(0, '127.0.0.1', async () => { + const child = spawn(process.execPath, ['/app/main.cjs'], { + env: { ...process.env, LIBRECHAT_INTERNAL_URL: `http://127.0.0.1:${backend.address().port}`, CHANNEL_SERVICE_TOKEN: 'fixture', CHANNEL_TENANT_ID: 'fixture', CHANNEL_ID: 'fixture', SLACK_TRANSPORT: 'webhook' }, + stdio: 'ignore', + }); + const exited = new Promise(resolve => child.once('exit', resolve)); + const deadline = setTimeout(() => child.kill('SIGKILL'), 10000); + try { + let ready = false; + for (let attempt = 0; attempt < 50; attempt++) { + try { ready = (await fetch('http://127.0.0.1:8080/healthz', { signal: AbortSignal.timeout(500) })).ok; } catch {} + if (ready) break; + await delay(100); + } + if (!ready) throw new Error('Gateway did not become healthy'); + child.kill('SIGTERM'); + if (await exited !== 0) throw new Error('Gateway did not exit cleanly'); + console.log('Standalone gateway became healthy and stopped cleanly.'); + } catch { + child.kill('SIGKILL'); process.exitCode = 1; + } finally { + clearTimeout(deadline); backend.closeAllConnections(); backend.close(); + } + }); + NODE diff --git a/docker-compose.channels.yml b/docker-compose.channels.yml new file mode 100644 index 00000000000..e72ff0d6c31 --- /dev/null +++ b/docker-compose.channels.yml @@ -0,0 +1,23 @@ +services: + channel-gateway: + profiles: [channels] + build: + context: . + dockerfile: packages/channel-gateway/Dockerfile + init: true + restart: unless-stopped + read_only: true + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + environment: + LIBRECHAT_INTERNAL_URL: http://api:${PORT:-3080} + CHANNEL_SERVICE_TOKEN: ${CHANNEL_SERVICE_TOKEN:-} + CHANNEL_TENANT_ID: ${CHANNEL_TENANT_ID:-} + CHANNEL_ID: ${CHANNEL_ID:-} + APP_TITLE: ${APP_TITLE:-LibreChat} + SLACK_TRANSPORT: ${SLACK_TRANSPORT:-socket_mode} + SLACK_APP_TOKEN: ${SLACK_APP_TOKEN:-} + SLACK_NATIVE_SESSIONS: ${SLACK_NATIVE_SESSIONS:-false} + SLACK_LIFECYCLE_REACTIONS: ${SLACK_LIFECYCLE_REACTIONS:-false} + stop_grace_period: 10s diff --git a/packages/channel-gateway/Dockerfile b/packages/channel-gateway/Dockerfile new file mode 100644 index 00000000000..ef76bf2f963 --- /dev/null +++ b/packages/channel-gateway/Dockerfile @@ -0,0 +1,35 @@ +FROM node:24.22.1-alpine AS build + +ENV SCARF_ANALYTICS=false HUSKY=0 +WORKDIR /build + +COPY package.json package-lock.json ./ +COPY api/package.json ./api/package.json +COPY client/package.json ./client/package.json +COPY packages/api/package.json ./packages/api/package.json +COPY packages/client/package.json ./packages/client/package.json +COPY packages/chat/package.json ./packages/chat/package.json +COPY packages/data-schemas/package.json ./packages/data-schemas/package.json +COPY packages/data-provider/package.json ./packages/data-provider/package.json +COPY packages/channel-gateway/package.json ./packages/channel-gateway/package.json +COPY vendor/code ./vendor/code +RUN npm ci --ignore-scripts --no-audit --workspace @librechat/channel-gateway --workspace librechat-data-provider --include-workspace-root=false + +COPY packages/data-provider ./packages/data-provider +COPY packages/channel-gateway ./packages/channel-gateway +COPY scripts/typecheck.mts ./scripts/typecheck.mts +RUN npm run build --workspace librechat-data-provider \ + && npm run typecheck --workspace @librechat/channel-gateway \ + && npm run build --workspace @librechat/channel-gateway + +FROM node:24.22.1-alpine AS runtime + +ENV NODE_ENV=production PORT=8080 LISTEN_HOST=0.0.0.0 +WORKDIR /app +COPY --from=build --chown=node:node /build/packages/channel-gateway/dist/main.cjs ./main.cjs +COPY --chown=node:node LICENSE ./LICENSE +USER node +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+process.env.PORT+'/healthz',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})" +CMD ["node", "main.cjs"] diff --git a/packages/channel-gateway/package.json b/packages/channel-gateway/package.json index 63c2015ee57..e8fb5868442 100644 --- a/packages/channel-gateway/package.json +++ b/packages/channel-gateway/package.json @@ -7,6 +7,7 @@ "scripts": { "build": "tsdown", "build:watch": "tsdown --watch", + "start": "node dist/main.cjs", "typecheck": "node ../../scripts/typecheck.mts", "test": "jest --watch", "test:ci": "jest --ci --runInBand" diff --git a/packages/channel-gateway/src/app.spec.ts b/packages/channel-gateway/src/app.spec.ts new file mode 100644 index 00000000000..25b837f7808 --- /dev/null +++ b/packages/channel-gateway/src/app.spec.ts @@ -0,0 +1,547 @@ +import { createHmac } from 'node:crypto'; +import { SocketModeClient } from '@slack/socket-mode'; +import { createServer, request as httpRequest } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import type { ChannelBundle } from './types'; +import { ChannelClient } from './client'; +import { connectSocket } from './socket'; +import { loadConfig } from './config'; +import { SlackClient } from './slack'; +import { GatewayApp } from './app'; + +const bundle: ChannelBundle = { + channel: { + channelId: 'binding', + tenantId: 'tenant', + agentId: 'agent', + createdBy: 'owner', + type: 'slack', + status: 'active', + credentialMode: 'per_user', + revision: 'r1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'B1' }, + }, + agent: { id: 'agent' }, + credentials: { botToken: 'test-bot', signingSecret: 'test-signing' }, +}; +const envelope = (event: object = {}) => ({ + type: 'event_callback', + event_id: 'E1', + team_id: 'T1', + api_app_id: 'A1', + event: { + type: 'app_mention', + user: 'U1', + channel: 'C1', + ts: '1793000000.123456', + text: '<@B1> hello', + ...event, + }, +}); +const callback = ( + action = 'channel_run_cancel', + value = JSON.stringify({ runId: 'run', revision: 'r1' }), +) => ({ + type: 'block_actions', + team: { id: 'T1' }, + api_app_id: 'A1', + user: { id: 'U1' }, + channel: { id: 'C1' }, + container: { channel_id: 'C1' }, + actions: [{ action_id: action, value }], +}); +const json = (body: object, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); +function setup(env: NodeJS.ProcessEnv = {}) { + const config = loadConfig({ + CHANNEL_SERVICE_TOKEN: 'test-service', + CHANNEL_TENANT_ID: 'tenant', + CHANNEL_ID: 'binding', + ...env, + }); + const backend = jest + .fn, Parameters>() + .mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ runId: 'run', duplicate: false }), + ); + const slack = jest + .fn, Parameters>() + .mockImplementation(async () => json({ ok: true })); + const log = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; + const app = new GatewayApp( + config, + new ChannelClient(config, backend), + new SlackClient(config, undefined, slack), + log, + ); + return { app, backend, slack, log, config }; +} +async function serve(app: GatewayApp, check: (base: string) => Promise) { + const server = createServer((request, response) => { + void app.http(request, response); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + try { + await check(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +} +function signed(body: string) { + const timestamp = String(Math.floor(Date.now() / 1000)); + return { + 'Content-Type': 'application/json', + 'x-slack-request-timestamp': timestamp, + 'x-slack-signature': + 'v0=' + createHmac('sha256', 'test-signing').update(`v0:${timestamp}:${body}`).digest('hex'), + }; +} + +describe('durable Slack ingress', () => { + test('does not acknowledge until admission has been durably confirmed', async () => { + const { app, backend } = setup(); + let accept: (response: Response) => void = () => {}; + const admission = new Promise((resolve) => { + accept = resolve; + }); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') ? json(bundle) : admission, + ); + let acknowledged = false; + const pending = app.dispatch(envelope(), false).then(() => { + acknowledged = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(acknowledged).toBe(false); + accept(json({ runId: 'run', duplicate: false })); + await pending; + expect(acknowledged).toBe(true); + expect(JSON.parse(String(backend.mock.calls[1][1]?.body))).toMatchObject({ + input: 'hello', + actor: { externalUserId: 'U1', teamId: 'T1', appId: 'A1' }, + }); + }); + + test.each(['team_id', 'api_app_id'])('does not admit a foreign %s', async (field) => { + const { app, backend } = setup(); + await expect(app.dispatch({ ...envelope(), [field]: 'foreign' }, false)).rejects.toThrow(); + expect(backend).toHaveBeenCalledTimes(1); + }); + + test('native Stop is constrained to its actor, thread and occurrence time', async () => { + const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + await app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.123456', + }), + false, + ); + expect(String(backend.mock.calls[1][0])).toContain('/runs/stop-thread'); + expect(JSON.parse(String(backend.mock.calls[1][1]?.body))).toEqual({ + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + externalThreadKey: 'C1:1792999999.5', + eventAt: 1793000001123, + }); + }); + + test('native Stop is not handled when its feature is disabled', async () => { + const { app, backend } = setup(); + await app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + false, + ); + expect(backend).toHaveBeenCalledTimes(1); + }); + + test.each([400, 403])('native Stop reports a safe diagnostic for backend %i', async (status) => { + const { app, backend, log } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'private stop content' }, status), + ); + const dispatch = app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + false, + ); + if (status === 400) await expect(dispatch).rejects.toThrow(); + else await dispatch; + expect(log.warn.mock.calls).toEqual([ + [status === 400 ? 'native_stop_invalid_request' : 'native_stop_forbidden'], + ]); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('private'); + }); + + test('native Stop malformed envelope and binding errors are distinguishable without payload logging', async () => { + const { app, log } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + const stop = envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }); + await expect(app.dispatch({ ...stop, team_id: 'foreign' }, false)).rejects.toThrow(); + await expect(app.dispatch({ ...stop, event_id: undefined }, false)).rejects.toThrow(); + await expect( + app.dispatch(envelope({ type: 'agent_session_stopped', event_ts: '1793000001.1' }), false), + ).rejects.toThrow(); + expect(log.warn.mock.calls).toEqual([ + ['native_stop_binding_rejected'], + ['native_stop_envelope_rejected'], + ['native_stop_event_rejected'], + ]); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('foreign'); + }); + + test('future Stop waits without clamping its original occurrence time', async () => { + const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + let now = 1793000000000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + const pending = app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000000.1', + }), + false, + ); + await new Promise((resolve) => setImmediate(resolve)); + expect(backend).toHaveBeenCalledTimes(1); + now += 100; + await pending; + expect(JSON.parse(String(backend.mock.calls[1][1]?.body)).eventAt).toBe(1793000000100); + }); + + test('future Stop outside the acknowledgement window is rejected without waiting or writing', async () => { + const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + jest.spyOn(Date, 'now').mockReturnValue(1793000000000); + await expect( + app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000010.1', + }), + false, + ), + ).rejects.toThrow('acknowledgement window'); + expect(backend).toHaveBeenCalledTimes(1); + }); + + test('future Stop waiting observes shutdown without sending a cancellation', async () => { + const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + jest.spyOn(Date, 'now').mockReturnValue(1793000000000); + const controller = new AbortController(); + const pending = app.dispatch( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + false, + controller.signal, + ); + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(); + await expect(pending).rejects.toThrow(); + expect(backend).toHaveBeenCalledTimes(1); + }); + + test('Socket Mode leaves rejected Stop unacknowledged for redelivery', async () => { + const { app, backend, log, config } = setup({ + SLACK_NATIVE_SESSIONS: 'true', + SLACK_TRANSPORT: 'socket_mode', + SLACK_APP_TOKEN: 'test-app', + }); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'invalid_run_action' }, 400), + ); + jest.spyOn(SocketModeClient.prototype, 'start').mockResolvedValue({ ok: true }); + jest.spyOn(SocketModeClient.prototype, 'disconnect').mockResolvedValue(undefined); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit( + 'ws_message', + JSON.stringify({ type: 'hello', connection_info: { app_id: 'A1' } }), + false, + ); + await new Promise((resolve) => setImmediate(resolve)); + const ack = jest.fn().mockResolvedValue(undefined); + socket.emit('slack_event', { + type: 'events_api', + envelope_id: 'stop', + body: envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + ack, + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(ack).not.toHaveBeenCalled(); + expect(log.warn.mock.calls).toContainEqual(['native_stop_invalid_request']); + expect(log.warn.mock.calls).toContainEqual(['slack_socket_delivery_not_accepted']); + controller.abort(); + }); + + test('only explicit link_required invokes private linking', async () => { + const { app, backend, slack } = setup(); + backend.mockImplementation(async (url) => { + if (String(url).includes('/service/slack/')) return json(bundle); + if (String(url).endsWith('/runs')) return json({ error: 'channel_link_required' }, 403); + if (String(url).endsWith('/identity/U1')) return json({ ok: true, value: null }); + return json({ + ok: true, + value: { + url: `https://chat.example/channels/link#${'a'.repeat(43)}`, + expiresAtMs: Date.now() + 60000, + }, + }); + }); + await app.dispatch(envelope(), false); + expect(slack).toHaveBeenCalledTimes(1); + expect(String(slack.mock.calls[0][0])).toContain('chat.postEphemeral'); + }); + + test.each([{ error: 'forbidden' }, { error: 'expired' }, { message: 'channel_link_required' }])( + 'does not reinterpret arbitrary 403 as linking %p', + async (failure) => { + const { app, backend, slack } = setup(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') ? json(bundle) : json(failure, 403), + ); + await expect(app.dispatch(envelope(), false)).rejects.toThrow(); + expect(slack).not.toHaveBeenCalled(); + expect(backend).toHaveBeenCalledTimes(2); + }, + ); + + test('rejects foreign interaction workspace/app and mismatched container', async () => { + const { app, backend } = setup(); + for (const body of [ + { ...callback(), team: { id: 'other' } }, + { ...callback(), api_app_id: 'other' }, + { ...callback(), container: { channel_id: 'other' } }, + ]) + await expect(app.dispatch(body, true)).rejects.toThrow(); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); + + test('stale controls cannot cancel work', async () => { + const { app, backend, slack } = setup(); + await app.dispatch( + callback('channel_run_cancel', JSON.stringify({ runId: 'run', revision: 'old' })), + true, + ); + expect(backend).toHaveBeenCalledTimes(1); + expect(slack).toHaveBeenCalledTimes(1); + }); + + test('keeps overload unacknowledged rather than queueing memory', async () => { + const { app, backend } = setup(); + let release: (response: Response) => void = () => {}; + const waiting = new Promise((resolve) => { + release = resolve; + }); + backend.mockImplementation(async () => waiting.then(() => json(bundle))); + const pending = Array.from({ length: 64 }, () => + app.dispatch(envelope({ bot_id: 'bot' }), false), + ); + await expect(app.dispatch(envelope(), false)).rejects.toThrow('capacity'); + expect(backend).toHaveBeenCalledTimes(64); + release(json(bundle)); + await Promise.all(pending); + backend.mockImplementation(async () => json(bundle)); + await expect(app.dispatch(envelope({ bot_id: 'bot' }), false)).resolves.toBeUndefined(); + }); +}); + +describe('signed webhook boundary', () => { + test('HTTP leaves rejected Stop retryable rather than acknowledging it', async () => { + const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'invalid_run_action' }, 400), + ); + await serve(app, async (base) => { + const body = JSON.stringify( + envelope({ + type: 'agent_session_stopped', + thread_ts: '1792999999.5', + event_ts: '1793000001.1', + }), + ); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ error: 'delivery_not_accepted' }); + }); + }); + test('bounds incomplete incoming bodies by the acknowledgement deadline', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + let request: ReturnType | undefined; + const response = new Promise((resolve) => { + request = httpRequest( + base + '/slack/events/tenant/binding', + { method: 'POST', headers: { 'Content-Type': 'application/json' } }, + (incoming) => { + incoming.resume(); + resolve(incoming.statusCode ?? 0); + }, + ); + request.on('error', () => resolve('closed')); + request.write('{'); + }); + const result = await Promise.race([ + response, + new Promise((resolve) => setTimeout(() => resolve('stalled'), 2600)), + ]); + request?.destroy(); + expect(result).not.toBe('stalled'); + expect(backend).not.toHaveBeenCalled(); + }); + }); + + test('bounds webhook capacity before backend channel lookup', async () => { + const { app, backend } = setup(); + let release: () => void = () => {}; + const waiting = new Promise((resolve) => { + release = resolve; + }); + backend.mockImplementation(async () => { + await waiting; + return json(bundle); + }); + await serve(app, async (base) => { + const body = JSON.stringify(envelope({ bot_id: 'bot' })); + const pending = Array.from({ length: 65 }, () => + fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }), + ); + for (let i = 0; i < 100 && backend.mock.calls.length < 64; i++) + await new Promise((resolve) => setTimeout(resolve, 5)); + await new Promise((resolve) => setTimeout(resolve, 50)); + const lookups = backend.mock.calls.length; + release(); + const results = await Promise.all(pending); + await Promise.all(results.map((result) => result.arrayBuffer())); + expect(lookups).toBeLessThanOrEqual(64); + expect(results.filter((result) => result.status === 503)).toHaveLength(1); + }); + }); + + test('rejects unsigned/tampered requests without admitting work', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body + 'tampered'), + body, + }); + expect(response.status).toBe(401); + expect(await response.json()).toEqual({ error: 'invalid_signature' }); + }); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); + + test('accepts exact signed payload only after durable admission', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({}); + }); + expect(backend.mock.calls.filter(([url]) => String(url).endsWith('/runs'))).toHaveLength(1); + }); + + test('requires exact configured route and rejects webhook use in Socket Mode', async () => { + const { app, backend } = setup({ SLACK_TRANSPORT: 'socket_mode', SLACK_APP_TOKEN: 'test-app' }); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + for (const path of ['/slack/events/tenant/binding', '/slack/events/other/binding']) + expect( + (await fetch(base + path, { method: 'POST', headers: signed(body), body })).status, + ).toBe(404); + }); + expect(backend).not.toHaveBeenCalled(); + }); + + test('failed admission returns retryable HTTP failure, not false success', async () => { + const { app, backend, log } = setup(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ error: 'private-secret-details' }, 500), + ); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ error: 'delivery_not_accepted' }); + }); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('private-secret-details'); + }); + + test('interactions require signed form encoding and one payload', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify(callback()); + expect( + ( + await fetch(base + '/slack/interactions/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }) + ).status, + ).toBe(400); + const form = new URLSearchParams({ + payload: JSON.stringify(callback('channel_link_open')), + }).toString(); + const response = await fetch(base + '/slack/interactions/tenant/binding', { + method: 'POST', + headers: { ...signed(form), 'Content-Type': 'application/x-www-form-urlencoded' }, + body: form, + }); + expect(response.status).toBe(200); + }); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); +}); diff --git a/packages/channel-gateway/src/app.ts b/packages/channel-gateway/src/app.ts new file mode 100644 index 00000000000..8ba6d8ca5ee --- /dev/null +++ b/packages/channel-gateway/src/app.ts @@ -0,0 +1,369 @@ +import { z } from 'zod'; +import { createHash } from 'node:crypto'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import type { ChannelBundle, Logger } from './types'; +import type { ReviewInteraction } from './review'; +import type { GatewayConfig } from './config'; +import { eventTimestampUs, normalizeEvent, parseEnvelope, verifySignature } from './events'; +import { ChannelClient, ServiceError } from './client'; +import { IdentityFlow } from './identity'; +import { ReviewFlow } from './review'; +import { SlackClient } from './slack'; + +const maxBody = 1024 * 1024; +const id = z.string().min(1).max(256); +const stateValue = z + .object({ + value: z.string().nullable().optional(), + selected_option: z.object({ value: z.string() }).nullable().optional(), + selected_options: z + .array(z.object({ value: z.string() })) + .nullable() + .optional(), + }) + .transform((value) => ({ + value: value.value ?? undefined, + selected_option: value.selected_option ?? undefined, + selected_options: value.selected_options ?? undefined, + })); +const interactionSchema = z.object({ + type: z.enum(['block_actions', 'view_submission', 'view_closed']), + team: z.object({ id }), + api_app_id: id, + user: z.object({ id }), + channel: z.object({ id }).optional(), + trigger_id: z.string().optional(), + container: z.object({ channel_id: id.optional(), message_ts: z.string().optional() }).optional(), + message: z.object({ ts: z.string().optional() }).optional(), + actions: z + .array( + z.object({ action_id: id, value: z.string().optional(), action_ts: z.string().optional() }), + ) + .max(1) + .optional(), + view: z + .object({ + callback_id: z.string(), + private_metadata: z.string().max(3000), + state: z + .object({ values: z.record(z.string(), z.record(z.string(), stateValue)) }) + .optional(), + }) + .optional(), +}); + +export class GatewayApp { + ready = false; + connected = false; + private active = 0; + private httpActive = 0; + private readonly identity: IdentityFlow; + private readonly review: ReviewFlow; + constructor( + private readonly config: GatewayConfig, + private readonly client: ChannelClient, + slack: SlackClient, + private readonly log: Logger, + ) { + this.identity = new IdentityFlow(client, slack); + this.review = new ReviewFlow(client, slack, config.APP_TITLE); + } + + /** No queue before admission: overload is left unacknowledged for Slack redelivery. */ + async dispatch( + body: unknown, + interactive: boolean, + parent?: AbortSignal, + verifiedBundle?: ChannelBundle, + ): Promise { + if (this.active >= 64) throw new Error('Gateway capacity exhausted'); + this.active++; + const signal = AbortSignal.any([AbortSignal.timeout(2000), ...(parent ? [parent] : [])]); + try { + const bundle = verifiedBundle ?? (await this.client.channel(signal)); + if (interactive) { + const callback = interactionSchema.parse(body); + if ( + callback.team.id !== bundle.channel.slack.teamId || + callback.api_app_id !== bundle.channel.slack.appId + ) + throw new Error('Interaction workspace or app changed'); + const channelId = callback.channel?.id ?? callback.container?.channel_id ?? ''; + if ( + callback.channel?.id && + callback.container?.channel_id && + callback.channel.id !== callback.container.channel_id + ) + throw new Error('Interaction channel mismatch'); + const interaction: ReviewInteraction = { + teamId: callback.team.id, + appId: callback.api_app_id, + actorId: callback.user.id, + channelId, + triggerId: callback.trigger_id, + view: callback.view + ? { + callbackId: callback.view.callback_id, + privateMetadata: callback.view.private_metadata, + values: callback.view.state?.values, + } + : undefined, + }; + if (callback.type === 'view_closed') return; + if (callback.type === 'view_submission') + return this.review.submit(interaction, bundle, signal); + const action = callback.actions?.[0]; + if (!action) throw new Error('Missing Slack action'); + if (action.action_id === 'channel_feedback') { + const target = callback.container?.message_ts; + if ( + !channelId || + !target || + (callback.message?.ts && callback.message.ts !== target) || + !action.action_ts || + !['+1', '-1'].includes(action.value ?? '') + ) + throw new Error('Invalid answer feedback'); + const eventId = + 'slack-feedback:' + + createHash('sha256') + .update( + [ + callback.team.id, + callback.api_app_id, + channelId, + target, + callback.user.id, + action.action_ts, + action.value, + ].join(':'), + ) + .digest('hex'); + await this.reaction( + { + eventId, + eventTimestampUs: eventTimestampUs(action.action_ts), + externalMessageKey: `${channelId}:${target}`, + externalUserId: callback.user.id, + reaction: action.value!, + action: 'added', + }, + signal, + ); + } else if (action.action_id === 'channel_link_confirm') + await this.identity.confirm( + channelId, + callback.user.id, + action.value ?? '', + bundle, + signal, + ); + else if (action.action_id.startsWith('channel_native_choice_')) + await this.review.choice(action.value ?? '', interaction, bundle, signal); + else if ( + ['channel_run_cancel', 'channel_run_review', 'channel_native_write'].includes( + action.action_id, + ) + ) + await this.review.control( + action.action_id, + action.value ?? '', + interaction, + bundle, + signal, + ); + else if (!['channel_link_open', 'channel_run_open'].includes(action.action_id)) + throw new Error('Unsupported Slack action'); + signal.throwIfAborted(); + return; + } + const nativeStop = z + .object({ event: z.object({ type: z.literal('agent_session_stopped') }) }) + .safeParse(body).success; + let envelope; + try { + envelope = parseEnvelope(body, bundle); + } catch (error) { + if (nativeStop) + this.log.warn( + error instanceof z.ZodError + ? 'native_stop_envelope_rejected' + : 'native_stop_binding_rejected', + ); + throw error; + } + let event; + try { + event = normalizeEvent(envelope, bundle); + } catch (error) { + if (nativeStop) this.log.warn('native_stop_event_rejected'); + throw error; + } + if (envelope.event.type === 'agent_session_stopped' && !event) + this.log.warn('native_stop_actor_ignored'); + if (!event) return; + if (event.kind === 'stop') { + if (event.eventAt > Date.now()) this.log.warn('native_stop_future_timestamp'); + if (this.config.SLACK_NATIVE_SESSIONS) { + // Preserve Slack's occurrence time: substituting receipt time could stop later work. + while (event.eventAt > Date.now()) { + const wait = event.eventAt - Date.now(); + if (wait >= 2000) + throw new Error('Native Stop timestamp exceeds acknowledgement window'); + await delay(wait, undefined, { signal }); + } + signal.throwIfAborted(); + try { + const result = await this.client.stop( + event.actor, + event.externalThreadKey, + event.eventAt, + signal, + ); + this.log.info( + result.cancellationRequested ? 'native_stop_requested' : 'native_stop_no_active_run', + ); + } catch (error) { + if (!(error instanceof ServiceError && [400, 403].includes(error.status))) throw error; + this.log.warn( + error.status === 400 ? 'native_stop_invalid_request' : 'native_stop_forbidden', + ); + if (error.status === 400) throw error; + } + } else this.log.warn('native_stop_disabled'); + } else if (event.kind === 'reaction') await this.reaction(event.reaction, signal); + else { + try { + const admitted = await this.client.admit(event.admission, signal); + if (!admitted.runId) throw new Error('Admission not confirmed'); + } catch (error) { + if (!(error instanceof ServiceError && error.code === 'channel_link_required')) + throw error; + await this.identity.prompt( + event.admission.externalChannelId, + event.admission.actor.externalUserId, + bundle, + signal, + ); + } + } + signal.throwIfAborted(); + } finally { + this.active--; + } + } + + private async reaction(record: object, signal: AbortSignal): Promise { + try { + await this.client.request(this.client.root + '/reactions', record, signal); + } catch (error) { + if (!(error instanceof ServiceError && error.status === 404)) throw error; + } + } + + async http(request: IncomingMessage, response: ServerResponse): Promise { + const path = request.url?.split('?')[0]; + const send = (status: number, body: object) => { + response.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' }); + response.end(JSON.stringify(body)); + }; + if (request.method === 'GET' && path === '/readyz') { + send(200, { status: 'ok' }); + return; + } + if (request.method === 'GET' && path === '/healthz') { + const ready = this.ready && (this.config.SLACK_TRANSPORT !== 'socket_mode' || this.connected); + send(ready ? 200 : 503, { status: ready ? 'ready' : 'not_ready' }); + return; + } + const route = path?.match(/^\/slack\/(events|interactions)\/([^/]+)\/([^/]+)$/); + if ( + !route || + request.method !== 'POST' || + this.config.SLACK_TRANSPORT !== 'webhook' || + route[2] !== encodeURIComponent(this.config.CHANNEL_TENANT_ID) || + route[3] !== encodeURIComponent(this.config.CHANNEL_ID) + ) { + send(404, { error: 'not_found' }); + return; + } + if (this.httpActive >= 64) { + send(503, { error: 'capacity_exhausted' }); + return; + } + this.httpActive++; + const controller = new AbortController(); + const stopReading = () => { + if (!request.complete) request.destroy(); + }; + controller.signal.addEventListener('abort', stopReading, { once: true }); + const timeout = setTimeout(() => controller.abort(), 2000); + request.on('aborted', () => controller.abort()); + try { + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of request) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += bytes.length; + if (size > maxBody) { + send(413, { error: 'payload_too_large' }); + return; + } + chunks.push(bytes); + controller.signal.throwIfAborted(); + } + const body = Buffer.concat(chunks); + const interactive = route[1] === 'interactions'; + if (!interactive) { + const challenge = z + .object({ type: z.literal('url_verification'), challenge: z.string().max(2048) }) + .safeParse(JSON.parse(body.toString())); + if (challenge.success) { + send(200, { challenge: challenge.data.challenge }); + return; + } + } + const bundle = await this.client.channel(controller.signal); + const signature = request.headers['x-slack-signature']; + const timestamp = request.headers['x-slack-request-timestamp']; + if ( + typeof signature !== 'string' || + typeof timestamp !== 'string' || + !verifySignature( + body, + signature, + timestamp, + bundle.credentials.signingSecret ?? '', + this.config.SLACK_SIGNATURE_TOLERANCE_SECS, + ) + ) { + send(401, { error: 'invalid_signature' }); + return; + } + let payload: unknown; + if (interactive) { + if ( + request.headers['content-type']?.split(';')[0].trim() !== + 'application/x-www-form-urlencoded' + ) { + send(400, { error: 'invalid_interaction' }); + return; + } + const form = new URLSearchParams(body.toString()); + if (form.getAll('payload').length !== 1) throw new Error('Invalid Slack payload'); + payload = JSON.parse(form.get('payload') ?? ''); + } else payload = JSON.parse(body.toString()); + const ack = await this.dispatch(payload, interactive, controller.signal, bundle); + controller.signal.throwIfAborted(); + send(200, ack ?? {}); + } catch { + this.log.warn('slack_delivery_not_accepted'); + if (!response.headersSent) send(503, { error: 'delivery_not_accepted' }); + } finally { + clearTimeout(timeout); + controller.signal.removeEventListener('abort', stopReading); + this.httpActive--; + } + } +} diff --git a/packages/channel-gateway/src/main.ts b/packages/channel-gateway/src/main.ts new file mode 100644 index 00000000000..6e481853552 --- /dev/null +++ b/packages/channel-gateway/src/main.ts @@ -0,0 +1,77 @@ +import { createServer } from 'node:http'; +import { Registry, collectDefaultMetrics, Counter } from 'prom-client'; +import { connectSocket } from './socket'; +import { ChannelWorker } from './worker'; +import { ChannelClient } from './client'; +import { loadConfig } from './config'; +import { SlackClient } from './slack'; +import { GatewayApp } from './app'; +import { logger } from './types'; + +async function main(): Promise { + const config = loadConfig(process.env); + const registry = new Registry(); + collectDefaultMetrics({ register: registry, prefix: 'channel_gateway_' }); + const httpRequests = new Counter({ + name: 'channel_gateway_http_requests_total', + help: 'Gateway HTTP requests by bounded outcome', + labelNames: ['status'], + registers: [registry], + }); + const controller = new AbortController(); + const client = new ChannelClient(config); + const slack = new SlackClient(config, logger); + const app = new GatewayApp(config, client, slack, logger); + const server = createServer((request, response) => { + response.on('finish', () => httpRequests.labels(String(response.statusCode)).inc()); + if (request.method === 'GET' && request.url === '/metrics') { + void registry + .metrics() + .then((metrics) => { + response.writeHead(200, { 'Content-Type': registry.contentType }); + response.end(metrics); + }) + .catch(() => { + response.writeHead(500); + response.end(); + }); + return; + } + void app.http(request, response).catch(() => { + logger.error('gateway_http_failed'); + if (!response.headersSent) response.writeHead(500); + response.end(); + }); + }); + server.requestTimeout = 5000; + server.headersTimeout = 5000; + server.maxConnections = 128; + const shutdown = () => { + app.ready = false; + controller.abort(); + server.close(); + const timer = setTimeout(() => server.closeAllConnections(), 5000); + timer.unref(); + }; + process.once('SIGTERM', shutdown); + process.once('SIGINT', shutdown); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(config.PORT, config.LISTEN_HOST, resolve); + }); + try { + const bundle = await client.channel(controller.signal); + if (config.SLACK_TRANSPORT === 'socket_mode') + await connectSocket(config, app, bundle.channel.slack.appId, logger, controller.signal); + app.ready = true; + logger.info('channel_gateway_started'); + await new ChannelWorker(config, client, slack, logger).run(controller.signal); + } finally { + shutdown(); + } +} + +void main().catch(() => { + logger.error('channel_gateway_startup_failed'); + process.exitCode = 1; +}); diff --git a/packages/channel-gateway/src/socket.spec.ts b/packages/channel-gateway/src/socket.spec.ts new file mode 100644 index 00000000000..9b078920310 --- /dev/null +++ b/packages/channel-gateway/src/socket.spec.ts @@ -0,0 +1,329 @@ +import { SocketModeClient } from '@slack/socket-mode'; +import type { ChannelBundle } from './types'; +import { BoundSocket, connectSocket } from './socket'; +import { ChannelClient } from './client'; +import { loadConfig } from './config'; +import { SlackClient } from './slack'; +import { GatewayApp } from './app'; + +const config = loadConfig({ + CHANNEL_SERVICE_TOKEN: 'test-service', + CHANNEL_TENANT_ID: 'tenant', + CHANNEL_ID: 'binding', + SLACK_TRANSPORT: 'socket_mode', + SLACK_APP_TOKEN: 'test-app', +}); +const bundle: ChannelBundle = { + channel: { + channelId: 'binding', + tenantId: 'tenant', + agentId: 'agent', + createdBy: 'owner', + type: 'slack', + status: 'active', + credentialMode: 'per_user', + revision: 'r1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'B1' }, + }, + agent: { id: 'agent' }, + credentials: { botToken: 'test-bot' }, +}; +const json = (body: object) => + new Response(JSON.stringify(body), { headers: { 'Content-Type': 'application/json' } }); +const envelope = () => ({ + type: 'event_callback', + event_id: 'E1', + team_id: 'T1', + api_app_id: 'A1', + event: { + type: 'app_mention', + user: 'U1', + channel: 'C1', + ts: '1793000000.123456', + text: '<@B1> private prompt', + }, +}); +const hello = (appId = 'A1') => + JSON.stringify({ type: 'hello', connection_info: { app_id: appId } }); +const tick = () => new Promise((resolve) => setImmediate(resolve)); +class TestSocket extends BoundSocket { + receive(data: string | ArrayBuffer, binary = false) { + return this.onWebSocketMessage(data, binary); + } +} +function setup() { + const backend = jest + .fn, Parameters>() + .mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : json({ runId: 'run', duplicate: false }), + ); + const log = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; + const slack = new SlackClient(config, log, async () => json({ ok: true })); + return { + backend, + log, + app: new GatewayApp(config, new ChannelClient(config, backend), slack, log), + }; +} +function event(ack = jest.fn().mockResolvedValue(undefined)) { + return { type: 'events_api', envelope_id: 'envelope', body: envelope(), ack }; +} + +describe('Socket Mode app binding', () => { + let start: jest.SpyInstance; + let disconnect: jest.SpyInstance; + beforeEach(() => { + start = jest.spyOn(SocketModeClient.prototype, 'start').mockResolvedValue({ ok: true }); + disconnect = jest.spyOn(SocketModeClient.prototype, 'disconnect').mockResolvedValue(undefined); + }); + + test('validates hello app ID before making the connection ready', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + await socket.receive(hello()); + expect(state).toHaveBeenCalledWith(true); + expect(disconnect).not.toHaveBeenCalled(); + }); + + test('SDK never receives application payloads before a verified hello', async () => { + const socket = new TestSocket('A1', jest.fn(), config); + const delivered = jest.fn(); + socket.on('slack_event', delivered); + const wire = JSON.stringify({ type: 'events_api', envelope_id: 'wire', payload: envelope() }); + await socket.receive(wire); + expect(delivered).not.toHaveBeenCalled(); + await socket.receive(hello()); + await socket.receive(wire); + expect(delivered).toHaveBeenCalledTimes(1); + socket.emit('reconnecting'); + await socket.receive(wire); + expect(delivered).toHaveBeenCalledTimes(1); + }); + + test('malformed hello removes an existing validation, not just readiness at startup', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + const delivered = jest.fn(); + socket.on('slack_event', delivered); + await socket.receive(hello()); + state.mockClear(); + await socket.receive(JSON.stringify({ type: 'hello', connection_info: {} })); + expect(state).toHaveBeenCalledWith(false); + await socket.receive( + JSON.stringify({ type: 'events_api', envelope_id: 'wire', payload: envelope() }), + ); + expect(delivered).not.toHaveBeenCalled(); + }); + + test('disconnect failure cannot become an unhandled SDK callback rejection', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + disconnect.mockRejectedValueOnce(new Error('private disconnect payload')); + await expect(socket.receive(hello('wrong'))).resolves.toBeUndefined(); + expect(state).toHaveBeenCalledWith(false); + }); + + test('SDK logging cannot print raw prompts, tokens or malformed payloads', async () => { + const prints = [ + jest.spyOn(console, 'debug'), + jest.spyOn(console, 'log'), + jest.spyOn(console, 'warn'), + jest.spyOn(console, 'error'), + ]; + const socket = new TestSocket('A1', jest.fn(), config); + await socket.receive(hello()); + await socket.receive( + JSON.stringify({ type: 'events_api', envelope_id: 'wire', payload: envelope() }), + ); + await socket.receive('{private token malformed'); + expect(prints.flatMap((print) => print.mock.calls)).toEqual([]); + }); + + test.each(['different', '', undefined])( + 'wrong or missing hello app ID %p disables the socket', + async (appId) => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + await socket.receive(JSON.stringify({ type: 'hello', connection_info: { app_id: appId } })); + expect(state).toHaveBeenCalledWith(false); + expect(state).not.toHaveBeenCalledWith(true); + expect(disconnect).toHaveBeenCalledTimes(1); + }, + ); + + test('malformed/binary/oversized input never authenticates the socket', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + for (const raw of [ + '{', + 'null', + 'true', + JSON.stringify({ type: 'hello' }).padEnd(1024 * 1024 + 1, ' '), + ]) + await socket.receive(raw); + await socket.receive(hello(), true); + expect(state).not.toHaveBeenCalledWith(true); + }); + + test('malformed events are contained rather than rejecting the SDK callback', async () => { + const state = jest.fn(); + const socket = new TestSocket('A1', state, config); + await socket.receive(hello()); + for (const payload of [ + { type: 'events_api' }, + { type: 'events_api', payload: {} }, + { type: 'events_api', payload: { event: null } }, + ]) + await expect(socket.receive(JSON.stringify(payload))).resolves.toBeUndefined(); + }); + + test('events before verified hello cannot reach application dispatch', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(backend).not.toHaveBeenCalled(); + expect(delivery.ack).not.toHaveBeenCalled(); + controller.abort(); + }); + + test('valid hello permits durable admission followed by acknowledgment', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(app.connected).toBe(true); + expect(backend).toHaveBeenCalledTimes(2); + expect(delivery.ack).toHaveBeenCalledWith(undefined); + controller.abort(); + expect(app.connected).toBe(false); + }); + + test('a wrong hello stops delivery and repeated wrong hello never becomes ready', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello('wrong'), false); + await tick(); + socket.emit('slack_event', event()); + await tick(); + socket.emit('ws_message', hello('wrong'), false); + await tick(); + socket.emit('slack_event', event()); + await tick(); + expect(app.connected).toBe(false); + expect(backend).not.toHaveBeenCalled(); + expect(disconnect).toHaveBeenCalledTimes(2); + controller.abort(); + }); + + test('reconnect requires another valid hello before accepting more events', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + socket.emit('reconnecting'); + socket.emit('slack_event', event()); + await tick(); + expect(app.connected).toBe(false); + expect(backend).not.toHaveBeenCalled(); + socket.emit('ws_message', hello(), false); + await tick(); + socket.emit('slack_event', event()); + await tick(); + expect(backend).toHaveBeenCalledTimes(2); + controller.abort(); + }); + + test('SDK error and rejected admission never log raw content or acknowledge failure', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + backend.mockImplementation(async (url) => + String(url).includes('/service/slack/') + ? json(bundle) + : Promise.reject(new Error('private provider credential')), + ); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + socket.emit('error', new Error('private Slack app token')); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(delivery.ack).not.toHaveBeenCalled(); + expect(log.warn.mock.calls).toEqual([ + ['slack_socket_transport_error'], + ['slack_socket_delivery_not_accepted'], + ]); + expect(JSON.stringify(log.warn.mock.calls)).not.toContain('private'); + controller.abort(); + }); + + test('invalid SDK deliveries are ignored without authority or acknowledgement', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + for (const delivery of [ + { ...event(), envelope_id: '' }, + { ...event(), type: 'other' }, + { ...event(), ack: 'not a function' }, + ]) + socket.emit('slack_event', delivery); + await tick(); + expect(backend).not.toHaveBeenCalled(); + controller.abort(); + }); + + test('pre-aborted startup never starts the SDK', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + controller.abort(); + await expect(connectSocket(config, app, 'A1', log, controller.signal)).rejects.toThrow(); + expect(start).not.toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); + + test('abort during startup rejects and cleans up a late successful start', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + let finish: (value: { ok: true }) => void = () => {}; + start.mockImplementation( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const pending = connectSocket(config, app, 'A1', log, controller.signal); + controller.abort(); + finish({ ok: true }); + await expect(pending).rejects.toThrow(); + expect(disconnect).toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); + + test('aborted active transport cannot dispatch or acknowledge later input', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + socket.emit('ws_message', hello(), false); + await tick(); + controller.abort(); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(backend).not.toHaveBeenCalled(); + expect(delivery.ack).not.toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); +}); diff --git a/packages/channel-gateway/src/socket.ts b/packages/channel-gateway/src/socket.ts new file mode 100644 index 00000000000..5b298f635d3 --- /dev/null +++ b/packages/channel-gateway/src/socket.ts @@ -0,0 +1,180 @@ +import { z } from 'zod'; +import { LogLevel } from '@slack/web-api'; +import { SocketModeClient } from '@slack/socket-mode'; +import type { GatewayConfig } from './config'; +import type { GatewayApp } from './app'; +import type { Logger } from './types'; + +const quiet = { + debug() {}, + info() {}, + warn() {}, + error() {}, + setLevel() {}, + setName() {}, + getLevel: () => LogLevel.ERROR, +}; +const event = z.object({ + type: z.enum(['events_api', 'interactive']), + body: z.unknown(), + envelope_id: z.string().min(1), + ack: z.custom<(value?: object) => Promise>((value) => typeof value === 'function'), +}); + +/** The SDK hides hello's connection_info; validate it before permitting its event dispatch. */ +export class BoundSocket extends SocketModeClient { + private validated = false; + constructor( + private readonly appId: string, + private readonly state: (connected: boolean) => void, + config: GatewayConfig, + signal?: AbortSignal, + ) { + super({ + appToken: config.SLACK_APP_TOKEN!, + logger: quiet, + clientOptions: { + slackApiUrl: config.SLACK_API_BASE_URL.replace(/\/?$/, '/'), + timeout: config.API_CALL_TIMEOUT_MS, + retryConfig: { retries: 0 }, + rejectRateLimitedCalls: true, + allowAbsoluteUrls: false, + fetch: (url, init) => + fetch(url, { + ...init, + redirect: 'error', + signal: AbortSignal.any([ + AbortSignal.timeout(config.API_CALL_TIMEOUT_MS), + ...(signal ? [signal] : []), + ]), + }), + }, + }); + for (const phase of ['connecting', 'reconnecting', 'disconnected']) + this.on(phase, () => { + this.validated = false; + this.state(false); + }); + } + + protected override async onWebSocketMessage( + data: string | ArrayBuffer, + binary: boolean, + ): Promise { + if (binary) return; + const raw = typeof data === 'string' ? data : new TextDecoder().decode(data); + if (Buffer.byteLength(raw) > 1024 * 1024) return; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return; + } + const envelope = z + .object({ + type: z.string(), + connection_info: z.unknown().optional(), + envelope_id: z.string().optional(), + payload: z.record(z.string(), z.unknown()).optional(), + }) + .safeParse(parsed); + if (!envelope.success) return; + const value = envelope.data; + if (value.type === 'hello') { + const connection = z + .object({ app_id: z.literal(this.appId) }) + .safeParse(value.connection_info); + if (!connection.success) { + this.state(false); + this.validated = false; + await this.disconnect().catch(() => {}); + return; + } + this.validated = true; + this.state(true); + } else if (value.type === 'disconnect') { + this.validated = false; + this.state(false); + } else { + if (!this.validated || !value.envelope_id || !value.payload) return; + if (value.type === 'events_api') { + if ( + !z.object({ event: z.object({ type: z.string().min(1) }) }).safeParse(value.payload) + .success + ) + return; + } else if (value.type !== 'interactive') return; + } + await super.onWebSocketMessage(data, binary); + } +} + +export async function connectSocket( + config: GatewayConfig, + app: GatewayApp, + appId: string, + log: Logger, + signal: AbortSignal, +): Promise { + app.connected = false; + const socket = new BoundSocket( + appId, + (connected) => { + app.connected = connected && !signal.aborted; + }, + config, + signal, + ); + socket.on('connecting', () => { + app.connected = false; + }); + socket.on('reconnecting', () => { + app.connected = false; + }); + socket.on('disconnected', () => { + app.connected = false; + }); + socket.on('error', () => log.warn('slack_socket_transport_error')); + socket.on('slack_event', (input: unknown) => { + if (!app.connected || signal.aborted) return; + const delivery = event.safeParse(input); + if (!delivery.success) return; + if ( + z + .object({ event: z.object({ type: z.literal('agent_session_stopped') }) }) + .safeParse(delivery.data.body).success + ) + log.info('slack_native_stop_received'); + void app + .dispatch(delivery.data.body, delivery.data.type === 'interactive', signal) + .then((payload) => delivery.data.ack(payload)) + .catch(() => log.warn('slack_socket_delivery_not_accepted')); + }); + if (signal.aborted) throw new Error('Gateway stopped'); + let rejectAborted: (error: Error) => void = () => {}; + const aborted = new Promise((_resolve, reject) => { + rejectAborted = reject; + }); + const stop = () => { + app.connected = false; + void socket.disconnect().catch(() => {}); + rejectAborted(new Error('Gateway stopped')); + }; + signal.addEventListener('abort', stop, { once: true }); + const started = socket.start(); + void started.then( + () => { + if (signal.aborted) void socket.disconnect().catch(() => {}); + }, + () => {}, + ); + try { + await Promise.race([started, aborted]); + } catch (error) { + signal.removeEventListener('abort', stop); + await socket.disconnect().catch(() => {}); + throw error; + } + signal.throwIfAborted(); + return socket; +} diff --git a/packages/channel-gateway/src/worker.spec.ts b/packages/channel-gateway/src/worker.spec.ts new file mode 100644 index 00000000000..559a4beadf3 --- /dev/null +++ b/packages/channel-gateway/src/worker.spec.ts @@ -0,0 +1,673 @@ +import { createHash } from 'node:crypto'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { + ChannelDeliveryPlan, + ChannelDeliveryReceipt, + ChannelRunView, +} from 'librechat-data-provider'; +import type { ChannelBundle, Lease, Logger, ResponseGrant } from './types'; +import { ChannelWorker, progressText } from './worker'; +import { ChannelClient } from './client'; +import { makeSlackPlan } from './render'; +import { loadConfig } from './config'; +import { SlackClient } from './slack'; + +const json = (body: object, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +const slackOK = (body: object = {}) => json({ ok: true, ...body }); + +interface MutationBody { + owner?: string; + fence?: Lease['fence']; + responseId?: string; + revision?: number; + rendererVersion?: string; + text?: string; + plan?: ChannelDeliveryPlan; + operationId?: string; + attempts?: number; + upload?: ChannelDeliveryReceipt['upload']; + receipt?: ChannelDeliveryReceipt; +} + +function fixture() { + const config = loadConfig({ + CHANNEL_SERVICE_TOKEN: 'service-token', + CHANNEL_ID: 'channel-1', + CHANNEL_TENANT_ID: 'tenant-1', + LIBRECHAT_INTERNAL_URL: 'http://backend.example', + SLACK_API_BASE_URL: 'http://127.0.0.1:8080/api/', + API_CALL_TIMEOUT_MS: '2000', + AGENT_INVOKE_TIMEOUT_MS: '2000', + }); + const bundle: ChannelBundle = { + channel: { + channelId: config.CHANNEL_ID, + tenantId: config.CHANNEL_TENANT_ID, + agentId: 'agent-1', + createdBy: 'user-1', + credentialMode: 'per_user', + type: 'slack', + status: 'active', + revision: 'revision-1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'BOT1' }, + }, + credentials: { botToken: 'bot-token' }, + agent: { id: 'agent-1', name: 'Test' }, + }; + const state: ChannelRunView = { + runId: 'run-1', + channelId: config.CHANNEL_ID, + revision: 'revision-1', + state: 'executing', + sequence: 1, + cancellationRequested: false, + receipts: [], + admission: { + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + eventId: 'event-1', + externalChannelId: 'C1', + externalThreadKey: 'C1:123.456', + externalMessageKey: 'C1:123.457', + input: 'hello', + }, + }; + const grant: ResponseGrant = { + apiKey: 'sk-bound-test', + requestKey: 'channel:run-1', + responseId: 'resp_test', + request: { + model: bundle.agent.id, + input: state.admission.input, + background: true, + store: true, + }, + }; + const lease: Lease = { + run: state, + fence: { runId: state.runId, owner: 'worker-1', epoch: 1 }, + leaseExpiresAt: Date.now() + 60_000, + }; + const requests: { + path: string; + body: MutationBody; + authorization: string | null; + key: string | null; + }[] = []; + const slackRequests: { method: string; body: URLSearchParams }[] = []; + const artifactBytes = Buffer.from('test artifact'); + const controller = new AbortController(); + let claimCount = 0; + const log: Logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; + const hooks: { + advance?: () => void; + renew?: () => Promise; + response?: () => Promise; + slack?: (method: string, body: URLSearchParams) => Promise; + prepare?: () => ResponseGrant; + checkpoint?: () => Promise; + } = {}; + const snapshot = () => structuredClone(state); + const backend = jest.fn, Parameters>( + async (input, init) => { + const path = new URL(String(input)).pathname; + const body: MutationBody = init?.body ? JSON.parse(String(init.body)) : {}; + const headers = new Headers(init?.headers); + requests.push({ + path, + body, + authorization: headers.get('authorization'), + key: headers.get('idempotency-key'), + }); + if (path.includes('/service/slack/')) return json(bundle); + if (path.includes('/api/agents/v1/responses')) { + if (hooks.response) return hooks.response(); + return json({ + id: grant.responseId ?? 'resp_test', + object: 'response', + model: bundle.agent.id, + background: true, + store: true, + status: 'completed', + }); + } + if (path.endsWith('/claim')) { + if (claimCount++) return new Response(null, { status: 204 }); + lease.fence.owner = body.owner ?? ''; + return json({ ...lease, run: snapshot() }); + } + if (path.endsWith('/renew')) { + if (hooks.renew) return hooks.renew(); + return json({ ...lease, run: snapshot(), leaseExpiresAt: Date.now() + 90 }); + } + if (path.endsWith('/yield')) { + controller.abort(); + return new Response(null, { status: 204 }); + } + if (path.endsWith('/response/prepare')) return json(hooks.prepare?.() ?? grant); + if (path.endsWith('/response/bind')) grant.responseId = body.responseId; + else if (path.endsWith('/advance')) { + if (hooks.advance) hooks.advance(); + else { + state.state = 'delivering'; + state.snapshot = { + status: 'completed', + text: 'Final answer', + responseId: grant.responseId, + elapsedMs: 200, + }; + } + } else if (path.endsWith('/progress/plan')) { + state.progress = { + revision: body.revision ?? 0, + rendererVersion: body.rendererVersion ?? '', + text: body.text ?? '', + outcome: 'planned', + attempts: 0, + externalMessageKey: state.progress?.externalMessageKey, + }; + } else if (path.endsWith('/progress/begin')) { + if (!state.progress) throw new Error('Missing progress fixture'); + state.progress.outcome = 'inflight'; + state.progress.attempts++; + } else if (path.endsWith('/progress/settle')) { + if (!state.progress || !body.receipt) + throw new Error('Missing progress settlement fixture'); + state.progress = { ...state.progress, ...body.receipt }; + } else if (path.endsWith('/plan')) { + state.plan = body.plan; + state.receipts = + body.plan?.parts.map((part) => ({ + operationId: part.operationId, + outcome: 'planned', + attempts: 0, + })) ?? []; + } else if (path.includes('/artifacts/')) { + const artifact = state.snapshot?.artifacts?.[0]; + if (!artifact) throw new Error('Missing artifact fixture'); + return new Response(artifactBytes, { + headers: { + 'Content-Type': 'application/octet-stream', + 'X-Artifact-Filename': encodeURIComponent(artifact.filename), + 'X-Artifact-SHA256': artifact.sha256, + 'X-Artifact-Media-Type': artifact.mediaType, + }, + }); + } else if (path.endsWith('/delivery/begin')) { + const receipt = state.receipts.find((item) => item.operationId === body.operationId); + if (!receipt) throw new Error('Missing begin fixture'); + receipt.outcome = 'inflight'; + receipt.attempts++; + } else if (path.endsWith('/delivery/checkpoint')) { + const receipt = state.receipts.find((item) => item.operationId === body.operationId); + if (!receipt) throw new Error('Missing checkpoint fixture'); + receipt.upload = body.upload; + if (hooks.checkpoint) return hooks.checkpoint(); + } else if (path.endsWith('/delivery/settle')) { + const receipt = state.receipts.find( + (item) => item.operationId === body.receipt?.operationId, + ); + if (!receipt || !body.receipt) throw new Error('Missing settle fixture'); + Object.assign(receipt, body.receipt); + } else if (path.endsWith('/commit')) state.state = 'delivered'; + else throw new Error('Unexpected backend fixture path'); + return json(snapshot()); + }, + ); + const slackTransport = jest.fn, Parameters>( + async (input, init) => { + const method = new URL(String(input)).pathname.split('/').at(-1) ?? ''; + const body = new URLSearchParams(String(init?.body)); + slackRequests.push({ method, body }); + if (hooks.slack) return hooks.slack(method, body); + if (method === 'files.getUploadURLExternal') + return slackOK({ file_id: 'F1', upload_url: 'http://127.0.0.1:8080/upload' }); + if (method === 'upload') return new Response('ok'); + if (method === 'files.completeUploadExternal') return slackOK({ files: [{ id: 'F1' }] }); + if (method === 'agents.sessions.setStatus') return slackOK(); + return slackOK({ channel: 'C1', ts: method === 'chat.update' ? body.get('ts') : '123.789' }); + }, + ); + const client = new ChannelClient(config, backend); + const slack = new SlackClient(config, log, slackTransport); + const worker = new ChannelWorker(config, client, slack, log); + const savedPlan = (artifacts = false) => { + state.state = 'delivering'; + state.snapshot = { + status: 'completed', + text: 'Saved final answer', + responseId: grant.responseId, + elapsedMs: 200, + }; + if (artifacts) + state.snapshot.artifacts = [ + { + artifactId: 'artifact-1', + filename: 'report.txt', + bytes: artifactBytes.length, + mediaType: 'text/plain', + sha256: createHash('sha256').update(artifactBytes).digest('hex'), + }, + ]; + state.plan = makeSlackPlan(state, true); + state.receipts = state.plan.parts.map((part) => ({ + operationId: part.operationId, + outcome: 'planned', + attempts: 0, + })); + }; + const execute = () => worker.process({ ...lease, run: snapshot() }, controller.signal); + const paths = () => requests.map((request) => request.path); + return { + config, + bundle, + state, + grant, + lease, + requests, + slackRequests, + artifactBytes, + controller, + log, + hooks, + backend, + slackTransport, + worker, + savedPlan, + execute, + paths, + }; +} + +describe('durable Channels worker', () => { + test('executes only through a bound Responses grant and persists progress before writes', async () => { + const f = fixture(); + delete f.grant.responseId; + await f.execute(); + const responseRequests = f.requests.filter((request) => + request.path.startsWith('/api/agents/v1/responses'), + ); + expect(responseRequests.map((request) => request.authorization)).toEqual([ + 'Bearer sk-bound-test', + 'Bearer sk-bound-test', + ]); + expect(responseRequests[0].key).toBe('channel:run-1'); + expect(responseRequests[0].body).toMatchObject({ + model: 'agent-1', + input: 'hello', + background: true, + store: true, + stream: false, + }); + expect(f.paths().some((path) => path.endsWith('/response/bind'))).toBe(true); + const writes = f.slackRequests.filter((request) => + ['chat.postMessage', 'chat.update'].includes(request.method), + ); + expect(writes.map((request) => request.method)).toEqual([ + 'chat.postMessage', + 'chat.update', + 'chat.postMessage', + ]); + expect(writes[0].body.get('text')).toBe(':hourglass_flowing_sand: Working…'); + expect(writes[1].body.get('text')).toBe(':white_check_mark: Completed.'); + expect(f.paths().filter((path) => path.endsWith('/progress/begin'))).toHaveLength(2); + expect(f.paths().filter((path) => path.endsWith('/progress/settle'))).toHaveLength(2); + expect(f.state.state).toBe('delivered'); + }); + + test('reuses a saved immutable plan without preparing or reexecuting Responses', async () => { + const f = fixture(); + f.savedPlan(); + f.state.snapshot!.text = 'Mutated snapshot must not render'; + await f.execute(); + expect( + f.slackRequests.find((request) => request.method === 'chat.postMessage')?.body.get('text'), + ).toContain('Saved final answer'); + expect(f.paths().some((path) => /response|advance|\/plan$/.test(path))).toBe(false); + expect(f.state.state).toBe('delivered'); + }); + + test.each(['unknown', 'inflight'] as const)( + 'never replays a required %s delivery', + async (outcome) => { + const f = fixture(); + f.savedPlan(); + f.state.receipts[0].outcome = outcome; + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /delivery\/begin|commit|yield/.test(path))).toBe(false); + }, + ); + + test('records an uncertain create once and holds it on recovery', async () => { + const f = fixture(); + f.savedPlan(); + f.hooks.slack = async () => { + throw new Error('Uncertain connection reset'); + }; + await expect(f.execute()).rejects.toThrow('not confirmed'); + expect(f.state.receipts[0]).toMatchObject({ outcome: 'unknown', attempts: 1 }); + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests).toHaveLength(1); + expect(f.paths().filter((path) => path.endsWith('/delivery/begin'))).toHaveLength(1); + }); + + test.each(['apiKey', 'requestKey', 'model', 'input'] as const)( + 'rejects a changed %s grant before Responses execution', + async (field) => { + const f = fixture(); + f.config.SLACK_NATIVE_SESSIONS = true; + if (field === 'apiKey' || field === 'requestKey') f.grant[field] = 'wrong'; + else f.grant.request[field] = 'wrong'; + await expect(f.execute()).rejects.toThrow('grant'); + expect(f.paths().some((path) => path.startsWith('/api/agents/'))).toBe(false); + }, + ); + + test('renews independently while a Slack create is slow', async () => { + const f = fixture(); + f.savedPlan(); + f.lease.leaseExpiresAt = Date.now() + 60; + f.hooks.slack = async () => { + await delay(120); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + await f.execute(); + expect(f.paths().filter((path) => path.endsWith('/renew')).length).toBeGreaterThan(1); + expect(f.state.state).toBe('delivered'); + expect(f.paths().some((path) => path.endsWith('/yield'))).toBe(false); + }); + + test('uncertain renewal stops settlement and does not blindly yield an inflight create', async () => { + const f = fixture(); + f.savedPlan(); + f.lease.leaseExpiresAt = Date.now() + 60; + f.hooks.renew = async () => json({ error: 'unavailable' }, 503); + f.hooks.slack = async () => { + await delay(100); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + f.log.warn = () => f.controller.abort(); + await f.worker.run(f.controller.signal); + expect(f.paths().some((path) => path.endsWith('/renew'))).toBe(true); + expect(f.state.receipts[0].outcome).toBe('inflight'); + expect(f.paths().some((path) => /delivery\/settle|commit|yield/.test(path))).toBe(false); + expect(f.slackRequests).toHaveLength(1); + }); + + test('unknown progress is not replayed and cannot safely yield a waiting run', async () => { + const f = fixture(); + f.state.progress = { + revision: 1, + rendererVersion: 'slack-progress-v1', + text: 'Working', + outcome: 'unknown', + attempts: 1, + }; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + responseId: 'resp_test', + elapsedMs: 10, + }; + }; + f.log.warn = () => f.controller.abort(); + await f.worker.run(f.controller.signal); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /progress\/(begin|settle)|yield/.test(path))).toBe(false); + }); + + test('cleans terminal progress before persisting a final plan', async () => { + const f = fixture(); + f.state.progress = { + revision: 1, + rendererVersion: 'slack-progress-v1', + text: progressText(f.state), + outcome: 'confirmed', + attempts: 1, + externalMessageKey: 'C1:123.700', + }; + await f.execute(); + expect(f.slackRequests[0].method).toBe('chat.update'); + expect(f.slackRequests[0].body.get('text')).toBe(':white_check_mark: Completed.'); + const cleanup = f.paths().findIndex((path) => path.endsWith('/progress/settle')); + const final = f.paths().findIndex((path) => path.endsWith('/run-1/plan')); + expect(cleanup).toBeLessThan(final); + expect(cleanup).toBeGreaterThan(-1); + }); + + test('checkpoint allocation and upload before completion, without a false failure notice', async () => { + const f = fixture(); + f.savedPlan(true); + await f.execute(); + expect(f.slackRequests.map((request) => request.method)).toEqual([ + 'chat.postMessage', + 'files.getUploadURLExternal', + 'upload', + 'files.completeUploadExternal', + ]); + expect( + f.requests + .filter((request) => request.path.endsWith('/delivery/checkpoint')) + .map((request) => request.body.upload), + ).toEqual([ + { fileId: 'F1', stage: 'allocated' }, + { fileId: 'F1', stage: 'uploaded' }, + ]); + expect( + f.state.receipts.find((receipt) => receipt.operationId.includes(':artifact:')), + ).toMatchObject({ + outcome: 'confirmed', + externalFileId: 'F1', + upload: { fileId: 'F1', stage: 'uploaded' }, + }); + expect(f.state.state).toBe('delivered'); + }); + + test('unknown artifact completion retains its checkpoint and is never allocated/uploaded again', async () => { + const f = fixture(); + f.savedPlan(true); + f.hooks.slack = async (method) => { + if (method === 'files.getUploadURLExternal') + return slackOK({ file_id: 'F1', upload_url: 'http://127.0.0.1:8080/upload' }); + if (method === 'upload') return new Response('ok'); + if (method === 'files.completeUploadExternal') + throw new Error('Connection lost after completion'); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + await f.execute(); + expect( + f.state.receipts.find((receipt) => receipt.operationId.includes(':artifact:')), + ).toMatchObject({ outcome: 'unknown', upload: { fileId: 'F1', stage: 'uploaded' } }); + const before = f.slackRequests.length; + await f.execute(); + expect(f.slackRequests).toHaveLength(before); + expect( + f.slackRequests.filter((request) => request.method === 'files.getUploadURLExternal'), + ).toHaveLength(1); + expect( + f.slackRequests.filter((request) => request.method === 'files.completeUploadExternal'), + ).toHaveLength(1); + }); + + test('an inflight artifact checkpoint is held, never resumed by allocating a second file', async () => { + const f = fixture(); + f.savedPlan(true); + f.state.receipts[0].outcome = 'confirmed'; + const artifact = f.state.receipts.find((receipt) => + receipt.operationId.includes(':artifact:'), + )!; + artifact.outcome = 'inflight'; + artifact.attempts = 1; + artifact.upload = { fileId: 'F-existing', stage: 'allocated' }; + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /delivery\/begin|commit/.test(path))).toBe(false); + }); + + test('an invalid actor binding cannot create Responses or send Slack output', async () => { + const f = fixture(); + f.state.admission.actor.appId = 'FOREIGN'; + await expect(f.execute()).rejects.toThrow('binding'); + expect(f.slackRequests).toHaveLength(0); + expect(f.paths().some((path) => /prepare|agents\/v1/.test(path))).toBe(false); + }); + + test('falls back to the persisted text only after an explicit rich-block rejection', async () => { + const f = fixture(); + f.savedPlan(); + f.hooks.slack = async (_method, body) => + body.has('blocks') && body.get('blocks') !== '[]' + ? slackOK({ ok: false, error: 'invalid_blocks' }) + : slackOK({ channel: 'C1', ts: '123.789' }); + await f.execute(); + expect(f.slackRequests).toHaveLength(2); + expect(f.slackRequests[0].body.get('text')).toBe(f.slackRequests[1].body.get('text')); + expect(f.slackRequests[1].body.get('blocks')).toBe('[]'); + expect(f.paths().filter((path) => path.endsWith('/delivery/begin'))).toHaveLength(1); + expect(f.state.receipts[0]).toMatchObject({ outcome: 'confirmed', attempts: 1 }); + }); + + test('exhausted optional artifact retries settle as failed instead of blocking final answer commit', async () => { + const f = fixture(); + f.savedPlan(true); + f.state.receipts[0].outcome = 'confirmed'; + const receipt = f.state.receipts.find((item) => item.operationId.includes(':artifact:'))!; + receipt.outcome = 'retryable'; + receipt.attempts = 5; + await f.execute(); + expect(receipt.outcome).toBe('permanent_failure'); + expect(f.slackRequests.some((request) => request.method === 'files.getUploadURLExternal')).toBe( + false, + ); + expect(f.state.state).toBe('delivered'); + }); + + test('does not yield when a renewal becomes uncertain while the independent renewer drains', async () => { + const f = fixture(); + f.config.SLACK_NATIVE_SESSIONS = true; + f.lease.leaseExpiresAt = Date.now() + 60; + f.hooks.renew = async () => { + await delay(30); + return json({ error: 'unavailable' }, 503); + }; + f.hooks.response = async () => { + await delay(25); + return json({ + id: 'resp_test', + object: 'response', + model: 'agent-1', + background: true, + store: true, + status: 'requires_action', + }); + }; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + responseId: 'resp_test', + elapsedMs: 10, + }; + }; + f.log.warn = () => f.controller.abort(); + await f.worker.run(f.controller.signal); + expect(f.paths().some((path) => path.endsWith('/renew'))).toBe(true); + expect(f.paths().some((path) => path.endsWith('/yield'))).toBe(false); + }); + + test('a lost allocation checkpoint acknowledgement cannot trigger binary upload or another allocation', async () => { + const f = fixture(); + f.savedPlan(true); + f.hooks.checkpoint = async () => { + throw new Error('Checkpoint acknowledgement lost'); + }; + await expect(f.execute()).rejects.toThrow('Checkpoint acknowledgement lost'); + const receipt = f.state.receipts.find((item) => item.operationId.includes(':artifact:'))!; + expect(receipt).toMatchObject({ + outcome: 'inflight', + upload: { fileId: 'F1', stage: 'allocated' }, + }); + await expect(f.execute()).rejects.toThrow('reconciliation'); + expect(f.slackRequests.map((request) => request.method)).toEqual([ + 'chat.postMessage', + 'files.getUploadURLExternal', + ]); + }); + + test('verified unstarted cancellation never creates or polls a Response', async () => { + const f = fixture(); + f.config.SLACK_NATIVE_SESSIONS = true; + f.state.cancellationRequested = true; + f.hooks.prepare = () => ({ + ...f.grant, + settled: { + ...f.state, + state: 'delivering', + snapshot: { text: '', status: 'cancelled', elapsedMs: 0 }, + }, + }); + await f.execute(); + expect(f.paths().some((path) => /agents\/v1|advance|response\/bind/.test(path))).toBe(false); + expect( + f.slackRequests.find((request) => request.method === 'chat.postMessage')?.body.get('text'), + ).toContain('This request was cancelled'); + }); + + test('unproved cancellation without a Response cursor is held instead of creating new work', async () => { + const f = fixture(); + f.config.SLACK_NATIVE_SESSIONS = true; + f.state.cancellationRequested = true; + delete f.grant.responseId; + await expect(f.execute()).rejects.toThrow('awaits reconciliation'); + expect(f.paths().some((path) => /agents\/v1|advance|response\/bind/.test(path))).toBe(false); + }); + + test('a canonical cancellation polls only the exact bound Response cancellation route', async () => { + const f = fixture(); + f.config.SLACK_NATIVE_SESSIONS = true; + f.state.cancellationRequested = true; + f.hooks.advance = () => { + f.state.state = 'delivering'; + f.state.snapshot = { text: '', status: 'cancelled', responseId: 'resp_test', elapsedMs: 0 }; + }; + await f.execute(); + expect( + f.requests + .filter((request) => request.path.startsWith('/api/agents/')) + .map((request) => request.path), + ).toEqual(['/api/agents/v1/responses/resp_test/cancel']); + expect( + f.slackRequests + .filter((request) => request.method === 'agents.sessions.setStatus') + .map((request) => request.body.get('status')), + ).toEqual(['processing', 'active']); + }); + + test('requires-action uses native suspended state and one safe actor-lane yield', async () => { + const f = fixture(); + f.config.SLACK_NATIVE_SESSIONS = true; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + responseId: 'resp_test', + elapsedMs: 10, + }; + }; + await f.worker.run(f.controller.signal); + expect(f.slackRequests.map((request) => [request.method, request.body.get('status')])).toEqual([ + ['agents.sessions.setStatus', 'processing'], + ['agents.sessions.setStatus', 'suspended'], + ]); + expect(f.paths().filter((path) => path.endsWith('/yield'))).toHaveLength(1); + expect(f.paths().some((path) => path.endsWith('/commit'))).toBe(false); + }); +}); diff --git a/packages/channel-gateway/src/worker.ts b/packages/channel-gateway/src/worker.ts new file mode 100644 index 00000000000..27667f989a9 --- /dev/null +++ b/packages/channel-gateway/src/worker.ts @@ -0,0 +1,680 @@ +import { randomBytes } from 'node:crypto'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { + ChannelDeliveryReceipt, + ChannelDeliveryPart, + ChannelRunView, + ChannelRunFence, +} from 'librechat-data-provider'; +import type { ChannelBundle, Lease, Logger, ResponseGrant } from './types'; +import type { GatewayConfig } from './config'; +import { durableSlackMessage, makeSlackPlan, supportedRenderer } from './render'; +import { SlackClient, SlackWriteError } from './slack'; +import { ChannelClient, ServiceError } from './client'; + +const progressRenderer = 'slack-progress-v1'; +const terminalStates = new Set(['delivered', 'failed', 'cancelled']); +class YieldRun extends Error {} + +export function terminalSnapshot(run: ChannelRunView): boolean { + return !!run.snapshot && ['completed', 'failed', 'cancelled'].includes(run.snapshot.status); +} + +export function pendingReview(run: ChannelRunView): boolean { + const pending = run.snapshot?.pendingAction; + return ( + run.snapshot?.status === 'requires_action' && + !!pending?.actionId && + pending.expiresAt > Date.now() && + ['tool_approval', 'ask_user_question', 'oauth'].includes(pending.kind) + ); +} + +export function progressText(run: ChannelRunView): string { + if (terminalSnapshot(run)) { + if (run.snapshot?.status === 'completed') return ':white_check_mark: Completed.'; + if (run.snapshot?.status === 'cancelled') return 'Cancelled.'; + return ':warning: Failed.'; + } + if (run.cancellationRequested) return 'Cancellation requested.'; + if (run.snapshot?.recovery === 'execution_state_unavailable') + return ':warning: This request’s execution status is unavailable, possibly after a restart. Sign-in may be saved, but continuation and completion are not confirmed. Check the result; do not repeat actions that may already have run. If this stays unavailable, ask an administrator to reconcile the request before retrying. Stop requests cancellation, not proof that work has stopped.'; + if (run.snapshot?.status === 'requires_action') + return pendingReview(run) + ? '⏸ Your input is needed. Open the private review to continue.' + : '⏸ Waiting for input. The review action is unavailable or expired.'; + return ':hourglass_flowing_sand: Working…'; +} + +export function deliveryReceipt(run: ChannelRunView, operationId: string): ChannelDeliveryReceipt { + const receipt = run.receipts.find((item) => item.operationId === operationId); + if (!receipt) throw new Error('Missing persisted delivery receipt'); + return { ...receipt }; +} + +export function writeOutcome( + error: unknown, + attempts: number, +): Omit { + if (error instanceof SlackWriteError && error.outcome === 'permanent') + return { attempts, outcome: 'permanent_failure', errorCode: 'transport_rejected' }; + if (error instanceof SlackWriteError && error.outcome === 'retryable') + return { + attempts, + outcome: 'retryable', + errorCode: 'transport_retryable', + retryAt: Date.now() + Math.max(1000, error.retryAfterMs), + }; + return { attempts, outcome: 'unknown', errorCode: 'transport_uncertain' }; +} + +function address(run: ChannelRunView): { channel: string; thread_ts: string } { + const channel = run.admission.externalChannelId; + const prefix = channel + ':'; + const thread = run.admission.externalThreadKey.slice(prefix.length); + if ( + !channel || + !run.admission.externalThreadKey.startsWith(prefix) || + !thread || + /[:\s]/.test(thread) + ) + throw new Error('Invalid durable thread address'); + return { channel, thread_ts: thread }; +} + +function assertView(run: ChannelRunView, fence: ChannelRunFence, channelId: string): void { + if (run.runId !== fence.runId || run.channelId !== channelId || !run.revision) + throw new Error('Durable identity changed'); + address(run); +} + +function canYield(run: ChannelRunView): void { + if ( + [run.progress, ...run.receipts].some( + (item) => item?.outcome === 'inflight' || item?.outcome === 'unknown', + ) + ) + throw new Error('Delivery awaits reconciliation'); + throw new YieldRun(); +} + +export class ChannelWorker { + private readonly owner = randomBytes(16).toString('hex'); + constructor( + private readonly config: GatewayConfig, + private readonly client: ChannelClient, + private readonly slack: SlackClient, + private readonly log: Logger, + ) {} + + async run(signal: AbortSignal): Promise { + while (!signal.aborted) { + let lease: Lease | null = null; + try { + lease = await this.client.claim(this.owner, signal); + if (lease) { + if (lease.fence.owner !== this.owner) throw new Error('Claim owner changed'); + await this.process(lease, signal); + } + } catch (error) { + if (signal.aborted) break; + if (error instanceof YieldRun && lease) { + try { + await this.client.yield(lease.fence, signal); + } catch { + this.log.warn('channel_yield_unconfirmed'); + } + } else this.log.warn('channel_run_held_for_recovery'); + } + try { + await delay(1000, undefined, { signal }); + } catch { + break; + } + } + } + + async process(lease: Lease, parent: AbortSignal): Promise { + const { fence } = lease; + assertView(lease.run, fence, this.config.CHANNEL_ID); + if (!fence.owner || fence.epoch <= 0 || lease.leaseExpiresAt <= Date.now()) + throw new Error('Invalid durable lease'); + const controller = new AbortController(); + const signal = AbortSignal.any([parent, controller.signal]); + let expiry = lease.leaseExpiresAt; + let renewalUncertain = false; + const check = () => { + if (signal.aborted || Date.now() >= expiry) throw new Error('Durable lease lost'); + }; + const renewer = (async () => { + while (!signal.aborted) { + const remaining = expiry - Date.now(); + if (remaining <= 0) { + renewalUncertain = true; + controller.abort(); + return; + } + try { + await delay(Math.max(1, Math.floor(remaining / 3)), undefined, { signal }); + } catch { + return; + } + try { + const renewed = await this.client.renew(fence, signal); + if ( + renewed.fence.runId !== fence.runId || + renewed.fence.owner !== fence.owner || + renewed.fence.epoch !== fence.epoch || + renewed.leaseExpiresAt <= Date.now() + ) + throw new Error('Renewal identity changed'); + expiry = renewed.leaseExpiresAt; + } catch { + renewalUncertain = true; + controller.abort(); + return; + } + } + })(); + let failure: unknown; + try { + await this.execute(lease.run, fence, signal, check); + } catch (error) { + failure = error; + } finally { + controller.abort(); + await renewer; + } + if (failure instanceof YieldRun && renewalUncertain) throw new Error('Renewal uncertain'); + if (failure) throw failure; + } + + private async execute( + initial: ChannelRunView, + fence: ChannelRunFence, + signal: AbortSignal, + check: () => void, + ): Promise { + const bundle = await this.client.channel(signal); + if ( + initial.revision !== bundle.channel.revision || + initial.admission.actor.teamId !== bundle.channel.slack.teamId || + initial.admission.actor.appId !== bundle.channel.slack.appId + ) + throw new Error('Run binding changed'); + const token = bundle.credentials.botToken; + let sessionStatus = ''; + let sessionUpdated = 0; + let sessionRetry = 0; + let lastProgressWrite = 0; + const syncProgress = async (current: ChannelRunView) => { + check(); + let native = false; + if (this.config.SLACK_NATIVE_SESSIONS) { + let status: 'active' | 'suspended' | 'processing' = 'processing'; + if (terminalSnapshot(current) || current.plan) status = 'active'; + else if (current.snapshot?.status === 'requires_action') status = 'suspended'; + if (!sessionRetry && sessionStatus === status && Date.now() - sessionUpdated < 60000) + native = true; + else if (terminalSnapshot(current) || Date.now() >= sessionRetry) { + const target = address(current); + try { + await this.slack.setSessionStatus( + token, + { + channel: target.channel, + threadTs: target.thread_ts, + status, + title: bundle.agent.name + ? `${bundle.agent.name} conversation`.slice(0, 200) + : undefined, + }, + signal, + ); + sessionStatus = status; + sessionUpdated = Date.now(); + sessionRetry = 0; + native = true; + } catch { + sessionRetry = Date.now() + 60000; + this.log.warn('native_status_unavailable'); + } + } + } + if (native && !current.progress && !pendingReview(current)) return current; + const updated = await this.progress(current, fence, bundle, signal, check, lastProgressWrite); + if ( + updated.progress?.revision !== current.progress?.revision || + updated.progress?.attempts !== current.progress?.attempts + ) + lastProgressWrite = Date.now(); + return updated; + }; + await this.lifecycle(initial, bundle, 'started', signal); + let run = await syncProgress(initial); + let grant: ResponseGrant | undefined; + let unstartedCancellation = false; + if (!run.plan) { + const prepared = await this.client.prepare(fence, signal); + if (prepared.settled) { + assertView(prepared.settled, fence, this.config.CHANNEL_ID); + if ( + prepared.settled.state !== 'delivering' || + !prepared.settled.cancellationRequested || + prepared.settled.snapshot?.status !== 'cancelled' || + prepared.settled.snapshot.responseId + ) + throw new Error('Invalid unstarted cancellation proof'); + run = prepared.settled; + unstartedCancellation = true; + } else { + if ( + !prepared.apiKey.startsWith('sk-bound-') || + prepared.requestKey !== `channel:${run.runId}` || + prepared.request.model !== bundle.agent.id || + prepared.request.input !== run.admission.input || + !prepared.request.background || + !prepared.request.store + ) + throw new Error('Invalid Responses grant'); + if (!prepared.responseId) { + if (run.cancellationRequested) + throw new Error('Unstarted cancellation awaits reconciliation'); + check(); + const response = await this.client.response(prepared, 'create', signal); + prepared.responseId = response.id; + run = await this.client.mutate( + fence, + 'response/bind', + { responseId: response.id }, + signal, + ); + } + grant = prepared; + } + } + for (let polls = 0; !run.plan; polls++) { + check(); + if (grant) + await this.client.response(grant, run.cancellationRequested ? 'cancel' : 'read', signal); + if (!unstartedCancellation) run = await this.client.mutate(fence, 'advance', {}, signal); + assertView(run, fence, this.config.CHANNEL_ID); + if (grant && run.snapshot?.responseId && run.snapshot.responseId !== grant.responseId) + throw new Error('Response projection changed'); + if (run.plan) break; + run = await syncProgress(run); + if ( + terminalSnapshot(run) && + run.progress?.outcome === 'confirmed' && + run.progress.text !== progressText(run) + ) + run = await syncProgress(run); + if (run.state === 'delivering' && run.snapshot) { + if ( + terminalSnapshot(run) && + run.progress?.externalMessageKey && + ['planned', 'retryable'].includes(run.progress.outcome) + ) + throw new Error('Terminal progress cleanup pending'); + run = await this.client.mutate( + fence, + 'plan', + { plan: makeSlackPlan(run, this.config.SLACK_NATIVE_SESSIONS) }, + signal, + ); + break; + } + if (terminalStates.has(run.state)) return; + if (run.state === 'blocked' || run.state === 'requires_action' || polls >= 4) canYield(run); + await delay(1000, undefined, { signal }); + } + if ( + !run.plan || + !supportedRenderer(run.plan.rendererVersion) || + !run.plan.parts.length || + run.plan.parts.length > 100 + ) + throw new Error('Unsupported delivery plan'); + const renderer = run.plan.rendererVersion; + for (const part of run.plan.parts) { + check(); + if (part.kind === 'artifact') + run = await this.artifact(run, fence, part, bundle, signal, check); + else { + if ( + part.operationId === `${run.runId}:artifact-status` && + !part.required && + !run.plan?.parts.some( + (item) => + item.kind === 'artifact' && + ['unknown', 'inflight', 'permanent_failure'].includes( + deliveryReceipt(run, item.operationId).outcome, + ), + ) + ) + continue; + let receipt = deliveryReceipt(run, part.operationId); + if ( + receipt.outcome === 'confirmed' || + (!part.required && ['unknown', 'permanent_failure'].includes(receipt.outcome)) + ) + continue; + this.retryable(receipt); + const message = durableSlackMessage(part, renderer); + run = await this.client.mutate( + fence, + 'delivery/begin', + { operationId: part.operationId }, + signal, + ); + receipt = deliveryReceipt(run, part.operationId); + if (receipt.outcome !== 'inflight') throw new Error('Delivery begin not persisted'); + check(); + try { + const result = await this.slack.deliverPart( + token, + { ...address(run), ...message }, + signal, + ); + receipt = { + ...receipt, + outcome: 'confirmed', + externalMessageKey: `${result.channel}:${result.ts}`, + }; + } catch (error) { + receipt = { operationId: part.operationId, ...writeOutcome(error, receipt.attempts) }; + } + check(); + run = await this.settle(run, fence, receipt, signal); + if (receipt.outcome !== 'confirmed' && (part.required || receipt.outcome === 'retryable')) + throw new Error('Delivery not confirmed'); + } + } + check(); + run = await this.client.mutate(fence, 'commit', {}, signal); + if (!terminalStates.has(run.state)) throw new Error('Commit not confirmed'); + await this.lifecycle(run, bundle, run.state === 'delivered' ? 'completed' : 'failed', signal); + } + + private retryable(receipt: ChannelDeliveryReceipt): void { + if ( + !['planned', 'retryable'].includes(receipt.outcome) || + receipt.upload || + receipt.attempts >= 5 || + (receipt.retryAt ?? 0) > Date.now() + ) + throw new Error('Delivery deferred or awaits reconciliation'); + } + + private async settle( + run: ChannelRunView, + fence: ChannelRunFence, + receipt: ChannelDeliveryReceipt, + signal: AbortSignal, + ): Promise { + const { upload: checkpoint, ...wire } = receipt; + const updated = await this.client.mutate(fence, 'delivery/settle', { receipt: wire }, signal); + const stored = deliveryReceipt(updated, receipt.operationId); + if ( + stored.outcome !== receipt.outcome || + stored.externalMessageKey !== receipt.externalMessageKey || + stored.externalFileId !== receipt.externalFileId || + (checkpoint && + (stored.upload?.fileId !== checkpoint.fileId || stored.upload.stage !== checkpoint.stage)) + ) + throw new Error('Delivery settlement not persisted'); + return updated; + } + + private async progress( + run: ChannelRunView, + fence: ChannelRunFence, + bundle: ChannelBundle, + signal: AbortSignal, + check: () => void, + lastWrite: number, + ): Promise { + if (run.plan || terminalStates.has(run.state) || run.state === 'blocked') return run; + let progress = run.progress; + if (terminalSnapshot(run) && !progress?.externalMessageKey) return run; + if ( + progress && + (progress.rendererVersion !== progressRenderer || + ['unknown', 'inflight', 'permanent_failure'].includes(progress.outcome)) + ) + return run; + const text = progressText(run); + if (!progress || progress.outcome === 'confirmed') { + if (progress?.text === text) return run; + const revision = (progress?.revision ?? 0) + 1; + check(); + run = await this.client.mutate( + fence, + 'progress/plan', + { revision, rendererVersion: progressRenderer, text }, + signal, + ); + progress = run.progress; + if (!progress || progress.revision !== revision || progress.text !== text) + throw new Error('Progress plan not persisted'); + } + if (!['planned', 'retryable'].includes(progress.outcome) || progress.text.length > 4000) + throw new Error('Unsupported progress state'); + if ((progress.retryAt ?? 0) > Date.now()) return run; + if (Date.now() < lastWrite + 1000) + await delay(lastWrite + 1000 - Date.now(), undefined, { signal }); + check(); + const revision = progress.revision; + run = await this.client.mutate(fence, 'progress/begin', { revision }, signal); + progress = run.progress; + if ( + !progress || + progress.revision !== revision || + progress.outcome !== 'inflight' || + progress.rendererVersion !== progressRenderer || + !progress.text || + progress.text.length > 4000 + ) + throw new Error('Progress begin not persisted'); + const target = address(run); + const blocks: Parameters[1]['blocks'] = [ + { type: 'section', text: { type: 'mrkdwn', text: progress.text }, expand: true }, + ]; + if (!terminalSnapshot(run) && !run.cancellationRequested) { + const value = JSON.stringify({ runId: run.runId, revision: run.revision }); + const elements = [ + { + type: 'button' as const, + text: { type: 'plain_text' as const, text: 'Stop' }, + action_id: 'channel_run_cancel', + value, + }, + ]; + if (pendingReview(run)) + elements.push({ + type: 'button', + text: { type: 'plain_text', text: 'Review request' }, + action_id: 'channel_run_review', + value, + }); + blocks.push({ type: 'actions', elements }); + } + let receipt: Omit; + check(); + try { + if (progress.attempts > 5) throw new SlackWriteError('permanent'); + let posted: { channel: string; ts: string }; + if (progress.externalMessageKey) { + const prefix = target.channel + ':'; + const ts = progress.externalMessageKey.slice(prefix.length); + if (!progress.externalMessageKey.startsWith(prefix) || !ts || /[:\s]/.test(ts)) + throw new Error('Invalid progress address'); + posted = await this.slack.updateMessageOnce( + bundle.credentials.botToken, + { channel: target.channel, ts, text: progress.text, blocks }, + signal, + ); + } else + posted = await this.slack.postMessage( + bundle.credentials.botToken, + { ...target, text: progress.text, blocks }, + signal, + ); + receipt = { + attempts: progress.attempts, + outcome: 'confirmed', + externalMessageKey: `${posted.channel}:${posted.ts}`, + }; + } catch (error) { + receipt = writeOutcome(error, progress.attempts); + } + check(); + run = await this.client.mutate(fence, 'progress/settle', { revision, receipt }, signal); + if ( + run.progress?.revision !== revision || + run.progress.outcome !== receipt.outcome || + (receipt.outcome === 'confirmed' && + run.progress.externalMessageKey !== receipt.externalMessageKey) + ) + throw new Error('Progress settlement not persisted'); + return run; + } + + private async artifact( + run: ChannelRunView, + fence: ChannelRunFence, + part: ChannelDeliveryPart, + bundle: ChannelBundle, + signal: AbortSignal, + check: () => void, + ): Promise { + const artifacts = run.snapshot?.artifacts ?? []; + const artifact = artifacts.find((item) => item.artifactId === part.artifactId); + if (!artifact || artifacts.length > 20 || part.required || part.presentation) + throw new Error('Unsupported artifact part'); + let receipt = deliveryReceipt(run, part.operationId); + if (['confirmed', 'unknown', 'permanent_failure'].includes(receipt.outcome)) return run; + if ( + !['planned', 'retryable'].includes(receipt.outcome) || + receipt.upload || + (receipt.retryAt ?? 0) > Date.now() + ) + throw new Error('Artifact deferred or awaits reconciliation'); + check(); + run = await this.client.mutate( + fence, + 'delivery/begin', + { operationId: part.operationId }, + signal, + ); + receipt = deliveryReceipt(run, part.operationId); + if (receipt.outcome !== 'inflight' || receipt.upload) + throw new Error('Artifact begin not persisted'); + const settle = async (outcome: ChannelDeliveryReceipt['outcome'], errorCode?: string) => { + check(); + run = await this.settle(run, fence, { ...receipt, outcome, errorCode }, signal); + if (outcome === 'retryable') throw new Error('Artifact retry deferred'); + return run; + }; + if (receipt.attempts > 5) return settle('permanent_failure', 'artifact_retry_exhausted'); + let bytes: Buffer; + try { + bytes = await this.client.artifact(fence, artifact, signal); + } catch (error) { + if (error instanceof ServiceError && [401, 403, 409].includes(error.status)) throw error; + return settle('permanent_failure', 'artifact_unavailable'); + } + check(); + let allocation: { fileId: string; uploadUrl: string }; + try { + allocation = await this.slack.allocateArtifact( + bundle.credentials.botToken, + artifact.filename, + artifact.bytes, + signal, + ); + } catch (error) { + const outcome = writeOutcome(error, receipt.attempts); + receipt.retryAt = outcome.retryAt; + return settle(outcome.outcome, outcome.errorCode); + } + const checkpoint = async (stage: 'allocated' | 'uploaded') => { + check(); + const upload = { fileId: allocation.fileId, stage }; + run = await this.client.mutate( + fence, + 'delivery/checkpoint', + { operationId: part.operationId, attempts: receipt.attempts, upload }, + signal, + ); + const stored = deliveryReceipt(run, part.operationId); + if ( + stored.outcome !== 'inflight' || + stored.attempts !== receipt.attempts || + stored.upload?.fileId !== upload.fileId || + stored.upload.stage !== stage + ) + throw new Error('Upload checkpoint not persisted'); + receipt = stored; + }; + await checkpoint('allocated'); + check(); + try { + await this.slack.uploadArtifact(allocation.uploadUrl, bytes, signal); + } catch { + return settle('unknown', 'artifact_upload_uncertain'); + } + await checkpoint('uploaded'); + check(); + const target = address(run); + try { + await this.slack.completeArtifact( + bundle.credentials.botToken, + { + channel: target.channel, + threadTs: target.thread_ts, + fileId: allocation.fileId, + filename: artifact.filename, + }, + signal, + ); + } catch { + return settle('unknown', 'artifact_completion_uncertain'); + } + receipt.externalFileId = allocation.fileId; + return settle('confirmed'); + } + + private async lifecycle( + run: ChannelRunView, + bundle: ChannelBundle, + phase: 'started' | 'completed' | 'failed', + signal: AbortSignal, + ): Promise { + if (!this.config.SLACK_LIFECYCLE_REACTIONS) return; + const prefix = run.admission.externalChannelId + ':'; + const ts = run.admission.externalMessageKey.slice(prefix.length); + if (!run.admission.externalMessageKey.startsWith(prefix) || !ts || /[:\s]/.test(ts)) return; + const limited = AbortSignal.any([signal, AbortSignal.timeout(2000)]); + const reaction = (operation: 'add' | 'remove', name: string) => + this.slack.reaction( + bundle.credentials.botToken, + operation, + run.admission.externalChannelId, + ts, + name, + limited, + ); + try { + if (phase === 'started') { + await reaction('add', 'eyes'); + await reaction('add', 'hourglass_flowing_sand'); + } else { + await reaction('remove', 'hourglass_flowing_sand'); + await reaction('add', phase === 'completed' ? 'white_check_mark' : 'warning'); + } + } catch { + this.log.warn('optional_lifecycle_unavailable'); + } + } +} diff --git a/packages/channel-gateway/tsdown.config.mjs b/packages/channel-gateway/tsdown.config.mjs index 5a66c1503a9..1295f517a9f 100644 --- a/packages/channel-gateway/tsdown.config.mjs +++ b/packages/channel-gateway/tsdown.config.mjs @@ -1,15 +1,7 @@ import { defineConfig } from 'tsdown'; export default defineConfig({ - entry: [ - 'src/client.ts', - 'src/config.ts', - 'src/events.ts', - 'src/slack.ts', - 'src/render.ts', - 'src/identity.ts', - 'src/review.ts', - ], + entry: ['src/main.ts'], platform: 'node', format: ['cjs'], dts: false, From 35a21fc2b7fb4c58981311c7fde5ad43b0e29a8b Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:33:33 +0100 Subject: [PATCH 02/12] fix: Preserve Gateway Recovery and Optional Delivery Boundaries --- packages/channel-gateway/src/app.spec.ts | 58 +++++++++++ packages/channel-gateway/src/app.ts | 3 +- packages/channel-gateway/src/client.spec.ts | 13 +++ packages/channel-gateway/src/client.ts | 5 +- packages/channel-gateway/src/worker.spec.ts | 103 +++++++++++++++++++- packages/channel-gateway/src/worker.ts | 45 +++++++-- 6 files changed, 211 insertions(+), 16 deletions(-) diff --git a/packages/channel-gateway/src/app.spec.ts b/packages/channel-gateway/src/app.spec.ts index 25b837f7808..3be186d514d 100644 --- a/packages/channel-gateway/src/app.spec.ts +++ b/packages/channel-gateway/src/app.spec.ts @@ -150,6 +150,7 @@ describe('durable Slack ingress', () => { expect(JSON.parse(String(backend.mock.calls[1][1]?.body))).toEqual({ actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, externalThreadKey: 'C1:1792999999.5', + externalChannelId: 'C1', eventAt: 1793000001123, }); }); @@ -545,3 +546,60 @@ describe('signed webhook boundary', () => { expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); }); }); + +test.each([false, true])( + 'pending modal work owns ingress capacity until settlement (failure=%s)', + async (fail) => { + const { app, backend } = setup(); + let release!: () => void; + const waiting = new Promise((resolve) => { + release = resolve; + }); + let reads = 0; + const reference = { + runId: 'run', + revision: 'r1', + actionId: 'action', + channelId: 'C1', + actorId: 'U1', + expiresAt: Date.now() + 60000, + }; + backend.mockImplementation(async () => { + reads++; + await waiting; + if (fail) throw new Error('outage'); + return json({ + actionId: 'action', + pendingAction: { + actionId: 'action', + expiresAt: reference.expiresAt, + payload: { type: 'ask_user_question', question: { question: 'Answer this' } }, + }, + }); + }); + const body = { + type: 'view_submission', + team: { id: 'T1' }, + api_app_id: 'A1', + user: { id: 'U1' }, + view: { + callback_id: 'channel_native_review', + private_metadata: JSON.stringify(reference), + state: { values: {} }, + }, + }; + const pending = Array.from({ length: 64 }, () => app.dispatch(body, true, undefined, bundle)); + for (let attempt = 0; attempt < 20 && reads < 64; attempt++) + await new Promise((resolve) => setTimeout(resolve, 0)); + try { + expect(reads).toBe(64); + await expect(app.dispatch(envelope(), false, undefined, bundle)).rejects.toThrow('capacity'); + } finally { + release(); + await Promise.all(pending); + } + await expect( + app.dispatch(envelope({ bot_id: 'bot' }), false, undefined, bundle), + ).resolves.toBeUndefined(); + }, +); diff --git a/packages/channel-gateway/src/app.ts b/packages/channel-gateway/src/app.ts index 8ba6d8ca5ee..eb39b4d62da 100644 --- a/packages/channel-gateway/src/app.ts +++ b/packages/channel-gateway/src/app.ts @@ -112,7 +112,7 @@ export class GatewayApp { }; if (callback.type === 'view_closed') return; if (callback.type === 'view_submission') - return this.review.submit(interaction, bundle, signal); + return await this.review.submit(interaction, bundle, signal); const action = callback.actions?.[0]; if (!action) throw new Error('Missing Slack action'); if (action.action_id === 'channel_feedback') { @@ -219,6 +219,7 @@ export class GatewayApp { event.actor, event.externalThreadKey, event.eventAt, + event.externalChannelId, signal, ); this.log.info( diff --git a/packages/channel-gateway/src/client.spec.ts b/packages/channel-gateway/src/client.spec.ts index 768506b853e..60bae5e7ab2 100644 --- a/packages/channel-gateway/src/client.spec.ts +++ b/packages/channel-gateway/src/client.spec.ts @@ -565,3 +565,16 @@ test('Stop and cancel retain the complete external-channel scope at the service { actor, revision: run.revision, externalChannelId: 'C1' }, ]); }); + +test('exposes only the approved trace-unavailable code for a409; other diagnostics stay private', async () => { + const transport = jest + .fn, Parameters>() + .mockResolvedValueOnce(json({ error: 'trace_unavailable', secret: 'private' }, 409)) + .mockResolvedValueOnce(json({ error: 'private-database-error', secret: 'private' }, 409)); + const client = new ChannelClient(config, transport); + await expect(client.claim('worker')).rejects.toMatchObject({ + status: 409, + code: 'trace_unavailable', + }); + await expect(client.claim('worker')).rejects.toMatchObject({ status: 409, code: '' }); +}); diff --git a/packages/channel-gateway/src/client.ts b/packages/channel-gateway/src/client.ts index 51348601d33..3c8cd2208f6 100644 --- a/packages/channel-gateway/src/client.ts +++ b/packages/channel-gateway/src/client.ts @@ -122,7 +122,10 @@ export class ChannelClient { let code = ''; try { const failure: { error?: string } = JSON.parse(bytes.toString()); - if (response.status === 403 && failure.error === 'channel_link_required') + if ( + (response.status === 403 && failure.error === 'channel_link_required') || + (response.status === 409 && failure.error === 'trace_unavailable') + ) code = failure.error; } catch { /* Untrusted error bodies are never exposed. */ diff --git a/packages/channel-gateway/src/worker.spec.ts b/packages/channel-gateway/src/worker.spec.ts index 559a4beadf3..c98d5010030 100644 --- a/packages/channel-gateway/src/worker.spec.ts +++ b/packages/channel-gateway/src/worker.spec.ts @@ -5,7 +5,7 @@ import type { ChannelDeliveryReceipt, ChannelRunView, } from 'librechat-data-provider'; -import type { ChannelBundle, Lease, Logger, ResponseGrant } from './types'; +import type { ChannelBundle, Lease, Logger, ResponseGrant, ResponsePreparation } from './types'; import { ChannelWorker, progressText } from './worker'; import { ChannelClient } from './client'; import { makeSlackPlan } from './render'; @@ -107,8 +107,9 @@ function fixture() { renew?: () => Promise; response?: () => Promise; slack?: (method: string, body: URLSearchParams) => Promise; - prepare?: () => ResponseGrant; + prepare?: () => ResponsePreparation; checkpoint?: () => Promise; + begin?: (body: MutationBody) => Response | undefined; } = {}; const snapshot = () => structuredClone(state); const backend = jest.fn, Parameters>( @@ -141,7 +142,7 @@ function fixture() { } if (path.endsWith('/renew')) { if (hooks.renew) return hooks.renew(); - return json({ ...lease, run: snapshot(), leaseExpiresAt: Date.now() + 90 }); + return json({ fence: lease.fence, leaseExpiresAt: Date.now() + 90 }); } if (path.endsWith('/yield')) { controller.abort(); @@ -197,6 +198,8 @@ function fixture() { }, }); } else if (path.endsWith('/delivery/begin')) { + const intercepted = hooks.begin?.(body); + if (intercepted) return intercepted; const receipt = state.receipts.find((item) => item.operationId === body.operationId); if (!receipt) throw new Error('Missing begin fixture'); receipt.outcome = 'inflight'; @@ -606,7 +609,6 @@ describe('durable Channels worker', () => { f.config.SLACK_NATIVE_SESSIONS = true; f.state.cancellationRequested = true; f.hooks.prepare = () => ({ - ...f.grant, settled: { ...f.state, state: 'delivering', @@ -671,3 +673,96 @@ describe('durable Channels worker', () => { expect(f.paths().some((path) => path.endsWith('/commit'))).toBe(false); }); }); + +describe('optional delivery and uncertain execution boundaries', () => { + function trace(f: ReturnType) { + const part = { + operationId: 'run-1:trace:0', + kind: 'message' as const, + required: false, + text: '', + traceUrl: 'https://chat.example.com/session', + }; + f.state.plan!.parts.push(part); + f.state.receipts.push({ operationId: part.operationId, outcome: 'planned', attempts: 0 }); + return part; + } + test('an exhausted optional message cannot hold a confirmed required answer or cause a repost', async () => { + const f = fixture(); + f.savedPlan(); + f.state.receipts[0].outcome = 'confirmed'; + f.state.plan!.parts.push({ + operationId: 'notice', + kind: 'message', + required: false, + text: 'Attachment unavailable', + }); + f.state.receipts.push({ + operationId: 'notice', + outcome: 'retryable', + attempts: 5, + retryAt: Date.now() + 60000, + }); + await f.execute(); + expect(f.state.state).toBe('delivered'); + expect(f.slackRequests).toHaveLength(0); + }); + test('a trace omitted by the latest view after answer delivery is skipped before receipt lookup', async () => { + const f = fixture(); + f.savedPlan(); + const part = trace(f); + f.hooks.slack = async () => { + f.state.plan!.parts = f.state.plan!.parts.filter((p) => p.operationId !== part.operationId); + f.state.receipts = f.state.receipts.filter((p) => p.operationId !== part.operationId); + return slackOK({ channel: 'C1', ts: '123.789' }); + }; + await f.execute(); + expect(f.state.state).toBe('delivered'); + expect(f.slackRequests).toHaveLength(1); + expect(f.slackRequests[0].body.get('text')).not.toContain(part.traceUrl); + }); + test.each(['trace_unavailable', 'stale_fence'])( + 'only the exact trace authority outcome is skippable: %s', + async (code) => { + const f = fixture(); + f.savedPlan(); + f.state.receipts[0].outcome = 'confirmed'; + const part = trace(f); + f.hooks.begin = (body) => + body.operationId === part.operationId ? json({ error: code }, 409) : undefined; + if (code === 'trace_unavailable') { + await f.execute(); + expect(f.state.state).toBe('delivered'); + } else { + await expect(f.execute()).rejects.toThrow('HTTP 409'); + expect(f.state.state).toBe('delivering'); + } + expect(f.slackRequests).toHaveLength(0); + }, + ); + test('native status success still publishes recovery guidance and suppresses stale review controls', async () => { + const f = fixture(); + f.config.SLACK_NATIVE_SESSIONS = true; + f.hooks.advance = () => { + f.state.state = 'requires_action'; + f.state.snapshot = { + status: 'requires_action', + text: '', + elapsedMs: 10, + responseId: 'resp_test', + recovery: 'execution_state_unavailable', + pendingAction: { actionId: 'stale', kind: 'tool_approval', expiresAt: Date.now() + 60000 }, + }; + }; + await f.worker.run(f.controller.signal); + const warning = f.slackRequests.find((r) => r.method === 'chat.postMessage')!; + expect(warning.body.get('text')).toContain('do not repeat actions'); + expect(warning.body.get('text')).toContain('reconcile'); + expect(warning.body.get('blocks')).not.toContain('channel_run_review'); + expect(f.slackRequests.some((r) => r.method === 'agents.sessions.setStatus')).toBe(true); + expect(f.paths().some((p) => p.endsWith('/yield'))).toBe(true); + f.state.cancellationRequested = true; + expect(progressText(f.state)).toContain('Cancellation requested.'); + expect(progressText(f.state)).toContain('do not repeat actions'); + }); +}); diff --git a/packages/channel-gateway/src/worker.ts b/packages/channel-gateway/src/worker.ts index 27667f989a9..4b9fe72062b 100644 --- a/packages/channel-gateway/src/worker.ts +++ b/packages/channel-gateway/src/worker.ts @@ -23,6 +23,7 @@ export function terminalSnapshot(run: ChannelRunView): boolean { export function pendingReview(run: ChannelRunView): boolean { const pending = run.snapshot?.pendingAction; return ( + !run.snapshot?.recovery && run.snapshot?.status === 'requires_action' && !!pending?.actionId && pending.expiresAt > Date.now() && @@ -36,9 +37,12 @@ export function progressText(run: ChannelRunView): string { if (run.snapshot?.status === 'cancelled') return 'Cancelled.'; return ':warning: Failed.'; } - if (run.cancellationRequested) return 'Cancellation requested.'; if (run.snapshot?.recovery === 'execution_state_unavailable') - return ':warning: This request’s execution status is unavailable, possibly after a restart. Sign-in may be saved, but continuation and completion are not confirmed. Check the result; do not repeat actions that may already have run. If this stays unavailable, ask an administrator to reconcile the request before retrying. Stop requests cancellation, not proof that work has stopped.'; + return ( + (run.cancellationRequested ? 'Cancellation requested. ' : '') + + ':warning: This request’s execution status is unavailable, possibly after a restart. Sign-in may be saved, but continuation and completion are not confirmed. Check the result; do not repeat actions that may already have run. If this stays unavailable, ask an administrator to reconcile the request before retrying. Stop requests cancellation, not proof that work has stopped.' + ); + if (run.cancellationRequested) return 'Cancellation requested.'; if (run.snapshot?.status === 'requires_action') return pendingReview(run) ? '⏸ Your input is needed. Open the private review to continue.' @@ -241,7 +245,8 @@ export class ChannelWorker { } } } - if (native && !current.progress && !pendingReview(current)) return current; + if (native && !current.progress && !pendingReview(current) && !current.snapshot?.recovery) + return current; const updated = await this.progress(current, fence, bundle, signal, check, lastProgressWrite); if ( updated.progress?.revision !== current.progress?.revision || @@ -338,6 +343,13 @@ export class ChannelWorker { const renderer = run.plan.rendererVersion; for (const part of run.plan.parts) { check(); + const traceAvailable = () => + !part.traceUrl || + run.plan?.parts.some( + (current) => + current.operationId === part.operationId && current.traceUrl === part.traceUrl, + ); + if (!traceAvailable()) continue; if (part.kind === 'artifact') run = await this.artifact(run, fence, part, bundle, signal, check); else { @@ -356,17 +368,30 @@ export class ChannelWorker { let receipt = deliveryReceipt(run, part.operationId); if ( receipt.outcome === 'confirmed' || - (!part.required && ['unknown', 'permanent_failure'].includes(receipt.outcome)) + (!part.required && + (['unknown', 'permanent_failure'].includes(receipt.outcome) || receipt.attempts >= 5)) ) continue; this.retryable(receipt); const message = durableSlackMessage(part, renderer); - run = await this.client.mutate( - fence, - 'delivery/begin', - { operationId: part.operationId }, - signal, - ); + try { + run = await this.client.mutate( + fence, + 'delivery/begin', + { operationId: part.operationId }, + signal, + ); + } catch (error) { + if ( + part.traceUrl && + error instanceof ServiceError && + error.status === 409 && + error.code === 'trace_unavailable' + ) + continue; + throw error; + } + if (!traceAvailable()) continue; receipt = deliveryReceipt(run, part.operationId); if (receipt.outcome !== 'inflight') throw new Error('Delivery begin not persisted'); check(); From dfbb14c41054d2b986d1075b639b598796d92e3c Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:56:03 +0100 Subject: [PATCH 03/12] Document Verified Channel Delivery Reconciliation --- packages/channel-gateway/README.md | 55 ++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index 1e5fb6e3e0d..a84cdbe18ea 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -1,22 +1,55 @@ -# Slack transport adapters +# Channel gateway delivery recovery -This private workspace contains transport, native presentation and private interaction adapters for Agent Channels. The executable consumer is the [standalone gateway runtime in #17618](https://github.com/LibreChat-AI/LibreChat/pull/17618), which composes these adapters with native presentation, HTTP and Socket Mode ingress, durable workers, process lifecycle and container packaging. +The gateway preserves each frozen delivery operation and its receipt. A required operation whose provider result is unknown holds its actor lane: retrying blindly could post the same answer twice. Stopping a worker or requesting cancellation does not prove whether Slack accepted an earlier request. -The adapter build produces seven internal modules: +## Reconcile a verified provider result -| Module | Responsibility | -| ---------- | -------------------------------------------------------------------------------------------- | -| `client` | Private service authentication, channel/run identity checks and bounded API responses | -| `config` | Process bindings and resolution of the server-owned gateway policy | -| `events` | Original-byte webhook authentication, installation challenges and scoped event normalization | -| `slack` | Bounded provider calls, conservative uncertain-write outcomes and artifact transport | +1. Pause gateway processing for the affected channel while investigating. Keep the channel enabled and preserve its actor link and source authority; disabling these is not a recovery bypass. +2. Inspect the frozen run plan and retained receipt. Verify in the provider that the exact operation reached the intended channel or thread. Record its message key or file ID, matching the retained attempt and any upload checkpoint. If the result cannot be verified, leave the receipt unknown and the lane held. +3. After the old lease expires, use `POST /api/channels/service/:channelId/runs/claim` with `{ "owner": "" }`, normal private service authentication and tenant binding. Verify that the returned run ID is the run being investigated; claims select eligible work and do not take a target run ID. Use the returned current fence, renewing it through `POST /renew` as needed. An expired or replaced worker's fence cannot reconcile the run. +4. Submit the confirmed receipt through the private channel API. These endpoints confirm existing attempts; they do not send messages, replace the frozen plan, reset attempts or accept a retryable outcome. + +All paths below are relative to `/api/channels/service/:channelId/runs/:runId`. Send the channel service bearer token and the configured tenant header, using the same internal service URL as the gateway. Preserve the complete fence returned by the claim. + +For a delivery operation, `POST /delivery/reconcile`: + +```json +{ + "fence": { "runId": "", "owner": "", "epoch": 1 }, + "receipt": { + "operationId": "", + "outcome": "confirmed", + "attempts": 1, + "externalMessageKey": ":" + } +} +``` + +Replace every illustrative value with the actual current fence and retained attempt count; do not assume the example epoch is current. Supply `externalFileId` for a verified file operation as required by its frozen operation and checkpoint. Existing provider identities are immutable; reconciliation cannot replace them with a different message or file. + +For a progress update, `POST /progress/reconcile` with the current `fence`, the retained progress `revision`, and a `receipt` containing `outcome: "confirmed"`, the retained `attempts` and verified `externalMessageKey`. The message identity remains fixed across progress revisions. + +Once every required delivery operation is confirmed, `POST /commit` with the current `fence` to complete delivery and release the actor lane. Resume normal gateway processing afterward. If any required result remains uncertain, keep the run held rather than resending or recreating it. Optional exhausted effects follow the gateway's bounded recovery policy and do not justify replaying a required answer. + +## Transport adapter boundary + +The standalone process composes private transport adapters with native presentation, HTTP and Socket Mode ingress, durable workers and process lifecycle. + +The production bundle incorporates these seven internal source modules: + +| Module | Responsibility | +| -------- | -------------------------------------------------------------------------------------------- | +| `client` | Private service authentication, channel/run identity checks and bounded API responses | +| `config` | Process bindings and resolution of the server-owned gateway policy | +| `events` | Original-byte webhook authentication, installation challenges and scoped event normalization | +| `slack` | Bounded provider calls, conservative uncertain-write outcomes and artifact transport | | `render` | Deterministic bounded answer/progress plans and durable presentation replay | | `identity` | Actor-private account-link tickets and exact-account confirmation | | `review` | Current actor/action validation, native forms and immutable saved-decision retry | -The application defaults `endpoints.agents.channels.enabled` to `false`. Enable the capability in a deployment that includes the matching executable consumer. These adapter artifacts are internal dependencies; deploying the runnable gateway also requires its entry point, presentation layer and durable worker. +The application defaults `endpoints.agents.channels.enabled` to `false`. Enable the capability in a deployment that includes the matching executable consumer. The deployed `main.cjs` bundle includes its entry point, presentation layer, durable worker and these internal modules. -## Consumer contract +### Consumer contract Before opening ingress, fetch the private channel bundle, validate its versioned gateway policy and call `resolveGatewayConfig`. The server selects Socket Mode or webhook transport; an optional process transport assertion must match. Missing or unsupported policy fails closed. Application and gateway must both support policy version 1. From 6507127231d82909c37b924aefb0f41089450ba6 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:37:24 +0100 Subject: [PATCH 04/12] test: Expect Transport Credential Rejection Before Execution --- packages/channel-gateway/src/worker.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/channel-gateway/src/worker.spec.ts b/packages/channel-gateway/src/worker.spec.ts index c98d5010030..d3393721637 100644 --- a/packages/channel-gateway/src/worker.spec.ts +++ b/packages/channel-gateway/src/worker.spec.ts @@ -365,7 +365,9 @@ describe('durable Channels worker', () => { f.config.SLACK_NATIVE_SESSIONS = true; if (field === 'apiKey' || field === 'requestKey') f.grant[field] = 'wrong'; else f.grant.request[field] = 'wrong'; - await expect(f.execute()).rejects.toThrow('grant'); + await expect(f.execute()).rejects.toThrow( + field === 'apiKey' ? 'Invalid run-bound response credential' : 'Invalid Responses grant', + ); expect(f.paths().some((path) => path.startsWith('/api/agents/'))).toBe(false); }, ); From 21e083a11d227115c9a172e505c5ce6db574bb58 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:32:37 +0100 Subject: [PATCH 05/12] fix: Bind Gateway Runtime to Authoritative Channel Policy --- .github/workflows/channel-gateway.yml | 17 +- docker-compose.channels.yml | 3 - packages/channel-gateway/README.md | 33 +++- packages/channel-gateway/src/app.spec.ts | 186 +++++++++++++------- packages/channel-gateway/src/app.ts | 130 +++++++------- packages/channel-gateway/src/main.ts | 11 +- packages/channel-gateway/src/socket.spec.ts | 11 +- packages/channel-gateway/src/worker.spec.ts | 39 +++- packages/channel-gateway/src/worker.ts | 6 +- 9 files changed, 282 insertions(+), 154 deletions(-) diff --git a/.github/workflows/channel-gateway.yml b/.github/workflows/channel-gateway.yml index bc2bca2fab4..f6c4e08c9aa 100644 --- a/.github/workflows/channel-gateway.yml +++ b/.github/workflows/channel-gateway.yml @@ -77,10 +77,12 @@ jobs: docker run --rm -i --read-only --entrypoint node channel-gateway:ci - <<'NODE' const http = require('node:http'); const { spawn } = require('node:child_process'); + const { createHmac } = require('node:crypto'); const { setTimeout: delay } = require('node:timers/promises'); const bundle = { - channel: { channelId: 'fixture', tenantId: 'fixture', agentId: 'agent_fixture', type: 'slack', status: 'active', revision: 'fixture', slack: { teamId: 'T1', appId: 'A1', botUserId: 'U1' } }, - credentials: { botToken: 'fixture' }, agent: { id: 'agent_fixture' }, + channel: { channelId: 'fixture', tenantId: 'fixture', agentId: 'agent_fixture', createdBy: 'owner', type: 'slack', status: 'active', revision: 'fixture', slack: { teamId: 'T1', appId: 'A1', botUserId: 'U1' } }, + gateway: { version: 1, transport: 'webhook', nativeSessions: false, lifecycleReactions: false, signatureToleranceSeconds: 300 }, + credentials: { botToken: 'fixture', signingSecret: 'fixture-signing' }, agent: { id: 'agent_fixture' }, }; const backend = http.createServer((req, res) => { if (req.url.endsWith('/runs/claim')) { req.resume(); res.writeHead(204); res.end(); return; } @@ -88,7 +90,7 @@ jobs: }); backend.listen(0, '127.0.0.1', async () => { const child = spawn(process.execPath, ['/app/main.cjs'], { - env: { ...process.env, LIBRECHAT_INTERNAL_URL: `http://127.0.0.1:${backend.address().port}`, CHANNEL_SERVICE_TOKEN: 'fixture', CHANNEL_TENANT_ID: 'fixture', CHANNEL_ID: 'fixture', SLACK_TRANSPORT: 'webhook' }, + env: { ...process.env, LIBRECHAT_INTERNAL_URL: `http://127.0.0.1:${backend.address().port}`, CHANNEL_SERVICE_TOKEN: 'fixture', CHANNEL_TENANT_ID: 'fixture', CHANNEL_ID: 'fixture' }, stdio: 'ignore', }); const exited = new Promise(resolve => child.once('exit', resolve)); @@ -101,6 +103,15 @@ jobs: await delay(100); } if (!ready) throw new Error('Gateway did not become healthy'); + const body = JSON.stringify({ type: 'url_verification', challenge: 'verified' }); + const url = 'http://127.0.0.1:8080/slack/events/fixture/fixture'; + const unsigned = await fetch(url, { method: 'POST', body }); + if (unsigned.status !== 401) throw new Error('Unsigned challenge accepted'); + const timestamp = String(Math.floor(Date.now() / 1000)); + const signature = 'v0=' + createHmac('sha256', 'fixture-signing').update(`v0:${timestamp}:${body}`).digest('hex'); + const signed = await fetch(url, { method: 'POST', body, headers: { 'x-slack-request-timestamp': timestamp, 'x-slack-signature': signature } }); + if (signed.status !== 200 || (await signed.json()).challenge !== 'verified') throw new Error('Signed challenge rejected'); + child.kill('SIGTERM'); if (await exited !== 0) throw new Error('Gateway did not exit cleanly'); console.log('Standalone gateway became healthy and stopped cleanly.'); diff --git a/docker-compose.channels.yml b/docker-compose.channels.yml index e72ff0d6c31..141aa3c023a 100644 --- a/docker-compose.channels.yml +++ b/docker-compose.channels.yml @@ -16,8 +16,5 @@ services: CHANNEL_TENANT_ID: ${CHANNEL_TENANT_ID:-} CHANNEL_ID: ${CHANNEL_ID:-} APP_TITLE: ${APP_TITLE:-LibreChat} - SLACK_TRANSPORT: ${SLACK_TRANSPORT:-socket_mode} SLACK_APP_TOKEN: ${SLACK_APP_TOKEN:-} - SLACK_NATIVE_SESSIONS: ${SLACK_NATIVE_SESSIONS:-false} - SLACK_LIFECYCLE_REACTIONS: ${SLACK_LIFECYCLE_REACTIONS:-false} stop_grace_period: 10s diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index a84cdbe18ea..b9e44f29629 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -1,4 +1,35 @@ -# Channel gateway delivery recovery +# Slack Channel gateway + +## Configure one server-owned policy + +Configure the application in `librechat.yaml`: + +```yaml +endpoints: + agents: + channels: + enabled: true + transport: socket_mode + nativeSessions: false + lifecycleReactions: false + signatureToleranceSeconds: 300 +``` + +Socket Mode, disabled optional native/lifecycle emission and a 300-second signature window are the defaults. The signature window accepts 1–300 seconds. For webhooks, set `transport: webhook` and `gatewayUrl` to the externally reachable HTTPS gateway base URL. Keep the private application service URL internal. + +The gateway loads this versioned policy with its private channel bundle before opening its listener or selecting ingress. Deploy matching application and gateway versions: policy version 1 is required, and absent or unsupported versions fail closed. Configure `CHANNEL_SERVICE_TOKEN`, `CHANNEL_TENANT_ID`, `CHANNEL_ID`, `LIBRECHAT_INTERNAL_URL` and, for Socket Mode, `SLACK_APP_TOKEN` in the gateway environment. These are process bindings and secrets. `APP_TITLE`, listening settings, provider/service addresses and process request timeouts also remain deployment settings. + +Remove legacy `SLACK_NATIVE_SESSIONS`, `SLACK_LIFECYCLE_REACTIONS` and `SLACK_SIGNATURE_TOLERANCE_SECS` environment variables; their presence now fails startup instead of overriding the server. Use the YAML fields above. `SLACK_TRANSPORT` is optional and only asserts the server-selected mode; a mismatch fails startup. Restart gateway replicas after changing ingress mode. A replica which observes a changed mode rejects ingress and becomes unhealthy. Workers read optional feature policy from the current private bundle, while already-frozen delivery plans retain their original operations. + +## Install a webhook app + +1. Select webhook mode and set the gateway base URL in the application configuration. Create the Slack app and apply the generated bot capabilities; install it to obtain its bot token. Keep its signing secret private. +2. Activate the Agent Channel with the app/workspace/bot identity, bot token and signing secret, and configure the matching gateway service/tenant/channel bindings. The private bundle requires an active channel with usable signing credentials before webhook verification can succeed. +3. Start the gateway, then apply the generated Events API and interaction URLs to the Slack app. Installation challenges authenticate the original request bytes with that channel's signing secret before echoing the challenge. Missing, stale or tampered signatures cannot pass URL verification. Socket Mode instead requires the app-level token and matching authenticated app identity. + +Native Stop remains available for an authenticated existing session even when optional native emission is disabled. The application accepts bounded future clock skew and durably pins each click's first observed target, lease epoch and cutoff, including no target. A retry cannot advance its cutoff or stop a newer run. Past occurrence times remain unchanged; clock skew does not establish a globally exact event order. + +# Delivery recovery The gateway preserves each frozen delivery operation and its receipt. A required operation whose provider result is unknown holds its actor lane: retrying blindly could post the same answer twice. Stopping a worker or requesting cancellation does not prove whether Slack accepted an earlier request. diff --git a/packages/channel-gateway/src/app.spec.ts b/packages/channel-gateway/src/app.spec.ts index 3be186d514d..86d7735eb3e 100644 --- a/packages/channel-gateway/src/app.spec.ts +++ b/packages/channel-gateway/src/app.spec.ts @@ -1,15 +1,17 @@ import { createHmac } from 'node:crypto'; import { SocketModeClient } from '@slack/socket-mode'; import { createServer, request as httpRequest } from 'node:http'; +import { DEFAULT_CHANNEL_GATEWAY_POLICY } from 'librechat-data-provider'; import type { AddressInfo } from 'node:net'; import type { ChannelBundle } from './types'; +import { loadConfig, resolveGatewayConfig } from './config'; import { ChannelClient } from './client'; import { connectSocket } from './socket'; -import { loadConfig } from './config'; import { SlackClient } from './slack'; import { GatewayApp } from './app'; const bundle: ChannelBundle = { + gateway: { version: 1, ...DEFAULT_CHANNEL_GATEWAY_POLICY, transport: 'webhook' }, channel: { channelId: 'binding', tenantId: 'tenant', @@ -53,17 +55,28 @@ const callback = ( const json = (body: object, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); function setup(env: NodeJS.ProcessEnv = {}) { - const config = loadConfig({ - CHANNEL_SERVICE_TOKEN: 'test-service', - CHANNEL_TENANT_ID: 'tenant', - CHANNEL_ID: 'binding', - ...env, - }); + const currentBundle: ChannelBundle = { + ...bundle, + gateway: { + ...bundle.gateway, + transport: env.SLACK_TRANSPORT === 'socket_mode' ? 'socket_mode' : 'webhook', + }, + }; + const config = resolveGatewayConfig( + loadConfig({ + CHANNEL_SERVICE_TOKEN: 'test-service', + SLACK_TRANSPORT: 'webhook', + CHANNEL_TENANT_ID: 'tenant', + CHANNEL_ID: 'binding', + ...env, + }), + currentBundle.gateway, + ); const backend = jest .fn, Parameters>() .mockImplementation(async (url) => String(url).includes('/service/slack/') - ? json(bundle) + ? json(currentBundle) : json({ runId: 'run', duplicate: false }), ); const slack = jest @@ -76,7 +89,7 @@ function setup(env: NodeJS.ProcessEnv = {}) { new SlackClient(config, undefined, slack), log, ); - return { app, backend, slack, log, config }; + return { app, backend, slack, log, config, bundle: currentBundle }; } async function serve(app: GatewayApp, check: (base: string) => Promise) { const server = createServer((request, response) => { @@ -137,7 +150,7 @@ describe('durable Slack ingress', () => { }); test('native Stop is constrained to its actor, thread and occurrence time', async () => { - const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + const { app, backend } = setup(); await app.dispatch( envelope({ type: 'agent_session_stopped', @@ -155,7 +168,7 @@ describe('durable Slack ingress', () => { }); }); - test('native Stop is not handled when its feature is disabled', async () => { + test('native Stop remains available when native emission is disabled', async () => { const { app, backend } = setup(); await app.dispatch( envelope({ @@ -165,11 +178,12 @@ describe('durable Slack ingress', () => { }), false, ); - expect(backend).toHaveBeenCalledTimes(1); + expect(backend).toHaveBeenCalledTimes(2); + expect(String(backend.mock.calls[1][0])).toContain('/runs/stop-thread'); }); test.each([400, 403])('native Stop reports a safe diagnostic for backend %i', async (status) => { - const { app, backend, log } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + const { app, backend, log } = setup(); backend.mockImplementation(async (url) => String(url).includes('/service/slack/') ? json(bundle) @@ -192,7 +206,7 @@ describe('durable Slack ingress', () => { }); test('native Stop malformed envelope and binding errors are distinguishable without payload logging', async () => { - const { app, log } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + const { app, log } = setup(); const stop = envelope({ type: 'agent_session_stopped', thread_ts: '1792999999.5', @@ -211,63 +225,41 @@ describe('durable Slack ingress', () => { expect(JSON.stringify(log.warn.mock.calls)).not.toContain('foreign'); }); - test('future Stop waits without clamping its original occurrence time', async () => { - const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); - let now = 1793000000000; - jest.spyOn(Date, 'now').mockImplementation(() => now); - const pending = app.dispatch( - envelope({ - type: 'agent_session_stopped', - thread_ts: '1792999999.5', - event_ts: '1793000000.1', - }), - false, - ); - await new Promise((resolve) => setImmediate(resolve)); - expect(backend).toHaveBeenCalledTimes(1); - now += 100; - await pending; - expect(JSON.parse(String(backend.mock.calls[1][1]?.body)).eventAt).toBe(1793000000100); - }); + test.each(['1793000000.1', '1793000010.1'])( + 'forwards future Stop %s immediately with its original occurrence time for durable service normalization', + async (event_ts) => { + const { app, backend } = setup(); + jest.spyOn(Date, 'now').mockReturnValue(1793000000000); + await app.dispatch( + envelope({ type: 'agent_session_stopped', thread_ts: '1792999999.5', event_ts }), + false, + ); + expect(JSON.parse(String(backend.mock.calls[1][1]?.body)).eventAt).toBe( + Math.floor(Number(event_ts) * 1000), + ); + }, + ); - test('future Stop outside the acknowledgement window is rejected without waiting or writing', async () => { - const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); - jest.spyOn(Date, 'now').mockReturnValue(1793000000000); + test('already-aborted Stop does not reach the backend', async () => { + const { app, backend } = setup(); + const controller = new AbortController(); + controller.abort(); await expect( app.dispatch( envelope({ type: 'agent_session_stopped', thread_ts: '1792999999.5', - event_ts: '1793000010.1', + event_ts: '1793000001.1', }), false, + controller.signal, ), - ).rejects.toThrow('acknowledgement window'); - expect(backend).toHaveBeenCalledTimes(1); - }); - - test('future Stop waiting observes shutdown without sending a cancellation', async () => { - const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); - jest.spyOn(Date, 'now').mockReturnValue(1793000000000); - const controller = new AbortController(); - const pending = app.dispatch( - envelope({ - type: 'agent_session_stopped', - thread_ts: '1792999999.5', - event_ts: '1793000001.1', - }), - false, - controller.signal, - ); - await new Promise((resolve) => setImmediate(resolve)); - controller.abort(); - await expect(pending).rejects.toThrow(); - expect(backend).toHaveBeenCalledTimes(1); + ).rejects.toThrow(); + expect(backend).not.toHaveBeenCalled(); }); test('Socket Mode leaves rejected Stop unacknowledged for redelivery', async () => { - const { app, backend, log, config } = setup({ - SLACK_NATIVE_SESSIONS: 'true', + const { app, backend, log, config, bundle } = setup({ SLACK_TRANSPORT: 'socket_mode', SLACK_APP_TOKEN: 'test-app', }); @@ -377,8 +369,82 @@ describe('durable Slack ingress', () => { }); describe('signed webhook boundary', () => { + test('does not echo a challenge after the ingress deadline expires during channel lookup', async () => { + const { app, backend } = setup(); + backend.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 2100)); + return json(bundle); + }); + await serve(app, async (base) => { + const body = JSON.stringify({ type: 'url_verification', challenge: 'too-late' }); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(503); + expect(await response.text()).not.toContain('too-late'); + }); + }); + test('echoes only an authenticated installation challenge without admitting a run', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const body = JSON.stringify({ type: 'url_verification', challenge: 'verified-challenge' }); + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ challenge: 'verified-challenge' }); + const tampered = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body + 'changed'), + body, + }); + expect(tampered.status).toBe(401); + }); + expect(backend.mock.calls.every(([url]) => String(url).includes('/service/slack/'))).toBe(true); + }); + + test('fails closed and becomes unhealthy if the current server policy changes ingress', async () => { + const { app, backend, bundle: current } = setup(); + app.ready = true; + backend.mockImplementation(async () => + json({ ...current, gateway: { ...current.gateway, transport: 'socket_mode' } }), + ); + await serve(app, async (base) => { + const body = JSON.stringify(envelope()); + expect( + ( + await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: signed(body), + body, + }) + ).status, + ).toBe(503); + expect((await fetch(base + '/healthz')).status).toBe(503); + }); + expect(backend).toHaveBeenCalledTimes(1); + }); + + test('rejects unsigned installation challenges before echoing their payload', async () => { + const { app, backend } = setup(); + await serve(app, async (base) => { + const response = await fetch(base + '/slack/events/tenant/binding', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ type: 'url_verification', challenge: 'must-authenticate-first' }), + }); + expect(response.status).toBe(401); + expect(await response.text()).not.toContain('must-authenticate-first'); + expect(backend).toHaveBeenCalledTimes(1); + }); + }); + test('HTTP leaves rejected Stop retryable rather than acknowledging it', async () => { - const { app, backend } = setup({ SLACK_NATIVE_SESSIONS: 'true' }); + const { app, backend } = setup(); backend.mockImplementation(async (url) => String(url).includes('/service/slack/') ? json(bundle) diff --git a/packages/channel-gateway/src/app.ts b/packages/channel-gateway/src/app.ts index eb39b4d62da..3eb91c420c8 100644 --- a/packages/channel-gateway/src/app.ts +++ b/packages/channel-gateway/src/app.ts @@ -1,17 +1,21 @@ import { z } from 'zod'; import { createHash } from 'node:crypto'; -import { setTimeout as delay } from 'node:timers/promises'; import type { IncomingMessage, ServerResponse } from 'node:http'; import type { ChannelBundle, Logger } from './types'; +import type { GatewayRuntimeConfig } from './config'; import type { ReviewInteraction } from './review'; -import type { GatewayConfig } from './config'; -import { eventTimestampUs, normalizeEvent, parseEnvelope, verifySignature } from './events'; +import { + eventTimestampUs, + normalizeEvent, + parseEnvelope, + parseWebhookRequest, + SLACK_REQUEST_MAX_BYTES, +} from './events'; import { ChannelClient, ServiceError } from './client'; import { IdentityFlow } from './identity'; import { ReviewFlow } from './review'; import { SlackClient } from './slack'; -const maxBody = 1024 * 1024; const id = z.string().min(1).max(256); const stateValue = z .object({ @@ -61,7 +65,7 @@ export class GatewayApp { private readonly identity: IdentityFlow; private readonly review: ReviewFlow; constructor( - private readonly config: GatewayConfig, + private readonly config: GatewayRuntimeConfig, private readonly client: ChannelClient, slack: SlackClient, private readonly log: Logger, @@ -81,7 +85,9 @@ export class GatewayApp { this.active++; const signal = AbortSignal.any([AbortSignal.timeout(2000), ...(parent ? [parent] : [])]); try { + signal.throwIfAborted(); const bundle = verifiedBundle ?? (await this.client.channel(signal)); + this.assertPolicy(bundle); if (interactive) { const callback = interactionSchema.parse(body); if ( @@ -205,34 +211,27 @@ export class GatewayApp { if (!event) return; if (event.kind === 'stop') { if (event.eventAt > Date.now()) this.log.warn('native_stop_future_timestamp'); - if (this.config.SLACK_NATIVE_SESSIONS) { - // Preserve Slack's occurrence time: substituting receipt time could stop later work. - while (event.eventAt > Date.now()) { - const wait = event.eventAt - Date.now(); - if (wait >= 2000) - throw new Error('Native Stop timestamp exceeds acknowledgement window'); - await delay(wait, undefined, { signal }); - } - signal.throwIfAborted(); - try { - const result = await this.client.stop( - event.actor, - event.externalThreadKey, - event.eventAt, - event.externalChannelId, - signal, - ); - this.log.info( - result.cancellationRequested ? 'native_stop_requested' : 'native_stop_no_active_run', - ); - } catch (error) { - if (!(error instanceof ServiceError && [400, 403].includes(error.status))) throw error; - this.log.warn( - error.status === 400 ? 'native_stop_invalid_request' : 'native_stop_forbidden', - ); - if (error.status === 400) throw error; - } - } else this.log.warn('native_stop_disabled'); + // Emission policy does not revoke cancellation for an already-created native session. + // The service binds clock-skew normalization to one durable target across retries. + signal.throwIfAborted(); + try { + const result = await this.client.stop( + event.actor, + event.externalThreadKey, + event.eventAt, + event.externalChannelId, + signal, + ); + this.log.info( + result.cancellationRequested ? 'native_stop_requested' : 'native_stop_no_active_run', + ); + } catch (error) { + if (!(error instanceof ServiceError && [400, 403].includes(error.status))) throw error; + this.log.warn( + error.status === 400 ? 'native_stop_invalid_request' : 'native_stop_forbidden', + ); + if (error.status === 400) throw error; + } } else if (event.kind === 'reaction') await this.reaction(event.reaction, signal); else { try { @@ -255,6 +254,13 @@ export class GatewayApp { } } + private assertPolicy(bundle: ChannelBundle): void { + if (bundle.gateway.transport !== this.config.SLACK_TRANSPORT) { + this.ready = false; + throw new Error('Gateway ingress policy changed; restart required'); + } + } + private async reaction(record: object, signal: AbortSignal): Promise { try { await this.client.request(this.client.root + '/reactions', record, signal); @@ -307,7 +313,7 @@ export class GatewayApp { for await (const chunk of request) { const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); size += bytes.length; - if (size > maxBody) { + if (size > SLACK_REQUEST_MAX_BYTES) { send(413, { error: 'payload_too_large' }); return; } @@ -316,45 +322,29 @@ export class GatewayApp { } const body = Buffer.concat(chunks); const interactive = route[1] === 'interactions'; - if (!interactive) { - const challenge = z - .object({ type: z.literal('url_verification'), challenge: z.string().max(2048) }) - .safeParse(JSON.parse(body.toString())); - if (challenge.success) { - send(200, { challenge: challenge.data.challenge }); - return; - } - } const bundle = await this.client.channel(controller.signal); - const signature = request.headers['x-slack-signature']; - const timestamp = request.headers['x-slack-request-timestamp']; - if ( - typeof signature !== 'string' || - typeof timestamp !== 'string' || - !verifySignature( - body, - signature, - timestamp, - bundle.credentials.signingSecret ?? '', - this.config.SLACK_SIGNATURE_TOLERANCE_SECS, - ) - ) { - send(401, { error: 'invalid_signature' }); + controller.signal.throwIfAborted(); + this.assertPolicy(bundle); + const parsed = parseWebhookRequest( + body, + { + signature: request.headers['x-slack-signature'], + timestamp: request.headers['x-slack-request-timestamp'], + contentType: request.headers['content-type'], + }, + bundle, + interactive, + ); + if (!parsed.ok) { + send(parsed.status, { error: parsed.code }); return; } - let payload: unknown; - if (interactive) { - if ( - request.headers['content-type']?.split(';')[0].trim() !== - 'application/x-www-form-urlencoded' - ) { - send(400, { error: 'invalid_interaction' }); - return; - } - const form = new URLSearchParams(body.toString()); - if (form.getAll('payload').length !== 1) throw new Error('Invalid Slack payload'); - payload = JSON.parse(form.get('payload') ?? ''); - } else payload = JSON.parse(body.toString()); + if (parsed.request.kind === 'challenge') { + send(200, { challenge: parsed.request.challenge }); + return; + } + const payload = + parsed.request.kind === 'event' ? parsed.request.envelope : parsed.request.payload; const ack = await this.dispatch(payload, interactive, controller.signal, bundle); controller.signal.throwIfAborted(); send(200, ack ?? {}); diff --git a/packages/channel-gateway/src/main.ts b/packages/channel-gateway/src/main.ts index 6e481853552..dc62ebe5da1 100644 --- a/packages/channel-gateway/src/main.ts +++ b/packages/channel-gateway/src/main.ts @@ -1,15 +1,19 @@ import { createServer } from 'node:http'; import { Registry, collectDefaultMetrics, Counter } from 'prom-client'; +import { loadConfig, resolveGatewayConfig } from './config'; import { connectSocket } from './socket'; import { ChannelWorker } from './worker'; import { ChannelClient } from './client'; -import { loadConfig } from './config'; import { SlackClient } from './slack'; import { GatewayApp } from './app'; import { logger } from './types'; async function main(): Promise { - const config = loadConfig(process.env); + const bootstrap = loadConfig(process.env); + const controller = new AbortController(); + const client = new ChannelClient(bootstrap); + const bundle = await client.channel(controller.signal); + const config = resolveGatewayConfig(bootstrap, bundle.gateway); const registry = new Registry(); collectDefaultMetrics({ register: registry, prefix: 'channel_gateway_' }); const httpRequests = new Counter({ @@ -18,8 +22,6 @@ async function main(): Promise { labelNames: ['status'], registers: [registry], }); - const controller = new AbortController(); - const client = new ChannelClient(config); const slack = new SlackClient(config, logger); const app = new GatewayApp(config, client, slack, logger); const server = createServer((request, response) => { @@ -60,7 +62,6 @@ async function main(): Promise { server.listen(config.PORT, config.LISTEN_HOST, resolve); }); try { - const bundle = await client.channel(controller.signal); if (config.SLACK_TRANSPORT === 'socket_mode') await connectSocket(config, app, bundle.channel.slack.appId, logger, controller.signal); app.ready = true; diff --git a/packages/channel-gateway/src/socket.spec.ts b/packages/channel-gateway/src/socket.spec.ts index 9b078920310..1db580543f3 100644 --- a/packages/channel-gateway/src/socket.spec.ts +++ b/packages/channel-gateway/src/socket.spec.ts @@ -1,8 +1,9 @@ import { SocketModeClient } from '@slack/socket-mode'; +import { DEFAULT_CHANNEL_GATEWAY_POLICY } from 'librechat-data-provider'; import type { ChannelBundle } from './types'; +import { loadConfig, resolveGatewayConfig } from './config'; import { BoundSocket, connectSocket } from './socket'; import { ChannelClient } from './client'; -import { loadConfig } from './config'; import { SlackClient } from './slack'; import { GatewayApp } from './app'; @@ -14,6 +15,7 @@ const config = loadConfig({ SLACK_APP_TOKEN: 'test-app', }); const bundle: ChannelBundle = { + gateway: { version: 1, ...DEFAULT_CHANNEL_GATEWAY_POLICY, transport: 'socket_mode' }, channel: { channelId: 'binding', tenantId: 'tenant', @@ -64,7 +66,12 @@ function setup() { return { backend, log, - app: new GatewayApp(config, new ChannelClient(config, backend), slack, log), + app: new GatewayApp( + resolveGatewayConfig(config, bundle.gateway), + new ChannelClient(config, backend), + slack, + log, + ), }; } function event(ack = jest.fn().mockResolvedValue(undefined)) { diff --git a/packages/channel-gateway/src/worker.spec.ts b/packages/channel-gateway/src/worker.spec.ts index d3393721637..e3452078087 100644 --- a/packages/channel-gateway/src/worker.spec.ts +++ b/packages/channel-gateway/src/worker.spec.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; +import { DEFAULT_CHANNEL_GATEWAY_POLICY } from 'librechat-data-provider'; import type { ChannelDeliveryPlan, ChannelDeliveryReceipt, @@ -44,6 +45,7 @@ function fixture() { AGENT_INVOKE_TIMEOUT_MS: '2000', }); const bundle: ChannelBundle = { + gateway: { version: 1, ...DEFAULT_CHANNEL_GATEWAY_POLICY, transport: 'socket_mode' }, channel: { channelId: config.CHANNEL_ID, tenantId: config.CHANNEL_TENANT_ID, @@ -286,6 +288,29 @@ function fixture() { } describe('durable Channels worker', () => { + test.each([false, true])( + 'uses the current server lifecycle-reaction policy (%s)', + async (enabled) => { + const f = fixture(); + f.bundle.gateway.lifecycleReactions = enabled; + await f.execute(); + const reactions = f.slackRequests.filter((request) => + request.method.startsWith('reactions.'), + ); + expect(reactions.map((request) => [request.method, request.body.get('name')])).toEqual( + enabled + ? [ + ['reactions.add', 'eyes'], + ['reactions.add', 'hourglass_flowing_sand'], + ['reactions.remove', 'hourglass_flowing_sand'], + ['reactions.add', 'white_check_mark'], + ] + : [], + ); + expect(f.state.state).toBe('delivered'); + }, + ); + test('executes only through a bound Responses grant and persists progress before writes', async () => { const f = fixture(); delete f.grant.responseId; @@ -362,7 +387,7 @@ describe('durable Channels worker', () => { 'rejects a changed %s grant before Responses execution', async (field) => { const f = fixture(); - f.config.SLACK_NATIVE_SESSIONS = true; + f.bundle.gateway.nativeSessions = true; if (field === 'apiKey' || field === 'requestKey') f.grant[field] = 'wrong'; else f.grant.request[field] = 'wrong'; await expect(f.execute()).rejects.toThrow( @@ -555,7 +580,7 @@ describe('durable Channels worker', () => { test('does not yield when a renewal becomes uncertain while the independent renewer drains', async () => { const f = fixture(); - f.config.SLACK_NATIVE_SESSIONS = true; + f.bundle.gateway.nativeSessions = true; f.lease.leaseExpiresAt = Date.now() + 60; f.hooks.renew = async () => { await delay(30); @@ -608,7 +633,7 @@ describe('durable Channels worker', () => { test('verified unstarted cancellation never creates or polls a Response', async () => { const f = fixture(); - f.config.SLACK_NATIVE_SESSIONS = true; + f.bundle.gateway.nativeSessions = true; f.state.cancellationRequested = true; f.hooks.prepare = () => ({ settled: { @@ -626,7 +651,7 @@ describe('durable Channels worker', () => { test('unproved cancellation without a Response cursor is held instead of creating new work', async () => { const f = fixture(); - f.config.SLACK_NATIVE_SESSIONS = true; + f.bundle.gateway.nativeSessions = true; f.state.cancellationRequested = true; delete f.grant.responseId; await expect(f.execute()).rejects.toThrow('awaits reconciliation'); @@ -635,7 +660,7 @@ describe('durable Channels worker', () => { test('a canonical cancellation polls only the exact bound Response cancellation route', async () => { const f = fixture(); - f.config.SLACK_NATIVE_SESSIONS = true; + f.bundle.gateway.nativeSessions = true; f.state.cancellationRequested = true; f.hooks.advance = () => { f.state.state = 'delivering'; @@ -656,7 +681,7 @@ describe('durable Channels worker', () => { test('requires-action uses native suspended state and one safe actor-lane yield', async () => { const f = fixture(); - f.config.SLACK_NATIVE_SESSIONS = true; + f.bundle.gateway.nativeSessions = true; f.hooks.advance = () => { f.state.state = 'requires_action'; f.state.snapshot = { @@ -744,7 +769,7 @@ describe('optional delivery and uncertain execution boundaries', () => { ); test('native status success still publishes recovery guidance and suppresses stale review controls', async () => { const f = fixture(); - f.config.SLACK_NATIVE_SESSIONS = true; + f.bundle.gateway.nativeSessions = true; f.hooks.advance = () => { f.state.state = 'requires_action'; f.state.snapshot = { diff --git a/packages/channel-gateway/src/worker.ts b/packages/channel-gateway/src/worker.ts index 4b9fe72062b..fd098695bd8 100644 --- a/packages/channel-gateway/src/worker.ts +++ b/packages/channel-gateway/src/worker.ts @@ -214,7 +214,7 @@ export class ChannelWorker { const syncProgress = async (current: ChannelRunView) => { check(); let native = false; - if (this.config.SLACK_NATIVE_SESSIONS) { + if (bundle.gateway.nativeSessions) { let status: 'active' | 'suspended' | 'processing' = 'processing'; if (terminalSnapshot(current) || current.plan) status = 'active'; else if (current.snapshot?.status === 'requires_action') status = 'suspended'; @@ -324,7 +324,7 @@ export class ChannelWorker { run = await this.client.mutate( fence, 'plan', - { plan: makeSlackPlan(run, this.config.SLACK_NATIVE_SESSIONS) }, + { plan: makeSlackPlan(run, bundle.gateway.nativeSessions) }, signal, ); break; @@ -676,7 +676,7 @@ export class ChannelWorker { phase: 'started' | 'completed' | 'failed', signal: AbortSignal, ): Promise { - if (!this.config.SLACK_LIFECYCLE_REACTIONS) return; + if (!bundle.gateway.lifecycleReactions) return; const prefix = run.admission.externalChannelId + ':'; const ts = run.admission.externalMessageKey.slice(prefix.length); if (!run.admission.externalMessageKey.startsWith(prefix) || !ts || /[:\s]/.test(ts)) return; From 1bb63f6a0cd454af4ca2c01eb8ab94d5508cd2bd Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:49:30 +0100 Subject: [PATCH 06/12] fix: Share Truthful Gateway Readiness Across Probes --- packages/channel-gateway/README.md | 2 ++ packages/channel-gateway/src/app.spec.ts | 23 +++++++++++++++++++++++ packages/channel-gateway/src/app.ts | 4 ++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index b9e44f29629..4872485ef40 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -23,6 +23,8 @@ Remove legacy `SLACK_NATIVE_SESSIONS`, `SLACK_LIFECYCLE_REACTIONS` and `SLACK_SI ## Install a webhook app +`/healthz` and `/readyz` share the actual readiness state: startup must complete, Socket Mode must be connected when selected, and an observed ingress-policy mismatch makes the replica unready. `/livez` reports only that the HTTP process is alive. The container healthcheck uses `/healthz`. + 1. Select webhook mode and set the gateway base URL in the application configuration. Create the Slack app and apply the generated bot capabilities; install it to obtain its bot token. Keep its signing secret private. 2. Activate the Agent Channel with the app/workspace/bot identity, bot token and signing secret, and configure the matching gateway service/tenant/channel bindings. The private bundle requires an active channel with usable signing credentials before webhook verification can succeed. 3. Start the gateway, then apply the generated Events API and interaction URLs to the Slack app. Installation challenges authenticate the original request bytes with that channel's signing secret before echoing the challenge. Missing, stale or tampered signatures cannot pass URL verification. Socket Mode instead requires the app-level token and matching authenticated app identity. diff --git a/packages/channel-gateway/src/app.spec.ts b/packages/channel-gateway/src/app.spec.ts index 86d7735eb3e..2f8cc5524d0 100644 --- a/packages/channel-gateway/src/app.spec.ts +++ b/packages/channel-gateway/src/app.spec.ts @@ -369,6 +369,28 @@ describe('durable Slack ingress', () => { }); describe('signed webhook boundary', () => { + test.each([ + ['webhook', false, false, 503], + ['webhook', true, false, 200], + ['socket_mode', false, true, 503], + ['socket_mode', true, false, 503], + ['socket_mode', true, true, 200], + ] as const)( + 'shares truthful readiness for %s ready=%s connected=%s', + async (transport, ready, connected, status) => { + const { app } = setup({ SLACK_TRANSPORT: transport, SLACK_APP_TOKEN: 'test-app' }); + app.ready = ready; + app.connected = connected; + await serve(app, async (base) => { + for (const path of ['/healthz', '/readyz']) { + const response = await fetch(base + path); + expect(response.status).toBe(status); + expect(await response.json()).toEqual({ status: status === 200 ? 'ready' : 'not_ready' }); + } + expect((await fetch(base + '/livez')).status).toBe(200); + }); + }, + ); test('does not echo a challenge after the ingress deadline expires during channel lookup', async () => { const { app, backend } = setup(); backend.mockImplementation(async () => { @@ -425,6 +447,7 @@ describe('signed webhook boundary', () => { ).status, ).toBe(503); expect((await fetch(base + '/healthz')).status).toBe(503); + expect((await fetch(base + '/readyz')).status).toBe(503); }); expect(backend).toHaveBeenCalledTimes(1); }); diff --git a/packages/channel-gateway/src/app.ts b/packages/channel-gateway/src/app.ts index 3eb91c420c8..df9f0b60f0e 100644 --- a/packages/channel-gateway/src/app.ts +++ b/packages/channel-gateway/src/app.ts @@ -275,11 +275,11 @@ export class GatewayApp { response.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' }); response.end(JSON.stringify(body)); }; - if (request.method === 'GET' && path === '/readyz') { + if (request.method === 'GET' && path === '/livez') { send(200, { status: 'ok' }); return; } - if (request.method === 'GET' && path === '/healthz') { + if (request.method === 'GET' && (path === '/healthz' || path === '/readyz')) { const ready = this.ready && (this.config.SLACK_TRANSPORT !== 'socket_mode' || this.connected); send(ready ? 200 : 503, { status: ready ? 'ready' : 'not_ready' }); return; From 2db8e8a0c7de27a219f6ca73c06ff912e63961cd Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:52:26 +0100 Subject: [PATCH 07/12] docs: Describe Runtime Bundle and Adapter Boundaries --- packages/channel-gateway/README.md | 16 ++++++++-------- packages/channel-gateway/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index 4872485ef40..b4360f641e1 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -31,11 +31,11 @@ Remove legacy `SLACK_NATIVE_SESSIONS`, `SLACK_LIFECYCLE_REACTIONS` and `SLACK_SI Native Stop remains available for an authenticated existing session even when optional native emission is disabled. The application accepts bounded future clock skew and durably pins each click's first observed target, lease epoch and cutoff, including no target. A retry cannot advance its cutoff or stop a newer run. Past occurrence times remain unchanged; clock skew does not establish a globally exact event order. -# Delivery recovery +## Delivery recovery The gateway preserves each frozen delivery operation and its receipt. A required operation whose provider result is unknown holds its actor lane: retrying blindly could post the same answer twice. Stopping a worker or requesting cancellation does not prove whether Slack accepted an earlier request. -## Reconcile a verified provider result +### Reconcile a verified provider result 1. Pause gateway processing for the affected channel while investigating. Keep the channel enabled and preserve its actor link and source authority; disabling these is not a recovery bypass. 2. Inspect the frozen run plan and retained receipt. Verify in the provider that the exact operation reached the intended channel or thread. Record its message key or file ID, matching the retained attempt and any upload checkpoint. If the result cannot be verified, leave the receipt unknown and the lane held. @@ -70,12 +70,12 @@ The standalone process composes private transport adapters with native presentat The production bundle incorporates these seven internal source modules: -| Module | Responsibility | -| -------- | -------------------------------------------------------------------------------------------- | -| `client` | Private service authentication, channel/run identity checks and bounded API responses | -| `config` | Process bindings and resolution of the server-owned gateway policy | -| `events` | Original-byte webhook authentication, installation challenges and scoped event normalization | -| `slack` | Bounded provider calls, conservative uncertain-write outcomes and artifact transport | +| Module | Responsibility | +| ---------- | -------------------------------------------------------------------------------------------- | +| `client` | Private service authentication, channel/run identity checks and bounded API responses | +| `config` | Process bindings and resolution of the server-owned gateway policy | +| `events` | Original-byte webhook authentication, installation challenges and scoped event normalization | +| `slack` | Bounded provider calls, conservative uncertain-write outcomes and artifact transport | | `render` | Deterministic bounded answer/progress plans and durable presentation replay | | `identity` | Actor-private account-link tickets and exact-account confirmation | | `review` | Current actor/action validation, native forms and immutable saved-decision retry | diff --git a/packages/channel-gateway/package.json b/packages/channel-gateway/package.json index e8fb5868442..106b3b500e2 100644 --- a/packages/channel-gateway/package.json +++ b/packages/channel-gateway/package.json @@ -2,7 +2,7 @@ "name": "@librechat/channel-gateway", "version": "0.1.0", "private": true, - "description": "Slack transport adapters for LibreChat Agent Channels", + "description": "Standalone Slack gateway for LibreChat Agent Channels", "type": "module", "scripts": { "build": "tsdown", From b0d4124fe02b21eaa344c053117cb337daf3d942 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:28:33 +0100 Subject: [PATCH 08/12] fix: Pin Runtime Readiness to the Socket App Binding --- packages/channel-gateway/README.md | 4 +-- packages/channel-gateway/src/app.spec.ts | 10 +++++++ packages/channel-gateway/src/app.ts | 26 ++++++++++++++++-- packages/channel-gateway/src/socket.spec.ts | 29 +++++++++++++++++++++ packages/channel-gateway/src/socket.ts | 1 + 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index b4360f641e1..fd569db9b11 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -19,11 +19,11 @@ Socket Mode, disabled optional native/lifecycle emission and a 300-second signat The gateway loads this versioned policy with its private channel bundle before opening its listener or selecting ingress. Deploy matching application and gateway versions: policy version 1 is required, and absent or unsupported versions fail closed. Configure `CHANNEL_SERVICE_TOKEN`, `CHANNEL_TENANT_ID`, `CHANNEL_ID`, `LIBRECHAT_INTERNAL_URL` and, for Socket Mode, `SLACK_APP_TOKEN` in the gateway environment. These are process bindings and secrets. `APP_TITLE`, listening settings, provider/service addresses and process request timeouts also remain deployment settings. -Remove legacy `SLACK_NATIVE_SESSIONS`, `SLACK_LIFECYCLE_REACTIONS` and `SLACK_SIGNATURE_TOLERANCE_SECS` environment variables; their presence now fails startup instead of overriding the server. Use the YAML fields above. `SLACK_TRANSPORT` is optional and only asserts the server-selected mode; a mismatch fails startup. Restart gateway replicas after changing ingress mode. A replica which observes a changed mode rejects ingress and becomes unhealthy. Workers read optional feature policy from the current private bundle, while already-frozen delivery plans retain their original operations. +Remove legacy `SLACK_NATIVE_SESSIONS`, `SLACK_LIFECYCLE_REACTIONS` and `SLACK_SIGNATURE_TOLERANCE_SECS` environment variables; their presence now fails startup instead of overriding the server. Use the YAML fields above. `SLACK_TRANSPORT` is optional and only asserts the server-selected mode; a mismatch fails startup. Restart gateway replicas after changing ingress mode or the Slack app identity used by Socket Mode. A replica which observes a changed mode or Socket app binding rejects that delivery and becomes unhealthy. Workers read optional feature policy from the current private bundle, while already-frozen delivery plans retain their original operations. ## Install a webhook app -`/healthz` and `/readyz` share the actual readiness state: startup must complete, Socket Mode must be connected when selected, and an observed ingress-policy mismatch makes the replica unready. `/livez` reports only that the HTTP process is alive. The container healthcheck uses `/healthz`. +`/healthz` and `/readyz` share the actual readiness state: startup must complete, Socket Mode must be connected to its selected authenticated app when selected, and an observed ingress-policy mismatch makes the replica unready. `/livez` reports only that the HTTP process is alive. The container healthcheck uses `/healthz`. 1. Select webhook mode and set the gateway base URL in the application configuration. Create the Slack app and apply the generated bot capabilities; install it to obtain its bot token. Keep its signing secret private. 2. Activate the Agent Channel with the app/workspace/bot identity, bot token and signing secret, and configure the matching gateway service/tenant/channel bindings. The private bundle requires an active channel with usable signing credentials before webhook verification can succeed. diff --git a/packages/channel-gateway/src/app.spec.ts b/packages/channel-gateway/src/app.spec.ts index 2f8cc5524d0..bbf91ed0920 100644 --- a/packages/channel-gateway/src/app.spec.ts +++ b/packages/channel-gateway/src/app.spec.ts @@ -89,6 +89,8 @@ function setup(env: NodeJS.ProcessEnv = {}) { new SlackClient(config, undefined, slack), log, ); + if (config.SLACK_TRANSPORT === 'socket_mode') + app.bindSocketApp(currentBundle.channel.slack.appId); return { app, backend, slack, log, config, bundle: currentBundle }; } async function serve(app: GatewayApp, check: (base: string) => Promise) { @@ -369,6 +371,14 @@ describe('durable Slack ingress', () => { }); describe('signed webhook boundary', () => { + test('a Socket process cannot replace its authenticated app binding in place', () => { + const { app } = setup({ SLACK_TRANSPORT: 'socket_mode', SLACK_APP_TOKEN: 'test-app' }); + app.ready = true; + app.connected = true; + expect(() => app.bindSocketApp('A2')).toThrow('restart required'); + expect(app.ready).toBe(false); + expect(app.connected).toBe(false); + }); test.each([ ['webhook', false, false, 503], ['webhook', true, false, 200], diff --git a/packages/channel-gateway/src/app.ts b/packages/channel-gateway/src/app.ts index df9f0b60f0e..6a8d7cc16fb 100644 --- a/packages/channel-gateway/src/app.ts +++ b/packages/channel-gateway/src/app.ts @@ -62,6 +62,7 @@ export class GatewayApp { connected = false; private active = 0; private httpActive = 0; + private socketAppId?: string; private readonly identity: IdentityFlow; private readonly review: ReviewFlow; constructor( @@ -254,9 +255,27 @@ export class GatewayApp { } } + bindSocketApp(appId: string): void { + if ( + !appId || + this.config.SLACK_TRANSPORT !== 'socket_mode' || + (this.socketAppId !== undefined && this.socketAppId !== appId) + ) { + this.ready = false; + this.connected = false; + throw new Error('Socket application binding changed; restart required'); + } + this.socketAppId = appId; + } + private assertPolicy(bundle: ChannelBundle): void { - if (bundle.gateway.transport !== this.config.SLACK_TRANSPORT) { + if ( + bundle.gateway.transport !== this.config.SLACK_TRANSPORT || + (this.config.SLACK_TRANSPORT === 'socket_mode' && + bundle.channel.slack.appId !== this.socketAppId) + ) { this.ready = false; + this.connected = false; throw new Error('Gateway ingress policy changed; restart required'); } } @@ -280,7 +299,10 @@ export class GatewayApp { return; } if (request.method === 'GET' && (path === '/healthz' || path === '/readyz')) { - const ready = this.ready && (this.config.SLACK_TRANSPORT !== 'socket_mode' || this.connected); + const ready = + this.ready && + (this.config.SLACK_TRANSPORT !== 'socket_mode' || + (this.connected && this.socketAppId !== undefined)); send(ready ? 200 : 503, { status: ready ? 'ready' : 'not_ready' }); return; } diff --git a/packages/channel-gateway/src/socket.spec.ts b/packages/channel-gateway/src/socket.spec.ts index 1db580543f3..642a3609182 100644 --- a/packages/channel-gateway/src/socket.spec.ts +++ b/packages/channel-gateway/src/socket.spec.ts @@ -198,6 +198,35 @@ describe('Socket Mode app binding', () => { controller.abort(); }); + test('marks the app unready when its current channel changes Socket application', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const socket = await connectSocket(config, app, 'A1', log, controller.signal); + try { + socket.emit('ws_message', hello(), false); + await tick(); + app.ready = true; + backend.mockImplementation(async () => + json({ + ...bundle, + channel: { + ...bundle.channel, + revision: 'r2', + slack: { ...bundle.channel.slack, appId: 'A2' }, + }, + }), + ); + const delivery = event(); + socket.emit('slack_event', delivery); + await tick(); + expect(delivery.ack).not.toHaveBeenCalled(); + expect(app.ready).toBe(false); + expect(backend).toHaveBeenCalledTimes(1); + } finally { + controller.abort(); + } + }); + test('valid hello permits durable admission followed by acknowledgment', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); diff --git a/packages/channel-gateway/src/socket.ts b/packages/channel-gateway/src/socket.ts index 5b298f635d3..bf76996c296 100644 --- a/packages/channel-gateway/src/socket.ts +++ b/packages/channel-gateway/src/socket.ts @@ -116,6 +116,7 @@ export async function connectSocket( log: Logger, signal: AbortSignal, ): Promise { + app.bindSocketApp(appId); app.connected = false; const socket = new BoundSocket( appId, From b692be1378d45752610ce2ff54ef33a8c45dd89f Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:59:06 +0100 Subject: [PATCH 09/12] fix: supervise gateway socket reconnect and shutdown --- packages/api/src/channels/service.spec.ts | 2 + .../src/endpoints/config/endpoints.spec.ts | 1 + packages/channel-gateway/README.md | 7 +- packages/channel-gateway/src/app.spec.ts | 2 +- packages/channel-gateway/src/config.spec.ts | 15 ++ packages/channel-gateway/src/config.ts | 9 +- packages/channel-gateway/src/main.ts | 128 +++++++++------ packages/channel-gateway/src/socket.spec.ts | 104 ++++++++++-- packages/channel-gateway/src/socket.ts | 151 ++++++++++++++---- packages/data-provider/src/config.spec.ts | 7 + packages/data-provider/src/config.ts | 10 ++ 11 files changed, 339 insertions(+), 97 deletions(-) diff --git a/packages/api/src/channels/service.spec.ts b/packages/api/src/channels/service.spec.ts index 7205b1fac3b..a73ab169186 100644 --- a/packages/api/src/channels/service.spec.ts +++ b/packages/api/src/channels/service.spec.ts @@ -349,6 +349,7 @@ it('uses one policy snapshot and supplies the verified webhook credentials only lifecycleReactions: true, signatureToleranceSeconds: 60, maxDeliveryParts: 1, + socketReconnectDelayMs: 100, })); deps.config = config; const id = await createChannel(); @@ -378,6 +379,7 @@ it('uses one policy snapshot and supplies the verified webhook credentials only nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }); expect(result.body.delivery).toEqual({ maxDeliveryParts: 1 }); expect(result.body.gateway).not.toHaveProperty('enabled'); diff --git a/packages/api/src/endpoints/config/endpoints.spec.ts b/packages/api/src/endpoints/config/endpoints.spec.ts index cd4e525859e..d5c07701d06 100644 --- a/packages/api/src/endpoints/config/endpoints.spec.ts +++ b/packages/api/src/endpoints/config/endpoints.spec.ts @@ -244,6 +244,7 @@ describe('createEndpointsConfigService', () => { nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }; const deps = createMockDeps({ loadDefaultEndpointsConfig: jest.fn().mockResolvedValue({ diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index fd569db9b11..73fe4dc22b2 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -13,11 +13,12 @@ endpoints: nativeSessions: false lifecycleReactions: false signatureToleranceSeconds: 300 + socketReconnectDelayMs: 5000 ``` -Socket Mode, disabled optional native/lifecycle emission and a 300-second signature window are the defaults. The signature window accepts 1–300 seconds. For webhooks, set `transport: webhook` and `gatewayUrl` to the externally reachable HTTPS gateway base URL. Keep the private application service URL internal. +Socket Mode, disabled optional native/lifecycle emission and a 300-second signature window are the defaults. The signature window accepts 1–300 seconds. Socket reconnect delay accepts 100–60,000 milliseconds and defaults to 5,000; consecutive failed attempts use linear backoff. For webhooks, set `transport: webhook` and `gatewayUrl` to the externally reachable HTTPS gateway base URL. Keep the private application service URL internal. -The gateway loads this versioned policy with its private channel bundle before opening its listener or selecting ingress. Deploy matching application and gateway versions: policy version 1 is required, and absent or unsupported versions fail closed. Configure `CHANNEL_SERVICE_TOKEN`, `CHANNEL_TENANT_ID`, `CHANNEL_ID`, `LIBRECHAT_INTERNAL_URL` and, for Socket Mode, `SLACK_APP_TOKEN` in the gateway environment. These are process bindings and secrets. `APP_TITLE`, listening settings, provider/service addresses and process request timeouts also remain deployment settings. +The gateway loads this versioned policy with its private channel bundle before opening its listener or selecting ingress. Deploy matching application and gateway versions: policy version 1 is required, and absent or unsupported versions fail closed. The four ingress/security/feature fields remain required; an older version-1 bundle may omit only the additive reconnect delay and receives its default. Configure `CHANNEL_SERVICE_TOKEN`, `CHANNEL_TENANT_ID`, `CHANNEL_ID`, `LIBRECHAT_INTERNAL_URL` and, for Socket Mode, `SLACK_APP_TOKEN` in the gateway environment. These are process bindings and secrets. `APP_TITLE`, listening settings, provider/service addresses and process request timeouts also remain deployment settings. Remove legacy `SLACK_NATIVE_SESSIONS`, `SLACK_LIFECYCLE_REACTIONS` and `SLACK_SIGNATURE_TOLERANCE_SECS` environment variables; their presence now fails startup instead of overriding the server. Use the YAML fields above. `SLACK_TRANSPORT` is optional and only asserts the server-selected mode; a mismatch fails startup. Restart gateway replicas after changing ingress mode or the Slack app identity used by Socket Mode. A replica which observes a changed mode or Socket app binding rejects that delivery and becomes unhealthy. Workers read optional feature policy from the current private bundle, while already-frozen delivery plans retain their original operations. @@ -91,3 +92,5 @@ Webhook consumers pass the original bounded bytes and signature headers to `pars Forward native Stop with its original occurrence time and complete actor/channel/thread scope. The application owns bounded clock-skew normalization and the durable first-observation target, epoch and cutoff. Retained observation identity follows the external actor/thread across account relinks and channel/agent revisions; it is separate from the versioned execution principal. A retry cannot select a later lane or run. The observation joins account-erasure admission and channel serialization fences; cancellation still requires current authority and the exact lease fence. Optional native-session emission does not revoke cancellation for an existing session. The runnable consumer preserves canonical execution identities and frozen delivery receipts across restarts. An uncertain required provider write requires reconciliation rather than a blind repeat. The standalone runtime's HTTP/Socket, worker, process and container checks validate that composition in addition to the adapter tests here. + +Socket connections use one gateway-owned supervision task. Connection loss immediately fails readiness; retries own every SDK startup result and require a fresh matching-app hello before ingress resumes. Shutdown aborts API requests and backoff, closes the SDK, destroys owned WebSocket transports and drains supervision alongside the worker. diff --git a/packages/channel-gateway/src/app.spec.ts b/packages/channel-gateway/src/app.spec.ts index bbf91ed0920..1e4f2651e6b 100644 --- a/packages/channel-gateway/src/app.spec.ts +++ b/packages/channel-gateway/src/app.spec.ts @@ -273,7 +273,7 @@ describe('durable Slack ingress', () => { jest.spyOn(SocketModeClient.prototype, 'start').mockResolvedValue({ ok: true }); jest.spyOn(SocketModeClient.prototype, 'disconnect').mockResolvedValue(undefined); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit( 'ws_message', JSON.stringify({ type: 'hello', connection_info: { app_id: 'A1' } }), diff --git a/packages/channel-gateway/src/config.spec.ts b/packages/channel-gateway/src/config.spec.ts index f2fb2eb23dc..d7578ae7994 100644 --- a/packages/channel-gateway/src/config.spec.ts +++ b/packages/channel-gateway/src/config.spec.ts @@ -25,6 +25,21 @@ describe('gateway configuration', () => { ).toBe(channelConfigSchema.parse({}).transport); }); + test('older v1 policies default only reconnect pacing and newer ones honor the override', () => { + const bootstrap = loadConfig({ ...required, SLACK_APP_TOKEN: 'test-app-token' }); + const older = { + transport: 'socket_mode' as const, + nativeSessions: false, + lifecycleReactions: false, + signatureToleranceSeconds: 300, + }; + expect(resolveGatewayConfig(bootstrap, older).socketReconnectDelayMs).toBe(5000); + expect( + resolveGatewayConfig(bootstrap, { ...older, socketReconnectDelayMs: 100 }) + .socketReconnectDelayMs, + ).toBe(100); + }); + test('supports legacy binding aliases and equal canonical/legacy values', () => { const legacy = { CHANNEL_SERVICE_TOKEN: required.CHANNEL_SERVICE_TOKEN, diff --git a/packages/channel-gateway/src/config.ts b/packages/channel-gateway/src/config.ts index 3dd26a35a67..02da3431324 100644 --- a/packages/channel-gateway/src/config.ts +++ b/packages/channel-gateway/src/config.ts @@ -36,6 +36,7 @@ const environment = z.object({ export type GatewayConfig = z.infer; export type GatewayRuntimeConfig = GatewayConfig & { SLACK_TRANSPORT: ChannelGatewayPolicy['transport']; + socketReconnectDelayMs: number; }; export function loadConfig(env: NodeJS.ProcessEnv): GatewayConfig { @@ -85,7 +86,7 @@ export function loadConfig(env: NodeJS.ProcessEnv): GatewayConfig { /** Bootstrap settings cannot override the server's current installation/ingress policy. */ export function resolveGatewayConfig( config: GatewayConfig, - value: ChannelGatewayPolicy, + value: z.input, ): GatewayRuntimeConfig { const parsed = channelGatewayPolicySchema.safeParse(value); if (!parsed.success) throw new Error('Invalid channel gateway policy'); @@ -94,5 +95,9 @@ export function resolveGatewayConfig( throw new Error('Gateway transport does not match channel policy'); if (policy.transport === 'socket_mode' && !config.SLACK_APP_TOKEN) throw new Error('SLACK_APP_TOKEN is required for Socket Mode'); - return { ...config, SLACK_TRANSPORT: policy.transport }; + return { + ...config, + SLACK_TRANSPORT: policy.transport, + socketReconnectDelayMs: policy.socketReconnectDelayMs, + }; } diff --git a/packages/channel-gateway/src/main.ts b/packages/channel-gateway/src/main.ts index dc62ebe5da1..9fba7c1edab 100644 --- a/packages/channel-gateway/src/main.ts +++ b/packages/channel-gateway/src/main.ts @@ -1,5 +1,7 @@ import { createServer } from 'node:http'; import { Registry, collectDefaultMetrics, Counter } from 'prom-client'; +import type { Server } from 'node:http'; +import type { SocketConnection } from './socket'; import { loadConfig, resolveGatewayConfig } from './config'; import { connectSocket } from './socket'; import { ChannelWorker } from './worker'; @@ -12,63 +14,95 @@ async function main(): Promise { const bootstrap = loadConfig(process.env); const controller = new AbortController(); const client = new ChannelClient(bootstrap); - const bundle = await client.channel(controller.signal); - const config = resolveGatewayConfig(bootstrap, bundle.gateway); - const registry = new Registry(); - collectDefaultMetrics({ register: registry, prefix: 'channel_gateway_' }); - const httpRequests = new Counter({ - name: 'channel_gateway_http_requests_total', - help: 'Gateway HTTP requests by bounded outcome', - labelNames: ['status'], - registers: [registry], - }); - const slack = new SlackClient(config, logger); - const app = new GatewayApp(config, client, slack, logger); - const server = createServer((request, response) => { - response.on('finish', () => httpRequests.labels(String(response.statusCode)).inc()); - if (request.method === 'GET' && request.url === '/metrics') { - void registry - .metrics() - .then((metrics) => { - response.writeHead(200, { 'Content-Type': registry.contentType }); - response.end(metrics); - }) - .catch(() => { - response.writeHead(500); - response.end(); - }); - return; - } - void app.http(request, response).catch(() => { - logger.error('gateway_http_failed'); - if (!response.headersSent) response.writeHead(500); - response.end(); - }); - }); - server.requestTimeout = 5000; - server.headersTimeout = 5000; - server.maxConnections = 128; + let app: GatewayApp | undefined; + let server: Server | undefined; + let connection: SocketConnection | undefined; + let worker: Promise | undefined; + let stopping = false; const shutdown = () => { - app.ready = false; + if (stopping) return; + stopping = true; + if (app) app.ready = false; controller.abort(); - server.close(); - const timer = setTimeout(() => server.closeAllConnections(), 5000); - timer.unref(); + if (server) { + server.close(); + const listener = server; + const timer = setTimeout(() => listener.closeAllConnections(), 5000); + timer.unref(); + } }; process.once('SIGTERM', shutdown); process.once('SIGINT', shutdown); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(config.PORT, config.LISTEN_HOST, resolve); - }); try { + const bundle = await client.channel(controller.signal); + const config = resolveGatewayConfig(bootstrap, bundle.gateway); + const registry = new Registry(); + collectDefaultMetrics({ register: registry, prefix: 'channel_gateway_' }); + const httpRequests = new Counter({ + name: 'channel_gateway_http_requests_total', + help: 'Gateway HTTP requests by bounded outcome', + labelNames: ['status'], + registers: [registry], + }); + const slack = new SlackClient(config, logger); + const application = new GatewayApp(config, client, slack, logger); + app = application; + const listener = createServer((request, response) => { + response.on('finish', () => httpRequests.labels(String(response.statusCode)).inc()); + if (request.method === 'GET' && request.url === '/metrics') { + void registry + .metrics() + .then((metrics) => { + response.writeHead(200, { 'Content-Type': registry.contentType }); + response.end(metrics); + }) + .catch(() => { + response.writeHead(500); + response.end(); + }); + return; + } + void application.http(request, response).catch(() => { + logger.error('gateway_http_failed'); + if (!response.headersSent) response.writeHead(500); + response.end(); + }); + }); + server = listener; + listener.requestTimeout = 5000; + listener.headersTimeout = 5000; + listener.maxConnections = 128; + await new Promise((resolve, reject) => { + listener.once('error', reject); + listener.listen(config.PORT, config.LISTEN_HOST, resolve); + }); + controller.signal.throwIfAborted(); if (config.SLACK_TRANSPORT === 'socket_mode') - await connectSocket(config, app, bundle.channel.slack.appId, logger, controller.signal); - app.ready = true; + connection = await connectSocket( + config, + application, + bundle.channel.slack.appId, + logger, + controller.signal, + ); + application.ready = true; logger.info('channel_gateway_started'); - await new ChannelWorker(config, client, slack, logger).run(controller.signal); + worker = new ChannelWorker(config, client, slack, logger).run(controller.signal); + if (connection) + await Promise.race([ + worker, + connection.closed.then(() => { + if (!controller.signal.aborted) throw new Error('Socket supervision ended'); + }), + ]); + else await worker; + } catch (error) { + if (!controller.signal.aborted) throw error; } finally { shutdown(); + await Promise.allSettled([worker, connection?.closed]); + process.removeListener('SIGTERM', shutdown); + process.removeListener('SIGINT', shutdown); } } diff --git a/packages/channel-gateway/src/socket.spec.ts b/packages/channel-gateway/src/socket.spec.ts index 642a3609182..9cb645ae67f 100644 --- a/packages/channel-gateway/src/socket.spec.ts +++ b/packages/channel-gateway/src/socket.spec.ts @@ -7,13 +7,16 @@ import { ChannelClient } from './client'; import { SlackClient } from './slack'; import { GatewayApp } from './app'; -const config = loadConfig({ - CHANNEL_SERVICE_TOKEN: 'test-service', - CHANNEL_TENANT_ID: 'tenant', - CHANNEL_ID: 'binding', - SLACK_TRANSPORT: 'socket_mode', - SLACK_APP_TOKEN: 'test-app', -}); +const config = resolveGatewayConfig( + loadConfig({ + CHANNEL_SERVICE_TOKEN: 'test-service', + CHANNEL_TENANT_ID: 'tenant', + CHANNEL_ID: 'binding', + SLACK_TRANSPORT: 'socket_mode', + SLACK_APP_TOKEN: 'test-app', + }), + DEFAULT_CHANNEL_GATEWAY_POLICY, +); const bundle: ChannelBundle = { gateway: { version: 1, ...DEFAULT_CHANNEL_GATEWAY_POLICY, transport: 'socket_mode' }, channel: { @@ -189,7 +192,7 @@ describe('Socket Mode app binding', () => { test('events before verified hello cannot reach application dispatch', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); const delivery = event(); socket.emit('slack_event', delivery); await tick(); @@ -201,7 +204,7 @@ describe('Socket Mode app binding', () => { test('marks the app unready when its current channel changes Socket application', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); try { socket.emit('ws_message', hello(), false); await tick(); @@ -230,7 +233,7 @@ describe('Socket Mode app binding', () => { test('valid hello permits durable admission followed by acknowledgment', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit('ws_message', hello(), false); await tick(); const delivery = event(); @@ -246,7 +249,7 @@ describe('Socket Mode app binding', () => { test('a wrong hello stops delivery and repeated wrong hello never becomes ready', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit('ws_message', hello('wrong'), false); await tick(); socket.emit('slack_event', event()); @@ -264,7 +267,7 @@ describe('Socket Mode app binding', () => { test('reconnect requires another valid hello before accepting more events', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit('ws_message', hello(), false); await tick(); socket.emit('reconnecting'); @@ -288,7 +291,7 @@ describe('Socket Mode app binding', () => { ? json(bundle) : Promise.reject(new Error('private provider credential')), ); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit('ws_message', hello(), false); await tick(); socket.emit('error', new Error('private Slack app token')); @@ -307,7 +310,7 @@ describe('Socket Mode app binding', () => { test('invalid SDK deliveries are ignored without authority or acknowledgement', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit('ws_message', hello(), false); await tick(); for (const delivery of [ @@ -321,6 +324,77 @@ describe('Socket Mode app binding', () => { controller.abort(); }); + test('initial startup failure is owned and closes the transport', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + start.mockRejectedValueOnce(new Error('private startup failure')); + await expect(connectSocket(config, app, 'A1', log, controller.signal)).rejects.toThrow(); + expect(disconnect).toHaveBeenCalled(); + expect(app.connected).toBe(false); + }); + + test('retries a failed reconnect and requires a fresh authenticated hello', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + const connection = await connectSocket( + { ...config, socketReconnectDelayMs: 100 }, + app, + 'A1', + log, + controller.signal, + ); + try { + connection.socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + start.mockRejectedValueOnce(new Error('private retry failure')); + connection.socket.emit('disconnected'); + await tick(); + expect(app.connected).toBe(false); + const deadline = Date.now() + 3000; + while (start.mock.calls.length < 3 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(start).toHaveBeenCalledTimes(3); + expect(app.connected).toBe(false); + expect(log.warn).toHaveBeenCalledWith('slack_socket_reconnect_failed'); + connection.socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + } finally { + controller.abort(); + await connection.closed; + } + }); + + test('abort during a pending reconnect drains without an unhandled rejection', async () => { + const { app, log } = setup(); + const controller = new AbortController(); + const connection = await connectSocket( + { ...config, socketReconnectDelayMs: 100 }, + app, + 'A1', + log, + controller.signal, + ); + let fail: (error: Error) => void = () => {}; + start.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + fail = reject; + }), + ); + connection.socket.emit('disconnected'); + const deadline = Date.now() + 3000; + while (start.mock.calls.length < 2 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(start).toHaveBeenCalledTimes(2); + controller.abort(); + fail(new Error('private late rejection')); + await expect(connection.closed).resolves.toBeUndefined(); + expect(app.connected).toBe(false); + expect(disconnect).toHaveBeenCalled(); + }); + test('pre-aborted startup never starts the SDK', async () => { const { app, log } = setup(); const controller = new AbortController(); @@ -351,7 +425,7 @@ describe('Socket Mode app binding', () => { test('aborted active transport cannot dispatch or acknowledge later input', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); - const socket = await connectSocket(config, app, 'A1', log, controller.signal); + const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit('ws_message', hello(), false); await tick(); controller.abort(); diff --git a/packages/channel-gateway/src/socket.ts b/packages/channel-gateway/src/socket.ts index bf76996c296..b3762f40375 100644 --- a/packages/channel-gateway/src/socket.ts +++ b/packages/channel-gateway/src/socket.ts @@ -1,6 +1,8 @@ import { z } from 'zod'; import { LogLevel } from '@slack/web-api'; import { SocketModeClient } from '@slack/socket-mode'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { GatewayRuntimeConfig } from './config'; import type { GatewayConfig } from './config'; import type { GatewayApp } from './app'; import type { Logger } from './types'; @@ -33,6 +35,8 @@ export class BoundSocket extends SocketModeClient { super({ appToken: config.SLACK_APP_TOKEN!, logger: quiet, + // SDK 3.1's automatic reconnect discards start rejections. The gateway owns that task. + autoReconnectEnabled: false, clientOptions: { slackApiUrl: config.SLACK_API_BASE_URL.replace(/\/?$/, '/'), timeout: config.API_CALL_TIMEOUT_MS, @@ -50,6 +54,13 @@ export class BoundSocket extends SocketModeClient { }), }, }); + this.on('connecting', () => { + // An aborted URL fetch may already have settled; close a late-created SDK socket too. + if (signal?.aborted) + queueMicrotask(() => { + void this.disconnect().catch(() => {}); + }); + }); for (const phase of ['connecting', 'reconnecting', 'disconnected']) this.on(phase, () => { this.validated = false; @@ -105,17 +116,46 @@ export class BoundSocket extends SocketModeClient { return; } else if (value.type !== 'interactive') return; } - await super.onWebSocketMessage(data, binary); + try { + await super.onWebSocketMessage(data, binary); + } catch { + // The SDK invokes this async callback without awaiting it. Contain malformed frames here. + this.validated = false; + this.state(false); + this.emit('error', new Error('Socket message rejected')); + await this.disconnect().catch(() => {}); + } + } +} + +export interface SocketConnection { + socket: BoundSocket; + /** Owned supervision/cleanup; rejects only an unexpected lifecycle failure. */ + closed: Promise; +} + +async function untilAbort(work: Promise, signal: AbortSignal): Promise { + let stop: () => void = () => {}; + const aborted = new Promise((_resolve, reject) => { + stop = () => reject(new Error('Gateway stopped')); + if (signal.aborted) stop(); + else signal.addEventListener('abort', stop, { once: true }); + }); + try { + return await Promise.race([work, aborted]); + } finally { + signal.removeEventListener('abort', stop); } } export async function connectSocket( - config: GatewayConfig, + config: GatewayRuntimeConfig, app: GatewayApp, appId: string, log: Logger, signal: AbortSignal, -): Promise { +): Promise { + signal.throwIfAborted(); app.bindSocketApp(appId); app.connected = false; const socket = new BoundSocket( @@ -126,15 +166,10 @@ export async function connectSocket( config, signal, ); - socket.on('connecting', () => { - app.connected = false; - }); - socket.on('reconnecting', () => { - app.connected = false; - }); - socket.on('disconnected', () => { - app.connected = false; - }); + for (const phase of ['connecting', 'reconnecting', 'disconnected']) + socket.on(phase, () => { + app.connected = false; + }); socket.on('error', () => log.warn('slack_socket_transport_error')); socket.on('slack_event', (input: unknown) => { if (!app.connected || signal.aborted) return; @@ -151,31 +186,87 @@ export async function connectSocket( .then((payload) => delivery.data.ack(payload)) .catch(() => log.warn('slack_socket_delivery_not_accepted')); }); - if (signal.aborted) throw new Error('Gateway stopped'); - let rejectAborted: (error: Error) => void = () => {}; - const aborted = new Promise((_resolve, reject) => { - rejectAborted = reject; + + let resolveReady: () => void = () => {}; + let rejectReady: (error: unknown) => void = () => {}; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; }); + let cleanup: Promise | undefined; + const close = () => { + if (!cleanup) + cleanup = (async () => { + app.connected = false; + let disconnect: Promise; + try { + disconnect = socket.disconnect(); + } catch (error) { + disconnect = Promise.reject(error); + } + const results = await Promise.allSettled([disconnect]); + if (results.some((result) => result.status === 'rejected')) + log.warn('slack_socket_shutdown_unconfirmed'); + })(); + return cleanup; + }; const stop = () => { app.connected = false; - void socket.disconnect().catch(() => {}); - rejectAborted(new Error('Gateway stopped')); + void close(); }; signal.addEventListener('abort', stop, { once: true }); - const started = socket.start(); - void started.then( - () => { - if (signal.aborted) void socket.disconnect().catch(() => {}); - }, - () => {}, - ); + const closed = (async () => { + let hasConnected = false; + let failures = 0; + try { + while (!signal.aborted) { + let disconnected: () => void = () => {}; + const ended = new Promise((resolve) => { + disconnected = resolve; + }); + socket.once('disconnected', disconnected); + try { + await untilAbort(socket.start(), signal); + signal.throwIfAborted(); + failures = 0; + if (!hasConnected) { + hasConnected = true; + resolveReady(); + } + await untilAbort(ended, signal); + } catch (error) { + if (signal.aborted) break; + if (!hasConnected) throw error; + log.warn('slack_socket_reconnect_failed'); + } finally { + socket.removeListener('disconnected', disconnected); + app.connected = false; + } + if (signal.aborted) break; + failures++; + // Node's timer ceiling is a platform constraint, not a retry policy limit. + const wait = Math.min(config.socketReconnectDelayMs * failures, 2_147_483_647); + try { + await delay(wait, undefined, { signal }); + } catch { + break; + } + socket.emit('reconnecting'); + } + } finally { + if (!hasConnected && signal.aborted) rejectReady(new Error('Gateway stopped')); + signal.removeEventListener('abort', stop); + await close(); + } + })(); + // Own failures even before the initial handle can be returned; main also observes completion. + void closed.catch(rejectReady); try { - await Promise.race([started, aborted]); + await ready; + signal.throwIfAborted(); } catch (error) { - signal.removeEventListener('abort', stop); - await socket.disconnect().catch(() => {}); + await closed.catch(() => {}); throw error; } - signal.throwIfAborted(); - return socket; + return { socket, closed }; } diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 66d382b8390..92584a1f179 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -3409,6 +3409,7 @@ describe('complete channel operator configuration', () => { nativeSessions: false, lifecycleReactions: false, signatureToleranceSeconds: 300, + socketReconnectDelayMs: 5000, }); expect( configSchema.parse({ @@ -3421,6 +3422,7 @@ describe('complete channel operator configuration', () => { nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }, }, }, @@ -3430,9 +3432,14 @@ describe('complete channel operator configuration', () => { nativeSessions: true, lifecycleReactions: true, signatureToleranceSeconds: 60, + socketReconnectDelayMs: 100, }); }); it.each([ + { socketReconnectDelayMs: 99 }, + { socketReconnectDelayMs: 60001 }, + { socketReconnectDelayMs: 100.5 }, + { socketReconnectDelayMs: '5000' }, { nativeSessions: 'true' }, { lifecycleReactions: 'false' }, { signatureToleranceSeconds: 0 }, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 6807cdedc99..6ca4d0f677b 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1234,11 +1234,19 @@ export const CHANNEL_ARTIFACT_FILES_HARD_MAX = 20; export const CHANNEL_ARTIFACT_READ_TIMEOUT_HARD_MAX_MS = 60_000; /** One policy controls installation, ingress and optional Slack effects across gateway replicas. */ +const SOCKET_RECONNECT_DELAY_DEFAULT_MS = 5000; export const channelGatewayPolicySchema = z.object({ transport: z.enum(['socket_mode', 'webhook']), nativeSessions: z.boolean(), lifecycleReactions: z.boolean(), signatureToleranceSeconds: z.number().int().min(1).max(300), + /** Additive v1 pacing setting; omitted older bundles retain the SDK's existing base delay. */ + socketReconnectDelayMs: z + .number() + .int() + .min(100) + .max(60_000) + .default(SOCKET_RECONNECT_DELAY_DEFAULT_MS), }); export type ChannelGatewayPolicy = z.infer; export const DEFAULT_CHANNEL_GATEWAY_POLICY: ChannelGatewayPolicy = { @@ -1246,6 +1254,7 @@ export const DEFAULT_CHANNEL_GATEWAY_POLICY: ChannelGatewayPolicy = { nativeSessions: false, lifecycleReactions: false, signatureToleranceSeconds: 300, + socketReconnectDelayMs: SOCKET_RECONNECT_DELAY_DEFAULT_MS, }; export const CHANNEL_GATEWAY_POLICY_VERSION = 1; export const channelGatewayWirePolicySchema = channelGatewayPolicySchema.extend({ @@ -1285,6 +1294,7 @@ export const channelConfigSchema = channelStorageConfigSchema.extend({ signatureToleranceSeconds: channelGatewayPolicySchema.shape.signatureToleranceSeconds.default( DEFAULT_CHANNEL_GATEWAY_POLICY.signatureToleranceSeconds, ), + socketReconnectDelayMs: channelGatewayPolicySchema.shape.socketReconnectDelayMs, gatewayUrl: z.string().url().optional(), ticketLifetimeMs: z .number() From 141a502f75c37f1ffdf117bcd0f76564e17fc340 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:06:46 +0100 Subject: [PATCH 10/12] fix: pass server delivery budget to gateway worker --- packages/channel-gateway/README.md | 4 ++- packages/channel-gateway/src/worker.spec.ts | 29 +++++++++++++++++++++ packages/channel-gateway/src/worker.ts | 8 +++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/channel-gateway/README.md b/packages/channel-gateway/README.md index 73fe4dc22b2..7cd21eaea66 100644 --- a/packages/channel-gateway/README.md +++ b/packages/channel-gateway/README.md @@ -93,4 +93,6 @@ Forward native Stop with its original occurrence time and complete actor/channel The runnable consumer preserves canonical execution identities and frozen delivery receipts across restarts. An uncertain required provider write requires reconciliation rather than a blind repeat. The standalone runtime's HTTP/Socket, worker, process and container checks validate that composition in addition to the adapter tests here. -Socket connections use one gateway-owned supervision task. Connection loss immediately fails readiness; retries own every SDK startup result and require a fresh matching-app hello before ingress resumes. Shutdown aborts API requests and backoff, closes the SDK, destroys owned WebSocket transports and drains supervision alongside the worker. +Socket connections use one gateway-owned supervision task. Connection loss immediately fails readiness; retries own every SDK startup result and require a fresh matching-app hello before ingress resumes. Shutdown aborts API requests and backoff, closes the SDK, lets the SDK close its WebSocket transport and drains supervision alongside the worker. + +The private bundle also supplies the server’s `maxDeliveryParts` storage limit. New plans reserve required answer output first, group attachment attempts with their failure notice, and explicitly report shortening or omitted optional effects. Already persisted plans retain their frozen operations. diff --git a/packages/channel-gateway/src/worker.spec.ts b/packages/channel-gateway/src/worker.spec.ts index e3452078087..ae6b26fa5c3 100644 --- a/packages/channel-gateway/src/worker.spec.ts +++ b/packages/channel-gateway/src/worker.spec.ts @@ -288,6 +288,35 @@ function fixture() { } describe('durable Channels worker', () => { + test('a reduced server delivery budget reaches the persisted plan and completes without optional uploads', async () => { + const f = fixture(); + f.bundle.delivery = { maxDeliveryParts: 1 }; + f.hooks.advance = () => { + f.state.state = 'delivering'; + f.state.snapshot = { + status: 'completed', + text: 'Final answer', + responseId: 'resp_test', + elapsedMs: 200, + artifacts: [ + { + artifactId: 'file-1', + filename: 'report.txt', + mediaType: 'text/plain', + bytes: 1, + sha256: 'a'.repeat(64), + }, + ], + }; + }; + await f.execute(); + expect(f.state.plan?.parts).toHaveLength(1); + expect(f.state.plan?.parts[0].text).toContain('Final answer'); + expect(f.state.plan?.parts[0].text).toContain('omitted'); + expect(f.slackRequests.some((request) => request.method.startsWith('files.'))).toBe(false); + expect(f.state.state).toBe('delivered'); + }); + test.each([false, true])( 'uses the current server lifecycle-reaction policy (%s)', async (enabled) => { diff --git a/packages/channel-gateway/src/worker.ts b/packages/channel-gateway/src/worker.ts index fd098695bd8..bfc1f6db66a 100644 --- a/packages/channel-gateway/src/worker.ts +++ b/packages/channel-gateway/src/worker.ts @@ -324,7 +324,13 @@ export class ChannelWorker { run = await this.client.mutate( fence, 'plan', - { plan: makeSlackPlan(run, bundle.gateway.nativeSessions) }, + { + plan: makeSlackPlan( + run, + bundle.gateway.nativeSessions, + bundle.delivery?.maxDeliveryParts, + ), + }, signal, ); break; From bbc6f49bb40516973728dc03a3280087348a82aa Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:17:01 +0100 Subject: [PATCH 11/12] fix: bind Socket hello authority to the current handshake --- packages/channel-gateway/src/socket.spec.ts | 32 +++++++++++++++++++-- packages/channel-gateway/src/socket.ts | 17 +++++------ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/packages/channel-gateway/src/socket.spec.ts b/packages/channel-gateway/src/socket.spec.ts index 9cb645ae67f..3f82198cc43 100644 --- a/packages/channel-gateway/src/socket.spec.ts +++ b/packages/channel-gateway/src/socket.spec.ts @@ -275,6 +275,7 @@ describe('Socket Mode app binding', () => { await tick(); expect(app.connected).toBe(false); expect(backend).not.toHaveBeenCalled(); + socket.emit('connecting'); socket.emit('ws_message', hello(), false); await tick(); socket.emit('slack_event', event()); @@ -283,6 +284,32 @@ describe('Socket Mode app binding', () => { controller.abort(); }); + test('transport failure revokes hello authority until the next connection attempt', async () => { + const { app, backend, log } = setup(); + const controller = new AbortController(); + const { socket, closed } = await connectSocket(config, app, 'A1', log, controller.signal); + try { + socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + socket.emit('error', new Error('private transport failure')); + expect(app.connected).toBe(false); + socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(false); + socket.emit('slack_event', event()); + await tick(); + expect(backend).not.toHaveBeenCalled(); + socket.emit('connecting'); + socket.emit('ws_message', hello(), false); + await tick(); + expect(app.connected).toBe(true); + } finally { + controller.abort(); + await closed; + } + }); + test('SDK error and rejected admission never log raw content or acknowledge failure', async () => { const { app, backend, log } = setup(); const controller = new AbortController(); @@ -294,14 +321,14 @@ describe('Socket Mode app binding', () => { const { socket } = await connectSocket(config, app, 'A1', log, controller.signal); socket.emit('ws_message', hello(), false); await tick(); - socket.emit('error', new Error('private Slack app token')); const delivery = event(); socket.emit('slack_event', delivery); await tick(); expect(delivery.ack).not.toHaveBeenCalled(); + socket.emit('error', new Error('private Slack app token')); expect(log.warn.mock.calls).toEqual([ - ['slack_socket_transport_error'], ['slack_socket_delivery_not_accepted'], + ['slack_socket_transport_error'], ]); expect(JSON.stringify(log.warn.mock.calls)).not.toContain('private'); controller.abort(); @@ -357,6 +384,7 @@ describe('Socket Mode app binding', () => { expect(start).toHaveBeenCalledTimes(3); expect(app.connected).toBe(false); expect(log.warn).toHaveBeenCalledWith('slack_socket_reconnect_failed'); + connection.socket.emit('connecting'); connection.socket.emit('ws_message', hello(), false); await tick(); expect(app.connected).toBe(true); diff --git a/packages/channel-gateway/src/socket.ts b/packages/channel-gateway/src/socket.ts index b3762f40375..fe9fcd474c6 100644 --- a/packages/channel-gateway/src/socket.ts +++ b/packages/channel-gateway/src/socket.ts @@ -25,7 +25,7 @@ const event = z.object({ /** The SDK hides hello's connection_info; validate it before permitting its event dispatch. */ export class BoundSocket extends SocketModeClient { - private validated = false; + private phase: 'handshaking' | 'verified' | 'inactive' = 'handshaking'; constructor( private readonly appId: string, private readonly state: (connected: boolean) => void, @@ -61,9 +61,9 @@ export class BoundSocket extends SocketModeClient { void this.disconnect().catch(() => {}); }); }); - for (const phase of ['connecting', 'reconnecting', 'disconnected']) + for (const phase of ['connecting', 'reconnecting', 'disconnecting', 'disconnected', 'error']) this.on(phase, () => { - this.validated = false; + this.phase = phase === 'connecting' ? 'handshaking' : 'inactive'; this.state(false); }); } @@ -97,17 +97,18 @@ export class BoundSocket extends SocketModeClient { .safeParse(value.connection_info); if (!connection.success) { this.state(false); - this.validated = false; + this.phase = 'inactive'; await this.disconnect().catch(() => {}); return; } - this.validated = true; + if (this.phase !== 'handshaking') return; + this.phase = 'verified'; this.state(true); } else if (value.type === 'disconnect') { - this.validated = false; + this.phase = 'inactive'; this.state(false); } else { - if (!this.validated || !value.envelope_id || !value.payload) return; + if (this.phase !== 'verified' || !value.envelope_id || !value.payload) return; if (value.type === 'events_api') { if ( !z.object({ event: z.object({ type: z.string().min(1) }) }).safeParse(value.payload) @@ -120,7 +121,7 @@ export class BoundSocket extends SocketModeClient { await super.onWebSocketMessage(data, binary); } catch { // The SDK invokes this async callback without awaiting it. Contain malformed frames here. - this.validated = false; + this.phase = 'inactive'; this.state(false); this.emit('error', new Error('Socket message rejected')); await this.disconnect().catch(() => {}); From f05d91057a7d03eb3537f0dead19fe27de3b2e58 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:08:38 +0100 Subject: [PATCH 12/12] refactor: use client-owned branding in gateway review adapter --- packages/channel-gateway/src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/channel-gateway/src/app.ts b/packages/channel-gateway/src/app.ts index 6a8d7cc16fb..5a0454b4f64 100644 --- a/packages/channel-gateway/src/app.ts +++ b/packages/channel-gateway/src/app.ts @@ -72,7 +72,7 @@ export class GatewayApp { private readonly log: Logger, ) { this.identity = new IdentityFlow(client, slack); - this.review = new ReviewFlow(client, slack, config.APP_TITLE); + this.review = new ReviewFlow(client, slack); } /** No queue before admission: overload is left unacknowledged for Slack redelivery. */