From c506b5cd90e0e1990a0d867e9500f343a5064513 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 17:33:11 -0800 Subject: [PATCH 1/6] chore(redis): require Redis and remove the configuration fallbacks --- .../auth/oauth2/token/route.integration.ts | 17 +- .../[identifier]/otp/route.integration.ts | 83 +--- .../api/chat/[identifier]/otp/route.test.ts | 9 - apps/sim/instrumentation-node.ts | 6 + .../core/config/env-capabilities.server.ts | 5 - .../lib/core/config/env-capabilities.test.ts | 45 +- apps/sim/lib/core/config/env.ts | 2 +- apps/sim/lib/core/config/redis.test.ts | 13 +- apps/sim/lib/core/config/redis.ts | 186 ++++----- apps/sim/lib/core/idempotency/service.test.ts | 7 - apps/sim/lib/core/idempotency/service.ts | 26 +- .../rate-limiter/provider-admission.test.ts | 46 +-- .../core/rate-limiter/provider-admission.ts | 10 +- .../provider-capacity-store.test.ts | 42 +- .../rate-limiter/provider-capacity-store.ts | 117 ++---- .../provider-capacity.integration.ts | 106 ++--- .../storage/db-token-bucket.test.ts | 59 --- .../rate-limiter/storage/db-token-bucket.ts | 385 ------------------ .../core/rate-limiter/storage/factory.test.ts | 74 +--- .../lib/core/rate-limiter/storage/factory.ts | 50 +-- .../lib/core/rate-limiter/storage/index.ts | 8 +- apps/sim/lib/core/security/otp.ts | 127 +----- apps/sim/lib/core/storage/index.ts | 7 - apps/sim/lib/core/storage/storage.ts | 72 ---- apps/sim/lib/execution/event-buffer.test.ts | 53 --- apps/sim/lib/execution/event-buffer.ts | 7 +- apps/sim/lib/execution/isolated-vm.ts | 3 - .../ocr-input-failures.integration.ts | 4 +- .../provider-admission.integration.ts | 161 +------- .../__integration__/provider-fixture-state.ts | 11 +- ...rovider-processing-recovery.integration.ts | 23 +- .../reranker-admission.integration.ts | 9 +- .../content-sharing.integration.ts | 9 +- apps/sim/lib/realtime/event-log.test.ts | 129 ------ apps/sim/lib/realtime/event-log.ts | 113 +---- apps/sim/lib/table/events.test.ts | 61 --- apps/sim/lib/table/events.ts | 4 +- apps/sim/vitest.integration.setup.ts | 28 +- packages/db/schema.ts | 1 + .../deployment-config/src/env-capabilities.ts | 47 ++- .../sim-setup/src/capability-config.test.ts | 8 +- packages/sim-setup/src/setup-status.test.ts | 1 + packages/testing/src/mocks/env.mock.ts | 4 +- .../testing/src/mocks/redis-config.mock.ts | 31 +- 44 files changed, 424 insertions(+), 1785 deletions(-) delete mode 100644 apps/sim/lib/core/rate-limiter/storage/db-token-bucket.test.ts delete mode 100644 apps/sim/lib/core/rate-limiter/storage/db-token-bucket.ts delete mode 100644 apps/sim/lib/core/storage/index.ts delete mode 100644 apps/sim/lib/core/storage/storage.ts delete mode 100644 apps/sim/lib/realtime/event-log.test.ts delete mode 100644 apps/sim/lib/table/events.test.ts diff --git a/apps/sim/app/api/auth/oauth2/token/route.integration.ts b/apps/sim/app/api/auth/oauth2/token/route.integration.ts index 8c324803623..7e7e82ab2a7 100644 --- a/apps/sim/app/api/auth/oauth2/token/route.integration.ts +++ b/apps/sim/app/api/auth/oauth2/token/route.integration.ts @@ -11,6 +11,7 @@ import { describe, expect, it, vi } from 'vitest' import { auth } from '@/lib/auth' import * as tokenStore from '@/lib/auth/oauth-access-token' import * as provider from '@/lib/auth/oauth-provider' +import { getRedisClient } from '@/lib/core/config/redis' import { isCapabilityWithheldForUser } from '@/lib/permission-groups/user-scope.server' import { POST as revokeRoute } from '@/app/api/auth/oauth2/revoke/route' import { POST as tokenRoute } from '@/app/api/auth/oauth2/token/route' @@ -251,9 +252,7 @@ describe('OAuth token route in PostgreSQL', () => { await db.delete(schema.verification).where(like(schema.verification.value, `%${userId}%`)) await db.delete(schema.oauthClient).where(eq(schema.oauthClient.clientId, clientId)) await db.delete(schema.user).where(eq(schema.user.id, userId)) - await db - .delete(schema.rateLimitBucket) - .where(eq(schema.rateLimitBucket.key, `route:oauth-provider-token:ip:${clientIp}`)) + await getRedisClient().del(`ratelimit:tb:route:oauth-provider-token:ip:${clientIp}`) requestUtilsMockFns.mockGetClientIp.mockReturnValue('127.0.0.1') } }, @@ -728,14 +727,10 @@ describe('OAuth token route in PostgreSQL', () => { .delete(schema.subscription) .where(eq(schema.subscription.referenceId, organizationId)) await db.delete(schema.organization).where(eq(schema.organization.id, organizationId)) - await db - .delete(schema.rateLimitBucket) - .where( - inArray(schema.rateLimitBucket.key, [ - `route:oauth-provider-token:ip:${clientIp}`, - `route:oauth-provider-revoke:ip:${clientIp}`, - ]) - ) + await getRedisClient().del( + `ratelimit:tb:route:oauth-provider-token:ip:${clientIp}`, + `ratelimit:tb:route:oauth-provider-revoke:ip:${clientIp}` + ) requestUtilsMockFns.mockGetClientIp.mockReset() requestUtilsMockFns.mockGetClientIp.mockReturnValue('127.0.0.1') } diff --git a/apps/sim/app/api/chat/[identifier]/otp/route.integration.ts b/apps/sim/app/api/chat/[identifier]/otp/route.integration.ts index 823e30426cc..85096b641e7 100644 --- a/apps/sim/app/api/chat/[identifier]/otp/route.integration.ts +++ b/apps/sim/app/api/chat/[identifier]/otp/route.integration.ts @@ -1,19 +1,17 @@ import { db } from '@sim/db' -import { chat, user, verification, workflow, workspace } from '@sim/db/schema' +import { chat, user, workflow, workspace } from '@sim/db/schema' import { readTestRedisUrl } from '@sim/db/testing/test-infrastructure' import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { createRouteContext } from '@sim/testing/helpers/http' import { createMockRequest } from '@sim/testing/mocks/request.mock' import { generateId } from '@sim/utils/id' -import { eq, like } from 'drizzle-orm' +import { eq } from 'drizzle-orm' import Redis from 'ioredis' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { storage, redisClient } = vi.hoisted(() => ({ - storage: { backend: 'database' as 'database' | 'redis' }, +const { redisClient } = vi.hoisted(() => ({ redisClient: { current: undefined as Redis | undefined }, })) -vi.mock('@/lib/core/storage', () => ({ getStorageMethod: () => storage.backend })) vi.mock('@/lib/core/config/redis', async (importOriginal) => ({ ...(await importOriginal()), getRedisClient: () => redisClient.current, @@ -91,22 +89,19 @@ beforeAll(async () => { }) afterAll(async () => { - await db.delete(verification).where(like(verification.identifier, `chat-otp:${chatId}:%`)) await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) await db.delete(user).where(eq(user.id, ownerId)) await redisClient.current?.quit() }) -describe.each(['database', 'redis'] as const)('%s chat OTP verification', (backend) => { - describe.runIf(backend === 'database' || Boolean(redisUrl))('real storage', () => { +describe('chat OTP verification', () => { + describe('real Redis storage', () => { beforeAll(async () => { - if (backend === 'redis' && redisUrl && !redisClient.current) { - redisClient.current = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) - await redisClient.current.connect() - } + if (!redisUrl) throw new Error('TEST_REDIS_URL is required') + redisClient.current = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) + await redisClient.current.connect() }) beforeEach(async () => { - storage.backend = backend await storeOTP('chat', chatId, email, CODE) }) afterEach(async () => { @@ -164,33 +159,20 @@ describe.each(['database', 'redis'] as const)('%s chat OTP verification', (backe it('keeps a newer code issued while an older correct submission is in flight', async () => { const replaceCode = () => storeOTP('chat', chatId, email, NEWER_CODE) const client = redisClient.current - if (backend === 'redis' && client) { - const evaluate = client.eval.bind(client) - let calls = 0 - vi.spyOn(client, 'eval').mockImplementation(async (...args) => { - calls += 1 - if (calls === 2) await replaceCode() - return evaluate(...args) - }) - } else { - const remove = db.delete.bind(db) - vi.spyOn(db, 'delete').mockImplementationOnce((table) => { - const query = remove(table) - return new Proxy(query, { - get(target, property, receiver) { - if (property !== 'then') return Reflect.get(target, property, receiver) - return (...settle: Parameters) => - replaceCode().then(() => target.then(...settle)) - }, - }) - }) - } + if (!client) throw new Error('Redis client was not connected') + const evaluate = client.eval.bind(client) + let calls = 0 + vi.spyOn(client, 'eval').mockImplementation(async (...args) => { + calls += 1 + if (calls === 2) await replaceCode() + return evaluate(...args) + }) expect(await verify(CODE)).toMatchObject({ status: 400, error: NO_CODE, cookie: null }) expect(await verify(NEWER_CODE)).toMatchObject({ status: 200 }) }) - it.runIf(backend === 'redis')('treats a code with no expiry as expired', async () => { + it('treats a code with no expiry as expired', async () => { const key = `otp:${email}:${chatId}` await redisClient.current?.set(key, `${CODE}:${MAX_OTP_ATTEMPTS - 1}`) expect(await verify(CODE)).toMatchObject({ status: 400, error: NO_CODE, cookie: null }) @@ -208,33 +190,12 @@ describe.each(['database', 'redis'] as const)('%s chat OTP verification', (backe }) it('keeps the code expiry when a failed attempt is recorded', async () => { - if (backend === 'redis') { - const key = `otp:${email}:${chatId}` - await redisClient.current?.pexpire(key, 900) - expect(await verify('999999')).toMatchObject({ status: 400, error: INVALID }) - const ttl = await redisClient.current?.pttl(key) - expect(ttl).toBeGreaterThan(0) - expect(ttl).toBeLessThanOrEqual(900) - return - } - const readExpiry = async () => - ( - await db - .select({ expiresAt: verification.expiresAt }) - .from(verification) - .where(eq(verification.identifier, `chat-otp:${chatId}:${email}`)) - )[0]?.expiresAt.getTime() - const before = await readExpiry() + const key = `otp:${email}:${chatId}` + await redisClient.current?.pexpire(key, 900) expect(await verify('999999')).toMatchObject({ status: 400, error: INVALID }) - expect(await readExpiry()).toBe(before) - }) - - it.runIf(backend === 'database')('rejects an expired code', async () => { - await db - .update(verification) - .set({ expiresAt: new Date(Date.now() - 1000) }) - .where(eq(verification.identifier, `chat-otp:${chatId}:${email}`)) - expect(await verify(CODE)).toMatchObject({ status: 400, error: NO_CODE, cookie: null }) + const ttl = await redisClient.current?.pttl(key) + expect(ttl).toBeGreaterThan(0) + expect(ttl).toBeLessThanOrEqual(900) }) }) }) diff --git a/apps/sim/app/api/chat/[identifier]/otp/route.test.ts b/apps/sim/app/api/chat/[identifier]/otp/route.test.ts index f4a48f6e6cf..e66096d0448 100644 --- a/apps/sim/app/api/chat/[identifier]/otp/route.test.ts +++ b/apps/sim/app/api/chat/[identifier]/otp/route.test.ts @@ -27,7 +27,6 @@ const { mockRedisClient, mockSetChatAuthCookie, mockIsEmailAllowed, - mockGetStorageMethod, mockZodParse, mockAfterResponse, } = vi.hoisted(() => { @@ -41,7 +40,6 @@ const { const domain = email.slice(email.indexOf('@') + 1) return allowedEmails.includes(`@${domain}`) }) - const mockGetStorageMethod = vi.fn() const mockZodParse = vi.fn() const mockAfterResponse = vi.fn() @@ -50,7 +48,6 @@ const { mockRedisClient, mockSetChatAuthCookie, mockIsEmailAllowed, - mockGetStorageMethod, mockZodParse, mockAfterResponse, } @@ -61,10 +58,6 @@ const mockGetEnv = envMockFns.getEnv const mockCreateSuccessResponse = workflowsApiUtilsMockFns.mockCreateSuccessResponse const mockCreateCodedErrorResponse = workflowsApiUtilsMockFns.mockCreateCodedErrorResponse -vi.mock('@/lib/core/storage', () => ({ - getStorageMethod: mockGetStorageMethod, -})) - vi.mock('@/lib/core/rate-limiter', () => rateLimiterMock) vi.mock('@/lib/core/utils/after-response', () => ({ @@ -167,8 +160,6 @@ describe('Chat OTP API Route', () => { mockGetRedisClient.mockReturnValue(mockRedisClient) mockRedisSet.mockResolvedValue('OK') - mockGetStorageMethod.mockReturnValue('redis') - mockSendEmail.mockResolvedValue({ success: true }) mockRenderOTPEmail.mockResolvedValue('OTP Email') diff --git a/apps/sim/instrumentation-node.ts b/apps/sim/instrumentation-node.ts index 276871980dd..cf0699ea71c 100644 --- a/apps/sim/instrumentation-node.ts +++ b/apps/sim/instrumentation-node.ts @@ -381,6 +381,12 @@ export async function register() { const { resolveEgressPolicy } = await import('./lib/core/security/egress/profiles') resolveEgressPolicy('requestTarget') + // Redis is required. Validating REDIS_URL here stops a misconfigured server at boot, naming the + // setting, instead of failing whichever request first needs Redis. Next.js never runs + // `register` during `next build`, so the build still works without REDIS_URL. + const { getConfiguredRedisUrl } = await import('@/lib/core/config/redis') + getConfiguredRedisUrl() + await initializeOpenTelemetry() const { reportAccountTokenSealing } = await import('@sim/db/sealed-text') diff --git a/apps/sim/lib/core/config/env-capabilities.server.ts b/apps/sim/lib/core/config/env-capabilities.server.ts index 28af2740442..43a85bd8a43 100644 --- a/apps/sim/lib/core/config/env-capabilities.server.ts +++ b/apps/sim/lib/core/config/env-capabilities.server.ts @@ -6,7 +6,6 @@ import { ASYNC_JOBS_CAPABILITY, - CACHE_CAPABILITY, type ConfiguredOAuthClient, type FallbackCapabilityDefinition, inspectCapability, @@ -42,10 +41,6 @@ export function getConfiguredAsyncJobsProvider() { return requireCapability(ASYNC_JOBS_CAPABILITY, env).providerId } -export function getConfiguredCacheProvider() { - return requireCapability(CACHE_CAPABILITY, env).providerId -} - export function inspectConfiguredOAuthClient(serviceId: string) { return inspectOAuthClientCapability(serviceId, env) } diff --git a/apps/sim/lib/core/config/env-capabilities.test.ts b/apps/sim/lib/core/config/env-capabilities.test.ts index bf35bb04237..5f3231f485e 100644 --- a/apps/sim/lib/core/config/env-capabilities.test.ts +++ b/apps/sim/lib/core/config/env-capabilities.test.ts @@ -1,10 +1,12 @@ import { ASYNC_JOBS_CAPABILITY, - CACHE_CAPABILITY, + CORE_CONFIGURATION_KEYS, defineCapability, EMAIL_CAPABILITY, + ENV_CAPABILITIES, EnvCapabilityConfigurationError, envField, + getRedisConfigurationError, inspectCapability, KNOWLEDGE_EMBEDDINGS_CAPABILITY, OCR_CAPABILITY, @@ -651,32 +653,35 @@ describe('env capabilities', () => { ).toThrow(/TRIGGER_SECRET_KEY/) }) - it('uses Redis only when REDIS_URL is present and valid', () => { - expect(requireCapability(CACHE_CAPABILITY, {}).providerId).toBe('database') - expect( - requireCapability(CACHE_CAPABILITY, { REDIS_URL: 'redis://cache.example.com:6379' }) - .providerId - ).toBe('redis') - expect(() => - requireCapability(CACHE_CAPABILITY, { REDIS_URL: 'https://cache.example.com' }) - ).toThrow(/redis:\/\/ or rediss:\/\//) + it('requires a redis:// or rediss:// REDIS_URL', () => { + expect(getRedisConfigurationError({})).toBe('REDIS_URL is required') + expect(getRedisConfigurationError({ REDIS_URL: 'redis://cache.example.com:6379' })).toBeNull() + expect(getRedisConfigurationError({ REDIS_URL: 'https://cache.example.com' })).toMatch( + /redis:\/\/ or rediss:\/\// + ) + expect(getRedisConfigurationError({ REDIS_URL: 'not a url' })).toMatch( + /redis:\/\/ or rediss:\/\// + ) }) it('requires a TLS server name for rediss IP addresses', () => { - expect(() => - requireCapability(CACHE_CAPABILITY, { REDIS_URL: 'rediss://10.0.0.1:6379' }) - ).toThrow(/REDIS_TLS_SERVERNAME/) + expect(getRedisConfigurationError({ REDIS_URL: 'rediss://10.0.0.1:6379' })).toMatch( + /REDIS_TLS_SERVERNAME/ + ) expect( - requireCapability(CACHE_CAPABILITY, { + getRedisConfigurationError({ REDIS_URL: 'rediss://10.0.0.1:6379', REDIS_TLS_SERVERNAME: 'cache.internal', - }).providerId - ).toBe('redis') + }) + ).toBeNull() expect( - requireCapability(CACHE_CAPABILITY, { - REDIS_URL: 'rediss://cache.example.com:6379', - }).providerId - ).toBe('redis') + getRedisConfigurationError({ REDIS_URL: 'rediss://cache.example.com:6379' }) + ).toBeNull() + }) + + it('treats REDIS_URL as core configuration rather than an optional capability', () => { + expect(CORE_CONFIGURATION_KEYS).toContain('REDIS_URL') + expect(ENV_CAPABILITIES.map((capability) => capability.id)).not.toContain('cache') }) }) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index ddd16537aef..c897e4869cf 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -174,7 +174,7 @@ export const env = createEnv({ COPILOT_STREAM_EVENT_LIMIT: z.number().optional(), // Max events retained per stream // Database & Storage - REDIS_URL: z.string().url().optional(), // Redis connection string for caching/sessions + REDIS_URL: z.string().url(), // Required Redis connection string (redis:// or rediss://); checked when the client is first created and at server boot REDIS_TLS_SERVERNAME: z.string().min(1).optional(), // TLS SNI override; required when REDIS_URL targets an IP over rediss:// (e.g. trigger.dev PrivateLink VPCE IP) so cert hostname verification matches the ElastiCache cert's CN /** Explicit file-storage backend; unset preserves Azure → S3 → GCS → local precedence. */ STORAGE_PROVIDER: z.enum(['local', 's3', 'azure', 'gcs']).optional(), diff --git a/apps/sim/lib/core/config/redis.test.ts b/apps/sim/lib/core/config/redis.test.ts index 1e32ffc31b1..2692a262fc0 100644 --- a/apps/sim/lib/core/config/redis.test.ts +++ b/apps/sim/lib/core/config/redis.test.ts @@ -305,10 +305,10 @@ describe('redis config', () => { ).rejects.toThrow('Command timed out') }) - it('returns true as a no-op when the cache capability selects the database', async () => { + it('fails instead of pretending to hold the lock when REDIS_URL is unset', async () => { mockEnv.REDIS_URL = undefined - expect(await acquireLock(lockKey, value, ttlSeconds)).toBe(true) + await expect(acquireLock(lockKey, value, ttlSeconds)).rejects.toThrow('REDIS_URL is required') expect(mockRedisInstance.set).not.toHaveBeenCalled() }) @@ -444,7 +444,14 @@ describe('redis config', () => { }) }) - describe('capability validation', () => { + describe('configuration validation', () => { + it('throws on call, not at import, when REDIS_URL is unset', () => { + mockEnv.REDIS_URL = undefined + + expect(() => getRedisClient()).toThrow('REDIS_URL is required') + expect(MockRedisConstructor).not.toHaveBeenCalled() + }) + it('rejects a non-Redis URL before constructing a client', () => { mockEnv.REDIS_URL = 'https://cache.example.com' diff --git a/apps/sim/lib/core/config/redis.ts b/apps/sim/lib/core/config/redis.ts index 7ca7889952d..fbb0abe235e 100644 --- a/apps/sim/lib/core/config/redis.ts +++ b/apps/sim/lib/core/config/redis.ts @@ -1,10 +1,10 @@ import { isIP } from 'node:net' +import { getRedisConfigurationError } from '@sim/deployment-config/env-capabilities' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { randomFloat } from '@sim/utils/random' import Redis from 'ioredis' import { env } from '@/lib/core/config/env' -import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { coldConnectionBudgetMs } from '@/lib/core/config/redis-budget' const logger = createLogger('Redis') @@ -244,12 +244,19 @@ const REDIS_WARMUP_TIMEOUT_MS = coldConnectionBudgetMs({ reconnectDelayMs: sharedReconnectDelayMs(1, 1), }) -export function getConfiguredRedisUrl(): string | null { - if (getConfiguredCacheProvider() === 'database') return null - +/** + * The configured Redis URL. Redis is required, so an unset or invalid REDIS_URL + * throws — on call, never at import: `next build` evaluates modules that build + * Redis channels at module scope without any runtime environment. + */ +export function getConfiguredRedisUrl(): string { const redisUrl = env.REDIS_URL if (!redisUrl) { - throw new Error('Cache capability selected Redis but REDIS_URL is missing') + throw new Error('REDIS_URL is required') + } + const configurationError = getRedisConfigurationError(env) + if (configurationError) { + throw new Error(configurationError) } return redisUrl } @@ -310,84 +317,83 @@ function startPingHealthCheck(redis: Redis): void { } /** - * Get a Redis client instance. - * Uses connection pooling to reuse connections across requests. + * Get the shared Redis client, creating it on first use. + * + * Redis is required: this throws when REDIS_URL is unset or the TLS options are + * invalid, and when called in a browser. It throws lazily, on call, so modules + * that reference it at import time still load during `next build`. * * ioredis handles command queuing internally via `enableOfflineQueue` (default: true), * so commands are queued and executed once connected. No manual connection checks needed. */ -export function getRedisClient(): Redis | null { - if (typeof window !== 'undefined') return null - const redisUrl = getConfiguredRedisUrl() - if (!redisUrl) return null +export function getRedisClient(): Redis { + if (typeof window !== 'undefined') { + throw new Error('The Redis client is server-only') + } if (state.client) return state.client + const redisUrl = getConfiguredRedisUrl() - // Outside the try/catch so config errors aren't silently swallowed. const defaults = getRedisConnectionDefaults(redisUrl) - try { - logger.info('Initializing Redis client') + logger.info('Initializing Redis client') - state.client = new Redis(redisUrl, { - ...defaults, - commandTimeout: SHARED_COMMAND_TIMEOUT_MS, - maxRetriesPerRequest: 5, + const client = new Redis(redisUrl, { + ...defaults, + commandTimeout: SHARED_COMMAND_TIMEOUT_MS, + maxRetriesPerRequest: 5, - retryStrategy: (times) => { - if (times > 10) { - logger.error(`Redis reconnection attempt ${times}`, { nextRetryMs: 30000 }) - return 30000 - } - const delay = sharedReconnectDelayMs(times, randomFloat()) - state.reconnects++ - logger.warn('Redis reconnecting', { attempt: times, nextRetryMs: delay }) - return delay - }, - - reconnectOnError: (err) => { - const targetErrors = ['READONLY', 'ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED'] - return targetErrors.some((e) => err.message.includes(e)) - }, - }) + retryStrategy: (times) => { + if (times > 10) { + logger.error(`Redis reconnection attempt ${times}`, { nextRetryMs: 30000 }) + return 30000 + } + const delay = sharedReconnectDelayMs(times, randomFloat()) + state.reconnects++ + logger.warn('Redis reconnecting', { attempt: times, nextRetryMs: delay }) + return delay + }, + + reconnectOnError: (err) => { + const targetErrors = ['READONLY', 'ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED'] + return targetErrors.some((e) => err.message.includes(e)) + }, + }) - state.clientCreatedAt = Date.now() - state.lastReadyAt = null - state.lastPingOkAt = null - - state.client.on('connect', () => { - state.connects++ - // Elapsed since construction, because the wait before a connection becomes - // usable is the number this path has never been able to produce: it is - // spent inside a command's deadline, where it surfaces as a command - // timeout rather than as connection latency. - // `connectCount`, not `attempt` — the retryStrategy above already logs an - // `attempt`, meaning the retry number. This is how many times this client - // has successfully connected, which is what separates a first connect from - // a reconnect. - logger.info('Redis connected', { - elapsedMs: elapsedSince(state.clientCreatedAt), - connectCount: state.connects, - }) - }) - state.client.on('ready', () => { - state.lastReadyAt = Date.now() - logger.info('Redis ready', { elapsedMs: elapsedSince(state.clientCreatedAt) }) - }) - state.client.on('error', (err: Error) => { - state.errors++ - state.lastErrorMessage = err.message - logger.error('Redis error', { error: err.message, code: (err as any).code }) + state.clientCreatedAt = Date.now() + state.lastReadyAt = null + state.lastPingOkAt = null + + client.on('connect', () => { + state.connects++ + // Elapsed since construction, because the wait before a connection becomes + // usable is the number this path has never been able to produce: it is + // spent inside a command's deadline, where it surfaces as a command + // timeout rather than as connection latency. + // `connectCount`, not `attempt` — the retryStrategy above already logs an + // `attempt`, meaning the retry number. This is how many times this client + // has successfully connected, which is what separates a first connect from + // a reconnect. + logger.info('Redis connected', { + elapsedMs: elapsedSince(state.clientCreatedAt), + connectCount: state.connects, }) - state.client.on('close', () => logger.warn('Redis connection closed')) - state.client.on('end', () => logger.error('Redis connection ended')) + }) + client.on('ready', () => { + state.lastReadyAt = Date.now() + logger.info('Redis ready', { elapsedMs: elapsedSince(state.clientCreatedAt) }) + }) + client.on('error', (err: Error) => { + state.errors++ + state.lastErrorMessage = err.message + logger.error('Redis error', { error: err.message, code: (err as NodeJS.ErrnoException).code }) + }) + client.on('close', () => logger.warn('Redis connection closed')) + client.on('end', () => logger.error('Redis connection ended')) - startPingHealthCheck(state.client) + state.client = client + startPingHealthCheck(client) - return state.client - } catch (error) { - logger.error('Failed to initialize Redis client', { error }) - return null - } + return client } /** @@ -398,23 +404,22 @@ export function getRedisClient(): Redis | null { * still shaking hands spends that budget waiting to connect and fails as a * command timeout from a server that never received it. * - * Resolves `true` once the shared connection is usable, `false` when Redis is - * not configured or the wait ran out. It never throws and never rejects: + * Resolves `true` once the shared connection is usable, `false` when the client + * could not be created (REDIS_URL unset or invalid) or the wait ran out. It never throws and never rejects: * callers run at process start — a Trigger.dev `init` hook fails the whole run * attempt if it throws — and a warm-up is an optimization, so failing to warm * must cost nothing beyond the connection staying cold. */ export function warmRedisConnection(timeoutMs = REDIS_WARMUP_TIMEOUT_MS): Promise { - let client: Redis | null = null + let client: Redis try { client = getRedisClient() } catch { - // A misconfigured URL belongs to the first real caller, which can report it - // against the operation that needed Redis. Warming must not turn it into a - // start-up failure. + // A missing or misconfigured URL belongs to the boot assertion and the first + // real caller, which can report it against the operation that needed Redis. + // Warming must not turn it into a start-up failure. return Promise.resolve(false) } - if (!client) return Promise.resolve(false) if (client.status === 'ready') return Promise.resolve(true) if (state.warmup?.client === client) return state.warmup.promise @@ -473,14 +478,6 @@ else end ` -/** - * Acquire a distributed lock using Redis SET NX. - * Returns true if lock acquired, false if already held. - * - * When Redis is not available, returns true (lock "acquired") to allow - * single-replica deployments to function without Redis. In multi-replica - * deployments without Redis, the idempotency layer prevents duplicate processing. - */ export interface AcquireLockOptions { /** * Release the lock this call may have taken when the SET itself rejects. @@ -502,6 +499,10 @@ export interface AcquireLockOptions { reclaimOnFailure?: boolean } +/** + * Acquire a distributed lock using Redis SET NX. + * Returns true if lock acquired, false if already held; throws when the SET fails. + */ export async function acquireLock( lockKey: string, value: string, @@ -509,9 +510,6 @@ export async function acquireLock( options?: AcquireLockOptions ): Promise { const redis = getRedisClient() - if (!redis) { - return true // No-op when Redis unavailable; idempotency layer handles duplicates - } try { const result = await redis.set(lockKey, value, 'EX', expirySeconds, 'NX') @@ -546,15 +544,9 @@ export async function acquireLock( * Release a distributed lock safely. * Only releases if the caller owns the lock (value matches). * Returns true if lock was released, false if not owned or already expired. - * - * When Redis is not available, returns true (no-op) since no lock was held. */ export async function releaseLock(lockKey: string, value: string): Promise { const redis = getRedisClient() - if (!redis) { - return true // No-op when Redis unavailable; no lock was actually held - } - const result = await redis.eval(RELEASE_LOCK_SCRIPT, 1, lockKey, value) return result === 1 } @@ -563,10 +555,6 @@ export async function releaseLock(lockKey: string, value: string): Promise { const redis = getRedisClient() - if (!redis) { - return true - } - const result = await redis.eval(EXTEND_LOCK_SCRIPT, 1, lockKey, value, expirySeconds) return result === 1 } diff --git a/apps/sim/lib/core/idempotency/service.test.ts b/apps/sim/lib/core/idempotency/service.test.ts index 91735f19eac..f8c4b6cce5f 100644 --- a/apps/sim/lib/core/idempotency/service.test.ts +++ b/apps/sim/lib/core/idempotency/service.test.ts @@ -9,13 +9,6 @@ const { redisDelMock, redisEvalMock, redisGetMock } = vi.hoisted(() => ({ redisGetMock: vi.fn(), })) -vi.mock('@/lib/core/storage', () => ({ - getStorageMethod: () => 'redis', - isRedisStorage: () => true, - isDatabaseStorage: () => false, - requireRedis: () => ({ del: redisDelMock, eval: redisEvalMock }), -})) - import { RetryableSetupError } from '@/lib/core/errors/retryable-infrastructure' import { IdempotencyService, diff --git a/apps/sim/lib/core/idempotency/service.ts b/apps/sim/lib/core/idempotency/service.ts index 4e288028289..7c9ec7340df 100644 --- a/apps/sim/lib/core/idempotency/service.ts +++ b/apps/sim/lib/core/idempotency/service.ts @@ -8,11 +8,16 @@ import { and, eq, lt, or, sql } from 'drizzle-orm' import { getRedisClient } from '@/lib/core/config/redis' import { isRetryableSetupError } from '@/lib/core/errors/retryable-infrastructure' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' -import { getStorageMethod, type StorageMethod } from '@/lib/core/storage' import { extractProviderIdentifierFromBody, isDeliveryIdOptional } from '@/lib/webhooks/providers' const logger = createLogger('IdempotencyService') +/** + * Where idempotency records live. Redis is the default; `'database'` keeps the + * record in Postgres so it can fate-share with a surrounding transaction. + */ +type StorageMethod = 'redis' | 'database' + export interface IdempotencyConfig { ttlSeconds?: number /** @@ -139,9 +144,7 @@ return redis.call('DEL', KEYS[1]) * Universal idempotency service for webhooks, triggers, and any other operations * that need duplicate prevention. * - * Storage is determined once based on configuration: - * - If `forceStorage` is set → that backend unconditionally - * - Else use the provider selected by the cache capability + * Records live in Redis unless `forceStorage: 'database'` keeps them in Postgres. */ export class IdempotencyService { private config: Required> @@ -155,7 +158,7 @@ export class IdempotencyService { retryFailures: config.retryFailures ?? false, storeResultBody: config.storeResultBody ?? true, } - this.storageMethod = config.forceStorage ?? getStorageMethod() + this.storageMethod = config.forceStorage ?? 'redis' logger.info(`IdempotencyService using ${this.storageMethod} storage`, { namespace: this.config.namespace, forced: Boolean(config.forceStorage), @@ -193,9 +196,6 @@ export class IdempotencyService { private async checkIdempotencyRedis(normalizedKey: string): Promise { const redis = getRedisClient() - if (!redis) { - throw new Error('Redis not available for idempotency check') - } const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}` const cachedResult = await redis.get(redisKey) @@ -286,9 +286,6 @@ export class IdempotencyService { inProgressResult: ProcessingResult ): Promise { const redis = getRedisClient() - if (!redis) { - throw new Error('Redis not available for atomic claim') - } const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}` const inProgressTtlSeconds = Math.max( @@ -415,9 +412,6 @@ export class IdempotencyService { if (storageMethod === 'redis') { const redis = getRedisClient() - if (!redis) { - throw new Error('Redis not available') - } const data = await redis.get(redisKey) currentResult = data ? JSON.parse(data) : null } else { @@ -466,9 +460,6 @@ export class IdempotencyService { claimToken: string ): Promise { const redis = getRedisClient() - if (!redis) { - throw new Error('Redis not available for storing result') - } const storedResult = { ...result, claimToken } const stored = await redis.eval( @@ -531,7 +522,6 @@ export class IdempotencyService { ): Promise { if (storageMethod === 'redis') { const redis = getRedisClient() - if (!redis) return false if (fence?.claimToken) { const deleted = await redis.eval( DELETE_REDIS_RESULT_IF_OWNER_SCRIPT, diff --git a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts index c31c01567da..17d2ca8e5c9 100644 --- a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts +++ b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts @@ -1,5 +1,4 @@ -import type { rateLimitBucket } from '@sim/db/schema' -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { resetDbChainMock } from '@sim/testing' import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -17,7 +16,6 @@ vi.mock('@/lib/core/rate-limiter/storage/factory', () => ({ })) import { waitForProviderAdmission } from '@/lib/core/rate-limiter/provider-admission' -import { DbTokenBucket } from '@/lib/core/rate-limiter/storage/db-token-bucket' import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils' const INPUT = { @@ -159,48 +157,6 @@ describe('provider admission', () => { }) }) - it('sustains 600 hosted reranks per minute through the real bucket refill calculation', async () => { - const input = { - ...INPUT, - operation: 'rerank' as const, - providerId: 'cohere', - isHostedCredential: true, - maxWaitMs: 1, - } - let stored: { key: string; tokens: string; lastRefillAt: Date } | undefined - dbChainMockFns.values.mockImplementation((row) => { - stored ??= { ...(row as typeof rateLimitBucket.$inferSelect) } - return { onConflictDoNothing: vi.fn().mockResolvedValue(undefined) } - }) - dbChainMockFns.limit.mockImplementation(async () => [stored]) - dbChainMockFns.set.mockImplementation((values) => { - Object.assign(stored!, values) - return { where: vi.fn().mockResolvedValue(undefined) } - }) - const bucket = new DbTokenBucket() - /** The single-dimension refill arithmetic the atomic statement mirrors in SQL. */ - consumeTokens.mockImplementation(async ([reservation]) => { - const result = await bucket.consumeTokens( - reservation.key, - reservation.cost, - reservation.config - ) - return { allowed: result.allowed, retryAfterMs: result.retryAfterMs ?? 0 } - }) - - for (let request = 0; request < 16; request++) await waitForProviderAdmission(input) - await expect(waitForProviderAdmission(input)).rejects.toMatchObject({ retryAfterMs: 1000 }) - for (let second = 0; second < 60; second++) { - await vi.advanceTimersByTimeAsync(1000) - for (let request = 0; request < 10; request++) await waitForProviderAdmission(input) - await expect(waitForProviderAdmission(input)).rejects.toMatchObject({ retryAfterMs: 1000 }) - } - expect(stored?.tokens).toBe('0') - expect(new Set(consumeTokens.mock.calls.map(([reservations]) => reservations[0].key))).toEqual( - new Set(['provider:rerank:cohere:hashed-credential:requests']) - ) - }) - it('stops waiting immediately when the caller aborts', async () => { consumeTokens.mockResolvedValue({ allowed: false, retryAfterMs: 5000 }) const controller = new AbortController() diff --git a/apps/sim/lib/core/rate-limiter/provider-admission.ts b/apps/sim/lib/core/rate-limiter/provider-admission.ts index 6d6860c920a..495afceb274 100644 --- a/apps/sim/lib/core/rate-limiter/provider-admission.ts +++ b/apps/sim/lib/core/rate-limiter/provider-admission.ts @@ -146,9 +146,7 @@ export async function waitForProviderAdmission(input: ProviderAdmissionInput): P throw new ProviderQuotaExhaustedError(input.providerId) let result: AtomicAdmissionResult try { - result = await createStorageAdapter({ - requireConfiguredBackend: true, - }).consumeTokensAtomically(reservations, { + result = await createStorageAdapter().consumeTokensAtomically(reservations, { cooldownKeys: [`${key}:cooldown`, `${key}:quota`], deadlineAt, signal: input.signal, @@ -202,9 +200,7 @@ function providerKey(identity: ProviderIdentity): string { export async function isProviderQuotaExhausted(identity: ProviderIdentity): Promise { try { - const until = await createStorageAdapter({ requireConfiguredBackend: true }).getCooldownUntil( - `${providerKey(identity)}:quota` - ) + const until = await createStorageAdapter().getCooldownUntil(`${providerKey(identity)}:quota`) return (until?.getTime() ?? 0) > Date.now() } catch (error) { throw new ProviderAdmissionStorageError(error) @@ -220,7 +216,7 @@ export async function recordProviderCooldown( if (!Number.isFinite(waitMs) || waitMs <= 0) throw new Error('Provider cooldown must be positive and finite') try { - await createStorageAdapter({ requireConfiguredBackend: true }).setCooldownUntil( + await createStorageAdapter().setCooldownUntil( `${providerKey(identity)}:${quotaExhausted ? 'quota' : 'cooldown'}`, new Date(Date.now() + waitMs) ) diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-store.test.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-store.test.ts index 297880cc247..6eafe55f93c 100644 --- a/apps/sim/lib/core/rate-limiter/provider-capacity-store.test.ts +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-store.test.ts @@ -1,16 +1,8 @@ -import { dbChainMockFns } from '@sim/testing/mocks/database.mock' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { getStorage } = vi.hoisted(() => ({ - getStorage: vi.fn(), -})) -vi.mock('@/lib/core/storage', () => ({ getStorageMethod: getStorage })) - import { mutateProviderCapacity } from '@/lib/core/rate-limiter/provider-capacity-store' const evalScript = vi.fn() -const transaction = dbChainMockFns.transaction const getRedis = redisConfigMockFns.mockGetRedisClient const CONFIG = { @@ -27,7 +19,6 @@ const RESULT = { allowed: true, retryAfterMs: 0, scale: 1, inFlight: 1 } describe('provider capacity storage bounds', () => { beforeEach(() => { vi.useFakeTimers() - getStorage.mockReturnValue('redis') getRedis.mockReturnValue({ eval: evalScript }) evalScript.mockResolvedValue(JSON.stringify(RESULT)) }) @@ -40,21 +31,16 @@ describe('provider capacity storage bounds', () => { expect(vi.getTimerCount()).toBe(0) }) - it.each(['redis', 'database'])( - 'bounds a stalled %s connection independently of the driver', - async (backend) => { - getStorage.mockReturnValue(backend) - evalScript.mockImplementation(() => new Promise(() => undefined)) - transaction.mockImplementation(() => new Promise(() => undefined)) - const pending = mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() + 5000) - const rejected = expect(pending).rejects.toThrow('deadline expired') - await vi.advanceTimersByTimeAsync(5000) - await rejected - expect(vi.getTimerCount()).toBe(0) - } - ) + it('bounds a stalled Redis connection independently of the driver', async () => { + evalScript.mockImplementation(() => new Promise(() => undefined)) + const pending = mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() + 5000) + const rejected = expect(pending).rejects.toThrow('deadline expired') + await vi.advanceTimersByTimeAsync(5000) + await rejected + expect(vi.getTimerCount()).toBe(0) + }) - it('honors caller cancellation during a stalled Redis command without falling back to DB', async () => { + it('honors caller cancellation during a stalled Redis command', async () => { evalScript.mockImplementation(() => new Promise(() => undefined)) const controller = new AbortController() const pending = mutateProviderCapacity( @@ -67,20 +53,20 @@ describe('provider capacity storage bounds', () => { const rejected = expect(pending).rejects.toThrow('cancelled') controller.abort(new Error('cancelled')) await rejected - expect(transaction).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) }) - it('fails closed when configured Redis is missing or gives malformed state', async () => { - getRedis.mockReturnValueOnce(null) + it('fails closed when Redis is not configured or gives malformed state', async () => { + getRedis.mockImplementationOnce(() => { + throw new Error('REDIS_URL is required') + }) await expect( mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() + 5000) - ).rejects.toThrow('Redis is unavailable') + ).rejects.toThrow('REDIS_URL is required') evalScript.mockResolvedValueOnce(JSON.stringify({ ...RESULT, scale: 100 })) await expect( mutateProviderCapacity('quota', CONFIG, ACTION, Date.now() + 5000) ).rejects.toThrow('Invalid provider capacity storage response') - expect(transaction).not.toHaveBeenCalled() }) it('never contacts storage for already aborted or expired calls', async () => { diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity-store.ts b/apps/sim/lib/core/rate-limiter/provider-capacity-store.ts index d4532062080..41d51a32960 100644 --- a/apps/sim/lib/core/rate-limiter/provider-capacity-store.ts +++ b/apps/sim/lib/core/rate-limiter/provider-capacity-store.ts @@ -1,19 +1,13 @@ -import { db } from '@sim/db' -import { rateLimitBucket } from '@sim/db/schema' -import { eq, sql } from 'drizzle-orm' import { getRedisClient } from '@/lib/core/config/redis' import { PROVIDER_CAPACITY_SCRIPT } from '@/lib/core/rate-limiter/provider-capacity-lua' -import { - type ProviderCapacityAction, - type ProviderCapacityConfig, - type ProviderCapacityResult, - type ProviderCapacityState, - updateProviderCapacity, +import type { + ProviderCapacityAction, + ProviderCapacityConfig, + ProviderCapacityResult, } from '@/lib/core/rate-limiter/provider-capacity-state' -import { getStorageMethod } from '@/lib/core/storage' import { withinDeadline } from '@/lib/core/utils/deadline' -/** One atomic state update; Redis outages never split provider capacity into another backend. */ +/** One atomic Redis state update, bounded by the caller's deadline and signal. */ export async function mutateProviderCapacity( key: string, config: ProviderCapacityConfig, @@ -24,82 +18,31 @@ export async function mutateProviderCapacity( return withinDeadline( async (operationSignal) => { operationSignal.throwIfAborted() - if (getStorageMethod() === 'redis') { - const redis = getRedisClient() - if (!redis) throw new Error('Configured provider capacity Redis is unavailable') - const result = await redis.eval( - PROVIDER_CAPACITY_SCRIPT, - 1, - `ratelimit:tb:${key}`, - JSON.stringify(config), - JSON.stringify(action), - String(deadlineAt) - ) - operationSignal.throwIfAborted() - const parsed: ProviderCapacityResult = JSON.parse(String(result)) - if ( - typeof parsed.allowed !== 'boolean' || - !Number.isFinite(parsed.retryAfterMs) || - parsed.retryAfterMs < 0 || - (parsed.cooldownRemainingMs !== undefined && - (!Number.isFinite(parsed.cooldownRemainingMs) || parsed.cooldownRemainingMs < 0)) || - !Number.isFinite(parsed.scale) || - parsed.scale <= 0 || - parsed.scale > 1 || - !Number.isSafeInteger(parsed.inFlight) || - parsed.inFlight < 0 - ) - throw new Error('Invalid provider capacity storage response') - return parsed - } - - return db.transaction(async (tx) => { - operationSignal.throwIfAborted() - const remainingMs = Math.max(1, deadlineAt - Date.now()) - await tx.execute( - sql`SELECT set_config('statement_timeout', ${String(remainingMs)}, true), set_config('lock_timeout', ${String(remainingMs)}, true)` - ) - operationSignal.throwIfAborted() - await tx - .insert(rateLimitBucket) - .values({ key, tokens: '0', lastRefillAt: new Date() }) - .onConflictDoNothing() - operationSignal.throwIfAborted() - const [row] = await tx - .select({ - state: rateLimitBucket.capacityState, - }) - .from(rateLimitBucket) - .where(eq(rateLimitBucket.key, key)) - .for('update') - .limit(1) - if (!row) throw new Error('Provider capacity state disappeared') - operationSignal.throwIfAborted() - if (Date.now() >= deadlineAt) throw new Error('Provider capacity storage deadline expired') - /** Read the backend clock after the row lock, including any contention wait. */ - const clock = await tx.execute<{ now: string }>( - sql`SELECT floor(extract(epoch from clock_timestamp()) * 1000)::bigint AS now` - ) - const now = Number(clock[0]?.now) - if (!Number.isFinite(now)) throw new Error('Provider capacity storage clock unavailable') - const stored = row.state as ProviderCapacityState | null - if ( - stored && - (stored.version !== 1 || - !Array.isArray(stored.leases) || - (stored.pageWindow !== undefined && !Array.isArray(stored.pageWindow))) - ) { - throw new Error('Unsupported provider capacity state') - } - const { state, result } = updateProviderCapacity(stored, config, action, now) - await tx - .update(rateLimitBucket) - .set({ capacityState: state, updatedAt: new Date(now) }) - .where(eq(rateLimitBucket.key, key)) - operationSignal.throwIfAborted() - if (Date.now() >= deadlineAt) throw new Error('Provider capacity storage deadline expired') - return result - }) + const redis = getRedisClient() + const result = await redis.eval( + PROVIDER_CAPACITY_SCRIPT, + 1, + `ratelimit:tb:${key}`, + JSON.stringify(config), + JSON.stringify(action), + String(deadlineAt) + ) + operationSignal.throwIfAborted() + const parsed: ProviderCapacityResult = JSON.parse(String(result)) + if ( + typeof parsed.allowed !== 'boolean' || + !Number.isFinite(parsed.retryAfterMs) || + parsed.retryAfterMs < 0 || + (parsed.cooldownRemainingMs !== undefined && + (!Number.isFinite(parsed.cooldownRemainingMs) || parsed.cooldownRemainingMs < 0)) || + !Number.isFinite(parsed.scale) || + parsed.scale <= 0 || + parsed.scale > 1 || + !Number.isSafeInteger(parsed.inFlight) || + parsed.inFlight < 0 + ) + throw new Error('Invalid provider capacity storage response') + return parsed }, deadlineAt, signal diff --git a/apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts b/apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts index 2058f883045..fcb70757ca7 100644 --- a/apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts +++ b/apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts @@ -1,19 +1,14 @@ import { createHash } from 'node:crypto' import { createServer } from 'node:http' -import { db } from '@sim/db' -import { rateLimitBucket } from '@sim/db/schema' import { readTestRedisUrl } from '@sim/db/testing/test-infrastructure' import { interruptibleSleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { eq, sql } from 'drizzle-orm' import Redis from 'ioredis' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { storage, redisClient } = vi.hoisted(() => ({ - storage: { backend: 'database' }, +const { redisClient } = vi.hoisted(() => ({ redisClient: { current: undefined as Redis | undefined }, })) -vi.mock('@/lib/core/storage', () => ({ getStorageMethod: () => storage.backend })) vi.mock('@/lib/core/config/redis', () => ({ getRedisClient: () => redisClient.current })) import { PROVIDER_CAPACITY_SCRIPT } from '@/lib/core/rate-limiter/provider-capacity-lua' @@ -59,24 +54,21 @@ function initial( } } -describe.each(['database', 'redis'] as const)('%s weighted provider capacity', (backend) => { - describe.runIf(backend === 'database' || Boolean(redisUrl))('real atomic storage', () => { +describe('weighted provider capacity', () => { + describe('real atomic Redis storage', () => { let key: string beforeAll(async () => { - if (backend === 'redis') { - redisClient.current = new Redis(redisUrl!, { lazyConnect: true, maxRetriesPerRequest: 0 }) - await redisClient.current.connect() - } + if (!redisUrl) throw new Error('TEST_REDIS_URL is required') + redisClient.current = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) + await redisClient.current.connect() }) beforeEach(() => { - storage.backend = backend key = `provider-capacity-test:${generateId()}` }) afterEach(async () => { vi.unstubAllGlobals() - if (backend === 'redis') await redisClient.current?.del(`ratelimit:tb:${key}`) - else await db.delete(rateLimitBucket).where(eq(rateLimitBucket.key, key)) + await redisClient.current?.del(`ratelimit:tb:${key}`) }) afterAll(async () => { await redisClient.current?.quit() @@ -84,44 +76,20 @@ describe.each(['database', 'redis'] as const)('%s weighted provider capacity', ( }) async function readClock(): Promise { - if (backend === 'redis') { - const [seconds, microseconds] = await redisClient.current!.time() - return Number(seconds) * 1000 + Math.floor(Number(microseconds) / 1000) - } - const clock = await db.execute<{ now: string }>( - sql`SELECT floor(extract(epoch from clock_timestamp()) * 1000)::bigint AS now` - ) - return Number(clock[0]?.now) + const [seconds, microseconds] = await redisClient.current!.time() + return Number(seconds) * 1000 + Math.floor(Number(microseconds) / 1000) } async function seed(state: ProviderCapacityState) { - if (backend === 'redis') { - await redisClient.current!.hset( - `ratelimit:tb:${key}`, - 'capacityState', - JSON.stringify(state) - ) - } else { - await db - .insert(rateLimitBucket) - .values({ key, tokens: '0', lastRefillAt: new Date(), capacityState: state }) - .onConflictDoUpdate({ target: rateLimitBucket.key, set: { capacityState: state } }) - } + await redisClient.current!.hset(`ratelimit:tb:${key}`, 'capacityState', JSON.stringify(state)) } async function read(): Promise { - if (backend === 'redis') { - const state: ProviderCapacityState = JSON.parse( - (await redisClient.current!.hget(`ratelimit:tb:${key}`, 'capacityState'))! - ) - if (!Array.isArray(state.leases)) state.leases = [] - if (!Array.isArray(state.pageWindow)) state.pageWindow = [] - return state - } - const [row] = await db - .select({ state: rateLimitBucket.capacityState }) - .from(rateLimitBucket) - .where(eq(rateLimitBucket.key, key)) - return row!.state as ProviderCapacityState + const state: ProviderCapacityState = JSON.parse( + (await redisClient.current!.hget(`ratelimit:tb:${key}`, 'capacityState'))! + ) + if (!Array.isArray(state.leases)) state.leases = [] + if (!Array.isArray(state.pageWindow)) state.pageWindow = [] + return state } const mutate = (action: ProviderCapacityAction, config = CONFIG) => mutateProviderCapacity(key, config, action, Date.now() + 10_000) @@ -439,30 +407,24 @@ describe.each(['database', 'redis'] as const)('%s weighted provider capacity', ( } ) - it.runIf(backend === 'redis')( - 'rejects a stale queued Redis command before creating any reservation', - async () => { - const cutoff = (await readClock()) - 1 - await expect( - redisClient.current!.eval( - PROVIDER_CAPACITY_SCRIPT, - 1, - `ratelimit:tb:${key}`, - JSON.stringify(CONFIG), - JSON.stringify(acquire('late')), - String(cutoff) - ) - ).rejects.toThrow('storage deadline expired') - expect(await redisClient.current!.exists(`ratelimit:tb:${key}`)).toBe(0) - } - ) + it('rejects a stale queued Redis command before creating any reservation', async () => { + const cutoff = (await readClock()) - 1 + await expect( + redisClient.current!.eval( + PROVIDER_CAPACITY_SCRIPT, + 1, + `ratelimit:tb:${key}`, + JSON.stringify(CONFIG), + JSON.stringify(acquire('late')), + String(cutoff) + ) + ).rejects.toThrow('storage deadline expired') + expect(await redisClient.current!.exists(`ratelimit:tb:${key}`)).toBe(0) + }) - it.runIf(backend === 'redis')( - 'retains state longer than every live lease and cooldown', - async () => { - await mutate(acquire('first')) - expect(await redisClient.current!.pttl(`ratelimit:tb:${key}`)).toBeGreaterThan(86_500_000) - } - ) + it('retains state longer than every live lease and cooldown', async () => { + await mutate(acquire('first')) + expect(await redisClient.current!.pttl(`ratelimit:tb:${key}`)).toBeGreaterThan(86_500_000) + }) }) }) diff --git a/apps/sim/lib/core/rate-limiter/storage/db-token-bucket.test.ts b/apps/sim/lib/core/rate-limiter/storage/db-token-bucket.test.ts deleted file mode 100644 index 443f3dc8100..00000000000 --- a/apps/sim/lib/core/rate-limiter/storage/db-token-bucket.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { DbTokenBucket } from '@/lib/core/rate-limiter/storage/db-token-bucket' - -const CONFIG = { maxTokens: 10, refillRate: 1, refillIntervalMs: 1000 } - -describe('PostgreSQL token bucket', () => { - beforeEach(() => { - vi.useFakeTimers() - vi.setSystemTime(new Date('2026-09-04T12:00:00Z')) - resetDbChainMock() - }) - - afterEach(() => vi.useRealTimers()) - - it('preserves an insufficient balance so a later refill can admit the request', async () => { - const stored = { tokens: '2', lastRefillAt: new Date() } - dbChainMockFns.limit.mockImplementation(async () => [stored]) - dbChainMockFns.set.mockImplementation((values) => { - Object.assign(stored, values) - return { where: vi.fn().mockResolvedValue(undefined) } - }) - const bucket = new DbTokenBucket() - expect(await bucket.consumeTokens('key', 3, CONFIG)).toMatchObject({ - allowed: false, - tokensRemaining: 2, - retryAfterMs: 1000, - }) - expect(stored.tokens).toBe('2') - await vi.advanceTimersByTimeAsync(1000) - expect(await bucket.consumeTokens('key', 3, CONFIG)).toMatchObject({ - allowed: true, - tokensRemaining: 0, - }) - }) - - it('normalizes legacy denial markers and computes the complete refill wait', async () => { - dbChainMockFns.limit.mockResolvedValue([{ tokens: '-1', lastRefillAt: new Date() }]) - expect(await new DbTokenBucket().consumeTokens('key', 3, CONFIG)).toMatchObject({ - allowed: false, - tokensRemaining: 0, - retryAfterMs: 3000, - }) - expect(dbChainMockFns.set).toHaveBeenCalledWith(expect.objectContaining({ tokens: '0' })) - }) - - it('accepts an empty reservation without attempting an empty insert', async () => { - dbChainMockFns.limit.mockResolvedValue([]) - - expect( - await new DbTokenBucket().consumeTokensAtomically([], { - cooldownKeys: [], - deadlineAt: Date.now() + 1000, - }) - ).toEqual({ allowed: true, retryAfterMs: 0 }) - expect(dbChainMockFns.insert).not.toHaveBeenCalled() - expect(dbChainMockFns.update).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/core/rate-limiter/storage/db-token-bucket.ts b/apps/sim/lib/core/rate-limiter/storage/db-token-bucket.ts deleted file mode 100644 index 097931a60aa..00000000000 --- a/apps/sim/lib/core/rate-limiter/storage/db-token-bucket.ts +++ /dev/null @@ -1,385 +0,0 @@ -import { db } from '@sim/db' -import { rateLimitBucket } from '@sim/db/schema' -import { eq, type SQL, sql } from 'drizzle-orm' -import type { - AtomicAdmissionOptions, - AtomicAdmissionResult, - ConsumeResult, - RateLimitStorageAdapter, - TokenBucketConfig, - TokenBucketReservation, - TokenStatus, -} from '@/lib/core/rate-limiter/storage/adapter' - -/** Thrown inside the admission transaction to roll back a spend that finished past its deadline. */ -class AdmissionExpiredError extends Error {} - -interface AdmissionRow extends Record { - kind: 'reservation' | 'cooldown' - cost: number | null - available: number | null - last_refill_ms: number | null - refill_rate: number | null - refill_interval_ms: number | null - blocked_until_ms: number | null - allowed: boolean - spent: number - now_ms: number -} - -/** An array of scalar binds; the pool cannot bind a JavaScript array as one parameter. */ -function sqlArray(values: readonly (string | number)[], type: 'text' | 'float8'): SQL { - return sql`ARRAY[${sql.join( - values.map((value) => sql`${value}`), - sql`, ` - )}]::${sql.raw(type)}[]` -} - -/** - * Locks every key in sorted order, refills each reservation to the moment every lock is held - * with the same float8 arithmetic {@link DbTokenBucket.consumeTokens} uses in JavaScript, and - * spends all reservations only when no cooldown is active and every balance covers its cost. - * Returns one row per reservation and per cooldown key, each carrying the decision, how many - * buckets were spent, and the clock the decision used. - * - * That clock is the caller's `sentAt` advanced by the time spent acquiring the locks, in whole - * milliseconds. `lock_start` is joined into `locked`, so it is read before the first row can - * reach the lock; `clock` counts every `locked` row before reading `clock_timestamp()` again, so - * that read happens only once the last lock is held. An uncontended admission therefore decides - * at exactly `sentAt`, and measuring the wait rather than reading the database's absolute time - * keeps every stored instant on the application clock that wrote it. - */ -function buildAdmissionStatement( - reservations: readonly TokenBucketReservation[], - cooldownKeys: readonly string[], - keys: readonly string[], - sentAt: number -): SQL { - return sql` - WITH reservation AS ( - SELECT * FROM unnest( - ${sqlArray( - reservations.map((item) => item.key), - 'text' - )}, - ${sqlArray( - reservations.map((item) => item.cost), - 'float8' - )}, - ${sqlArray( - reservations.map((item) => item.config.maxTokens), - 'float8' - )}, - ${sqlArray( - reservations.map((item) => item.config.refillRate), - 'float8' - )}, - ${sqlArray( - reservations.map((item) => item.config.refillIntervalMs), - 'float8' - )} - ) AS r(key, cost, max_tokens, refill_rate, refill_interval_ms) - ), - lock_start AS ( - SELECT clock_timestamp() AS started_at - ), - locked AS ( - SELECT bucket.key, bucket.tokens, bucket.last_refill_at, bucket.blocked_until - FROM ${rateLimitBucket} AS bucket - CROSS JOIN lock_start - WHERE bucket.key = ANY(${sqlArray(keys, 'text')}) - ORDER BY bucket.key - FOR UPDATE OF bucket - ), - clock AS ( - SELECT ${sentAt}::float8 - + GREATEST(0, floor(extract(epoch FROM (held.locked_at - lock_start.started_at)) * 1000)) - AS now_ms - FROM (SELECT clock_timestamp() AS locked_at FROM (SELECT count(*) FROM locked) every_lock) held - CROSS JOIN lock_start - ), - balance AS ( - SELECT - r.key, - r.cost, - r.refill_rate, - r.refill_interval_ms, - LEAST(r.max_tokens, GREATEST(0, l.tokens::float8) + e.elapsed * r.refill_rate) AS available, - m.last_ms + e.elapsed * r.refill_interval_ms AS last_refill_ms, - l.last_refill_at + (e.elapsed * r.refill_interval_ms) * interval '1 millisecond' AS last_refill_at - FROM reservation r - JOIN locked l ON l.key = r.key - CROSS JOIN clock - CROSS JOIN LATERAL ( - SELECT (extract(epoch FROM l.last_refill_at) * 1000)::float8 AS last_ms - ) m - CROSS JOIN LATERAL ( - SELECT GREATEST(0, floor((clock.now_ms - m.last_ms) / r.refill_interval_ms)) AS elapsed - ) e - ), - cooldown AS ( - SELECT (extract(epoch FROM blocked_until) * 1000)::float8 AS blocked_until_ms - FROM locked - WHERE key = ANY(${sqlArray(cooldownKeys, 'text')}) - ), - decision AS ( - SELECT - NOT EXISTS (SELECT 1 FROM balance WHERE available < cost) - AND NOT EXISTS ( - SELECT 1 FROM cooldown, clock WHERE cooldown.blocked_until_ms > clock.now_ms - ) AS allowed - ), - spent AS ( - UPDATE ${rateLimitBucket} AS bucket - SET - tokens = (balance.available - balance.cost)::text::numeric, - last_refill_at = balance.last_refill_at, - updated_at = timestamp '1970-01-01' + clock.now_ms * interval '1 millisecond' - FROM balance, decision, clock - WHERE decision.allowed AND bucket.key = balance.key - RETURNING bucket.key - ), - outcome AS ( - SELECT decision.allowed, (SELECT count(*) FROM spent)::int AS spent, clock.now_ms - FROM decision, clock - ) - SELECT 'reservation' AS kind, b.cost, b.available, b.last_refill_ms, b.refill_rate, - b.refill_interval_ms, NULL::float8 AS blocked_until_ms, o.allowed, o.spent, o.now_ms - FROM balance b CROSS JOIN outcome o - UNION ALL - SELECT 'cooldown', NULL, NULL, NULL, NULL, NULL, c.blocked_until_ms, o.allowed, o.spent, - o.now_ms - FROM cooldown c CROSS JOIN outcome o - ` -} - -/** - * Derives the wait from the locked balances and cross-checks it against the statement's own - * decision, so the JavaScript reading and the SQL spend can never disagree silently: a mismatch, - * or a reservation whose bucket vanished, throws and rolls the spend back. - */ -function readAdmissionResult( - rows: readonly AdmissionRow[], - reservations: readonly TokenBucketReservation[], - now: number -): AtomicAdmissionResult { - let retryAfterMs = 0 - let reserved = 0 - for (const row of rows) { - if (row.kind === 'cooldown') { - if (row.blocked_until_ms !== null) { - retryAfterMs = Math.max(retryAfterMs, Number(row.blocked_until_ms) - now) - } - continue - } - reserved++ - const available = Number(row.available) - const cost = Number(row.cost) - if (available < cost) { - retryAfterMs = Math.max( - retryAfterMs, - Number(row.last_refill_ms) + - Math.ceil((cost - available) / Number(row.refill_rate)) * Number(row.refill_interval_ms) - - now - ) - } - } - if (reserved !== reservations.length) { - throw new Error('Rate limit bucket disappeared during admission') - } - const allowed = rows[0]?.allowed ?? true - const spent = rows[0]?.spent ?? 0 - const spentExpected = allowed ? new Set(reservations.map((item) => item.key)).size : 0 - if (allowed !== retryAfterMs <= 0 || spent !== spentExpected) { - throw new Error('Rate limit admission decision is inconsistent') - } - return allowed ? { allowed: true, retryAfterMs: 0 } : { allowed: false, retryAfterMs } -} - -export class DbTokenBucket implements RateLimitStorageAdapter { - async consumeTokens( - key: string, - requestedTokens: number, - config: TokenBucketConfig - ): Promise { - return db.transaction(async (tx) => { - const createdAt = new Date() - await tx - .insert(rateLimitBucket) - .values({ - key, - tokens: String(config.maxTokens), - lastRefillAt: createdAt, - updatedAt: createdAt, - }) - .onConflictDoNothing() - - const [record] = await tx - .select({ tokens: rateLimitBucket.tokens, lastRefillAt: rateLimitBucket.lastRefillAt }) - .from(rateLimitBucket) - .where(eq(rateLimitBucket.key, key)) - .for('update') - .limit(1) - if (!record) throw new Error('Rate limit bucket disappeared during admission') - - const now = new Date() - const intervalsElapsed = Math.max( - 0, - Math.floor((now.getTime() - record.lastRefillAt.getTime()) / config.refillIntervalMs) - ) - /** Old denied writes used -1; normalize them without granting a new burst. */ - const available = Math.min( - config.maxTokens, - Math.max(0, Number(record.tokens)) + intervalsElapsed * config.refillRate - ) - const allowed = available >= requestedTokens - const tokensRemaining = allowed ? available - requestedTokens : available - const lastRefillAt = new Date( - record.lastRefillAt.getTime() + intervalsElapsed * config.refillIntervalMs - ) - await tx - .update(rateLimitBucket) - .set({ tokens: String(tokensRemaining), lastRefillAt, updatedAt: now }) - .where(eq(rateLimitBucket.key, key)) - - const intervalsUntilAdmission = Math.max( - 1, - Math.ceil((requestedTokens - tokensRemaining) / config.refillRate) - ) - const resetAt = new Date(lastRefillAt.getTime() + config.refillIntervalMs) - return { - allowed, - tokensRemaining, - resetAt, - retryAfterMs: allowed - ? undefined - : Math.max( - 0, - lastRefillAt.getTime() + - intervalsUntilAdmission * config.refillIntervalMs - - now.getTime() - ), - } - }) - } - - /** - * Locks all dimensions and provider gates before spending any capacity. - * - * The lock, refill, decision, and spend run as one statement, so the bucket rows stay locked - * for that statement and the commit rather than for a round trip per dimension: every waiter - * on a hot provider key queues behind that whole hold. Refill and cooldowns are judged at the - * moment the locks are held, as before, so a wait for a contended lock is refilled. - */ - async consumeTokensAtomically( - reservations: readonly TokenBucketReservation[], - options: AtomicAdmissionOptions - ): Promise { - options.signal?.throwIfAborted() - if (Date.now() >= options.deadlineAt) return { allowed: false, retryAfterMs: 0 } - const keys = [ - ...new Set([...options.cooldownKeys, ...reservations.map((item) => item.key)]), - ].sort() - if (keys.length === 0) return { allowed: true, retryAfterMs: 0 } - try { - return await db.transaction(async (tx) => { - const remainingMs = Math.max(1, options.deadlineAt - Date.now()) - await tx.execute( - sql`SELECT set_config('statement_timeout', ${String(remainingMs)}, true), set_config('lock_timeout', ${String(remainingMs)}, true)` - ) - const createdAt = new Date() - await tx - .insert(rateLimitBucket) - .values( - keys.map((key) => { - const reservation = reservations.find((item) => item.key === key) - return { - key, - tokens: String(reservation?.config.maxTokens ?? 0), - lastRefillAt: createdAt, - updatedAt: createdAt, - } - }) - ) - .onConflictDoNothing() - options.signal?.throwIfAborted() - const sentAt = Date.now() - const rows = await tx.execute( - buildAdmissionStatement(reservations, options.cooldownKeys, keys, sentAt) - ) - const now = rows[0]?.now_ms ?? sentAt - options.signal?.throwIfAborted() - if (Math.max(now, Date.now()) >= options.deadlineAt) throw new AdmissionExpiredError() - return readAdmissionResult(rows, reservations, now) - }) - } catch (error) { - if (error instanceof AdmissionExpiredError) return { allowed: false, retryAfterMs: 0 } - throw error - } - } - - async getCooldownUntil(key: string): Promise { - const [row] = await db - .select({ until: rateLimitBucket.blockedUntil }) - .from(rateLimitBucket) - .where(eq(rateLimitBucket.key, key)) - .limit(1) - return row?.until ?? null - } - - async setCooldownUntil(key: string, until: Date): Promise { - const now = new Date() - await db - .insert(rateLimitBucket) - .values({ key, tokens: '0', lastRefillAt: now, updatedAt: now, blockedUntil: until }) - .onConflictDoUpdate({ - target: rateLimitBucket.key, - set: { - blockedUntil: sql`GREATEST(${rateLimitBucket.blockedUntil}, excluded.blocked_until)`, - updatedAt: now, - }, - }) - } - - async getTokenStatus(key: string, config: TokenBucketConfig): Promise { - const now = new Date() - - const [record] = await db - .select({ - tokens: rateLimitBucket.tokens, - lastRefillAt: rateLimitBucket.lastRefillAt, - }) - .from(rateLimitBucket) - .where(eq(rateLimitBucket.key, key)) - .limit(1) - - if (!record) { - return { - tokensAvailable: config.maxTokens, - maxTokens: config.maxTokens, - lastRefillAt: now, - nextRefillAt: new Date(now.getTime() + config.refillIntervalMs), - } - } - - const tokens = Math.max(0, Number.parseFloat(record.tokens)) - const elapsed = now.getTime() - record.lastRefillAt.getTime() - const intervalsElapsed = Math.max(0, Math.floor(elapsed / config.refillIntervalMs)) - const refillAmount = intervalsElapsed * config.refillRate - const tokensAvailable = Math.min(config.maxTokens, tokens + refillAmount) - const lastRefillAt = new Date( - record.lastRefillAt.getTime() + intervalsElapsed * config.refillIntervalMs - ) - - return { - tokensAvailable, - maxTokens: config.maxTokens, - lastRefillAt, - nextRefillAt: new Date(lastRefillAt.getTime() + config.refillIntervalMs), - } - } - - async resetBucket(key: string): Promise { - await db.delete(rateLimitBucket).where(eq(rateLimitBucket.key, key)) - } -} diff --git a/apps/sim/lib/core/rate-limiter/storage/factory.test.ts b/apps/sim/lib/core/rate-limiter/storage/factory.test.ts index bf11142a9d6..5909271b1c8 100644 --- a/apps/sim/lib/core/rate-limiter/storage/factory.test.ts +++ b/apps/sim/lib/core/rate-limiter/storage/factory.test.ts @@ -1,33 +1,17 @@ import { redisConfigMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetStorageMethod, reconnectCallbacks } = vi.hoisted(() => { - const callbacks: Array<() => void> = [] - return { - mockGetStorageMethod: vi.fn(() => 'db'), - reconnectCallbacks: callbacks, - } -}) +const { reconnectCallbacks } = vi.hoisted(() => ({ + reconnectCallbacks: [] as Array<() => void>, +})) const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient const mockOnRedisReconnect = redisConfigMockFns.mockOnRedisReconnect -vi.mock('@/lib/core/storage', () => ({ - getStorageMethod: mockGetStorageMethod, -})) - -vi.mock('@/lib/core/rate-limiter/storage/db-token-bucket', () => ({ - DbTokenBucket: vi.fn().mockImplementation( - class { - type = 'db' - } - ), -})) - vi.mock('@/lib/core/rate-limiter/storage/redis-token-bucket', () => ({ RedisTokenBucket: vi.fn().mockImplementation( class { - type = 'redis' + constructor(readonly redis: unknown) {} } ), })) @@ -36,50 +20,34 @@ import { createStorageAdapter, resetStorageAdapter } from '@/lib/core/rate-limit describe('rate limit storage factory', () => { beforeEach(() => { - mockGetRedisClient.mockReset().mockReturnValue(null) - mockGetStorageMethod.mockReset().mockReturnValue('db') mockOnRedisReconnect.mockImplementation((cb: () => void) => { reconnectCallbacks.push(cb) }) resetStorageAdapter() }) - it('should fall back to DbTokenBucket when Redis is configured but client unavailable', () => { - mockGetStorageMethod.mockReturnValue('redis') - mockGetRedisClient.mockReturnValue(null) + it('surfaces a missing Redis configuration instead of falling back to another backend', () => { + mockGetRedisClient.mockImplementationOnce(() => { + throw new Error('REDIS_URL is required') + }) - const adapter = createStorageAdapter() - expect(adapter).toEqual({ type: 'db' }) - }) + expect(() => createStorageAdapter()).toThrow('REDIS_URL is required') - it('provider admission refuses a configured Redis outage instead of opening a second budget', () => { - mockGetStorageMethod.mockReturnValue('redis') - expect(createStorageAdapter()).toEqual({ type: 'db' }) - expect(() => createStorageAdapter({ requireConfiguredBackend: true })).toThrow( - 'Configured Redis rate limit storage is unavailable' - ) + const recovered = { ping: vi.fn() } + mockGetRedisClient.mockReturnValue(recovered) + expect(createStorageAdapter()).toMatchObject({ redis: recovered }) }) - it('strict admission uses recovered Redis even when the ordinary fallback adapter is cached', () => { - mockGetStorageMethod.mockReturnValue('redis') - createStorageAdapter() - mockGetRedisClient.mockReturnValue({ ping: vi.fn() } as never) - expect(createStorageAdapter({ requireConfiguredBackend: true })).toEqual({ type: 'redis' }) - }) - - it('should re-evaluate storage on next call after reconnect resets cache', () => { - mockGetStorageMethod.mockReturnValue('redis') - mockGetRedisClient.mockReturnValue(null) - - const adapter1 = createStorageAdapter() - expect(adapter1).toEqual({ type: 'db' }) - - const latestCallback = reconnectCallbacks[reconnectCallbacks.length - 1] - latestCallback() + it('rebuilds the adapter against the replacement client after a forced reconnect', () => { + const first = { ping: vi.fn() } + mockGetRedisClient.mockReturnValue(first) + expect(createStorageAdapter()).toMatchObject({ redis: first }) - mockGetRedisClient.mockReturnValue({ ping: vi.fn() } as never) + const replacement = { ping: vi.fn() } + mockGetRedisClient.mockReturnValue(replacement) + expect(createStorageAdapter()).toMatchObject({ redis: first }) - const adapter2 = createStorageAdapter() - expect(adapter2).toEqual({ type: 'redis' }) + reconnectCallbacks[reconnectCallbacks.length - 1]() + expect(createStorageAdapter()).toMatchObject({ redis: replacement }) }) }) diff --git a/apps/sim/lib/core/rate-limiter/storage/factory.ts b/apps/sim/lib/core/rate-limiter/storage/factory.ts index 64a9a071b5c..11d0b89a24e 100644 --- a/apps/sim/lib/core/rate-limiter/storage/factory.ts +++ b/apps/sim/lib/core/rate-limiter/storage/factory.ts @@ -1,11 +1,6 @@ -import { createLogger } from '@sim/logger' import { getRedisClient, onRedisReconnect } from '@/lib/core/config/redis' import type { RateLimitStorageAdapter } from '@/lib/core/rate-limiter/storage/adapter' -import { DbTokenBucket } from '@/lib/core/rate-limiter/storage/db-token-bucket' import { RedisTokenBucket } from '@/lib/core/rate-limiter/storage/redis-token-bucket' -import { getStorageMethod, type StorageMethod } from '@/lib/core/storage' - -const logger = createLogger('RateLimitStorage') type FactoryGlobal = typeof globalThis & { _rlCachedAdapter?: RateLimitStorageAdapter | null @@ -18,16 +13,11 @@ if (!('_rlCachedAdapter' in g)) { g._rlReconnectListenerRegistered = false } -export function createStorageAdapter( - options: { requireConfiguredBackend?: boolean } = {} -): RateLimitStorageAdapter { - /** Provider quotas cannot split across two backends when only some workers lose Redis. */ - if (options.requireConfiguredBackend) { - if (getStorageMethod() !== 'redis') return new DbTokenBucket() - const redis = getRedisClient() - if (!redis) throw new Error('Configured Redis rate limit storage is unavailable') - return new RedisTokenBucket(redis) - } +/** + * The shared Redis token-bucket adapter. It is cached until the Redis health + * check replaces the client, so every caller reaches the current connection. + */ +export function createStorageAdapter(): RateLimitStorageAdapter { if (g._rlCachedAdapter) { return g._rlCachedAdapter } @@ -39,35 +29,11 @@ export function createStorageAdapter( g._rlReconnectListenerRegistered = true } - const storageMethod = getStorageMethod() - - if (storageMethod === 'redis') { - const redis = getRedisClient() - if (!redis) { - logger.warn( - 'Redis configured but client unavailable - falling back to PostgreSQL for rate limiting' - ) - g._rlCachedAdapter = new DbTokenBucket() - } else { - logger.info('Rate limiting: Using Redis') - g._rlCachedAdapter = new RedisTokenBucket(redis) - } - } else { - logger.info('Rate limiting: Using PostgreSQL') - g._rlCachedAdapter = new DbTokenBucket() - } - - return g._rlCachedAdapter! -} - -export function getAdapterType(): StorageMethod { - return getStorageMethod() + const adapter = new RedisTokenBucket(getRedisClient()) + g._rlCachedAdapter = adapter + return adapter } export function resetStorageAdapter(): void { g._rlCachedAdapter = null } - -export function setStorageAdapter(adapter: RateLimitStorageAdapter): void { - g._rlCachedAdapter = adapter -} diff --git a/apps/sim/lib/core/rate-limiter/storage/index.ts b/apps/sim/lib/core/rate-limiter/storage/index.ts index 1bc16f3b31e..c08965b7f69 100644 --- a/apps/sim/lib/core/rate-limiter/storage/index.ts +++ b/apps/sim/lib/core/rate-limiter/storage/index.ts @@ -4,11 +4,5 @@ export type { TokenBucketConfig, TokenStatus, } from './adapter' -export { DbTokenBucket } from './db-token-bucket' -export { - createStorageAdapter, - getAdapterType, - resetStorageAdapter, - setStorageAdapter, -} from './factory' +export { createStorageAdapter, resetStorageAdapter } from './factory' export { RedisTokenBucket } from './redis-token-bucket' diff --git a/apps/sim/lib/core/security/otp.ts b/apps/sim/lib/core/security/otp.ts index 732d2aea2e0..cf893fc4baa 100644 --- a/apps/sim/lib/core/security/otp.ts +++ b/apps/sim/lib/core/security/otp.ts @@ -1,12 +1,7 @@ import { randomInt } from 'crypto' -import { db } from '@sim/db' -import { verification } from '@sim/db/schema' import { safeCompare } from '@sim/security/compare' -import { generateId } from '@sim/utils/id' -import { and, eq, gt, lt, sql } from 'drizzle-orm' import { getRedisClient } from '@/lib/core/config/redis' import type { TokenBucketConfig } from '@/lib/core/rate-limiter' -import { getStorageMethod } from '@/lib/core/storage' export type DeploymentKind = 'chat' | 'file' @@ -37,26 +32,13 @@ export const OTP_RESOURCE_RATE_LIMIT: TokenBucketConfig = { } /** - * Key formats are kept per-kind to preserve any in-flight OTPs already issued - * against existing chat deployments. The chat Redis key uses the legacy `otp:` - * prefix; the chat DB identifier uses `chat-otp:`. + * Redis key per deployment kind. The chat key keeps the legacy `otp:` prefix so + * codes already issued against existing chat deployments stay valid. */ const OTP_KEYS = { - chat: { - redisKey: (email: string, deploymentId: string) => `otp:${email}:${deploymentId}`, - dbIdentifier: (email: string, deploymentId: string) => `chat-otp:${deploymentId}:${email}`, - }, - file: { - redisKey: (email: string, deploymentId: string) => `otp:file:${email}:${deploymentId}`, - dbIdentifier: (email: string, deploymentId: string) => `file-otp:${deploymentId}:${email}`, - }, -} as const satisfies Record< - DeploymentKind, - { - redisKey: (email: string, deploymentId: string) => string - dbIdentifier: (email: string, deploymentId: string) => string - } -> + chat: (email: string, deploymentId: string) => `otp:${email}:${deploymentId}`, + file: (email: string, deploymentId: string) => `otp:file:${email}:${deploymentId}`, +} as const satisfies Record string> /** Returns a cryptographically random 6-digit OTP code. */ export function generateOTP(): string { @@ -65,7 +47,7 @@ export function generateOTP(): string { /** * OTP values are stored as `"code:attempts"` (e.g. `"654321:0"`). - * This keeps the attempt counter in the same key/row as the OTP itself. + * This keeps the attempt counter in the same key as the OTP itself. */ function encodeOTPValue(otp: string, attempts: number): string { return `${otp}:${attempts}` @@ -78,42 +60,20 @@ function decodeOTPValue(value: string): { otp: string; attempts: number } { return { otp: value.slice(0, lastColon), attempts: Number.isNaN(attempts) ? 0 : attempts } } -/** - * Stores an OTP for a deployment+email pair, choosing Redis or the - * `verification` table based on the configured storage method. - */ +/** Stores an OTP for a deployment+email pair in Redis, replacing any previous code. */ export async function storeOTP( kind: DeploymentKind, deploymentId: string, email: string, otp: string ): Promise { - const keys = OTP_KEYS[kind] - const value = encodeOTPValue(otp, 0) - const storageMethod = getStorageMethod() - - if (storageMethod === 'redis') { - const redis = getRedisClient() - if (!redis) throw new Error('Redis configured but client unavailable') - await redis.set(keys.redisKey(email, deploymentId), value, 'EX', OTP_EXPIRY_SECONDS) - return - } - - const now = new Date() - const expiresAt = new Date(now.getTime() + OTP_EXPIRY_MS) - const identifier = keys.dbIdentifier(email, deploymentId) - - await db.transaction(async (tx) => { - await tx.delete(verification).where(eq(verification.identifier, identifier)) - await tx.insert(verification).values({ - id: generateId(), - identifier, - value, - expiresAt, - createdAt: now, - updatedAt: now, - }) - }) + const redis = getRedisClient() + await redis.set( + OTP_KEYS[kind](email, deploymentId), + encodeOTPValue(otp, 0), + 'EX', + OTP_EXPIRY_SECONDS + ) } /** @@ -150,14 +110,6 @@ if not val or val:match('^(.*):%d+$') ~= ARGV[1] then return 0 end return redis.call('DEL', KEYS[1]) ` -/** - * The code and attempt count of a stored `code:attempts` value. Pattern - * matching rather than split + cast: other `verification` rows sharing the - * table must never fail a cast. - */ -const STORED_CODE = sql`substring(${verification.value} from '^(.*):[0-9]+$')` -const STORED_ATTEMPTS = sql`substring(${verification.value} from ':([0-9]+)$')::int` - /** * Atomically spends one of the code's {@link MAX_OTP_ATTEMPTS} attempts and * returns the stored code with its reserved attempt count, or `null` when no @@ -168,33 +120,10 @@ async function reserveOTPAttempt( deploymentId: string, email: string ): Promise<{ otp: string; attempts: number } | null> { - const keys = OTP_KEYS[kind] - const storageMethod = getStorageMethod() - - if (storageMethod === 'redis') { - const redis = getRedisClient() - if (!redis) throw new Error('Redis configured but client unavailable') - const key = keys.redisKey(email, deploymentId) - const result = await redis.eval(RESERVE_ATTEMPT_SCRIPT, 1, key, MAX_OTP_ATTEMPTS) - return typeof result === 'string' ? decodeOTPValue(result) : null - } - - const [reserved] = await db - .update(verification) - .set({ - value: sql`${STORED_CODE} || ':' || (${STORED_ATTEMPTS} + 1)`, - updatedAt: new Date(), - }) - .where( - and( - eq(verification.identifier, keys.dbIdentifier(email, deploymentId)), - gt(verification.expiresAt, new Date()), - lt(STORED_ATTEMPTS, MAX_OTP_ATTEMPTS) - ) - ) - .returning({ value: verification.value }) - - return reserved ? decodeOTPValue(reserved.value) : null + const redis = getRedisClient() + const key = OTP_KEYS[kind](email, deploymentId) + const result = await redis.eval(RESERVE_ATTEMPT_SCRIPT, 1, key, MAX_OTP_ATTEMPTS) + return typeof result === 'string' ? decodeOTPValue(result) : null } export type OTPVerification = 'valid' | 'invalid' | 'locked' | 'missing' @@ -234,21 +163,7 @@ async function consumeOTP( email: string, otp: string ): Promise { - const keys = OTP_KEYS[kind] - const storageMethod = getStorageMethod() - - if (storageMethod === 'redis') { - const redis = getRedisClient() - if (!redis) throw new Error('Redis configured but client unavailable') - const key = keys.redisKey(email, deploymentId) - return (await redis.eval(CONSUME_SCRIPT, 1, key, otp)) === 1 - } - - const consumed = await db - .delete(verification) - .where( - and(eq(verification.identifier, keys.dbIdentifier(email, deploymentId)), eq(STORED_CODE, otp)) - ) - .returning({ id: verification.id }) - return consumed.length > 0 + const redis = getRedisClient() + const key = OTP_KEYS[kind](email, deploymentId) + return (await redis.eval(CONSUME_SCRIPT, 1, key, otp)) === 1 } diff --git a/apps/sim/lib/core/storage/index.ts b/apps/sim/lib/core/storage/index.ts deleted file mode 100644 index 5917c484486..00000000000 --- a/apps/sim/lib/core/storage/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { - getStorageMethod, - isDatabaseStorage, - isRedisStorage, - requireRedis, - type StorageMethod, -} from './storage' diff --git a/apps/sim/lib/core/storage/storage.ts b/apps/sim/lib/core/storage/storage.ts deleted file mode 100644 index 0edd4044f06..00000000000 --- a/apps/sim/lib/core/storage/storage.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' -import { getRedisClient } from '@/lib/core/config/redis' - -const logger = createLogger('Storage') - -export type StorageMethod = 'redis' | 'database' - -let cachedStorageMethod: StorageMethod | null = null - -/** - * Determine the storage method once based on configuration. - * This decision is made at first call and cached for the lifetime of the process. - * - * - If the cache capability selects Redis and the client initializes → 'redis' - * - If the cache capability selects the built-in provider → 'database' - * - * Transient failures do NOT change the storage method. - * If Redis is configured but fails, operations will fail (not fallback to DB). - */ -export function getStorageMethod(): StorageMethod { - if (cachedStorageMethod) { - return cachedStorageMethod - } - - if (getConfiguredCacheProvider() === 'redis') { - const redis = getRedisClient() - if (!redis) throw new Error('REDIS_URL is configured but the Redis client is unavailable') - cachedStorageMethod = 'redis' - logger.info('Storage method: Redis') - } else { - cachedStorageMethod = 'database' - logger.info('Storage method: PostgreSQL') - } - - return cachedStorageMethod -} - -/** - * Check if Redis is the configured storage method. - * Use this for conditional logic that depends on storage type. - */ -export function isRedisStorage(): boolean { - return getStorageMethod() === 'redis' -} - -/** - * Check if database is the configured storage method. - */ -export function isDatabaseStorage(): boolean { - return getStorageMethod() === 'database' -} - -/** - * Get Redis client, but only if Redis is the configured storage method. - * Throws if Redis is configured but client is unavailable. - * - * Use this instead of getRedisClient() directly when you need to ensure - * consistency with the storage method decision. - */ -export function requireRedis() { - if (!isRedisStorage()) { - throw new Error('Redis storage not configured') - } - - const redis = getRedisClient() - if (!redis) { - throw new Error('Redis client unavailable') - } - - return redis -} diff --git a/apps/sim/lib/execution/event-buffer.test.ts b/apps/sim/lib/execution/event-buffer.test.ts index b8ce4cee781..1a6c722eb7a 100644 --- a/apps/sim/lib/execution/event-buffer.test.ts +++ b/apps/sim/lib/execution/event-buffer.test.ts @@ -164,33 +164,6 @@ describe('execution event buffer', () => { }) }) - it('uses the configured in-memory provider when database cache mode is selected', async () => { - setEnv({ REDIS_URL: undefined }) - mockGetRedisClient.mockReturnValue(null) - const executionId = 'exec-database-provider' - - await expect( - initializeExecutionStreamMeta(executionId, { - workflowId: 'wf-1', - userId: 'user-1', - }) - ).resolves.toBe(true) - const writer = createExecutionEventWriter(executionId) - const entry = await writer.write({ - ...makeEvent('function-local'), - executionId, - }) - - await expect(readExecutionEventsState(executionId, 0)).resolves.toMatchObject({ - status: 'ok', - events: [{ eventId: entry.eventId, event: { type: 'block:started' } }], - }) - await expect(readExecutionMetaState(executionId)).resolves.toMatchObject({ - status: 'found', - meta: { status: 'active', protocolVersion: 1 }, - }) - }) - it('retains active runs through the execution ceiling but expires terminal streams after an hour', async () => { await setExecutionMeta('exec-1', { status: 'active' }) await setExecutionMeta('exec-1', { status: 'complete' }) @@ -243,32 +216,6 @@ describe('execution event buffer', () => { expect(ttlSeconds).toBeGreaterThan(60 * 60) }) - it('does not replace a terminal process-local snapshot with a late block event', async () => { - setEnv({ REDIS_URL: undefined }) - mockGetRedisClient.mockReturnValue(null) - const executionId = 'exec-terminal-snapshot' - - await expect(markExecutionStreamTerminal(executionId, 'cancelled')).resolves.toBe(true) - await expect( - setExecutionActiveBlockStarts(executionId, [ - { - eventId: 9, - data: { - blockId: 'function-late', - blockName: 'Late', - blockType: 'function', - executionOrder: 3, - }, - }, - ]) - ).resolves.toBe(true) - - await expect(readExecutionMetaState(executionId)).resolves.toMatchObject({ - status: 'found', - meta: { status: 'cancelled', activeBlockStarts: [] }, - }) - }) - it('serializes event id reservation so reconnect replay preserves write order', async () => { let releaseReservation: ((value: number) => void) | undefined mockRedis.incrby.mockReturnValueOnce( diff --git a/apps/sim/lib/execution/event-buffer.ts b/apps/sim/lib/execution/event-buffer.ts index 79b15b45005..2bb06bf238a 100644 --- a/apps/sim/lib/execution/event-buffer.ts +++ b/apps/sim/lib/execution/event-buffer.ts @@ -1,7 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { randomInt } from '@sim/utils/random' -import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' import { getRedisBudgetKeys, getRedisBudgetLimits } from '@/lib/core/redis/byte-budget.server' @@ -388,8 +387,12 @@ async function compactEventForBuffer( const memoryExecutionStreams = new Map() +/** + * Redis is required, so the in-memory buffer is never selected; its branches are + * unreachable behind `getRedisClient()`, which throws instead of returning null. + */ function canUseMemoryEventBuffer(): boolean { - return typeof window === 'undefined' && getConfiguredCacheProvider() === 'database' + return false } function pruneExpiredMemoryStreams(now = Date.now()): void { diff --git a/apps/sim/lib/execution/isolated-vm.ts b/apps/sim/lib/execution/isolated-vm.ts index edd1a0ce5da..b05e9d9320b 100644 --- a/apps/sim/lib/execution/isolated-vm.ts +++ b/apps/sim/lib/execution/isolated-vm.ts @@ -7,7 +7,6 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { filterUndefined } from '@sim/utils/object' import { randomFloat } from '@sim/utils/random' import { env, envNumber } from '@/lib/core/config/env' -import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { getRedisClient } from '@/lib/core/config/redis' import { captureOutboundScope } from '@/lib/core/network/context.server' @@ -459,8 +458,6 @@ async function tryAcquireDistributedLease( leaseId: string, timeoutMs: number ): Promise { - if (getConfiguredCacheProvider() === 'database') return 'acquired' - const redis = getRedisClient() if (!redis) { logger.warn('No Redis client for distributed lease acquisition; using local limits', { diff --git a/apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts b/apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts index 23fe514e266..e2aaf291d36 100644 --- a/apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts @@ -10,7 +10,6 @@ import { knowledgeBase, organization, outboxEvent, - rateLimitBucket, user, workspace, } from '@sim/db/schema' @@ -30,6 +29,7 @@ vi.mock('@/lib/uploads/core/setup.server', () => ({ import { resolveBillingAttribution } from '@/lib/billing/core/attribution' import { env } from '@/lib/core/config/env' +import { getRedisClient } from '@/lib/core/config/redis' import * as egress from '@/lib/core/security/input-validation.server' import { getMistralCapacityScope } from '@/lib/internal/mistral/capacity' import { @@ -81,7 +81,7 @@ describe('OCR input failures stop without partial indexing or futile retries', ( vi.unstubAllGlobals() Object.assign(env, prior) if (capacityKeys.length) - await db.delete(rateLimitBucket).where(inArray(rateLimitBucket.key, capacityKeys)) + await getRedisClient().del(...capacityKeys.map((key) => `ratelimit:tb:${key}`)) for (const ids of seeded) { await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) diff --git a/apps/sim/lib/knowledge/__integration__/provider-admission.integration.ts b/apps/sim/lib/knowledge/__integration__/provider-admission.integration.ts index 421650dea1c..e8484c1e3f4 100644 --- a/apps/sim/lib/knowledge/__integration__/provider-admission.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/provider-admission.integration.ts @@ -1,22 +1,14 @@ -import { db } from '@sim/db' -import { rateLimitBucket } from '@sim/db/schema' import { readTestRedisUrl } from '@sim/db/testing/test-infrastructure' -import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { eq } from 'drizzle-orm' import Redis from 'ioredis' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -import type { - AtomicAdmissionResult, - RateLimitStorageAdapter, -} from '@/lib/core/rate-limiter/storage/adapter' -import { DbTokenBucket } from '@/lib/core/rate-limiter/storage/db-token-bucket' +import type { RateLimitStorageAdapter } from '@/lib/core/rate-limiter/storage/adapter' import { RedisTokenBucket } from '@/lib/core/rate-limiter/storage/redis-token-bucket' const redisUrl = readTestRedisUrl() -describe.each(['PostgreSQL', 'Redis'] as const)('%s provider admission', (backend) => { - describe.runIf(backend === 'PostgreSQL' || Boolean(redisUrl))('real shared buckets', () => { +describe('Redis provider admission', () => { + describe('real shared buckets', () => { let redis: Redis | undefined let first: RateLimitStorageAdapter let second: RateLimitStorageAdapter @@ -24,15 +16,11 @@ describe.each(['PostgreSQL', 'Redis'] as const)('%s provider admission', (backen let now: Date beforeAll(async () => { - if (backend === 'Redis') { - redis = new Redis(redisUrl!, { lazyConnect: true, maxRetriesPerRequest: 0 }) - await redis.connect() - first = new RedisTokenBucket(redis) - second = new RedisTokenBucket(redis) - } else { - first = new DbTokenBucket() - second = new DbTokenBucket() - } + if (!redisUrl) throw new Error('TEST_REDIS_URL is required') + redis = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) + await redis.connect() + first = new RedisTokenBucket(redis) + second = new RedisTokenBucket(redis) }) beforeEach(() => { @@ -256,136 +244,3 @@ describe.each(['PostgreSQL', 'Redis'] as const)('%s provider admission', (backen }) }) }) - -describe('PostgreSQL admission under a held bucket lock', () => { - const bucket = new DbTokenBucket() - let key: string - let now: number - - /** How long the competing transaction holds the row; long enough to cross a refill interval. */ - const LOCK_HOLD_MS = 1200 - - beforeEach(() => { - now = Date.now() - vi.useFakeTimers({ toFake: ['Date'] }) - vi.setSystemTime(now) - key = `provider-integration:${generateId()}` - }) - - afterEach(async () => { - vi.useRealTimers() - await bucket.resetBucket(key) - }) - - /** Locks the bucket row from another connection, runs `whileHeld`, then commits the holder. */ - async function withRowLockHeld(whileHeld: () => Promise): Promise { - let held: () => void = () => {} - let release: () => void = () => {} - const acquired = new Promise((resolve) => { - held = resolve - }) - const released = new Promise((resolve) => { - release = resolve - }) - const holder = db.transaction(async (tx) => { - await tx.select().from(rateLimitBucket).where(eq(rateLimitBucket.key, key)).for('update') - held() - await released - }) - await acquired - try { - await whileHeld() - } finally { - release() - await holder - } - } - - /** - * Starts an admission and resolves once it is queued behind the held row lock. The result is - * wrapped so awaiting this does not also await the blocked admission. - */ - async function queuedBehindLock( - admit: () => Promise - ): Promise<{ result: Promise }> { - const result = admit() - result.catch(() => {}) - await sleep(300) - return { result } - } - - async function storedTokens(): Promise { - const [row] = await db - .select({ tokens: rateLimitBucket.tokens }) - .from(rateLimitBucket) - .where(eq(rateLimitBucket.key, key)) - return Number(row?.tokens) - } - - it('refills the time spent waiting for the lock and measures the wait from then', async () => { - const reservation = { - key, - cost: 2, - config: { maxTokens: 2, refillRate: 1, refillIntervalMs: 1000 }, - } - const options = { cooldownKeys: [], deadlineAt: now + 60_000 } - expect(await bucket.consumeTokensAtomically([reservation], options)).toEqual({ - allowed: true, - retryAfterMs: 0, - }) - let denied: { result: Promise } | undefined - await withRowLockHeld(async () => { - denied = await queuedBehindLock(() => bucket.consumeTokensAtomically([reservation], options)) - await sleep(LOCK_HOLD_MS - 300) - }) - const result = await denied?.result - expect(result?.allowed).toBe(false) - expect(result?.retryAfterMs).toBeGreaterThan(0) - expect(result?.retryAfterMs).toBeLessThan(1000) - - let admitted: { result: Promise } | undefined - await withRowLockHeld(async () => { - admitted = await queuedBehindLock(() => - bucket.consumeTokensAtomically([{ ...reservation, cost: 1 }], options) - ) - await sleep(LOCK_HOLD_MS - 300) - }) - expect(await admitted?.result).toEqual({ allowed: true, retryAfterMs: 0 }) - }) - - it.each(['deadline', 'abort'] as const)( - 'rolls back a spend whose %s arrives while the statement runs', - async (interruption) => { - const reservation = { - key, - cost: 5, - config: { maxTokens: 10, refillRate: 1, refillIntervalMs: 600_000 }, - } - expect( - await bucket.consumeTokensAtomically([{ ...reservation, cost: 0 }], { - cooldownKeys: [], - deadlineAt: now + 60_000, - }) - ).toEqual({ allowed: true, retryAfterMs: 0 }) - const controller = new AbortController() - let pending: { result: Promise } | undefined - await withRowLockHeld(async () => { - pending = await queuedBehindLock(() => - bucket.consumeTokensAtomically([reservation], { - cooldownKeys: [], - deadlineAt: now + 5000, - signal: controller.signal, - }) - ) - if (interruption === 'deadline') vi.setSystemTime(now + 6000) - else controller.abort(new Error('cancelled')) - }) - if (interruption === 'deadline') { - expect(await pending?.result).toEqual({ allowed: false, retryAfterMs: 0 }) - } else { - await expect(pending?.result).rejects.toThrow('cancelled') - } - expect(await storedTokens()).toBe(10) - } - ) -}) diff --git a/apps/sim/lib/knowledge/__integration__/provider-fixture-state.ts b/apps/sim/lib/knowledge/__integration__/provider-fixture-state.ts index 40bb79fc5c3..939aa04ef6c 100644 --- a/apps/sim/lib/knowledge/__integration__/provider-fixture-state.ts +++ b/apps/sim/lib/knowledge/__integration__/provider-fixture-state.ts @@ -1,6 +1,4 @@ -import { db } from '@sim/db' -import { rateLimitBucket } from '@sim/db/schema' -import { inArray } from 'drizzle-orm' +import { getRedisClient } from '@/lib/core/config/redis' /** Matches OpenAI's requested Float32 base64 transport with a deterministic unit vector. */ export function createFixtureOpenAIEmbedding(dimensions = 1536): string { @@ -12,10 +10,9 @@ export function createFixtureOpenAIEmbedding(dimensions = 1536): string { /** Prevents simulated processing clocks from leaking future shared balances into later fixtures. */ export async function resetHostedEmbeddingFixtureAdmission(): Promise { const prefix = 'provider:embedding:openai:hosted:openai' - await db.delete(rateLimitBucket).where( - inArray( - rateLimitBucket.key, - ['requests', 'tokens', 'cooldown', 'quota'].map((dimension) => `${prefix}:${dimension}`) + await getRedisClient().del( + ...['requests', 'tokens', 'cooldown', 'quota'].map( + (dimension) => `ratelimit:tb:${prefix}:${dimension}` ) ) } diff --git a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts index 9fd971d6e28..b6d1f8b5253 100644 --- a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts @@ -13,7 +13,6 @@ import { knowledgeBase, organization, outboxEvent, - rateLimitBucket, usageLog, user, workspace, @@ -21,7 +20,7 @@ import { import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { sha256Hex } from '@sim/security/hash' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, sql } from 'drizzle-orm' +import { and, eq, inArray } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const fixtureStorage = vi.hoisted(() => ({ root: '' })) @@ -33,6 +32,7 @@ vi.mock('@/lib/uploads/core/setup.server', () => ({ import { resolveBillingAttribution } from '@/lib/billing/core/attribution' import { env } from '@/lib/core/config/env' +import { getRedisClient } from '@/lib/core/config/redis' import { processOutboxEventById } from '@/lib/core/outbox/service' import * as egress from '@/lib/core/security/input-validation.server' import { getMistralCapacityScope } from '@/lib/internal/mistral/capacity' @@ -93,7 +93,7 @@ describe('provider throttling resumes the shared indexing pipeline', () => { Object.assign(env, prior) if (events.length) await db.delete(outboxEvent).where(inArray(outboxEvent.id, events)) if (capacityKeys.length) - await db.delete(rateLimitBucket).where(inArray(rateLimitBucket.key, capacityKeys)) + await getRedisClient().del(...capacityKeys.map((key) => `ratelimit:tb:${key}`)) for (const ids of seeded) { await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) @@ -337,13 +337,16 @@ describe('provider throttling resumes the shared indexing pipeline', () => { vi.useFakeTimers({ toFake: ['Date'] }) vi.setSystemTime(event.availableAt.getTime() + 1) - /** PostgreSQL owns the capacity clock; model its elapsed cooldown without a ten-minute test sleep. */ - await db - .update(rateLimitBucket) - .set({ - capacityState: sql`${rateLimitBucket.capacityState} || '{"cooldownUntil":0,"nextRequestAt":0,"pageTokens":30}'::jsonb`, - }) - .where(eq(rateLimitBucket.key, capacityKey)) + /** Redis owns the capacity clock; model its elapsed cooldown without a ten-minute test sleep. */ + const capacityRedisKey = `ratelimit:tb:${capacityKey}` + const capacityState = JSON.parse( + (await getRedisClient().hget(capacityRedisKey, 'capacityState')) ?? '{}' + ) + await getRedisClient().hset( + capacityRedisKey, + 'capacityState', + JSON.stringify({ ...capacityState, cooldownUntil: 0, nextRequestAt: 0, pageTokens: 30 }) + ) expect( await processOutboxEventById(eventId, knowledgeDocumentProcessingOutboxHandlers) ).toBe('completed') diff --git a/apps/sim/lib/knowledge/__integration__/reranker-admission.integration.ts b/apps/sim/lib/knowledge/__integration__/reranker-admission.integration.ts index 7388314221d..c1794f2a612 100644 --- a/apps/sim/lib/knowledge/__integration__/reranker-admission.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/reranker-admission.integration.ts @@ -1,16 +1,17 @@ -/** Real PostgreSQL admission with a synthetic provider response; no Cohere credential is used. */ +/** Real Redis admission with a synthetic provider response; no Cohere credential is used. */ import { db } from '@sim/db' import { sha256Hex } from '@sim/security/hash' import { generateId } from '@sim/utils/id' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import { env } from '@/lib/core/config/env' -import { DbTokenBucket } from '@/lib/core/rate-limiter/storage/db-token-bucket' +import { getRedisClient } from '@/lib/core/config/redis' +import { RedisTokenBucket } from '@/lib/core/rate-limiter/storage/redis-token-bucket' import { rerank } from '@/lib/knowledge/reranker' -describe('reranker shared PostgreSQL admission', () => { +describe('reranker shared Redis admission', () => { const keys = [generateId(), generateId()] const originalKey = env.COHERE_API_KEY - const storage = new DbTokenBucket() + const storage = new RedisTokenBucket(getRedisClient()) const upstream = vi.fn() const options = { model: 'rerank-v4.0-fast' } const items = [{ id: 'fixture-chunk', text: 'Synthetic Orion release checklist' }] diff --git a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts index 3201a0c2af7..f406e9f0573 100644 --- a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts +++ b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts @@ -16,7 +16,6 @@ import { publicShare, subscription, user, - verification, workspace, workspaceFileSecretProvenance, workspaceFiles, @@ -575,12 +574,10 @@ describe('Public file shares against PostgreSQL and private storage', () => { await ( await requestPublicFileShareOtp({ token: share.token, email: 'outsider@shared.invalid' }) ).deliver() + const { getRedisClient } = await import('@/lib/core/config/redis') const storedCode = async (address: string) => { - const [row] = await db - .select({ value: verification.value }) - .from(verification) - .where(eq(verification.identifier, `file-otp:${share.id}:${address}`)) - return row?.value.split(':')[0] + const value = await getRedisClient().get(`otp:file:${address}:${share.id}`) + return value?.split(':')[0] } expect(await storedCode('outsider@shared.invalid')).toBeUndefined() const failure = { authorized: false, status: 400, error: 'Invalid verification code' } diff --git a/apps/sim/lib/realtime/event-log.test.ts b/apps/sim/lib/realtime/event-log.test.ts deleted file mode 100644 index 34037f8c073..00000000000 --- a/apps/sim/lib/realtime/event-log.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' -import { afterAll, beforeEach, describe, expect, it } from 'vitest' -import { - appendEvent, - type EventLogConfig, - type EventLogEntry, - readEventsSince, - resetEventLogMemoryForTesting, -} from '@/lib/realtime/event-log' - -interface TestEntry extends EventLogEntry { - eventId: number - streamId: string - value: string -} - -const config: EventLogConfig = { - prefix: 'test:stream:', - ttlSeconds: 3600, - cap: 3, - maxBytes: 0, - readChunk: 500, -} - -function serializerFor(streamId: string, value: string) { - return { - entryPrefix: '{"eventId":', - entrySuffix: `,"streamId":${JSON.stringify(streamId)},"value":${JSON.stringify(value)}}`, - buildEntry: (eventId: number): TestEntry => ({ eventId, streamId, value }), - } -} - -afterAll(resetEnvMock) - -describe('event-log (memory fallback)', () => { - beforeEach(() => { - setEnv({ REDIS_URL: undefined, REDIS_TLS_SERVERNAME: undefined }) - resetEventLogMemoryForTesting() - }) - - it('reads only events after the cursor', async () => { - await appendEvent(config, 's1', serializerFor('s1', 'a')) - await appendEvent(config, 's1', serializerFor('s1', 'b')) - const result = await readEventsSince(config, 's1', 1) - expect(result.status).toBe('ok') - if (result.status === 'ok') { - expect(result.events).toHaveLength(1) - expect(result.events[0].eventId).toBe(2) - expect(result.events[0].value).toBe('b') - } - }) - - it('reports pruned when the cursor falls behind the cap-trimmed buffer', async () => { - // cap = 3; append 5, so the earliest retained id is 3. - for (const v of ['a', 'b', 'c', 'd', 'e']) { - await appendEvent(config, 's1', serializerFor('s1', v)) - } - const result = await readEventsSince(config, 's1', 1) - expect(result.status).toBe('pruned') - if (result.status === 'pruned') expect(result.earliestEventId).toBe(3) - }) - - it('reports pruned for a non-zero cursor against a never-seen stream', async () => { - const result = await readEventsSince(config, 'missing', 5) - expect(result.status).toBe('pruned') - }) - - it('does not use memory when Redis is selected but its client is unavailable', async () => { - setEnv({ REDIS_URL: 'redis://localhost:6379' }) - - await expect(appendEvent(config, 's1', serializerFor('s1', 'a'))).resolves.toBeNull() - await expect(readEventsSince(config, 's1', 0)).resolves.toEqual({ - status: 'unavailable', - error: 'Redis client unavailable', - }) - }) - - it('fails fast instead of using memory for an invalid Redis configuration', async () => { - setEnv({ REDIS_URL: 'https://cache.example.com' }) - - await expect(appendEvent(config, 's1', serializerFor('s1', 'a'))).rejects.toThrow( - /valid redis:\/\/ or rediss:\/\/ URL/ - ) - }) -}) - -describe('event-log byte ceiling', () => { - beforeEach(() => { - setEnv({ REDIS_URL: undefined, REDIS_TLS_SERVERNAME: undefined }) - resetEventLogMemoryForTesting() - }) - - /** - * The entry cap is what let one writer hold hundreds of megabytes: `cap` bounds how - * many entries a stream keeps and nothing about how large each one is. - */ - it('drops oldest entries once the buffer exceeds maxBytes, under the entry cap', async () => { - const bounded: EventLogConfig = { ...config, cap: 1000, maxBytes: 400 } - const big = 'x'.repeat(150) - - for (let i = 0; i < 6; i++) { - await appendEvent(bounded, 's1', serializerFor('s1', big)) - } - - const fromStart = await readEventsSince(bounded, 's1', 0) - expect(fromStart.status).toBe('pruned') - const earliest = fromStart.status === 'pruned' ? fromStart.earliestEventId : undefined - expect(earliest).toBeGreaterThan(1) - - const retained = await readEventsSince(bounded, 's1', (earliest as number) - 1) - expect(retained.status).toBe('ok') - const events = retained.status === 'ok' ? retained.events : [] - expect(events.at(-1)?.eventId).toBe(6) - const bytes = events.reduce((total, e) => total + JSON.stringify(e).length, 0) - expect(bytes).toBeLessThanOrEqual(400) - }) - - it('keeps the newest entry even when it alone exceeds maxBytes', async () => { - const bounded: EventLogConfig = { ...config, cap: 1000, maxBytes: 10 } - await appendEvent(bounded, 's1', serializerFor('s1', 'a')) - await appendEvent(bounded, 's1', serializerFor('s1', 'b'.repeat(500))) - - const result = await readEventsSince(bounded, 's1', 1) - expect(result.status).toBe('ok') - const events = result.status === 'ok' ? result.events : [] - expect(events).toHaveLength(1) - expect(events[0]?.eventId).toBe(2) - }) -}) diff --git a/apps/sim/lib/realtime/event-log.ts b/apps/sim/lib/realtime/event-log.ts index f002b6a1c89..d7d32278fc9 100644 --- a/apps/sim/lib/realtime/event-log.ts +++ b/apps/sim/lib/realtime/event-log.ts @@ -1,6 +1,5 @@ /** - * Generic durable event log over Redis (sorted set + monotonic id + TTL), with an - * in-memory fallback for dev/tests. This is the reusable core extracted from the + * Generic durable event log over Redis (sorted set + monotonic id + TTL). This is the reusable core extracted from the * Tables cell-event buffer; a domain adapter (e.g. `lib/table/events.ts`) supplies * its Redis key prefix and how to serialize an entry, and gets append/read/tail * semantics for free — including replay-on-reconnect and prune detection. @@ -16,7 +15,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' const logger = createLogger('EventLog') @@ -116,11 +114,10 @@ export interface EventLogEntry { /** * How an adapter serializes one entry. `entryPrefix`/`entrySuffix` are spliced * around the minted `eventId` in Lua (`prefix + eventId + suffix`) for the Redis - * write; `buildEntry` returns the equivalent object. It is the canonical entry - * builder for BOTH paths — the in-memory fallback AND the Redis success path, - * which returns `buildEntry(eventId)` rather than re-parsing the stored string — - * so it MUST produce the byte-identical shape to the spliced JSON, or dev/no-Redis - * behaves differently from prod. + * write; `buildEntry` returns the equivalent object. The append path returns + * `buildEntry(eventId)` rather than re-parsing the stored string, while readers + * parse the stored string — so it MUST produce the byte-identical shape to the + * spliced JSON, or a live append and its replay disagree. */ export interface EntrySerializer { entryPrefix: string @@ -143,41 +140,6 @@ function metaKey(config: EventLogConfig, streamId: string) { return `${config.prefix}${streamId}:meta` } -interface MemoryStream { - events: E[] - earliestEventId?: number - nextEventId: number - expiresAt: number -} - -/** In-memory fallback keyed by `${prefix}${streamId}`, shared across all families. */ -const memoryStreams = new Map>() - -function memoryKey(config: EventLogConfig, streamId: string) { - return `${config.prefix}${streamId}` -} - -function canUseMemoryBuffer(): boolean { - return typeof window === 'undefined' && getConfiguredCacheProvider() === 'database' -} - -function pruneExpiredMemoryStreams(now = Date.now()): void { - for (const [key, stream] of memoryStreams) { - if (stream.expiresAt <= now) memoryStreams.delete(key) - } -} - -function getMemoryStream(config: EventLogConfig, streamId: string): MemoryStream { - pruneExpiredMemoryStreams() - const key = memoryKey(config, streamId) - let stream = memoryStreams.get(key) - if (!stream) { - stream = { events: [], nextEventId: 1, expiresAt: Date.now() + config.ttlSeconds * 1000 } - memoryStreams.set(key, stream) - } - return stream -} - /** * Append an event. Fire-and-forget from the caller — never throws, returns null on * failure. A Redis blip must not fail the originating mutation. @@ -188,39 +150,6 @@ export async function appendEvent( serializer: EntrySerializer ): Promise { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - try { - const stream = getMemoryStream(config, streamId) - const entry = serializer.buildEntry(stream.nextEventId++) - stream.events.push(entry) - if (stream.events.length > config.cap) { - stream.events = stream.events.slice(-config.cap) - } - if (config.maxBytes > 0) { - // UTF-8 bytes, so this path bounds a stream identically to the Lua's `string.len`; - // `String.length` counts UTF-16 units and under-reports every non-ASCII event. - const entryBytes = (event: EventLogEntry) => - Buffer.byteLength(JSON.stringify(event), 'utf8') - let bytes = stream.events.reduce((total, event) => total + entryBytes(event), 0) - while (bytes > config.maxBytes && stream.events.length > 1) { - bytes -= entryBytes(stream.events[0]) - stream.events = stream.events.slice(1) - } - } - stream.earliestEventId = stream.events[0]?.eventId - stream.expiresAt = Date.now() + config.ttlSeconds * 1000 - return entry - } catch (error) { - logger.warn('appendEvent: memory append failed', { - streamId, - error: toError(error).message, - }) - return null - } - } - return null - } try { const result = await redis.eval( APPEND_EVENT_SCRIPT, @@ -252,13 +181,6 @@ export async function appendEvent( */ export async function getLatestEventId(config: EventLogConfig, streamId: string): Promise { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - const stream = memoryStreams.get(memoryKey(config, streamId)) - return stream ? stream.nextEventId - 1 : 0 - } - return 0 - } const raw = await redis.get(seqKey(config, streamId)) if (!raw) return 0 const parsed = Number.parseInt(raw, 10) @@ -276,26 +198,6 @@ export async function readEventsSince( afterEventId: number ): Promise> { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryBuffer()) { - pruneExpiredMemoryStreams() - const stream = memoryStreams.get(memoryKey(config, streamId)) - if (!stream) { - if (afterEventId > 0) return { status: 'pruned', earliestEventId: undefined } - return { status: 'ok', events: [] } - } - if (stream.earliestEventId !== undefined && afterEventId + 1 < stream.earliestEventId) { - return { status: 'pruned', earliestEventId: stream.earliestEventId } - } - return { - status: 'ok', - events: stream.events - .filter((entry) => entry.eventId > afterEventId) - .slice(0, config.readChunk) as E[], - } - } - return { status: 'unavailable', error: 'Redis client unavailable' } - } try { const meta = await redis.hgetall(metaKey(config, streamId)) const earliestEventId = @@ -335,8 +237,3 @@ export async function readEventsSince( return { status: 'unavailable', error: message } } } - -/** Test-only: clear the in-memory streams between cases. */ -export function resetEventLogMemoryForTesting(): void { - memoryStreams.clear() -} diff --git a/apps/sim/lib/table/events.test.ts b/apps/sim/lib/table/events.test.ts deleted file mode 100644 index df11e25784a..00000000000 --- a/apps/sim/lib/table/events.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { resetEnvMock, setEnv } from '@sim/testing' -import { afterAll, beforeAll, describe, expect, it } from 'vitest' - -beforeAll(() => { - setEnv({ REDIS_URL: undefined }) -}) - -afterAll(resetEnvMock) - -import type { TableEvent } from '@/lib/table/events' -import { appendTableEvent, getLatestTableEventId, readTableEventsSince } from '@/lib/table/events' - -/** Module-level memory buffer can't be reset without vi.resetModules — use a - * unique tableId per test to avoid cross-test bleed. */ -let seq = 0 -function uniqueTableId(): string { - seq++ - return `table-events-test-${seq}` -} - -function cellEvent(tableId: string): TableEvent { - return { - kind: 'cell', - tableId, - rowId: 'row-1', - groupId: 'group-1', - status: 'running', - } -} - -describe('getLatestTableEventId (memory buffer)', () => { - it('returns the latest assigned eventId after appends', async () => { - const tableId = uniqueTableId() - await appendTableEvent(cellEvent(tableId)) - const second = await appendTableEvent(cellEvent(tableId)) - expect(second?.eventId).toBe(2) - expect(await getLatestTableEventId(tableId)).toBe(2) - }) - - it('tailing from the latest id yields no replayed events', async () => { - const tableId = uniqueTableId() - await appendTableEvent(cellEvent(tableId)) - await appendTableEvent(cellEvent(tableId)) - const latest = await getLatestTableEventId(tableId) - const result = await readTableEventsSince(tableId, latest) - expect(result).toEqual({ status: 'ok', events: [] }) - }) - - it('a subsequent append is visible to a reader tailing from the prior latest', async () => { - const tableId = uniqueTableId() - await appendTableEvent(cellEvent(tableId)) - const latest = await getLatestTableEventId(tableId) - await appendTableEvent(cellEvent(tableId)) - const result = await readTableEventsSince(tableId, latest) - expect(result.status).toBe('ok') - if (result.status === 'ok') { - expect(result.events).toHaveLength(1) - expect(result.events[0].eventId).toBe(latest + 1) - } - }) -}) diff --git a/apps/sim/lib/table/events.ts b/apps/sim/lib/table/events.ts index d1c507ec60b..5eefab4553e 100644 --- a/apps/sim/lib/table/events.ts +++ b/apps/sim/lib/table/events.ts @@ -188,8 +188,8 @@ export type TableEventsReadResult = EventLogReadResult /** * Append an event to the table's buffer. Fire-and-forget — never throws, returns - * null on failure (a Redis blip must not fail a cell-write). The Redis (Lua splice) - * and in-memory paths are built to produce byte-identical entries. + * null on failure (a Redis blip must not fail a cell-write). The returned entry is + * built to match the Lua-spliced stored entry byte for byte. */ export async function appendTableEvent(event: TableEvent): Promise { return appendEvent(TABLE_EVENT_LOG, event.tableId, { diff --git a/apps/sim/vitest.integration.setup.ts b/apps/sim/vitest.integration.setup.ts index b28eb7f8000..9a204df41f2 100644 --- a/apps/sim/vitest.integration.setup.ts +++ b/apps/sim/vitest.integration.setup.ts @@ -5,15 +5,21 @@ import { readTestDatabaseUrl, readTestRedisUrl, } from '@sim/db/testing/test-infrastructure' +import Redis from 'ioredis' import { afterAll } from 'vitest' /** - * Setup for `vitest run --mode integration`. Suites run against real PostgreSQL (and Redis when - * `TEST_REDIS_URL` is set) with real application modules: none of the unit-test global mocks are - * installed here, so a suite that needs a fixture declares it itself. + * Setup for `vitest run --mode integration`. Suites run against real PostgreSQL and Redis with + * real application modules: none of the unit-test global mocks are installed here, so a suite + * that needs a fixture declares it itself. */ const databaseUrl = readTestDatabaseUrl() -readTestRedisUrl() +const redisUrl = readTestRedisUrl() +if (!redisUrl) { + throw new Error( + 'Set TEST_REDIS_URL: Redis is required. `bun run test:integration` provisions a disposable one.' + ) +} const encryptionKey = process.env.TEST_ENCRYPTION_KEY ?? '0'.repeat(64) if (!/^[a-f0-9]{64}$/i.test(encryptionKey)) { throw new Error('TEST_ENCRYPTION_KEY must be a 64-character hexadecimal fixture key') @@ -28,6 +34,7 @@ clearInheritedDatabaseUrls() Object.assign(process.env, { NODE_ENV: 'test', DATABASE_URL: databaseUrl, + REDIS_URL: redisUrl, DB_TX_TRIPWIRE: 'throw', NEXT_PUBLIC_APP_URL: 'http://localhost:3000', NEXT_PUBLIC_FORCE_HOSTED: 'false', @@ -49,6 +56,19 @@ Object.assign(process.env, { EGRESS_ALLOWED_HOSTS: process.env.GITLAB_LIVE_FIXTURE_FILE ? 'localhost' : '', }) +/** + * Every file starts from an empty Redis, so keys one file leaves behind never decide another's + * outcome. Files run one at a time (`fileParallelism: false`), and `readTestRedisUrl` only accepts + * a credential-free loopback URL, which is the disposable instance the harness provisions. + */ +const redisReset = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 1 }) +try { + await redisReset.connect() + await redisReset.flushdb() +} finally { + redisReset.disconnect() +} + /** Local stand-in for the realtime process; deployments still perform their real HTTP notification. */ const realtime = createServer(async (request, response) => { for await (const _chunk of request) { diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 9c2a878fe7b..6de022a60ca 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -1861,6 +1861,7 @@ export const subscription = pgTable( }) ) +// contract-pending(after the release that stops using it is deployed): drop rate_limit_bucket — Redis is required and the Postgres token bucket and provider-capacity store were removed, so no deployed release reads or writes it. export const rateLimitBucket = pgTable('rate_limit_bucket', { key: text('key').primaryKey(), tokens: decimal('tokens').notNull(), diff --git a/packages/deployment-config/src/env-capabilities.ts b/packages/deployment-config/src/env-capabilities.ts index 16841554756..9d464e1e87b 100644 --- a/packages/deployment-config/src/env-capabilities.ts +++ b/packages/deployment-config/src/env-capabilities.ts @@ -22,6 +22,7 @@ export const CORE_CONFIGURATION_KEYS = [ 'NEXT_PUBLIC_APP_URL', 'ENCRYPTION_KEY', 'INTERNAL_API_SECRET', + 'REDIS_URL', ] as const export type EnvCapabilityValues = @@ -1141,31 +1142,42 @@ export const ASYNC_JOBS_CAPABILITY = defineCapability({ ], } as const) -/** Validates the one provider dependency that cannot be expressed as a field-shape rule. */ -function validateRedisProvider(values: EnvCapabilityValues): readonly EnvProviderValidationIssue[] { - if (!hasValue(values, 'REDIS_URL')) return [] +const REDIS_URL_PROTOCOLS = ['redis:', 'rediss:'] as const + +/** + * Validates the required Redis configuration: REDIS_URL must be a `redis://` or + * `rediss://` URL, and a `rediss://` URL targeting a bare IPv4 address needs + * REDIS_TLS_SERVERNAME, because certificate hostname verification cannot match + * an IP. Returns the first problem, or `null` when the configuration is usable. + */ +export function getRedisConfigurationError(values: EnvCapabilityValues): string | null { + if (!hasValue(values, 'REDIS_URL')) return 'REDIS_URL is required' let redisUrl: URL try { redisUrl = new URL(String(readValue(values, 'REDIS_URL'))) } catch { - return [] + return 'REDIS_URL must be a valid redis:// or rediss:// URL' + } + if (!(REDIS_URL_PROTOCOLS as readonly string[]).includes(redisUrl.protocol)) { + return 'REDIS_URL must be a valid redis:// or rediss:// URL' } if ( redisUrl.protocol === 'rediss:' && /^\d+\.\d+\.\d+\.\d+$/.test(redisUrl.hostname) && !hasValue(values, 'REDIS_TLS_SERVERNAME') ) { - return [ - { - kind: 'missing', - fields: ['REDIS_TLS_SERVERNAME'], - message: 'REDIS_TLS_SERVERNAME is required for rediss:// IP addresses', - }, - ] + return 'REDIS_TLS_SERVERNAME is required for rediss:// IP addresses' } - return [] + return null } +/** + * The pre-required-Redis cache capability, kept only for the `sim-setup` wizard + * until its setup flow treats REDIS_URL as a core value. It is not part of + * {@link ENV_CAPABILITIES} and the app never reads it: Redis is required. + * + * @deprecated Use {@link getRedisConfigurationError}; REDIS_URL is a core configuration key. + */ export const CACHE_CAPABILITY = defineCapability({ strategy: 'selected', id: 'cache', @@ -1180,12 +1192,17 @@ export const CACHE_CAPABILITY = defineCapability({ requires: envField('REDIS_URL', { validation: { kind: 'url', - protocols: ['redis:', 'rediss:'], + protocols: REDIS_URL_PROTOCOLS, message: 'must be a valid redis:// or rediss:// URL', }, }), optionalFields: [envField('REDIS_TLS_SERVERNAME')], - validate: validateRedisProvider, + validate: (values) => { + const error = getRedisConfigurationError(values) + return error?.startsWith('REDIS_TLS_SERVERNAME') + ? [{ kind: 'missing', fields: ['REDIS_TLS_SERVERNAME'], message: error }] + : [] + }, }, ], } as const) @@ -1516,7 +1533,6 @@ export const ENV_CAPABILITIES = [ STORAGE_CAPABILITY, SANDBOX_CAPABILITY, ASYNC_JOBS_CAPABILITY, - CACHE_CAPABILITY, OCR_CAPABILITY, KNOWLEDGE_EMBEDDINGS_CAPABILITY, ] as const @@ -1556,6 +1572,7 @@ export const DEPLOYMENT_CONFIGURATION_KEYS: readonly string[] = [ ...new Set([ ...CORE_CONFIGURATION_KEYS, ...ENV_CAPABILITIES.flatMap(capabilityKeys), + 'REDIS_TLS_SERVERNAME', 'COPILOT_API_KEY', 'EMAIL_VERIFICATION_ENABLED', 'NEXT_PUBLIC_CHAT_DISABLED', diff --git a/packages/sim-setup/src/capability-config.test.ts b/packages/sim-setup/src/capability-config.test.ts index 785e407c982..92e200774f4 100644 --- a/packages/sim-setup/src/capability-config.test.ts +++ b/packages/sim-setup/src/capability-config.test.ts @@ -8,9 +8,11 @@ import { CAPABILITY_SETUPS, defineCapabilitySetup } from './capability-config' describe('capability setup configuration', () => { it('maps every runtime capability and provider exactly once', () => { - expect(CAPABILITY_SETUPS.map((setup) => setup.definition.id)).toEqual( - ENV_CAPABILITIES.map((capability) => capability.id) - ) + // The cache setup outlives the runtime capability until the wizard prompts for + // REDIS_URL as a required core value. + expect( + CAPABILITY_SETUPS.map((setup) => setup.definition.id).filter((id) => id !== 'cache') + ).toEqual(ENV_CAPABILITIES.map((capability) => capability.id)) for (const setup of CAPABILITY_SETUPS) { expect(Object.keys(setup.providers).sort()).toEqual( setup.definition.providers.map((provider) => provider.id).sort() diff --git a/packages/sim-setup/src/setup-status.test.ts b/packages/sim-setup/src/setup-status.test.ts index ed2625e59be..73cacf08eb9 100644 --- a/packages/sim-setup/src/setup-status.test.ts +++ b/packages/sim-setup/src/setup-status.test.ts @@ -19,6 +19,7 @@ const CORE_VALUES = { NEXT_PUBLIC_APP_URL: 'http://localhost:3000', ENCRYPTION_KEY: 'b'.repeat(64), INTERNAL_API_SECRET: 'c'.repeat(32), + REDIS_URL: 'redis://localhost:6379', } describe('setup status', () => { diff --git a/packages/testing/src/mocks/env.mock.ts b/packages/testing/src/mocks/env.mock.ts index 6e555c4e05b..44b9e68a7c1 100644 --- a/packages/testing/src/mocks/env.mock.ts +++ b/packages/testing/src/mocks/env.mock.ts @@ -18,6 +18,7 @@ export const defaultMockEnv = { BETTER_AUTH_SECRET: 'test-secret-that-is-at-least-32-chars-long', ENCRYPTION_KEY: 'test-encryption-key-32-chars-long!', INTERNAL_API_SECRET: 'test-internal-api-secret-32-chars!', + REDIS_URL: 'redis://localhost:6379', // Email RESEND_API_KEY: 'test-resend-key', @@ -25,9 +26,6 @@ export const defaultMockEnv = { EMAIL_DOMAIN: 'test.sim.ai', PERSONAL_EMAIL_FROM: 'Test ', - // Cache - REDIS_URL: undefined, - // Storage STORAGE_PROVIDER: 'local', diff --git a/packages/testing/src/mocks/redis-config.mock.ts b/packages/testing/src/mocks/redis-config.mock.ts index 4103f5ce377..7803653c9f8 100644 --- a/packages/testing/src/mocks/redis-config.mock.ts +++ b/packages/testing/src/mocks/redis-config.mock.ts @@ -1,5 +1,6 @@ import { vi } from 'vitest' import { envMockFns } from './env.mock' +import { createMockRedis } from './redis.mock' /** * Mirrors the real `resolveRedisTlsOptions`: `rediss://` URLs targeting a raw @@ -45,14 +46,15 @@ function getRedisConnectionDefaultsImpl(url?: string): { } } +const DEFAULT_REDIS_URL = 'redis://localhost:6379' + /** - * Mirrors the real `describeRedisConnection` under its Redis-unavailable - * default: no client, no lifecycle history, and nothing derivable from an - * unset REDIS_URL. + * Mirrors the real `describeRedisConnection` for the default mock client: a + * ready connection to a DNS-named, non-TLS host with no lifecycle history. */ function describeRedisConnectionImpl() { return { - status: 'no-client', + status: 'ready', clientAgeMs: null, readyAgeMs: null, msSinceLastPingOk: null, @@ -60,7 +62,7 @@ function describeRedisConnectionImpl() { reconnects: 0, errors: 0, lastErrorMessage: null, - hostKind: 'unknown' as const, + hostKind: 'dns' as const, tls: false, sniOverride: false, } @@ -68,11 +70,12 @@ function describeRedisConnectionImpl() { /** * Controllable mock functions for `@/lib/core/config/redis`. - * Default: `getConfiguredRedisUrl` and `getRedisClient` return `null` (tests - * that need Redis override them), matching the real module's database-cache - * behavior. - * `acquireLock`/`releaseLock`/`extendLock` default to succeeding (`true`), - * matching the real module's Redis-unavailable no-op path. + * Redis is required, so by default `getConfiguredRedisUrl` returns a local URL + * and `getRedisClient` returns one {@link createMockRedis} client (the same + * instance on every call until {@link resetRedisConfigMock} replaces it). Tests + * that assert on Redis traffic override it with their own client. + * `acquireLock`/`releaseLock`/`extendLock` default to succeeding (`true`): the + * lock is acquired, still owned, and released. * {@link resetRedisConfigMock} restores the default behaviors. * * @example @@ -83,8 +86,8 @@ function describeRedisConnectionImpl() { * ``` */ export const redisConfigMockFns = { - mockGetConfiguredRedisUrl: vi.fn().mockReturnValue(null), - mockGetRedisClient: vi.fn().mockReturnValue(null), + mockGetConfiguredRedisUrl: vi.fn().mockReturnValue(DEFAULT_REDIS_URL), + mockGetRedisClient: vi.fn().mockReturnValue(createMockRedis()), mockGetRedisConnectionDefaults: vi.fn(getRedisConnectionDefaultsImpl), mockOnRedisReconnect: vi.fn(), mockAcquireLock: vi.fn().mockResolvedValue(true), @@ -101,8 +104,8 @@ export const redisConfigMockFns = { * Restores every redis-config mock function to its default behavior. */ export function resetRedisConfigMock(): void { - redisConfigMockFns.mockGetConfiguredRedisUrl.mockReset().mockReturnValue(null) - redisConfigMockFns.mockGetRedisClient.mockReset().mockReturnValue(null) + redisConfigMockFns.mockGetConfiguredRedisUrl.mockReset().mockReturnValue(DEFAULT_REDIS_URL) + redisConfigMockFns.mockGetRedisClient.mockReset().mockReturnValue(createMockRedis()) redisConfigMockFns.mockGetRedisConnectionDefaults .mockReset() .mockImplementation(getRedisConnectionDefaultsImpl) From 0eaae612487fc73f7d34fc5968a94620db546e00 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 17:55:44 -0800 Subject: [PATCH 2/6] chore(redis): remove the remaining no-Redis paths across the app, realtime, setup and deploy config --- .claude/rules/sim-testing.md | 2 +- .cursor/rules/sim-testing.mdc | 2 +- .github/scripts/http-e2e.sh | 14 +- .github/workflows/checks.yml | 14 +- .github/workflows/docs-negotiation.yml | 12 + README.md | 1 - apps/docs/MACHINE_ACCESS.md | 2 +- .../platform/self-hosting/architecture.mdx | 6 +- .../docs/platform/self-hosting/docker.mdx | 6 +- .../self-hosting/environment-variables.mdx | 6 +- .../docs/platform/self-hosting/kubernetes.mdx | 2 +- .../platform/self-hosting/observability.mdx | 2 +- .../docs/platform/self-hosting/redis.mdx | 30 +- .../self-hosting/reference-architectures.mdx | 4 +- .../docs/platform/self-hosting/scaling.mdx | 10 +- .../platform/self-hosting/troubleshooting.mdx | 6 +- apps/realtime/.env.example | 5 +- apps/realtime/src/config/socket.ts | 73 ++- apps/realtime/src/env.ts | 2 +- apps/realtime/src/handlers/connection.test.ts | 13 +- .../file-doc-capabilities.integration.ts | 12 +- .../handlers/file-doc-presence.integration.ts | 12 +- .../handlers/file-doc-project.integration.ts | 9 +- .../src/handlers/file-doc-store.test.ts | 422 ++------------ apps/realtime/src/handlers/file-doc-store.ts | 173 ++---- .../handlers/file-doc.multireplica.test.ts | 22 +- .../src/handlers/file-doc.redis-store.test.ts | 2 +- apps/realtime/src/handlers/file-doc.test.ts | 37 +- apps/realtime/src/handlers/file-doc.ts | 167 ++---- .../src/handlers/fixtures/redis.fixture.ts | 344 ++++++++++++ apps/realtime/src/index.ts | 21 +- apps/realtime/src/rooms/index.ts | 3 +- .../realtime/src/rooms/memory-manager.test.ts | 137 ----- apps/realtime/src/rooms/memory-manager.ts | 200 ------- apps/realtime/src/rooms/types.ts | 12 +- apps/realtime/vitest.setup.ts | 1 + apps/sim/.env.example | 5 + apps/sim/app/api/auth/oauth/utils.test.ts | 4 +- apps/sim/app/api/copilot/chat/delete/route.ts | 2 +- apps/sim/app/api/copilot/chats/route.ts | 2 +- apps/sim/app/api/link-preview/route.ts | 34 +- apps/sim/app/api/mcp/events/route.ts | 5 +- apps/sim/app/api/mothership/events/route.ts | 9 +- .../workspace/[workspaceId]/layout.test.tsx | 19 +- .../app/workspace/[workspaceId]/layout.tsx | 10 +- .../w/components/sidebar/sidebar.tsx | 7 +- apps/sim/ee/scim/TESTING.md | 16 +- .../hooks/queries/desktop-activity.test.tsx | 2 +- apps/sim/hooks/queries/desktop-activity.ts | 7 +- .../sim/lib/api/contracts/desktop-executor.ts | 4 +- apps/sim/lib/api/contracts/v2/workflows.ts | 1 + apps/sim/lib/api/contracts/workflows.ts | 2 + .../calculations/usage-reservation.test.ts | 8 - .../billing/calculations/usage-reservation.ts | 15 +- .../core/reporting-usage-cache.integration.ts | 10 - .../lib/billing/core/reporting-usage-cache.ts | 4 +- .../lib/billing/core/reporting-usage-epoch.ts | 7 +- .../billing/core/usage-segment-cache.test.ts | 21 +- .../lib/billing/core/usage-segment-cache.ts | 17 - apps/sim/lib/cli-auth/approval-store.test.ts | 8 - apps/sim/lib/cli-auth/approval-store.ts | 16 +- apps/sim/lib/core/config/appconfig.ts | 2 - .../lib/core/outbox/retention.integration.ts | 3 +- apps/sim/lib/core/outbox/retention.ts | 9 +- .../rate-limiter/hosted-key/queue.test.ts | 7 - .../lib/core/rate-limiter/hosted-key/queue.ts | 12 +- .../hosted-key/rate-limiter.test.ts | 2 +- .../rate-limiter/hosted-key/rate-limiter.ts | 4 +- .../credential-groups/mcp-oauth-state.test.ts | 8 - .../lib/credential-groups/mcp-oauth-state.ts | 14 +- .../lib/credential-groups/oauth-state.test.ts | 22 - apps/sim/lib/credential-groups/oauth-state.ts | 12 +- .../search-connection-completion.ts | 2 - .../credential-groups/slack-managed-users.ts | 12 +- apps/sim/lib/demo-booking/bookers.ts | 10 +- .../lib/desktop/application/activity.test.ts | 20 +- apps/sim/lib/desktop/application/activity.ts | 9 +- .../application/executor.integration.ts | 31 +- apps/sim/lib/desktop/application/executor.ts | 56 +- .../desktop/application/source-requests.ts | 15 +- apps/sim/lib/desktop/executor/availability.ts | 11 - .../executor/bound-turn.integration.ts | 21 +- apps/sim/lib/desktop/executor/presence.ts | 9 +- apps/sim/lib/events/pubsub.test.ts | 39 +- apps/sim/lib/events/pubsub.ts | 82 ++- apps/sim/lib/execution/blocked-run-log.ts | 12 +- .../cancel-workflow-execution.test.ts | 58 +- .../execution/cancel-workflow-execution.ts | 22 +- apps/sim/lib/execution/cancellation.test.ts | 50 -- apps/sim/lib/execution/cancellation.ts | 98 +--- .../lib/execution/event-buffer.integration.ts | 60 ++ apps/sim/lib/execution/event-buffer.ts | 213 +------- apps/sim/lib/execution/isolated-vm.test.ts | 35 +- apps/sim/lib/execution/isolated-vm.ts | 9 - .../session-file-provenance.integration.ts | 2 +- .../session-file-provenance.test.ts | 21 +- .../remote-sandbox/session-file-provenance.ts | 11 +- .../sim/lib/execution/removal-stop-records.ts | 5 +- .../execution/removal-stops.integration.ts | 7 +- apps/sim/lib/execution/removal-stops.test.ts | 8 +- apps/sim/lib/execution/signal.test.ts | 20 +- apps/sim/lib/execution/signal.ts | 54 +- apps/sim/lib/favicon/resolve.server.ts | 2 - .../workspace-lifecycle.integration.ts | 3 +- apps/sim/lib/knowledge/github-setup-state.ts | 16 +- .../lib/logs/execution/progress-markers.ts | 15 +- .../mcp/application/workflow-deployments.ts | 8 +- apps/sim/lib/mcp/connection-manager.ts | 31 +- .../orchestration/workflow-mcp-lifecycle.ts | 10 +- apps/sim/lib/mcp/pubsub.ts | 64 +-- apps/sim/lib/mcp/service-pool.test.ts | 1 - apps/sim/lib/mcp/service.test.ts | 1 - apps/sim/lib/mcp/service.ts | 7 +- apps/sim/lib/mcp/storage/factory.ts | 31 +- apps/sim/lib/mcp/storage/index.ts | 2 +- apps/sim/lib/mcp/storage/memory-cache.test.ts | 82 --- apps/sim/lib/mcp/storage/memory-cache.ts | 103 ---- apps/sim/lib/mcp/storage/redis-cache.ts | 10 +- apps/sim/lib/mcp/workflow-mcp-sync.test.ts | 3 +- apps/sim/lib/mcp/workflow-mcp-sync.ts | 2 +- .../content-delivery.integration.ts | 9 +- ...roject-file-write-transport.integration.ts | 3 +- .../agent-cli/saved-run-read.live.test.ts | 7 +- .../agent-cli/workbench-file-provenance.ts | 8 +- .../async-runs/orphaned-runs.integration.ts | 7 +- .../mothership/async-runs/orphaned-runs.ts | 3 +- apps/sim/lib/mothership/chat-status.test.ts | 11 +- apps/sim/lib/mothership/chat-status.ts | 34 +- apps/sim/lib/mothership/inbox/executor.ts | 8 +- .../lib/mothership/jobs/lane.integration.ts | 3 +- apps/sim/lib/mothership/jobs/lane.test.ts | 7 +- apps/sim/lib/mothership/jobs/lane.ts | 8 +- .../memory/application/spaces.integration.ts | 1 - .../persistence/tool-confirm/index.ts | 42 +- .../application/abandoned-runs.integration.ts | 3 +- .../request/application/abandoned-runs.ts | 6 +- .../removal-stops-runs.integration.ts | 8 +- .../request/go/file-preview-adapter.test.ts | 14 + .../lifecycle/unattended.integration.ts | 3 +- .../mothership/request/session/abort.test.ts | 21 +- .../lib/mothership/request/session/abort.ts | 140 ++--- .../request/session/buffer-ttl.integration.ts | 8 +- .../lib/mothership/request/session/buffer.ts | 7 +- .../request/session/controller-lease.ts | 3 +- .../request/session/file-preview-session.ts | 6 +- .../session/recovery-storm.integration.ts | 3 +- .../session/replay-budget.integration.ts | 1 - .../request/session/replay-gap.integration.ts | 4 +- .../session/stream-recovery.integration.ts | 8 +- .../lib/mothership/request/session/viewer.ts | 11 +- .../workflow-client-settlement.integration.ts | 1 - .../application/prepare-wake.integration.ts | 4 +- apps/sim/lib/mothership/tasks/wake.ts | 4 +- .../tools/hosted-workbench.live.test.ts | 2 +- .../mothership/tools/sandbox-project-files.ts | 9 +- .../lib/mothership/tools/sandbox-resources.ts | 28 +- .../tools/server/files/edit-roundtrip.test.ts | 34 +- ...re.test.ts => intent-store.integration.ts} | 52 +- .../tools/server/files/intent-store.ts | 56 +- apps/sim/lib/oauth/terminal-errors.test.ts | 12 +- apps/sim/lib/oauth/terminal-errors.ts | 3 - .../sim/lib/realtime/event-log.integration.ts | 150 +++++ apps/sim/lib/slack-search/oauth-state.ts | 10 +- apps/sim/lib/slack-search/onboarding-state.ts | 10 +- .../utils/user-file-base64.server.test.ts | 25 +- .../uploads/utils/user-file-base64.server.ts | 37 +- .../lib/webhooks/pending-verification.test.ts | 12 +- apps/sim/lib/webhooks/pending-verification.ts | 69 +-- .../webhooks/polling/admission-refusals.ts | 2 - .../sim/lib/webhooks/slack-stream-sessions.ts | 14 +- apps/sim/lib/workflows/lifecycle.ts | 2 +- apps/sim/lib/workspaces/lifecycle.ts | 18 +- apps/sim/scripts/backfill-projects.ts | 2 +- ...ackfill-table-workflow-deployments.test.ts | 230 -------- .../backfill-table-workflow-deployments.ts | 515 ------------------ .../scripts/test-docs-search-provider-e2e.ts | 19 +- apps/sim/scripts/test-scim-e2e.ts | 35 +- docker-compose.local.yml | 7 +- docker-compose.prod.yml | 7 +- helm/sim/Chart.yaml | 2 +- helm/sim/README.md | 10 +- helm/sim/examples/values-aws.yaml | 4 +- helm/sim/examples/values-azure.yaml | 4 +- helm/sim/examples/values-existing-secret.yaml | 10 +- helm/sim/examples/values-external-db.yaml | 9 +- .../sim/examples/values-external-secrets.yaml | 9 +- helm/sim/examples/values-gcp.yaml | 4 +- helm/sim/examples/values-production.yaml | 11 +- helm/sim/templates/_helpers.tpl | 18 + helm/sim/tests/redis_test.yaml | 81 +++ helm/sim/values.schema.json | 4 +- helm/sim/values.yaml | 27 +- .../deployment-config/src/env-capabilities.ts | 36 -- packages/sim-setup/README.md | 1 - .../sim-setup/src/capability-config.test.ts | 8 +- packages/sim-setup/src/capability-config.ts | 32 -- .../sim-setup/src/capability-status.test.ts | 2 - packages/sim-setup/src/capability-status.ts | 31 -- packages/sim-setup/src/checks.ts | 37 +- .../sim-setup/src/configuration-sources.ts | 1 + packages/sim-setup/src/env-files.ts | 1 + packages/sim-setup/src/index.ts | 2 +- packages/sim-setup/src/modes/dev.ts | 25 +- packages/sim-setup/src/redis.ts | 19 +- packages/sim-setup/src/setup-status.test.ts | 18 + packages/sim-setup/src/setup-status.ts | 18 +- packages/sim-setup/src/wizard.ts | 2 +- packages/testing/src/mocks/mcp-pubsub.mock.ts | 5 +- .../src/mocks/mothership-chat-status.mock.ts | 4 +- packages/testing/src/mocks/redis.mock.ts | 40 +- scripts/check-explicit-any.baseline.json | 12 +- .../check-source-conventions.baseline.json | 5 +- scripts/check-unused-exports.baseline.json | 14 - scripts/test-patterns-baseline.json | 4 - 214 files changed, 1888 insertions(+), 3897 deletions(-) create mode 100644 apps/realtime/src/handlers/fixtures/redis.fixture.ts delete mode 100644 apps/realtime/src/rooms/memory-manager.test.ts delete mode 100644 apps/realtime/src/rooms/memory-manager.ts create mode 100644 apps/sim/lib/execution/event-buffer.integration.ts delete mode 100644 apps/sim/lib/mcp/storage/memory-cache.test.ts delete mode 100644 apps/sim/lib/mcp/storage/memory-cache.ts rename apps/sim/lib/mothership/tools/server/files/{intent-store.test.ts => intent-store.integration.ts} (68%) create mode 100644 apps/sim/lib/realtime/event-log.integration.ts delete mode 100644 apps/sim/scripts/backfill-table-workflow-deployments.test.ts delete mode 100644 apps/sim/scripts/backfill-table-workflow-deployments.ts diff --git a/.claude/rules/sim-testing.md b/.claude/rules/sim-testing.md index 867f6393387..a3a736b0501 100644 --- a/.claude/rules/sim-testing.md +++ b/.claude/rules/sim-testing.md @@ -34,7 +34,7 @@ contracts, and demonstrated regressions. | Suffix | Needs | Run with | In CI | |--------|-------|----------|-------| | `*.test.ts(x)` | nothing; global mocks from `vitest.setup.ts` | `bun run --cwd test` | `test` jobs | -| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`; optional locally, required in CI) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | +| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`, required) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | | `*.live.test.ts` | provider APIs, hosted sandboxes, local runtimes, or sibling checkouts | `bun run --cwd apps/sim test --mode live ` | never | | `apps/desktop/e2e/*.spec.ts` | the packaged Electron app | Playwright | desktop E2E workflow | | `apps/sim/scripts/test-*-e2e.ts` | a running app over HTTP | its `package.json` script when one exists (`bun run test:scim:e2e`; `test:workflow-version-compare:e2e` adds `--no-env-file`), else `bun scripts/test--e2e.ts` from apps/sim | `e2e` jobs (`.github/scripts/http-e2e.sh`) | diff --git a/.cursor/rules/sim-testing.mdc b/.cursor/rules/sim-testing.mdc index e8ab580643a..46778c442a0 100644 --- a/.cursor/rules/sim-testing.mdc +++ b/.cursor/rules/sim-testing.mdc @@ -31,7 +31,7 @@ contracts, and demonstrated regressions. | Suffix | Needs | Run with | In CI | |--------|-------|----------|-------| | `*.test.ts(x)` | nothing; global mocks from `vitest.setup.ts` | `bun run --cwd test` | `test` jobs | -| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`; optional locally, required in CI) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | +| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`, required) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | | `*.live.test.ts` | provider APIs, hosted sandboxes, local runtimes, or sibling checkouts | `bun run --cwd apps/sim test --mode live ` | never | | `apps/desktop/e2e/*.spec.ts` | the packaged Electron app | Playwright | desktop E2E workflow | | `apps/sim/scripts/test-*-e2e.ts` | a running app over HTTP | its `package.json` script when one exists (`bun run test:scim:e2e`; `test:workflow-version-compare:e2e` adds `--no-env-file`), else `bun scripts/test--e2e.ts` from apps/sim | `e2e` jobs (`.github/scripts/http-e2e.sh`) | diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index d1ac13c2546..a27b9acd6bb 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -5,7 +5,8 @@ # # Usage: http-e2e.sh (run from apps/sim) # -# The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of +# The job provides DATABASE_URL, BETTER_AUTH_SECRET, ENCRYPTION_KEY and a Redis on 127.0.0.1:6379 +# (Redis is required, so every app and realtime server gets REDIS_URL); each group sets the rest of # its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e. # # Under `next dev` the first request cold-compiles the app under Turbopack, which takes 60-185s on @@ -23,6 +24,7 @@ set -euo pipefail group=${1:?usage: http-e2e.sh } report_dir="$RUNNER_TEMP/e2e" ready_timeout_seconds=600 +export REDIS_URL=${REDIS_URL:-redis://127.0.0.1:6379} mkdir -p "$report_dir" server_pid='' @@ -204,6 +206,7 @@ case "$group" in export PEOPLE_E2E_SCREENSHOT_DIR="$report_dir/people-ui-report.json.artifacts" SCIM_E2E_BASE_URL="$local_app_url" \ SCIM_E2E_DATABASE_URL="$DATABASE_URL" \ + SCIM_E2E_REDIS_URL="$REDIS_URL" \ SCIM_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \ SCIM_E2E_REPORT_PATH="$report_dir/scim-e2e-report.json" \ bun run test:scim:e2e @@ -238,8 +241,7 @@ case "$group" in ;; # The search suites serve their own fixtures in-process and need no app. They share this group - # because they finish in seconds. Self-hosted without billing: hosted billing admits runs through - # Redis, which the CLI app is not given. + # because they finish in seconds. Self-hosted, so runs are not admitted through hosted billing. cli) for search in google-content lucid zoom google-meet; do report_var="SEARCH_$(echo "$search" | tr 'a-z-' 'A-Z_')_REPORT_PATH" @@ -262,7 +264,7 @@ case "$group" in bun run test:cli-run-latency:e2e ;; - # Self-hosted: hosted billing admits a run only through a Redis usage reservation. + # Self-hosted, so runs are not admitted through hosted billing's usage reservation. stop-after) export NEXT_PUBLIC_FORCE_HOSTED=false export CRON_SECRET=stop-after-e2e-local-cron-secret @@ -278,7 +280,6 @@ case "$group" in ;; desktop-inbox|assistant-actions|organization-search|search-permissions) - export REDIS_URL=redis://127.0.0.1:6379 export NEXT_PUBLIC_FORCE_HOSTED=false export COPILOT_TOOL_PERMISSIONS_ENABLED=true export KNOWLEDGE_MEMBER_ACCESS=true CREDENTIAL_GROUPS=true ACCESS_CONTROL_ENABLED=true @@ -330,7 +331,6 @@ case "$group" in browser-extension-composer) extension_group=composer ;; browser-extension-access) extension_group=access ;; esac - export REDIS_URL=redis://127.0.0.1:6379 export NEXT_PUBLIC_FORCE_HOSTED=false export KNOWLEDGE_MEMBER_ACCESS=true CREDENTIAL_GROUPS=true ACCESS_CONTROL_ENABLED=true export SIM_LINKS=true @@ -358,7 +358,7 @@ JS project-files) PROJECT_FILES_E2E_ADMIN_DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ - PROJECT_FILES_E2E_REDIS_URL=redis://127.0.0.1:6379 \ + PROJECT_FILES_E2E_REDIS_URL="$REDIS_URL" \ PROJECT_FILES_E2E_BASE_URL=http://127.0.0.1:3050 \ PROJECT_FILES_E2E_RELAY_URL=http://127.0.0.1:3052 \ PROJECT_FILES_E2E_REPORT_PATH="$report_dir/project-files/orchestration.json" \ diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 0b04f0fa081..7254a04149d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -236,9 +236,9 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 10 - # Desktop inbox, browser extension, and Project Files use Redis. + # Redis is required: every app server the suites start connects to it. redis: - image: ${{ (matrix.group == 'desktop-inbox' || matrix.group == 'assistant-actions' || matrix.group == 'organization-search' || matrix.group == 'search-permissions' || startsWith(matrix.group, 'browser-extension-') || matrix.group == 'project-files') && 'redis:8.2-alpine' || '' }} + image: redis:8.2-alpine ports: - 6379:6379 options: >- @@ -408,6 +408,16 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 10 + # Redis is required: the app server refuses to start without it. + redis: + image: redis:8.2-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 env: DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters diff --git a/.github/workflows/docs-negotiation.yml b/.github/workflows/docs-negotiation.yml index 89e96fc8f44..d1098a5f4f9 100644 --- a/.github/workflows/docs-negotiation.yml +++ b/.github/workflows/docs-negotiation.yml @@ -334,6 +334,17 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 10 + # The platform consumer runs the app's docs search in-process, and its provider admission + # needs Redis. + search-redis: + image: ${{ (matrix.group == 'http' || matrix.group == 'platform') && 'redis:8.2-alpine' || '' }} + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 env: DATABASE_URL: postgresql://docs_test:docs_test@127.0.0.1:1/docs_test DOCS_E2E_BASE_URL: http://127.0.0.1:3001 @@ -348,6 +359,7 @@ jobs: DOCS_SAMPLES_SYNTAX_LANGUAGES: javascript,python,bash,go,java,csharp DOCS_SAMPLES_HTTP_BASE_URL: http://127.0.0.1:3001 DOCS_PROVIDER_E2E_DATABASE_URL: postgresql://docs_test:docs_test@127.0.0.1:5432/docs_search_test + DOCS_PROVIDER_E2E_REDIS_URL: redis://127.0.0.1:6379 DOCS_PROVIDER_E2E_BASE_URL: http://127.0.0.1:3001 DOCS_PROVIDER_E2E_REPORT_PATH: ${{ github.workspace }}/apps/docs/test-results/platform-consumer/report.json steps: diff --git a/README.md b/README.md index de4131f0e0b..390f646094b 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,6 @@ npx sim-setup add email npx sim-setup add storage npx sim-setup add sandbox npx sim-setup add jobs -npx sim-setup add cache npx sim-setup add knowledge npx sim-setup add chat npx sim-setup add llm diff --git a/apps/docs/MACHINE_ACCESS.md b/apps/docs/MACHINE_ACCESS.md index 2e2ea1f2b7f..1c6e9c0c64d 100644 --- a/apps/docs/MACHINE_ACCESS.md +++ b/apps/docs/MACHINE_ACCESS.md @@ -79,7 +79,7 @@ The suite exercises real HTTP and the official MCP client, including discovery p The PostgreSQL suite uses a new, empty, loopback database and no embedding key. It verifies real keyword search, citation freshness, graceful database failure, repeated index publication, retirement and failed-publication preservation. It refuses databases containing existing tables. CI provisions its own `pgvector/pgvector:pg17` service and uploads the report with the other docs checks. -The platform consumer harness also uses an empty loopback database and the running docs build. From `apps/sim`, run `bun --no-env-file scripts/test-docs-search-provider-e2e.ts` with `DOCS_PROVIDER_E2E_DATABASE_URL`, `DOCS_PROVIDER_E2E_BASE_URL` and `DOCS_PROVIDER_E2E_REPORT_PATH`. Its default mode uses the existing Azure endpoint configuration with a local HTTP fixture, deterministic vectors and the real PostgreSQL provider-admission store. It verifies the platform's direct-table search, scope exclusions, VFS reads and public citations, with no paid requests. This verifies the transport and consumer contract, not semantic relevance. +The platform consumer harness also uses an empty loopback database and the running docs build. From `apps/sim`, run `bun --no-env-file scripts/test-docs-search-provider-e2e.ts` with `DOCS_PROVIDER_E2E_DATABASE_URL`, `DOCS_PROVIDER_E2E_REDIS_URL` (a loopback Redis), `DOCS_PROVIDER_E2E_BASE_URL` and `DOCS_PROVIDER_E2E_REPORT_PATH`. Its default mode uses the existing Azure endpoint configuration with a local HTTP fixture, deterministic vectors and the real Redis provider-admission store. It verifies the platform's direct-table search, scope exclusions, VFS reads and public citations, with no paid requests. This verifies the transport and consumer contract, not semantic relevance. Live provider verification is opt-in: set `DOCS_PROVIDER_E2E_ALLOW_LIVE=1` and supply `OPENAI_API_KEY`, using a fresh database named `docs_search_provider_test`. The harness limits publication to the introduction page, rejects plans above 15 chunks, and sends at most one batch with up to three attempts. The consumer makes at most one query embedding request, for a total cap of four provider requests. Provider failure stops the run and remains a failed report. The default fixture mode also accepts the empty `docs_search_test` database used by the PostgreSQL suite. Both modes remove their tables when finished. diff --git a/apps/docs/content/docs/platform/self-hosting/architecture.mdx b/apps/docs/content/docs/platform/self-hosting/architecture.mdx index e43e1a1abb5..8d504dfa975 100644 --- a/apps/docs/content/docs/platform/self-hosting/architecture.mdx +++ b/apps/docs/content/docs/platform/self-hosting/architecture.mdx @@ -15,7 +15,7 @@ Sim runs an app, a realtime server, PostgreSQL, Redis, and a scheduler. This pag | **realtime** | `ghcr.io/simstudioai/realtime` | 3002 | In-flight collaboration state | Yes | | **migrations** | `ghcr.io/simstudioai/migrations` | — | One-shot schema and data migrations | Yes | | **postgresql** | `pgvector/pgvector:pg17` | 5432 | Durable application data | Yes | -| **redis** | `redis:7-alpine` | 6379 | Cache, coordination, and live events | Bundled by default; required across replicas | +| **redis** | `redis:7-alpine` | 6379 | Rate limits, coordination, and live events | Yes (bundled by default) | | **cron** | `ghcr.io/simstudioai/cron` (Compose) / `curlimages/curl` (Kubernetes) | — | None | Required for scheduled work | | **pii** | `ghcr.io/simstudioai/pii` | 5001 | None | Optional | | **ollama** | `ollama/ollama` | 11434 | Model cache | Optional | @@ -36,7 +36,7 @@ The Bun Socket.IO server carries collaborative editing, live execution updates, Realtime shares PostgreSQL and `BETTER_AUTH_SECRET` with the app, using Better Auth's shared-database-session pattern. It authenticates the same accounts without a separate login. - More than one realtime replica requires `REDIS_URL`. The Socket.IO Redis adapter carries events between replicas; without it, members connected to different pods do not see each other's edits. + Realtime requires `REDIS_URL`, pointing at the same Redis as the app. The Socket.IO Redis adapter carries events between replicas, so members connected to different pods see each other's edits. A realtime server must exist for editing and live updates. Set `realtime.enabled: false` in Helm only when `app.env.SOCKET_SERVER_URL` points to a realtime instance you run separately. @@ -55,7 +55,7 @@ PostgreSQL holds workflows, runs, logs, accounts, organizations, credentials, kn ### Redis -Redis carries live events and stores shared coordination state. Some stores fall back to PostgreSQL or process-local state, but cross-pod collaboration has no local fallback. CLI API-key pairing also requires Redis, even with one replica; OAuth login uses a separate path. See [Redis](/platform/self-hosting/redis) for the full fallback matrix. +Redis is required, even with one replica. It holds rate-limit buckets, one-time passcodes, and shared coordination state, and carries live events between processes. There is no PostgreSQL or process-local fallback, and the app refuses to start without `REDIS_URL`. See [Redis](/platform/self-hosting/redis) for everything it backs. ### Scheduler diff --git a/apps/docs/content/docs/platform/self-hosting/docker.mdx b/apps/docs/content/docs/platform/self-hosting/docker.mdx index 11135d005ec..6ce50047526 100644 --- a/apps/docs/content/docs/platform/self-hosting/docker.mdx +++ b/apps/docs/content/docs/platform/self-hosting/docker.mdx @@ -89,7 +89,7 @@ curl -fsS http://localhost:3000/api/health | `simstudio` | 3000 | App; 8G memory limit. | | `realtime` | 3002 | WebSocket service; 1G memory limit. | | `db` | Not published | PostgreSQL 17 with pgvector. | -| `redis` | Not published | Shared coordination, pub/sub, and caches. | +| `redis` | Not published | Required. Rate limits, shared coordination, pub/sub, and caches. | | `cron` | None | Calls background job endpoints. | | `migrations` | None | Applies migrations once, then exits. | @@ -238,14 +238,14 @@ Invoke commands as `bunx sim-setup ` unless the package is installed gl | `reset` | Confirms before archiving env files and removing managed Compose/dev data. On Helm it uninstalls the release; PostgreSQL StatefulSet PVCs remain and must be deleted separately to erase data. | | `config` | Reports configured capabilities and integrations. | | `doctor` | Checks env files and reachable services. `--fix` writes supported repairs; `--json` emits structured findings. | -| `add ` | Configures `email`, `storage`, `sandbox`, `jobs`, `cache`, `knowledge`, `knowledge-embeddings`, `chat`, `llm`, or `integration `. Requires one writable configuration; refuses ambiguous destinations and Helm releases. | +| `add ` | Configures `email`, `storage`, `sandbox`, `jobs`, `knowledge`, `knowledge-embeddings`, `chat`, `llm`, or `integration `. Requires one writable configuration; refuses ambiguous destinations and Helm releases. | | `desktop` | Prints this deployment's installer/server links. `--url ` skips discovery; `--no-open` skips the browser-opening prompt. | `--dir ` works with every command and scopes standalone lifecycle commands to that installation. For Compose, `reset` archives `.env` beside the Compose file. For dev, Kubernetes, or no detected install, it archives `apps/sim/.env`, `apps/realtime/.env`, `packages/db/.env`, and root `.env`. Preserve any backups before intentionally removing volumes. -`add` writes beside the Compose file or to `apps/sim/.env` on a source checkout. Recreate containers that read the change: `cache` changes `REDIS_URL` for both app and realtime. For a source checkout, mirror shared values into `apps/realtime/.env`. `restart` alone does not apply changed container environment. Desktop discovery refuses configurations naming different origins; pass `--url` to choose. +`add` writes beside the Compose file or to `apps/sim/.env` on a source checkout. Recreate containers that read the change. To point both app and realtime at a different Redis, set `REDIS_URL` in `.env` and recreate both containers. For a source checkout, mirror shared values into `apps/realtime/.env`. `restart` alone does not apply changed container environment. Desktop discovery refuses configurations naming different origins; pass `--url` to choose. ## Back up and update [#commands] diff --git a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx index e39b0556278..5b8f079567a 100644 --- a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx +++ b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx @@ -19,6 +19,7 @@ Production Compose derives `DATABASE_URL` and `BETTER_AUTH_URL` on its service d | `ENCRYPTION_KEY` | Encryption key, exactly 64 hex characters (32 bytes): `openssl rand -hex 32`. A key in any other format leaves [account token sealing](/platform/self-hosting/security#account-token-sealing) off | | `INTERNAL_API_SECRET` | Internal API secret, at least 32 characters: `openssl rand -hex 32` | | `NEXT_PUBLIC_APP_URL` | Public app URL | +| `REDIS_URL` | Redis connection string (`redis://` or `rediss://`), shared by the app and realtime services. The app refuses to start without it; there is no PostgreSQL or in-memory fallback. See [Redis](/platform/self-hosting/redis) | `openssl rand -hex 32` prints 64 hex characters. Encryption keys require that format. The app skips its env schema validation; individual consumers can reject invalid values when first used. Realtime validates `BETTER_AUTH_SECRET` and `INTERNAL_API_SECRET` at boot and rejects values shorter than 32 characters. @@ -32,8 +33,7 @@ Production Compose derives `DATABASE_URL` and `BETTER_AUTH_URL` on its service d |----------|-------------| | `API_ENCRYPTION_KEY` | Encrypts stored Sim API-key material; use exactly 64 hex characters (`openssl rand -hex 32`). When unset, legacy API keys can be stored in plain text; the key getter logs a warning when called | | `CRON_SECRET` | Bearer token for the background job endpoints: `openssl rand -hex 32`. **Required whenever background jobs are enabled** — the Helm chart's default, and the chart will not render without it, unless it comes from `app.secrets.existingSecret` or External Secrets. See [Background jobs](/platform/self-hosting/background-jobs) | -| `REDIS_URL` | Redis connection string. Required for cross-replica events and features with no fallback, such as CLI pairing approval. Some single-process storage paths can fall back to PostgreSQL or local state; see [Redis](/platform/self-hosting/redis) | -| `REDIS_TLS_SERVERNAME` | TLS SNI override. Required when `REDIS_URL` uses `rediss://` with a bare IP, or the app throws the first time it opens a Redis connection | +| `REDIS_TLS_SERVERNAME` | TLS SNI override. Required when `REDIS_URL` uses `rediss://` with a bare IP, or the app refuses to start | | `NEXT_PUBLIC_SOCKET_URL` | Public WebSocket URL. Proxied deployments default to the page origin; a localhost page uses port 3002. Set it for a separate realtime host. | | `TRUSTED_ORIGINS` | Comma-separated additional origins to trust for auth (apex + `www`, alias domains) | | `AUTH_TRUSTED_PROXIES` | Comma-separated reverse-proxy IPs/CIDRs so the client IP cannot be forged through `X-Forwarded-For` | @@ -340,7 +340,7 @@ INTERNAL_API_SECRET= API_ENCRYPTION_KEY= CRON_SECRET= -# Coordination (required past one replica) +# Redis (required) REDIS_URL=redis://redis:6379 # Models — OPENAI_API_KEY also powers knowledge base embeddings diff --git a/apps/docs/content/docs/platform/self-hosting/kubernetes.mdx b/apps/docs/content/docs/platform/self-hosting/kubernetes.mdx index e0bc7e2b824..5005d81efc7 100644 --- a/apps/docs/content/docs/platform/self-hosting/kubernetes.mdx +++ b/apps/docs/content/docs/platform/self-hosting/kubernetes.mdx @@ -18,7 +18,7 @@ The examples pin published chart `1.11.8`, whose published app version is `v0.9. - `metrics-server` if you enable autoscaling. - External Secrets Operator and a configured SecretStore/ClusterSecretStore if you choose that secret strategy. -Redis is bundled by default. A non-empty `app.env.REDIS_URL` overrides its URL and suppresses the bundled Redis Deployment. `redis.enabled=false` is the explicit opt-out; supply an external URL if you need cross-replica events. See [Redis](/platform/self-hosting/redis). +Sim requires Redis, and the chart bundles it by default. A non-empty `app.env.REDIS_URL` overrides its URL and suppresses the bundled Redis Deployment. With `redis.enabled=false` you must supply `REDIS_URL` for a managed instance (through `app.env`, `app.secrets.existingSecret`, or `externalSecrets.remoteRefs.app.REDIS_URL`); the chart fails to render otherwise. See [Redis](/platform/self-hosting/redis). ## Installation diff --git a/apps/docs/content/docs/platform/self-hosting/observability.mdx b/apps/docs/content/docs/platform/self-hosting/observability.mdx index 7a308463faf..4878dcd22a3 100644 --- a/apps/docs/content/docs/platform/self-hosting/observability.mdx +++ b/apps/docs/content/docs/platform/self-hosting/observability.mdx @@ -95,7 +95,7 @@ Sim's hosted-key metrics recorder arms itself whenever `AWS_ACCESS_KEY_ID` is pr | Ingress 5xx rate above baseline | Broad user impact | | Postgres connections above 80% of `max_connections` | Next replica or traffic spike will start failing | | Postgres disk above 80% | Knowledge base embeddings grow steadily | -| Redis unreachable | Live collaboration and status updates can stop; inspect app and realtime connection errors | +| Redis unreachable | Redis is required: rate limiting, one-time passcodes, live collaboration, and status updates depend on it. Inspect app and realtime connection errors | | Certificate expiry within 14 days | Especially with manually managed certs | | Object storage 4xx/5xx rate | Broken uploads usually show here first | diff --git a/apps/docs/content/docs/platform/self-hosting/redis.mdx b/apps/docs/content/docs/platform/self-hosting/redis.mdx index 251e782af89..a1390af8f2f 100644 --- a/apps/docs/content/docs/platform/self-hosting/redis.mdx +++ b/apps/docs/content/docs/platform/self-hosting/redis.mdx @@ -1,27 +1,31 @@ --- title: Redis -description: When Redis is optional, when it is required, and how to configure it +description: Why Sim requires Redis, what it backs, and how to configure it --- import { Tab, Tabs } from '@/components/ui/tabs' import { Callout } from '@/components/ui/callout' -Configure Redis for shared coordination and live events. Compose and Helm bundle it by default; use this page to replace that instance or understand the fallback behavior. +Redis is required. Every Sim deployment needs a reachable Redis, even with a single app replica. Compose and Helm bundle one by default; use this page to replace that instance with your own. ## What it backs -| Use | Without Redis | +There are no PostgreSQL or in-memory fallbacks: each of these runs only on Redis. + +| Use | Purpose | |---|---| -| **Pub/sub** — live Chat task status, table events, execution cancellation, MCP tool-change notifications, Chat tool confirmations | Falls back to a **process-local** emitter: events never leave the pod that produced them | -| **Socket.IO adapter** (realtime) | Collaboration events are not delivered across realtime pods | -| Idempotency store | Falls back to PostgreSQL | -| Execution progress markers | Falls back to PostgreSQL | -| Distributed execution limits | Enforced per-pod instead of per-deployment | -| CLI pairing-code approval store | **No fallback** — API-key login through pairing requires Redis regardless of replica count; OAuth login uses a separate path | -| Collaborative document store (realtime) | Falls back to in-process state | +| **Rate limiting** | API and execution rate-limit token buckets, shared across every app replica | +| **Pub/sub** | Live Chat task status, table events, execution cancellation, MCP tool-change notifications, Chat tool confirmations | +| **Socket.IO adapter** (realtime) | Delivers collaboration events across realtime pods | +| One-time passcodes | Email OTPs for deployed chats and public file shares | +| Idempotency store | Deduplicates webhook deliveries and retried operations | +| Execution progress markers | Event buffers and progress for running executions | +| Distributed execution limits | Per-deployment concurrency and provider admission limits | +| CLI pairing-code approval store | API-key login through pairing; OAuth login uses a separate path | +| Collaborative document store (realtime) | Shared state for collaboratively edited documents | - Configure Redis before running more than one app or realtime replica. Without it, process-local fallbacks cannot deliver edits and status across pods. A healthy HTTP response does not prove cross-pod collaboration works. + The app checks `REDIS_URL` when it starts and refuses to boot when it is missing or is not a `redis://` or `rediss://` URL. `next build` does not need Redis, so an image builds without it; the running server does not. Everything Sim keeps in Redis is cache, coordination state, or an in-flight event — never committed data, which lives in PostgreSQL and object storage. Persistence is therefore not required. Losing or restarting the instance is not free, though: cancellation markers and the cross-pod half of execution streaming live here, so active runs stop streaming and a cancellation issued across the gap may not land. Completed work is unaffected. @@ -34,7 +38,7 @@ REDIS_URL=redis://:password@redis-host:6379 REDIS_URL=rediss://:password@redis-host:6380 ``` -Both the app and the realtime service need it — they use it for different things. +Both the app and the realtime service need it and must point at the same instance — they use it for different things. @@ -48,7 +52,7 @@ The port is deliberately not published to the host, so a Redis already running l The chart deploys Redis by default, matching the Compose stack. No additional Redis URL is needed for that bundled instance. -For production, prefer a managed instance — disable the bundled one and supply a URL: +For production, prefer a managed instance — disable the bundled one and supply a URL. With `redis.enabled: false`, the chart refuses to render unless `REDIS_URL` comes from `app.env`, a pre-created `app.secrets.existingSecret`, or `externalSecrets.remoteRefs.app.REDIS_URL`: ```yaml redis: diff --git a/apps/docs/content/docs/platform/self-hosting/reference-architectures.mdx b/apps/docs/content/docs/platform/self-hosting/reference-architectures.mdx index d1a1b6b15aa..1df7c02cce3 100644 --- a/apps/docs/content/docs/platform/self-hosting/reference-architectures.mdx +++ b/apps/docs/content/docs/platform/self-hosting/reference-architectures.mdx @@ -51,7 +51,7 @@ Use `helm/sim/examples/values-aws.yaml` as a starting point. It configures EBS G | ACM certificate | ALB discovers it from the ingress hosts, or pin it with `alb.ingress.kubernetes.io/certificate-arn` | | RDS PostgreSQL | With the `vector` extension available. Reachable from the cluster subnets | | S3 buckets | One per file purpose — see [Object storage](/platform/self-hosting/object-storage) | -| ElastiCache | Optional; the bundled Redis is a supported alternative | +| ElastiCache | Recommended for production; Sim requires Redis, and the bundled one is a supported alternative | ALB does not consume a Kubernetes TLS Secret directly. Supply an ACM certificate or a supported certificate-discovery configuration. A cert-manager-created Secret alone is not sufficient. @@ -75,7 +75,7 @@ The Azure example provides `managed-csi` / `managed-csi-premium` storage, node t | TLS certificate mechanism | Match your selected ingress controller. For one that consumes TLS Secrets, cert-manager with a ClusterIssuer can issue the Secret | | Azure Database for PostgreSQL | With `pgvector` enabled | | Azure Blob Storage containers | One per file purpose | -| Managed Redis | Optional; select a configuration compatible with Sim's single-endpoint clients. Existing Azure Cache for Redis deployments need [retirement/migration planning](https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/retirement-faq) | +| Managed Redis | Recommended for production (Sim requires Redis; the bundled one is a supported alternative); select a configuration compatible with Sim's single-endpoint clients. Existing Azure Cache for Redis deployments need [retirement/migration planning](https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/retirement-faq) | | GPU node pool | Only for Ollama. Taint it `sku=gpu:NoSchedule` to match the example tolerations | **Identity.** Azure Blob authenticates with a storage account key or connection string — Sim has no Azure workload-identity path for Blob. Keep the key out of `app.env` by supplying `AZURE_ACCOUNT_KEY` or `AZURE_CONNECTION_STRING` through `app.secrets.existingSecret` or External Secrets. diff --git a/apps/docs/content/docs/platform/self-hosting/scaling.mdx b/apps/docs/content/docs/platform/self-hosting/scaling.mdx index 05c0606f5b9..0b98104a6f6 100644 --- a/apps/docs/content/docs/platform/self-hosting/scaling.mdx +++ b/apps/docs/content/docs/platform/self-hosting/scaling.mdx @@ -5,12 +5,12 @@ description: Replicas, connection pooling, autoscaling, and where the limits act import { Callout } from '@/components/ui/callout' -Configure shared storage, Redis, and database capacity before adding replicas. Measure your execution workload before choosing memory limits or autoscaling targets. +Configure shared storage and database capacity, and size Redis, before adding replicas. Measure your execution workload before choosing memory limits or autoscaling targets. ## Prerequisites before scaling past one replica - Verify Redis connectivity before raising `replicaCount`. If you disable or remove bundled Redis, supply a working `REDIS_URL` for app and realtime. Process-local fallbacks do not deliver cross-pod events; see [Redis](/platform/self-hosting/redis). + Every replica must reach the same Redis. Redis is required at any replica count, and it is what carries rate limits and live events across pods. If you disable or remove bundled Redis, supply a working `REDIS_URL` for app and realtime; see [Redis](/platform/self-hosting/redis). You also need shared object storage — local-disk storage is per-pod, so a file uploaded through one replica is invisible to the others. See [Object Storage](/platform/self-hosting/object-storage). @@ -19,8 +19,8 @@ You also need shared object storage — local-disk storage is per-pod, so a file | Component | Scaling | Notes | |---|---|---| -| **app** | Horizontal | Stateless once object storage is configured. **Requires Redis** past one replica | -| **realtime** | Horizontal | **Requires Redis** past one replica (Socket.IO adapter) | +| **app** | Horizontal | Stateless once object storage is configured. All replicas share one Redis | +| **realtime** | Horizontal | All replicas share one Redis (Socket.IO adapter) | | **postgresql** | Vertical + read replicas | The eventual bottleneck | | **redis** | Vertical / HA pair | Coordination only; small | | **cronjobs** | Fixed | One call per tick regardless of replica count | @@ -64,7 +64,7 @@ Requires `metrics-server`. When enabled, the chart omits `spec.replicas` so the Scale-down terminates pods that may be running workflows. Set a conservative `minReplicas`, and consider a `behavior` block with a long `stabilizationWindowSeconds` on scale-down so long executions are not repeatedly interrupted. -Realtime gets the same HPA unless you disable it — and again, only scale it past one replica with Redis configured: +Realtime gets the same HPA unless you disable it; its replicas bridge events through the shared Redis: ```yaml autoscaling: diff --git a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx index 8924875a920..13ed133d51c 100644 --- a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx +++ b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx @@ -114,7 +114,7 @@ Your reverse proxy is buffering the response stream. Set `proxy_buffering off` ( ## Collaboration breaks with multiple replicas -If members on different replicas do not see edits or live status, check Redis configuration and connectivity in both services. Pub/sub and the Socket.IO adapter have no cross-pod local fallback: +If members on different replicas do not see edits or live status, check that both services reach the same Redis: ```bash kubectl exec -n simstudio deploy/sim-app -- sh -c 'test -n "$REDIS_URL" && echo "REDIS_URL is set"' @@ -123,6 +123,10 @@ kubectl exec -n simstudio deploy/sim-realtime -- sh -c 'test -n "$REDIS_URL" && Both pods must have `REDIS_URL`. On Helm they share one Secret, so setting it under `app.env` covers both. See [Redis](/platform/self-hosting/redis). +## App refuses to start: REDIS_URL is required + +Redis is required. The app validates `REDIS_URL` at boot and exits when it is missing or is not a `redis://` or `rediss://` URL. Set it for both the app and realtime services: Compose wires the bundled `redis` service by default, and Helm deploys a bundled Redis unless `redis.enabled=false`, in which case supply `REDIS_URL` through `app.env`, your existing Secret, or External Secrets. See [Redis](/platform/self-hosting/redis). + ## App crashes at startup with a REDIS_TLS_SERVERNAME error The app requires `REDIS_TLS_SERVERNAME` for an IPv4-literal `rediss://` URL. That override is not read by realtime and does not cover IPv6 literals. Prefer a DNS hostname matching the certificate for the shared deployment; see [Redis](/platform/self-hosting/redis#tls-to-an-ip-address). diff --git a/apps/realtime/.env.example b/apps/realtime/.env.example index 26ef119c01c..c9668b0c040 100644 --- a/apps/realtime/.env.example +++ b/apps/realtime/.env.example @@ -19,9 +19,8 @@ INTERNAL_API_SECRET=your_internal_api_secret_min_32_chars # Public app URL — used for CORS allow-list and base URL resolution NEXT_PUBLIC_APP_URL=http://localhost:3000 -# Optional: Redis for cross-pod room management -# Leave unset for single-pod / in-memory rooms -# REDIS_URL=redis://localhost:6379 +# Redis (required) — room presence, the Socket.IO adapter, and collaborative document streams +REDIS_URL=redis://localhost:6379 # Optional: extra Socket.IO CORS allow-list (comma-separated) # ALLOWED_ORIGINS=https://embed.example.com,https://admin.example.com diff --git a/apps/realtime/src/config/socket.ts b/apps/realtime/src/config/socket.ts index 2345b16ce2b..891440ff7be 100644 --- a/apps/realtime/src/config/socket.ts +++ b/apps/realtime/src/config/socket.ts @@ -35,7 +35,7 @@ function getAllowedOrigins(): string[] { /** * Create and configure a Socket.IO server instance. - * If REDIS_URL is configured, adds Redis adapter for cross-pod broadcasting. + * Attaches the Redis adapter for cross-pod broadcasting. */ export async function createSocketIOServer(httpServer: HttpServer): Promise { const allowedOrigins = getAllowedOrigins() @@ -61,52 +61,48 @@ export async function createSocketIOServer(httpServer: HttpServer): Promise { - if (retries > 10) { - logger.error('Redis adapter reconnection failed after 10 attempts') - return new Error('Redis adapter reconnection failed') - } - const delay = Math.min(retries * 100, 3000) - logger.warn(`Redis adapter reconnecting in ${delay}ms (attempt ${retries})`) - return delay - }, + logger.info('Configuring Socket.IO Redis adapter...') + + const redisOptions = { + url: env.REDIS_URL, + socket: { + reconnectStrategy: (retries: number) => { + if (retries > 10) { + logger.error('Redis adapter reconnection failed after 10 attempts') + return new Error('Redis adapter reconnection failed') + } + const delay = Math.min(retries * 100, 3000) + logger.warn(`Redis adapter reconnecting in ${delay}ms (attempt ${retries})`) + return delay }, - } + }, + } - // Create separate clients for pub and sub (recommended for reliability) - adapterPubClient = createClient(redisOptions) - adapterSubClient = createClient(redisOptions) + // Create separate clients for pub and sub (recommended for reliability) + adapterPubClient = createClient(redisOptions) + adapterSubClient = createClient(redisOptions) - adapterPubClient.on('error', (err) => { - logger.error('Redis adapter pub client error:', err) - }) + adapterPubClient.on('error', (err) => { + logger.error('Redis adapter pub client error:', err) + }) - adapterSubClient.on('error', (err) => { - logger.error('Redis adapter sub client error:', err) - }) + adapterSubClient.on('error', (err) => { + logger.error('Redis adapter sub client error:', err) + }) - adapterPubClient.on('ready', () => { - logger.info('Redis adapter pub client ready') - }) + adapterPubClient.on('ready', () => { + logger.info('Redis adapter pub client ready') + }) - adapterSubClient.on('ready', () => { - logger.info('Redis adapter sub client ready') - }) + adapterSubClient.on('ready', () => { + logger.info('Redis adapter sub client ready') + }) - await Promise.all([adapterPubClient.connect(), adapterSubClient.connect()]) + await Promise.all([adapterPubClient.connect(), adapterSubClient.connect()]) - io.adapter(createAdapter(adapterPubClient, adapterSubClient)) + io.adapter(createAdapter(adapterPubClient, adapterSubClient)) - logger.info('Socket.IO Redis adapter connected - cross-pod broadcasting enabled') - } else { - logger.warn('REDIS_URL not configured - running in single-pod mode') - } + logger.info('Socket.IO Redis adapter connected - cross-pod broadcasting enabled') logger.info('Socket.IO server configured with:', { allowedOrigins: allowedOrigins.length, @@ -116,7 +112,6 @@ export async function createSocketIOServer(httpServer: HttpServer): Promise (typeof value === 'string' && value.trim() === '' ? undefined : value), - z.string().url().optional() + z.string({ error: 'REDIS_URL is required' }).url() ), BETTER_AUTH_URL: z.string().url(), BETTER_AUTH_SECRET: z.string().min(32), diff --git a/apps/realtime/src/handlers/connection.test.ts b/apps/realtime/src/handlers/connection.test.ts index 9aad06f9f21..b1b185b18da 100644 --- a/apps/realtime/src/handlers/connection.test.ts +++ b/apps/realtime/src/handlers/connection.test.ts @@ -5,7 +5,7 @@ import { io as connect, type Socket } from 'socket.io-client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { setupConnectionHandlers, waitForConnectionCleanup } from '@/handlers/connection' import type { AuthenticatedSocket } from '@/middleware/auth' -import { MemoryRoomManager } from '@/rooms' +import type { IRoomManager } from '@/rooms' vi.mock('@/handlers/file-doc', () => ({ cleanupFileDocForSocket: vi.fn() })) vi.mock('@/handlers/subblocks', () => ({ cleanupPendingSubblocksForSocket: vi.fn() })) @@ -14,14 +14,18 @@ vi.mock('@/handlers/variables', () => ({ cleanupPendingVariablesForSocket: vi.fn describe('server shutdown connection drain', () => { let httpServer: HttpServer let io: Server - let manager: MemoryRoomManager + let manager: IRoomManager let client: Socket beforeEach(async () => { httpServer = createServer() io = new Server(httpServer, { transports: ['websocket'] }) - manager = new MemoryRoomManager(io) - await manager.initialize() + manager = { + io, + removeSocketFromAllRooms: vi.fn(async () => []), + removeUserFromRoom: vi.fn(async () => true), + broadcastPresenceUpdate: vi.fn(async () => {}), + } as unknown as IRoomManager io.on('connection', (socket) => setupConnectionHandlers(socket as AuthenticatedSocket, manager)) await new Promise((resolve) => httpServer.listen(0, '127.0.0.1', resolve)) const port = (httpServer.address() as AddressInfo).port @@ -35,7 +39,6 @@ describe('server shutdown connection drain', () => { client.disconnect() await io.close() await waitForConnectionCleanup() - await manager.shutdown() }) it('keeps automatic reconnection active after transport shutdown', async () => { diff --git a/apps/realtime/src/handlers/file-doc-capabilities.integration.ts b/apps/realtime/src/handlers/file-doc-capabilities.integration.ts index fee4db2ccb0..50a3303a61a 100644 --- a/apps/realtime/src/handlers/file-doc-capabilities.integration.ts +++ b/apps/realtime/src/handlers/file-doc-capabilities.integration.ts @@ -32,14 +32,14 @@ import { flushAllFileDocRooms, setupWorkspaceFileDocHandlers, } from '@/handlers/file-doc' -import { getFileDocStore } from '@/handlers/file-doc-store' +import { getFileDocStore, initFileDocStore } from '@/handlers/file-doc-store' import type { AuthenticatedSocket } from '@/middleware/auth' import { beginRoomPermissionRead, commitRoomPermission, ROLE_REVALIDATION_TTL_MS, } from '@/middleware/permissions' -import { MemoryRoomManager } from '@/rooms' +import { RedisRoomManager } from '@/rooms' const workspaceId = generateId() const projectId = generateId() @@ -54,7 +54,7 @@ const checks: { name: string; status: string; durationMs: number; error?: string let app: HttpServer let http: HttpServer let io: Server -let manager: MemoryRoomManager +let manager: RedisRoomManager let socketUrl: string let failAccessStatus: number | null = null /** Users whose permission group withholds the Files module, as the app's access route decides. */ @@ -100,7 +100,7 @@ beforeAll(async () => { permissionType: 'write', }, ]) - await getFileDocStore().init() + await initFileDocStore(env.REDIS_URL) app = createServer(async (request, response) => { response.setHeader('Content-Type', 'application/json') if (request.headers['x-api-key'] !== env.INTERNAL_API_SECRET) { @@ -166,7 +166,8 @@ beforeAll(async () => { env.NEXT_PUBLIC_APP_URL = await listen(app) http = createServer() io = new Server(http) - manager = new MemoryRoomManager(io) + manager = new RedisRoomManager(io, env.REDIS_URL) + await manager.initialize() io.on('connection', (socket) => { const authed = socket as AuthenticatedSocket authed.userId = socket.handshake.auth.actor @@ -475,6 +476,7 @@ afterAll(async () => { if (io) await new Promise((done) => io.close(() => done())) await flushAllFileDocRooms() await getFileDocStore().shutdown() + await manager?.shutdown() if (app) await new Promise((done) => app.close(() => done())) await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) for (const id of [readerId, writerId]) await db.delete(user).where(eq(user.id, id)) diff --git a/apps/realtime/src/handlers/file-doc-presence.integration.ts b/apps/realtime/src/handlers/file-doc-presence.integration.ts index 536983fabfc..44a94388cca 100644 --- a/apps/realtime/src/handlers/file-doc-presence.integration.ts +++ b/apps/realtime/src/handlers/file-doc-presence.integration.ts @@ -18,7 +18,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' import type { AuthenticatedSocket } from '@/middleware/auth' -const redisUrl = process.env.REDIS_URL +const redisUrl = process.env.REDIS_URL ?? '' if (!redisUrl) throw new Error('Integration setup must provide disposable Redis') const internalSecret = process.env.INTERNAL_API_SECRET @@ -60,7 +60,7 @@ async function startReplica(): Promise { vi.resetModules() const fileDoc = await import('@/handlers/file-doc') const fileDocStore = await import('@/handlers/file-doc-store') - const { MemoryRoomManager } = await import('@/rooms') + const { RedisRoomManager } = await import('@/rooms') await fileDocStore.initFileDocStore(redisUrl) const pub: RedisClientType = createClient({ url: redisUrl }) const sub: RedisClientType = createClient({ url: redisUrl }) @@ -69,7 +69,8 @@ async function startReplica(): Promise { const io = new Server(http) io.adapter(createAdapter(pub, sub)) fileDoc.registerFileDocPresenceRelay(io) - const manager = new MemoryRoomManager(io) + const manager = new RedisRoomManager(io, redisUrl) + await manager.initialize() io.on('connection', (socket) => { const authed = socket as AuthenticatedSocket authed.userId = socket.handshake.auth.actor @@ -83,7 +84,10 @@ async function startReplica(): Promise { url: await listen(http), pub, sub, - shutdownStore: () => fileDocStore.getFileDocStore().shutdown(), + shutdownStore: async () => { + await fileDocStore.getFileDocStore().shutdown() + await manager.shutdown() + }, } replicas.push(replica) return replica diff --git a/apps/realtime/src/handlers/file-doc-project.integration.ts b/apps/realtime/src/handlers/file-doc-project.integration.ts index 4bbe641af25..0d8e33c1730 100644 --- a/apps/realtime/src/handlers/file-doc-project.integration.ts +++ b/apps/realtime/src/handlers/file-doc-project.integration.ts @@ -39,7 +39,7 @@ import { commitRoomPermission, ROLE_REVALIDATION_TTL_MS, } from '@/middleware/permissions' -import { MemoryRoomManager } from '@/rooms' +import { RedisRoomManager } from '@/rooms' import { createHttpHandler } from '@/routes/http' const projectId = generateId() @@ -66,6 +66,7 @@ const delayedAccess = new Map< let app: HttpServer let http: HttpServer let io: Server +let manager: RedisRoomManager let socketUrl: string const capturedActors: string[] = [] const seedFailures = new Map() @@ -162,7 +163,8 @@ beforeAll(async () => { env.NEXT_PUBLIC_APP_URL = await listen(app) http = createServer() io = new Server(http) - const manager = new MemoryRoomManager(io) + manager = new RedisRoomManager(io, env.REDIS_URL) + await manager.initialize() http.on('request', createHttpHandler(manager, createLogger('OwnerProtocolFixture'))) io.on('connection', (socket) => { const authed = socket as AuthenticatedSocket @@ -613,7 +615,7 @@ describe('Project documents across the Socket.IO and internal HTTP boundary', () accessGates.set(actor, gate) const now = Date.now() const clock = vi.spyOn(Date, 'now').mockReturnValue(now + ROLE_REVALIDATION_TTL_MS + 1) - const sweep = startAccessRevalidationSweep(new MemoryRoomManager(io)) + const sweep = startAccessRevalidationSweep(manager) sweep.stop() const room = `project-file-doc:${projectId}/${fileId}` try { @@ -884,6 +886,7 @@ afterAll(async () => { await new Promise((resolve) => io.close(() => resolve())) await new Promise((resolve) => app.close(() => resolve())) await getFileDocStore().shutdown() + await manager?.shutdown() document.destroy() const reportPath = process.env.PROJECT_FILE_SOCKET_REPORT_PATH ?? resolve('test-results/project-file-socket.json') diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index 96a2e32a94b..c192028aeb6 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -1,324 +1,14 @@ import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' -import { sleep } from '@sim/utils/helpers' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' -interface TestStreamEntry { - id: string - message: Record -} - -interface TestRedisClient { - isOpen: boolean - connect(): Promise - quit(): Promise - on(): TestRedisClient - duplicate(): TestRedisClient - xAdd(key: string, id: string, fields: Record): Promise - xRange( - key: string, - start: string, - end: string, - options?: { COUNT?: number } - ): Promise - xRevRange( - key: string, - start: string, - end: string, - options?: { COUNT?: number } - ): Promise - xLen(key: string): Promise - xTrim(key: string, strategy: string, minId: string): Promise - xRead( - streams: { key: string; id: string }[], - options?: { BLOCK?: number; COUNT?: number } - ): Promise<{ name: string; messages: TestStreamEntry[] }[] | null> - set(key: string, value: string, options?: { NX?: boolean }): Promise - get(key: string): Promise - del(keys: string | string[]): Promise - eval( - script: string, - options: { keys: string[]; arguments: string[] } - ): Promise - expire(): Promise -} - -/** - * One shared in-memory Redis backing per test, so several {@link FileDocStore} instances (modelling - * several ECS tasks) all talk to the "same Redis". A minimal fake of just the stream/lock ops the - * store uses. - */ -interface Backing { - streams: Map - kv: Map - dedupe: Map - seq: number - /** Override generated IDs to model a recreated stream restarting its same-millisecond sequence. */ - nextIds?: string[] - /** Number of upcoming xAdd calls to fail with a transient error (to exercise publish retry). */ - failXAdd: number - /** Set to fail every xRead the way node-redis does once a client has been closed. */ - readerClosed: boolean - /** Failed reads served, so a test can prove the loop is not spinning at the read cadence. */ - reads: number - /** When each FAILED read was attempted, so a test can measure one backoff interval exactly. */ - failedReadTimes: number[] - /** Reads that returned (the idle steady state) — the event that ends a failure streak. */ - idleReads: number - /** `connect()` calls, so a test can prove a closed reader is re-opened rather than abandoned. */ - connects: number - /** Largest stream range response requested, proving replay is paginated. */ - maxRangeCount: number - /** Optional deterministic compaction hook invoked before each range page is read. */ - onRange?: (call: number, key: string, start: string) => void - rangeCalls: number - /** Largest multiplexed XREAD request and COUNT observed. */ - maxReadStreams: number - maxReadCount: number - onSnapshot?: () => Promise - failSnapshotTrim?: boolean - onLength?: () => Promise -} - -const state = vi.hoisted(() => ({ backing: null as Backing | null })) - -function compareStreamIds(left: string, right: string): bigint { - const [leftMs, leftSequence] = left.split('-').map(BigInt) - const [rightMs, rightSequence] = right.split('-').map(BigInt) - return leftMs === rightMs ? leftSequence - rightSequence : leftMs - rightMs -} - -function makeClient(): TestRedisClient { - const b = () => { - if (!state.backing) throw new Error('backing not initialized') - return state.backing - } - const nextId = () => b().nextIds?.shift() ?? `${++b().seq}-0` - const client: TestRedisClient = { - isOpen: true, - connect: async () => { - client.isOpen = true - b().connects++ - }, - quit: async () => {}, - on: () => client, - duplicate: () => makeClient(), - xAdd: async (key: string, _star: string, fields: Record) => { - if (b().failXAdd > 0) { - b().failXAdd-- - throw new Error('transient xAdd failure') - } - const id = nextId() - const arr = b().streams.get(key) ?? [] - arr.push({ id, message: { ...fields } }) - b().streams.set(key, arr) - return id - }, - xRange: async (key: string, start: string, end: string, options?: { COUNT?: number }) => { - b().rangeCalls++ - b().onRange?.(b().rangeCalls, key, start) - const startId = start.startsWith('(') ? start.slice(1) : start - const entries = (b().streams.get(key) ?? []).filter( - (entry) => - (start === '-' || compareStreamIds(entry.id, startId) > 0n) && - (end === '+' || compareStreamIds(entry.id, end) <= 0n) - ) - const count = options?.COUNT ?? entries.length - b().maxRangeCount = Math.max(b().maxRangeCount, count) - return entries.slice(0, count).map((entry) => ({ ...entry })) - }, - xRevRange: async (key: string, _start: string, _end: string, options?: { COUNT?: number }) => - [...(b().streams.get(key) ?? [])] - .reverse() - .slice(0, options?.COUNT) - .map((entry) => ({ ...entry })), - xLen: async (key: string) => { - await b().onLength?.() - return (b().streams.get(key) ?? []).length - }, - xTrim: async (key: string, _strategy: string, minid: string) => { - const arr = b().streams.get(key) ?? [] - b().streams.set( - key, - arr.filter((e) => compareStreamIds(e.id, minid) >= 0n) - ) - }, - xRead: async ( - streams: { key: string; id: string }[], - options?: { BLOCK?: number; COUNT?: number } - ) => { - b().reads++ - b().maxReadStreams = Math.max(b().maxReadStreams, streams.length) - b().maxReadCount = Math.max(b().maxReadCount, options?.COUNT ?? 0) - if (b().readerClosed) { - b().failedReadTimes.push(Date.now()) - client.isOpen = false - throw new Error('The client is closed') - } - const res: { name: string; messages: TestStreamEntry[] }[] = [] - for (const { key, id } of streams) { - const after = (b().streams.get(key) ?? []) - .filter((e) => compareStreamIds(e.id, id) > 0n) - .slice(0, options?.COUNT) - if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) - } - if (res.length) { - b().idleReads++ - return res - } - await sleep(5) - b().idleReads++ - return null - }, - set: async (key: string, val: string, opts?: { NX?: boolean }) => { - if (opts?.NX && b().kv.has(key)) return null - b().kv.set(key, val) - return 'OK' - }, - get: async (key: string) => b().kv.get(key) ?? null, - del: async (keys: string | string[]) => { - const targets = Array.isArray(keys) ? keys : [keys] - for (const key of targets) { - b().kv.delete(key) - b().streams.delete(key) - b().dedupe.delete(key) - } - return targets.length - }, - eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { - const [key] = opts.keys - if (script.startsWith('for _, key in ipairs(KEYS)')) return 1 - if (script.includes("redis.call('exists', KEYS[1])") && !b().streams.has(key)) { - return script.includes('zscore') ? -1 : false - } - if (script.includes('return ARGV[1]')) { - const generation = b().kv.get(opts.keys[1]) - if (generation !== undefined) return generation - if (!b().streams.get(key)?.length) return false - b().kv.set(opts.keys[1], opts.arguments[0]) - return opts.arguments[0] - } - if (script.includes("redis.call('del', KEYS[1], KEYS[4], KEYS[5])")) { - const [, generationKey, versionKey, dedupeKey, agentKey, invalidationKey] = opts.keys - const [version, , marker] = opts.arguments - const current = b().kv.get(versionKey) - const invalidated = b().kv.get(invalidationKey) - if (invalidated && Number(invalidated) >= Number(version)) return null - if (current && Number(current) > Number(version)) return null - if (current === version && b().kv.get(generationKey) === marker) return null - const generation = b().kv.get(generationKey) ?? '' - b().kv.set(generationKey, marker) - b().kv.set(versionKey, version) - b().kv.set(invalidationKey, version) - b().streams.delete(key) - b().dedupe.delete(dedupeKey) - b().kv.delete(agentKey) - return generation - } - if (script.includes('zscore')) { - const [, dedupeKey, generationKey] = opts.keys - const [member, field, value, capacityText, , expectedGeneration] = opts.arguments - const generation = b().kv.get(generationKey) - if ((generation ?? '') !== expectedGeneration) return -1 - const members = b().dedupe.get(dedupeKey) ?? [] - if (members.includes(member)) return 0 - const id = nextId() - const arr = b().streams.get(key) ?? [] - arr.push({ id, message: { [field]: value, eu: opts.arguments[6], ec: opts.arguments[7] } }) - b().streams.set(key, arr) - members.push(member) - const capacity = Number(capacityText) - if (members.length > capacity) members.splice(0, members.length - capacity) - b().dedupe.set(dedupeKey, members) - return 1 - } - // Atomic seed-if-empty (SEED_IF_EMPTY_SCRIPT): append the entry iff the stream is empty, in one - // synchronous step — mirroring Redis's atomic Lua execution, so two concurrent evals can never both - // append (the second sees a non-empty stream). - if (script.includes('xlen')) { - const [, generationKey, versionKey] = opts.keys - const [field, value, generation, , generationField, version] = opts.arguments - if (Number(b().kv.get(versionKey) ?? 0) > Number(version)) return 0 - const arr = b().streams.get(key) ?? [] - if (arr.length > 0) return 0 - b().kv.set(generationKey, generation) - if (version !== '0') b().kv.set(versionKey, version) - const id = nextId() - arr.push({ id, message: { [field]: value, [generationField]: generation } }) - b().streams.set(key, arr) - return 1 - } - if (script.includes('ARGV[5], ARGV[4]')) { - const [, generationKey] = opts.keys - const [field, value, marker, expectedGeneration, generationField, upTo, compactionField] = - opts.arguments - const generation = b().kv.get(generationKey) - if ((generation ?? '') !== expectedGeneration) return false - const id = nextId() - const arr = b().streams.get(key) ?? [] - arr.push({ - id, - message: { - [field]: value, - [marker]: '1', - [generationField]: expectedGeneration, - ...(compactionField ? { [compactionField]: '1' } : {}), - eu: opts.arguments[7], - ec: opts.arguments[8], - ei: opts.arguments[9], - }, - }) - b().streams.set(key, arr) - if (script.includes("redis.call('xtrim'")) { - if (b().failSnapshotTrim) throw new Error('snapshot trim failed') - b().streams.set( - key, - arr.filter((entry) => compareStreamIds(entry.id, upTo) >= 0n) - ) - } - await b().onSnapshot?.() - return id - } - if (script.includes("ARGV[3] ~= ''")) { - const [, generationKey] = opts.keys - const generation = b().kv.get(generationKey) - const expectedGeneration = opts.arguments[3] - if ((generation ?? '') !== expectedGeneration) return false - if (b().failXAdd > 0) { - b().failXAdd-- - throw new Error('transient xAdd failure') - } - const [field, value, marker] = opts.arguments - const id = nextId() - const arr = b().streams.get(key) ?? [] - arr.push({ id, message: { [field]: value, ...(marker ? { [marker]: '1' } : {}) } }) - b().streams.set(key, arr) - return id - } - if (script.includes('tonumber(c)')) { - const [value, , expectedGeneration] = opts.arguments - const generation = b().kv.get(opts.keys[1]) - if ((generation ?? '') !== expectedGeneration) return 0 - const current = b().kv.get(key) - if (current === undefined || Number(current) < Number(value)) b().kv.set(key, value) - return 1 - } - // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. - const [token] = opts.arguments - if (b().kv.get(key) === token) { - b().kv.delete(key) - return 1 - } - return 0 - }, - expire: async () => 1, - } - return client -} - -vi.mock('redis', () => ({ createClient: () => makeClient() })) +vi.mock('redis', async () => { + const { makeClient } = await import('@/handlers/fixtures/redis.fixture') + return { createClient: () => makeClient() } +}) import { FileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN } from '@/handlers/file-doc-store' +import { createBacking, fakeRedis } from '@/handlers/fixtures/redis.fixture' const REDIS_URL = 'redis://fake' const NAME = 'workspace-file-doc:file-1' @@ -336,7 +26,6 @@ interface StoreRoomTestAccess { } interface StoreTestAccess { - localInvalidations: Map rooms: Map maybeCompact(name: string): Promise appendUpdate(name: string, update: Uint8Array): Promise @@ -382,7 +71,7 @@ let stores: FileDocStore[] = [] /** An existing stream from a relay predating generation markers; modern seeds use seedIfEmpty. */ function seedLegacyStream(update = updateFor('')): void { - const backing = state.backing! + const backing = fakeRedis.backing! backing.streams.set(`filedoc:stream:${NAME}`, [ { id: `${++backing.seq}-0`, message: { u: Buffer.from(update).toString('base64') } }, ]) @@ -451,22 +140,7 @@ describe('FileDocStore', () => { }) beforeEach(() => { - state.backing = { - streams: new Map(), - kv: new Map(), - dedupe: new Map(), - seq: 0, - failXAdd: 0, - readerClosed: false, - reads: 0, - failedReadTimes: [], - idleReads: 0, - connects: 0, - maxRangeCount: 0, - rangeCalls: 0, - maxReadStreams: 0, - maxReadCount: 0, - } + fakeRedis.backing = createBacking() stores = [] }) @@ -486,19 +160,19 @@ describe('FileDocStore', () => { const store = await newStore() const doc = new Y.Doc() await store.attachRoom(NAME, doc) - state.backing!.readerClosed = true + fakeRedis.backing!.readerClosed = true - state.backing!.connects = 0 // ignore the two `init` connects; count only recovery attempts - const before = state.backing!.reads + fakeRedis.backing!.connects = 0 // ignore the two `init` connects; count only recovery attempts + const before = fakeRedis.backing!.reads await vi.advanceTimersByTimeAsync(3000) - const attempts = state.backing!.reads - before + const attempts = fakeRedis.backing!.reads - before // A fixed 500ms retry manages 6 attempts in this window; backing off (500 → 1s → 2s → …) // manages about 3. The ±20% jitter moves the exact count, so assert that it slowed down. expect(attempts).toBeGreaterThan(0) expect(attempts).toBeLessThanOrEqual(4) // …and it tried to bring the connection back rather than leaving the tailer dead forever. - expect(state.backing!.connects).toBeGreaterThan(0) + expect(fakeRedis.backing!.connects).toBeGreaterThan(0) doc.destroy() } finally { vi.useRealTimers() @@ -561,7 +235,7 @@ describe('FileDocStore', () => { userId: 'editor', connectionId: 'socket-editor', }) - state.backing!.streams.delete(`filedoc:stream:${NAME}`) + fakeRedis.backing!.streams.delete(`filedoc:stream:${NAME}`) await expect(store.publishAndWait(NAME, delta, generation)).rejects.toThrow('replaced') await expect( @@ -576,14 +250,14 @@ describe('FileDocStore', () => { connectionId: 'socket-editor', }) ).rejects.toThrow('replaced') - expect(state.backing!.streams.has(`filedoc:stream:${NAME}`)).toBe(false) + expect(fakeRedis.backing!.streams.has(`filedoc:stream:${NAME}`)).toBe(false) }) it('rejects a shared replay if the document generation changes between pages', async () => { const store = await newStore() await store.seedIfEmpty(NAME, seedFor('old generation'), 10) - state.backing!.onRange = () => { - state.backing!.kv.set(`filedoc:generation:${NAME}`, 'new generation') + fakeRedis.backing!.onRange = () => { + fakeRedis.backing!.kv.set(`filedoc:generation:${NAME}`, 'new generation') } await expect(store.getStreamState(NAME)).rejects.toThrow('replaced') }) @@ -591,14 +265,14 @@ describe('FileDocStore', () => { it('fails safely when an uncompacted stream exceeds the replay entry budget', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(updateFor('')).toString('base64') - state.backing!.streams.set( + fakeRedis.backing!.streams.set( streamKey, Array.from({ length: 2_001 }, (_, index) => ({ id: `${index + 1}-0`, message: { u: noop }, })) ) - state.backing!.seq = 2_001 + fakeRedis.backing!.seq = 2_001 const store = await newStore() await expect(store.getStreamState(NAME)).rejects.toThrow('replay exceeded its safety limit') @@ -607,14 +281,14 @@ describe('FileDocStore', () => { it('never exposes a partially replayed document when room attachment exceeds its budget', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(updateFor('')).toString('base64') - state.backing!.streams.set( + fakeRedis.backing!.streams.set( streamKey, Array.from({ length: 2_001 }, (_, index) => ({ id: `${index + 1}-0`, message: { u: noop }, })) ) - state.backing!.seq = 2_001 + fakeRedis.backing!.seq = 2_001 const store = await newStore() const doc = new Y.Doc() @@ -628,24 +302,24 @@ describe('FileDocStore', () => { it('recovers from compaction that trims unread pages during replay', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(updateFor('')).toString('base64') - state.backing!.streams.set( + fakeRedis.backing!.streams.set( streamKey, Array.from({ length: 8 }, (_, index) => ({ id: `${index + 1}-0`, message: { u: noop }, })) ) - state.backing!.seq = 8 - state.backing!.onRange = (call, key) => { + fakeRedis.backing!.seq = 8 + fakeRedis.backing!.onRange = (call, key) => { if (call !== 2 || key !== streamKey) return - state.backing!.streams.set(streamKey, [ + fakeRedis.backing!.streams.set(streamKey, [ { id: '9-0', message: { u: Buffer.from(updateFor('compacted')).toString('base64'), s: '1' }, }, ]) - state.backing!.seq = 9 - state.backing!.onRange = undefined + fakeRedis.backing!.seq = 9 + fakeRedis.backing!.onRange = undefined } const store = await newStore() @@ -668,16 +342,16 @@ describe('FileDocStore', () => { for (let i = 1; i <= 8; i++) source.getText('body').insert(source.getText('body').length, String(i)) const snapshot = Y.encodeStateAsUpdate(source) - state.backing!.streams.set(streamKey, entries.slice()) + fakeRedis.backing!.streams.set(streamKey, entries.slice()) for (let i = 9; i <= 11; i++) source.getText('body').insert(source.getText('body').length, String(i)) - state.backing!.onRange = (_call, key, start) => { + fakeRedis.backing!.onRange = (_call, key, start) => { if (key !== streamKey || start !== '(4-0') return - state.backing!.streams.set(streamKey, [ + fakeRedis.backing!.streams.set(streamKey, [ ...entries.slice(7), { id: '12-0', message: { u: Buffer.from(snapshot).toString('base64'), s: '1' } }, ]) - state.backing!.onRange = undefined + fakeRedis.backing!.onRange = undefined } const store = await newStore() const recovered = new Y.Doc() @@ -709,8 +383,8 @@ describe('FileDocStore', () => { })) entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) - state.backing!.streams.set(streamKey, entries) - state.backing!.seq = 402 + fakeRedis.backing!.streams.set(streamKey, entries) + fakeRedis.backing!.seq = 402 const a = await newStore() // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the @@ -736,7 +410,7 @@ describe('FileDocStore', () => { // The appended snapshot entry must carry the snapshot marker, so a fresh catch-up task treats it as // edited content (not a bare seed) and persists on last-disconnect. - const stream = state.backing!.streams.get(streamKey)! + const stream = fakeRedis.backing!.streams.get(streamKey)! expect(stream[stream.length - 1].message.s).toBe('1') }) @@ -759,7 +433,7 @@ describe('FileDocStore', () => { // the peer tailer applies it as REDIS_AGENT_ORIGIN — excluded from the relay's edited/persist gate. a.publish(NAME, updateFor('agent frame'), true) await vi.waitFor(() => expect(origins).toContain(REDIS_AGENT_ORIGIN), { timeout: 2000 }) - const stream = state.backing!.streams.get(streamKey)! + const stream = fakeRedis.backing!.streams.get(streamKey)! expect(stream.some((e) => e.message.a === '1')).toBe(true) // The normal edit's entry carries no agent marker. expect(stream.filter((e) => e.message.a === '1')).toHaveLength(1) @@ -797,7 +471,7 @@ describe('FileDocStore', () => { } await vi.waitFor(() => { - const stream = state.backing!.streams.get(`filedoc:stream:${NAME}`)! + const stream = fakeRedis.backing!.streams.get(`filedoc:stream:${NAME}`)! expect(stream.length).toBeLessThan(400) expect(stream.some((entry) => entry.message.c === '1')).toBe(true) }) @@ -813,7 +487,7 @@ describe('FileDocStore', () => { it('retries a transient append failure so the edit is not lost from the shared log', async () => { seedLegacyStream() const a = await newStore() - state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed + fakeRedis.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed a.publish(NAME, updateFor('resilient')) await vi.waitFor( async () => { @@ -840,7 +514,7 @@ describe('FileDocStore', () => { connectionId: 'socket-editor', }) - expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + expect(fakeRedis.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) }) it('does not drop different payloads that reuse an acknowledged update id', async () => { @@ -856,7 +530,7 @@ describe('FileDocStore', () => { connectionId: 'socket-editor', }) - expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) + expect(fakeRedis.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) }) it('preserves the inclusive barrier and delta bytes observed during compaction', async () => { @@ -868,7 +542,7 @@ describe('FileDocStore', () => { room.uncompactedDeltaBytes = 12 * 1024 * 1024 const retainedBarrierBytes = room.lastDeltaBytes const lateUpdate = updateFor('concurrent edit') - state.backing!.onSnapshot = async () => { + fakeRedis.backing!.onSnapshot = async () => { await store.publishAndWait(NAME, lateUpdate) await store.catchUp(NAME) } @@ -886,7 +560,7 @@ describe('FileDocStore', () => { const replacer = await newStore() const oldDoc = docWithText('old') oldDoc.getMap('config').set('docId', 'old-generation') - state.backing!.nextIds = ['1000-0', '1000-1', '1000-2'] + fakeRedis.backing!.nextIds = ['1000-0', '1000-1', '1000-2'] await store.seedIfEmpty(NAME, Y.encodeStateAsUpdate(oldDoc), 10) await store.publishAndWait(NAME, updateFor('first edit'), 'old-generation') await store.publishAndWait(NAME, updateFor('second edit'), 'old-generation') @@ -900,15 +574,15 @@ describe('FileDocStore', () => { freshDoc.getText('body').insert(5, ' accepted edit') const freshEdit = Y.encodeStateAsUpdate(freshDoc, beforeEdit) const streamKey = `filedoc:stream:${NAME}` - state.backing!.nextIds = ['1000-3', '1000-0', '1000-1'] - state.backing!.onSnapshot = async () => { + fakeRedis.backing!.nextIds = ['1000-3', '1000-0', '1000-1'] + fakeRedis.backing!.onSnapshot = async () => { await replacer.invalidateDocument(NAME, 20) await replacer.seedIfEmpty(NAME, freshSeed, 20) await replacer.publishClientUpdateAndWait(NAME, 'fresh-edit', freshEdit, 'new-generation', { userId: 'editor', connectionId: 'socket-editor', }) - expect(state.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ + expect(fakeRedis.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ '1000-0', '1000-1', ]) @@ -916,7 +590,7 @@ describe('FileDocStore', () => { await storeInternals(store).maybeCompact(NAME) - expect(state.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ + expect(fakeRedis.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ '1000-0', '1000-1', ]) @@ -924,7 +598,7 @@ describe('FileDocStore', () => { userId: 'editor', connectionId: 'socket-editor', }) - expect(state.backing!.streams.get(streamKey)).toHaveLength(2) + expect(fakeRedis.backing!.streams.get(streamKey)).toHaveLength(2) const persisted = await replacer.getStreamState(NAME) expect(persisted).not.toBeNull() const replayed = new Y.Doc() @@ -980,7 +654,7 @@ describe('FileDocStore', () => { const tokenA = await a.shouldSeed(NAME) expect(tokenA).toBeTruthy() // Simulate A's lock expiring mid-seed so B also wins the freed lock over a still-empty stream. - state.backing!.kv.delete(`filedoc:seedlock:${NAME}`) + fakeRedis.backing!.kv.delete(`filedoc:seedlock:${NAME}`) const tokenB = await b.shouldSeed(NAME) expect(tokenB).toBeTruthy() // Both tasks now race to seed with distinct client ids. @@ -1040,8 +714,8 @@ describe('FileDocStore', () => { })) entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) - state.backing!.streams.set(streamKey, entries) - state.backing!.seq = 402 + fakeRedis.backing!.streams.set(streamKey, entries) + fakeRedis.backing!.seq = 402 // Two tasks whose local docs lag at DIFFERENT points (400 and 401): both cross the threshold and // compact concurrently. Each must only trim what its own snapshot subsumes, so the union of both diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 1694416b774..1652b6e59a6 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -29,9 +29,6 @@ * tasks race. {@link shouldSeed} is a Redis lock + empty-stream check layered on top ONLY as an * efficiency gate (so tasks don't all run the seed fetch); correctness does not depend on it. * - * When `REDIS_URL` is unset (single-pod dev) the store is DISABLED and every method degrades to the - * relay's original single-replica behavior: seed locally, no stream, no tailer. - * * @module */ @@ -170,11 +167,6 @@ const COMPACTION_FIELD = 'c' const GENERATION_FIELD = 'g' const INVALIDATED_GENERATION = '__invalidated__' -/** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed - * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it - * can never collide with a {@link generateId} token. */ -const DISABLED_LOCK_TOKEN = '__disabled__' - /** How long a blocking multiplexed read waits before re-snapshotting the live room set. Also bounds * how long a room attached mid-block waits for its first cross-task update (updates are not lost — the * next read resumes from its last id — only briefly delayed). */ @@ -362,31 +354,21 @@ interface StoreRoom { realEdited: boolean } -/** - * The Redis-Streams shared Yjs backend. A single instance per process. `enabled` is false when there - * is no `REDIS_URL`, in which case every method is a no-op and the relay runs single-replica. - */ +/** The Redis-Streams shared Yjs backend. A single instance per process. */ export class FileDocStore { - readonly enabled: boolean /** Command connection: XADD / locks / XLEN / XTRIM / EXPIRE. */ private write: RedisClientType | null = null /** Dedicated connection for blocking XREAD (a blocking command monopolizes its connection). */ private read: RedisClientType | null = null private readonly rooms = new Map() - private readonly localInvalidations = new Map< - string, - { version: number; expiresAt: number; replacementDocId?: string } - >() private running = false private heartbeat: ReturnType | null = null - constructor(private readonly redisUrl: string | undefined) { - this.enabled = Boolean(redisUrl) - } + constructor(private readonly redisUrl: string) {} /** Connect the two Redis clients and start the multiplexed reader + TTL heartbeat. Idempotent. */ async init(): Promise { - if (!this.enabled || this.running || !this.redisUrl) return + if (this.running) return const options = { url: this.redisUrl, socket: { @@ -426,16 +408,15 @@ export class FileDocStore { this.write = null this.read = null this.rooms.clear() - this.localInvalidations.clear() } /** * Register a locally-opened room and load the shared state into its doc ({@link catchUp}). A * brand-new file has an empty stream and loads nothing (it is seeded shortly after, via - * {@link shouldSeed}). Single-replica rooms are tracked only for invalidation lifecycle. + * {@link shouldSeed}). */ async attachRoom(name: string, doc: Y.Doc): Promise { - if (this.enabled && !this.write) throw new Error('FileDocStore is not initialized') + if (!this.write) throw new Error('FileDocStore is not initialized') // Register BEFORE the async read so a concurrent publish/tailer for this room can't be missed — // the tailer resumes from `lastId`, which the catch-up advances. const room: StoreRoom = { @@ -453,7 +434,6 @@ export class FileDocStore { realEdited: false, } this.rooms.set(name, room) - if (!this.enabled) return try { await this.catchUp(name) } catch (error) { @@ -464,10 +444,9 @@ export class FileDocStore { /** * Completes shared replay before applying entries, so joins never receive a partial document. - * Repeated calls skip integrated entries; detached rooms and disabled stores are ignored. + * Repeated calls skip integrated entries; detached rooms are ignored. */ async catchUp(name: string): Promise { - if (!this.enabled) return if (!this.write) throw new Error('FileDocStore is not initialized') const room = this.rooms.get(name) if (!room) return @@ -568,11 +547,11 @@ export class FileDocStore { /** * Fire-and-forget append for the hot keystroke path (`doc.on('update')`): converges peers without - * blocking the relay. Retries internally; never throws. No-op when disabled. Pass `agent: true` for + * blocking the relay. Retries internally; never throws. Pass `agent: true` for * a copilot preview frame so peer tasks tail it as {@link REDIS_AGENT_ORIGIN} and never persist it. */ publish(name: string, update: Uint8Array, agent = false): void { - if (!this.enabled || !this.write) return + if (!this.write) return void this.appendUpdate(name, update, agent).catch((error) => { logger.warn(`FileDocStore rejected a non-durable legacy update for ${name}`, { error: getErrorMessage(error), @@ -583,14 +562,13 @@ export class FileDocStore { /** * Awaitable append for callers that must know the update is durably in the stream before proceeding * — the copilot merge, so the cross-task merge lock is not released before the diff is committed - * (else the next task would diff a stale base). Throws on ultimate failure. No-op when disabled. + * (else the next task would diff a stale base). Throws on ultimate failure. */ async publishAndWait( name: string, update: Uint8Array, expectedGeneration = this.rooms.get(name)?.generation ?? '' ): Promise { - if (!this.enabled) return if (!this.write) throw new Error('FileDocStore is not initialized') await this.appendUpdate(name, update, false, expectedGeneration) } @@ -609,7 +587,6 @@ export class FileDocStore { if (!editor?.userId.trim() || !editor.connectionId.trim()) { throw new Error('Document updates require an authenticated editor') } - if (!this.enabled) return if (!this.write) throw new Error('FileDocStore is not initialized') assertUpdateWithinLimit(update) const encoded = Buffer.from(update).toString('base64') @@ -674,8 +651,7 @@ export class FileDocStore { * there is no check-then-append window, and two tasks racing (even both past an expired seed lock) can * never both write a seed. Returns true if THIS call wrote the seed (apply it locally), false if the * stream was already seeded (the tailer will deliver the peer's seed — do NOT apply a second one). - * Retries a transient Redis error like {@link appendUpdate}; throws if it ultimately fails. Disabled → - * true (single-replica: seed locally, no stream). + * Retries a transient Redis error like {@link appendUpdate}; throws if it ultimately fails. */ async seedIfEmpty( name: string, @@ -685,22 +661,6 @@ export class FileDocStore { ): Promise { assertUpdateWithinLimit(update) const generation = generationOfSeed(update) - if (!this.enabled) { - const invalidation = this.localInvalidations.get(name) - if (invalidation && invalidation.expiresAt > Date.now() && invalidation.version > version) - return false - if ( - strictGeneration && - invalidation && - invalidation.expiresAt > Date.now() && - invalidation.replacementDocId && - invalidation.replacementDocId !== generation - ) - return false - const room = this.rooms.get(name) - if (room) room.generationInvalidated = false - return true - } if (!this.write) throw new Error('FileDocStore is not initialized') const encoded = Buffer.from(update).toString('base64') for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { @@ -744,23 +704,6 @@ export class FileDocStore { name: string, version: number ): Promise<{ status: 'applied'; docId?: string } | { status: 'stale' }> { - if (!this.enabled) { - const now = Date.now() - const previous = this.localInvalidations.get(name) - if (previous && previous.expiresAt > now && previous.version >= version) - return { status: 'stale' } - this.localInvalidations.set(name, { version, expiresAt: now + STREAM_TTL_SEC * 1_000 }) - const room = this.rooms.get(name) - const docId = room?.generationInvalidated - ? undefined - : room?.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) - if (room) room.generationInvalidated = true - if (!this.heartbeat) { - this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) - this.heartbeat.unref() - } - return { status: 'applied', ...(typeof docId === 'string' ? { docId } : {}) } - } if (!this.write) throw new Error('FileDocStore is not initialized') const generation = await this.write.eval(INVALIDATE_DOCUMENT_SCRIPT, { keys: [ @@ -781,27 +724,11 @@ export class FileDocStore { } async getDocumentGeneration(name: string): Promise { - if (!this.enabled) { - const invalidation = this.localInvalidations.get(name) - if (invalidation && invalidation.expiresAt > Date.now()) - return invalidation.replacementDocId ?? INVALIDATED_GENERATION - const generation = this.rooms - .get(name) - ?.doc.getMap(FILE_DOC_SEED.configMap) - .get(FILE_DOC_SEED.docIdKey) - return typeof generation === 'string' ? generation : '' - } if (!this.write) throw new Error('FileDocStore is not initialized') return (await this.write.get(generationKey(name))) ?? '' } async isDocumentGenerationCurrent(name: string, generation?: string): Promise { - if (!this.enabled) { - const invalidation = this.localInvalidations.get(name) - if (invalidation?.replacementDocId && invalidation.expiresAt > Date.now()) - return generation === invalidation.replacementDocId - return !this.rooms.get(name)?.generationInvalidated - } if (!this.write) throw new Error('FileDocStore is not initialized') const current = await this.write.get(generationKey(name)) return current === null ? !generation : current === generation @@ -815,23 +742,6 @@ export class FileDocStore { ): Promise<{ status: 'applied'; docId: string } | { status: 'stale' }> { if (!retiredDocId || !replacementDocId || retiredDocId === replacementDocId) throw new Error('Document retirement requires distinct identities') - if (!this.enabled) { - const generation = await this.getDocumentGeneration(name) - if (generation && generation !== retiredDocId) return { status: 'stale' } - const previous = this.localInvalidations.get(name) - this.localInvalidations.set(name, { - version: previous?.version ?? 0, - expiresAt: Date.now() + STREAM_TTL_SEC * 1_000, - replacementDocId, - }) - const room = this.rooms.get(name) - if (room) room.generationInvalidated = true - if (!this.heartbeat) { - this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) - this.heartbeat.unref() - } - return { status: 'applied', docId: retiredDocId } - } if (!this.write) throw new Error('FileDocStore is not initialized') const result = await this.write.eval(RETIRE_DOCUMENT_SCRIPT, { keys: [ @@ -852,10 +762,10 @@ export class FileDocStore { * skips the seed fetch when a prior holder already seeded (the split-brain guard itself is the atomic * {@link SEED_IF_EMPTY_SCRIPT}, not this check). Treats `true` as "already seeded", so it fails CLOSED: * a Redis `xLen` error returns `true` (cannot confirm empty → skip the redundant fetch; the atomic seed - * would no-op anyway). `false` only when genuinely empty, or when disabled (single-replica). + * would no-op anyway). `false` only when genuinely empty, or before the store is initialized. */ async streamHasContent(name: string): Promise { - if (!this.enabled || !this.write) return false + if (!this.write) return false try { return (await this.write.xLen(streamKey(name))) > 0 } catch (error) { @@ -869,11 +779,9 @@ export class FileDocStore { /** * Acquire a distributed lock with a unique ownership TOKEN (`SET key NX PX`). Returns the * token to release with, or `null` if not won. Fails CLOSED (null) on a Redis error — a lock we can't - * prove we hold must not be treated as held. The special sentinel {@link DISABLED_LOCK_TOKEN} lets a - * disabled store return a truthy token so callers proceed single-replica without special-casing. + * prove we hold must not be treated as held. */ private async acquireLock(key: string, ttlMs: number): Promise { - if (!this.enabled) return DISABLED_LOCK_TOKEN if (!this.write) return null const token = generateId() try { @@ -888,7 +796,7 @@ export class FileDocStore { /** Release a lock via compare-and-delete, so it is only dropped if we still hold our token. */ private async releaseLock(key: string, token: string): Promise { - if (!this.write || token === DISABLED_LOCK_TOKEN) return + if (!this.write) return await this.write.eval(RELEASE_LOCK_SCRIPT, { keys: [key], arguments: [token] }).catch(() => {}) } @@ -898,12 +806,11 @@ export class FileDocStore { * EFFICIENCY gate — it stops every task that opens the file at once from each fetching the seed. It does * NOT by itself guarantee a single seed: exactly-once is enforced by the atomic {@link seedIfEmpty} the * token-holder then calls (the split-brain guard), so a lock that expires mid-seed cannot cause a - * double-seed. Release the token with {@link releaseSeedLock}. Disabled → always a token (single-replica: - * seed locally). + * double-seed. Release the token with {@link releaseSeedLock}. */ async shouldSeed(name: string): Promise { const token = await this.acquireLock(`${SEED_LOCK_PREFIX}${name}`, SEED_LOCK_TTL_MS) - if (!token || token === DISABLED_LOCK_TOKEN) return token + if (!token) return null // The lock could be free yet the stream already seeded (a prior holder seeded then its lock // expired). Re-check so we skip the redundant seed fetch — the atomic seedIfEmpty would no-op anyway, // but this avoids the wasted app round-trip. @@ -918,7 +825,7 @@ export class FileDocStore { * Build the file's current shared state from the stream, headless (no registered room), for a merge * that must reach the live doc regardless of which task holds it. Returns the encoded Yjs state, or * `null` when the stream is empty — i.e. no doc is (or was recently) live, so there is nothing to - * merge into and the caller should fall back to a direct file write. Disabled → always null. + * merge into and the caller should fall back to a direct file write. */ async getStreamState(name: string, expectedGeneration?: string): Promise { return (await this.getStreamSnapshot(name, expectedGeneration))?.docState ?? null @@ -929,7 +836,6 @@ export class FileDocStore { name: string, expectedGeneration?: string ): Promise<{ docState: Uint8Array; editor: FileDocEditor | null } | null> { - if (!this.enabled) return null if (!this.write) throw new Error('FileDocStore is not initialized') const doc = new Y.Doc() try { @@ -1016,7 +922,7 @@ export class FileDocStore { * on this — it must always write. */ async tryClaimPersistWindow(name: string, ttlMs: number): Promise { - if (!this.enabled || !this.write) return true + if (!this.write) return true try { const won = await this.write.set(`${PERSIST_LOCK_PREFIX}${name}`, '1', { NX: true, @@ -1033,7 +939,7 @@ export class FileDocStore { * merges per task; this extends that across tasks so two copilot edits to the same file landing on * different tasks don't each diff the SAME shared base and publish conflicting full-document rewrites. * The loser waits and retries so it diffs against the winner's RESULT (correct sequential merge). - * Returns a lock TOKEN (proceed) when disabled or once won; `null` otherwise (fails CLOSED on error, so + * Returns a lock TOKEN (proceed) once won; `null` otherwise (fails CLOSED on error, so * a merge never races when exclusivity can't be proven). Release with {@link releaseMergeSlot}. */ async acquireMergeSlot(name: string, ttlMs: number): Promise { @@ -1048,7 +954,7 @@ export class FileDocStore { * another task. Returns `null` when unset/expired (persist then falls back to the local room's value). */ async getSyncedVersion(name: string): Promise { - if (!this.enabled || !this.write) return null + if (!this.write) return null try { const value = await this.write.get(`${SYNC_VERSION_PREFIX}${name}`) const parsed = value === null ? Number.NaN : Number(value) @@ -1064,13 +970,13 @@ export class FileDocStore { /** Record the durable version the shared live doc is now synced to. MONOTONIC — writes only when the * new value exceeds the stored one ({@link SET_VERSION_IF_NEWER_SCRIPT}), so an out-of-order * fire-and-forget write can't regress the token. Best-effort; TTL-bounded like the stream so an idle - * file's key can't outlive its room. No-op when disabled (single-pod fallback). */ + * file's key can't outlive its room. */ async setSyncedVersion( name: string, version: number, expectedGeneration = this.rooms.get(name)?.generation ?? '' ): Promise { - if (!this.enabled || !this.write) return + if (!this.write) return // Retry a transient failure (bounded) rather than swallow it: this token is the ONLY way a // peer-seeded task learns the durable version, so a dropped write would leave that peer's persists // deferring forever with the session's edits stranded in the TTL'd stream. The monotonic script makes @@ -1095,9 +1001,9 @@ export class FileDocStore { } /** Mark (or refresh) that a client is actively streaming an agent edit into this live doc — a plain - * `SET key "1" PX ttl`, so it self-clears when streaming stops. Best-effort; no-op when disabled. */ + * `SET key "1" PX ttl`, so it self-clears when streaming stops. Best-effort. */ async markAgentStreaming(name: string, ttlMs: number): Promise { - if (!this.enabled || !this.write) return + if (!this.write) return try { await this.write.set(`${AGENT_STREAM_PREFIX}${name}`, '1', { PX: ttlMs }) } catch (error) { @@ -1108,10 +1014,10 @@ export class FileDocStore { } /** Whether a client is currently streaming an agent edit into this live doc (see - * {@link markAgentStreaming}). Best-effort; treats an error/disabled store as "not streaming" so a - * merge never blocks on this check. */ + * {@link markAgentStreaming}). Best-effort; treats an error as "not streaming" so a merge never + * blocks on this check. */ async isAgentStreaming(name: string): Promise { - if (!this.enabled || !this.write) return false + if (!this.write) return false try { return (await this.write.exists(`${AGENT_STREAM_PREFIX}${name}`)) === 1 } catch (error) { @@ -1349,18 +1255,6 @@ export class FileDocStore { } private async refreshTtls(): Promise { - if (!this.write) { - const now = Date.now() - for (const [name, invalidation] of this.localInvalidations) { - if (this.rooms.has(name)) invalidation.expiresAt = now + STREAM_TTL_SEC * 1_000 - else if (invalidation.expiresAt <= now) this.localInvalidations.delete(name) - } - if (this.localInvalidations.size === 0 && this.heartbeat) { - clearInterval(this.heartbeat) - this.heartbeat = null - } - return - } for (const name of this.rooms.keys()) { await this.refreshDocumentTtls(name).catch(() => {}) } @@ -1371,19 +1265,16 @@ let store: FileDocStore | null = null /** * Initialize the process-wide store from the realtime server bootstrap (alongside the socket adapter). - * Authoritative: if a disabled placeholder was lazily created by an early {@link getFileDocStore} call, - * this REPLACES it with the real, connected store — so the bootstrap can never silently no-op. A second - * call once already initialized is a no-op. + * Idempotent: a second call reuses the existing store and re-connects it only if it was shut down. */ -export async function initFileDocStore(redisUrl: string | undefined): Promise { - if (store?.enabled) return store - store = new FileDocStore(redisUrl) +export async function initFileDocStore(redisUrl: string): Promise { + store ??= new FileDocStore(redisUrl) await store.init() return store } -/** The process-wide store. Returns a disabled instance if init was never called (e.g. in unit tests). */ +/** The process-wide store. Throws if {@link initFileDocStore} has not run. */ export function getFileDocStore(): FileDocStore { - if (!store) store = new FileDocStore(undefined) + if (!store) throw new Error('FileDocStore is not initialized') return store } diff --git a/apps/realtime/src/handlers/file-doc.multireplica.test.ts b/apps/realtime/src/handlers/file-doc.multireplica.test.ts index 15dacefc678..ab9f4bc967e 100644 --- a/apps/realtime/src/handlers/file-doc.multireplica.test.ts +++ b/apps/realtime/src/handlers/file-doc.multireplica.test.ts @@ -1,10 +1,8 @@ /** - * Multi-replica (store-enabled) coverage for the copilot live-merge stale-check. The main - * `file-doc.test.ts` runs with the store DISABLED (single-replica fallback); this file mocks an ENABLED - * store so the cross-process branch of `mergeMarkdownIntoRoom` — staleness against the SHARED synced - * version under the merge lock, and `recordVersion` writing `setSyncedVersion` — is exercised directly. - * The enabled merge path reads its base from the shared store (not an in-memory room), so no JOIN/seed - * is needed: calling `applyMarkdownToLiveFileDoc` against the fake store drives the branch on its own. + * Multi-replica coverage for the copilot live-merge stale-check: staleness against the SHARED synced + * version under the merge lock, and `recordVersion` writing `setSyncedVersion`. The merge path reads its + * base from the shared store (not an in-memory room), so no JOIN/seed is needed: calling + * `applyMarkdownToLiveFileDoc` against the fake store drives it on its own. */ import { beforeEach, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' @@ -14,12 +12,11 @@ const { mockFetchFileDocMerge } = vi.hoisted(() => ({ })) /** - * A minimal ENABLED store: in-memory monotonic synced version (mirrors SET_VERSION_IF_NEWER_SCRIPT), a - * non-null stream state so the merge has a base, and no-op locks/publish. Only the surface the - * store-enabled merge path touches is implemented. + * A minimal store: in-memory monotonic synced version (mirrors SET_VERSION_IF_NEWER_SCRIPT), a + * non-null stream state so the merge has a base, and no-op locks/publish. Only the surface the merge + * path touches is implemented. */ const fakeStore = { - enabled: true, versions: new Map(), acquireMergeSlot: vi.fn(async () => 'token'), releaseMergeSlot: vi.fn(async () => {}), @@ -53,7 +50,7 @@ import { applyMarkdownToLiveFileDoc } from '@/handlers/file-doc' const ROOM_NAME = 'workspace-file-doc:file-1' -describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering', () => { +describe('applyMarkdownToLiveFileDoc — multi-replica ordering', () => { beforeEach(() => { fakeStore.versions.clear() fakeStore.acquireMergeSlot.mockResolvedValue('token') @@ -102,8 +99,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' }) it('defers the content merge cluster-wide when a client is streaming (records version, no diff)', async () => { - // The cluster-wide counterpart of the single-replica deferral: while the shared `isAgentStreaming` - // flag is set (a client on ANY replica is streaming this agent edit), the durable merge must record + // While the shared `isAgentStreaming` flag is set (a client on ANY replica is streaming this agent edit), the durable merge must record // the version but skip the content diff — the streaming client owns the bytes, so a whole-document // merge here would double-write them. fakeStore.isAgentStreaming.mockResolvedValue(true) diff --git a/apps/realtime/src/handlers/file-doc.redis-store.test.ts b/apps/realtime/src/handlers/file-doc.redis-store.test.ts index 83c53f750de..4c86bcda1b2 100644 --- a/apps/realtime/src/handlers/file-doc.redis-store.test.ts +++ b/apps/realtime/src/handlers/file-doc.redis-store.test.ts @@ -1,5 +1,5 @@ /** - * The join's readiness contract, with the shared store ENABLED (`file-doc.test.ts` runs it disabled). + * The join's readiness contract against the shared store. * * A room loads its document from the file's Redis stream one entry at a time, into the same `Y.Doc` * that fans every update out to the room. So a client attached while that is happening is not sent the diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 03355f85eef..5d225d1d1b6 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -30,6 +30,11 @@ const { vi.mock('@sim/platform-authz/rooms', () => ({ authorizeRoom: vi.fn() })) +vi.mock('redis', async () => { + const { makeClient } = await import('@/handlers/fixtures/redis.fixture') + return { createClient: () => makeClient() } +}) + vi.mock('@/handlers/file-doc-app', () => ({ fetchFileDocAccess: mockFetchFileDocAccess, fetchFileDocSeed: mockFetchFileDocSeed, @@ -44,7 +49,12 @@ import { invalidateLiveFileDocument, setupWorkspaceFileDocHandlers, } from '@/handlers/file-doc' -import { FileDocInvalidatedError, getFileDocStore } from '@/handlers/file-doc-store' +import { + FileDocInvalidatedError, + getFileDocStore, + initFileDocStore, +} from '@/handlers/file-doc-store' +import { createBacking, fakeRedis } from '@/handlers/fixtures/redis.fixture' import { publishRoomPermission } from '@/handlers/room-permission' import * as permissions from '@/middleware/permissions' import { @@ -87,7 +97,7 @@ function createIo( left.push({ socketId, room }) }, })) - // With the shared store disabled, local and adapter broadcasts reach the same clients. + // A single replica: local and adapter broadcasts reach the same clients. return { io: { to, @@ -158,9 +168,8 @@ function setup(id: string, io: IRoomManager['io'], socketOverrides?: Record }) { } describe('setupWorkspaceFileDocHandlers', () => { - beforeEach(() => { + beforeEach(async () => { + fakeRedis.backing = createBacking() + await initFileDocStore('redis://fake') mockFetchFileDocAccess.mockResolvedValue({ allowed: true, status: 200, @@ -554,12 +565,18 @@ describe('setupWorkspaceFileDocHandlers', () => { const { io } = createIo() const { handlers } = setup('socket-closing-update', io) await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const store = getFileDocStore() + const appendToStream = store.publishClientUpdateAndWait.bind(store) let finishAppend!: () => void - const append = vi.spyOn(getFileDocStore(), 'publishClientUpdateAndWait').mockReturnValueOnce( - new Promise((resolve) => { - finishAppend = resolve + const appendGate = new Promise((resolve) => { + finishAppend = resolve + }) + const append = vi + .spyOn(store, 'publishClientUpdateAndWait') + .mockImplementationOnce(async (...args) => { + await appendGate + return appendToStream(...args) }) - ) const edit = new Y.Doc() edit.getText(FILE_DOC_FIELD).insert(0, 'accepted before socket close') const ack = vi.fn() diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index b22b720b9bf..901d6d5e651 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -13,9 +13,7 @@ * messages are broadcast LOCALLY ({@link io.local}) and cross-task delivery rides the stream — NOT the * Socket.IO adapter (that would double-deliver). Awareness stays on the adapter: it is ephemeral and * needs neither convergence nor replay. The presence roster is read across replicas with `fetchSockets`, - * and each replica (nudged by `serverSideEmit`) emits it to its own clients locally. When `REDIS_URL` - * is unset the store is disabled and this falls back to the original single-replica behavior (local - * doc, local seed, local fan-out). + * and each replica (nudged by `serverSideEmit`) emits it to its own clients locally. * * Durability: the live doc is projected back to the file's markdown server-side — debounced while it is * edited and flushed when the last collaborator leaves — via the app's `/persist` endpoint (the app @@ -185,15 +183,6 @@ interface FileDocRoom { * edit the live doc hasn't incorporated. `null` before the first seed (persist writes unconditionally). */ syncedVersion: number | null - /** - * Epoch ms until which a client is treated as actively streaming an agent edit into this doc — - * refreshed on every agent frame ({@link AGENT_SYNC_ORIGIN}/{@link REDIS_AGENT_ORIGIN}). While it is in - * the future, a durable {@link applyMarkdownToLiveFileDoc} merge DEFERS its content diff to that client - * (which applies the same content into the shared doc), preventing the two-writer duplication; the merge - * still records the durable version. The single-replica counterpart of the cluster-wide - * {@link FileDocStore.isAgentStreaming} flag. `0` when no agent stream is active. - */ - agentStreamingUntil: number /** * Resolves once this room's doc reflects the file's shared stream (see {@link FileDocStore.catchUp}). * Rejects when replay cannot complete so the join fails closed rather than serving partial state. @@ -311,8 +300,7 @@ function broadcast(io: Server, name: string, payload: Uint8Array, exceptSocketId /** * Broadcast a DOC-SYNC frame to this task's LOCAL clients only. Cross-task delivery rides the shared * Redis stream (each task's tailer applies the update and runs its OWN local fan-out), so using the - * adapter here would double-deliver and amplify. With no adapter (single-pod dev) `io.local` is the - * whole room, so behavior is unchanged. + * adapter here would double-deliver and amplify. */ function broadcastLocal( io: Server, @@ -331,7 +319,6 @@ function broadcastLocal( * last-disconnect is separate ({@link flushPersist} with `final`). */ function schedulePersist(name: string, room: FileDocRoom): void { - if (!getFileDocStore().enabled && (!room.lastEditorUserId || !room.lastEditorConnectionId)) return const now = Date.now() if (room.persistDeadline === null) room.persistDeadline = now + PERSIST_MAX_WAIT_MS if (room.persistTimer) clearTimeout(room.persistTimer) @@ -355,7 +342,7 @@ function schedulePersist(name: string, room: FileDocRoom): void { * holds content even when THIS task's doc was never locally seeded), so a last-disconnect flush can't * clobber the durable file with a lagging projection. The local doc is captured SYNCHRONOUSLY as a * fallback before any await, so a `void flushPersist(name, room, true)` fired immediately before the - * caller destroys `room.doc` never encodes a destroyed doc, and the disabled path stays authoritative. + * caller destroys `room.doc` never encodes a destroyed doc. */ function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { const pending = persistRoom(name, room, final).finally(() => @@ -367,11 +354,7 @@ function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise< async function persistRoom(name: string, room: FileDocRoom, final: boolean): Promise { // Never project a doc no user actually edited back over the file (see {@link FileDocRoom.edited}). - if ( - !room.edited || - (!getFileDocStore().enabled && (!room.lastEditorUserId || !room.lastEditorConnectionId)) - ) - return + if (!room.edited) return const store = getFileDocStore() const generation = docIdOf(room.doc) const userId = room.lastEditorUserId @@ -384,25 +367,10 @@ async function persistRoom(name: string, room: FileDocRoom, final: boolean): Pro // this yields. Only meaningful once seeded; used only when the authoritative stream state is absent. const localState = isDocSeeded(room.doc) ? Y.encodeStateAsUpdate(room.doc) : null - // Capture the AUTHORITATIVE doc state: the shared stream when enabled (a copilot merge or a peer's - // edit published by another task may not be integrated into THIS task's `room.doc` yet), else the - // local snapshot. Re-read each attempt so a post-reconcile retry projects the converged state. + // Capture the AUTHORITATIVE doc state: the shared stream (a copilot merge or a peer's edit published + // by another task may not be integrated into THIS task's `room.doc` yet). Re-read each attempt so a + // post-reconcile retry projects the converged state. const captureState = async (): Promise => { - if (!store.enabled) { - // Single-pod: re-read the live doc so a post-reconcile retry projects the CONVERGED state, not the - // snapshot captured before the reconcile mutated it (which — with the version already advanced — - // would let the If-Match pass and clobber the reconciled edit). Fall back to that snapshot once the - // room is torn down (last-leave), where the doc is gone and there is nothing left to reconcile. - if (fileDocRooms.get(name) === room && isDocSeeded(room.doc)) { - snapshotEditor = - room.lastEditorUserId && room.lastEditorConnectionId - ? { userId: room.lastEditorUserId, connectionId: room.lastEditorConnectionId } - : null - return Y.encodeStateAsUpdate(room.doc) - } - snapshotEditor = localEditor - return localState - } try { const snapshot = await store.getStreamSnapshot(name, generation) /** Entries without an author (older relays, durable merges) keep workspace attribution local. */ @@ -433,7 +401,7 @@ async function persistRoom(name: string, room: FileDocRoom, final: boolean): Pro // value (versions are monotonic epoch-ms). Cached back into the room so a later transient Redis failure // — or a peer-seeded/tail-only task that never set it locally — still resolves it. `undefined` if unknown. const currentVersion = async (): Promise => { - const shared = store.enabled ? await store.getSyncedVersion(name) : null + const shared = await store.getSyncedVersion(name) const best = Math.max(shared ?? 0, room.syncedVersion ?? 0) if (best > 0) room.syncedVersion = best return best > 0 ? best : undefined @@ -452,11 +420,7 @@ async function persistRoom(name: string, room: FileDocRoom, final: boolean): Pro // peer-seeded/tail-only task that never cached it, retry briefly rather than defer and strand the // edits in the TTL'd stream (the version is cluster-wide + heartbeat-refreshed). Bounded — a genuinely // unset version never appears, and the flush must not stall teardown. - for ( - let i = 0; - ifMatch === undefined && final && store.enabled && i < FINAL_VERSION_RETRIES; - i++ - ) { + for (let i = 0; ifMatch === undefined && final && i < FINAL_VERSION_RETRIES; i++) { await sleep(FINAL_VERSION_RETRY_MS) ifMatch = await currentVersion() } @@ -590,7 +554,7 @@ async function refreshFileDocPresence(io: Server, name: string, fileId: string) * the client-set awareness — so a peer cannot spoof or suppress an entry. */ function broadcastFileDocPresence(io: Server, name: string, room: FileDocRoom) { - if (getFileDocStore().enabled) io.serverSideEmit(FILE_DOC_PRESENCE_CHANGED, name, room.fileId) + io.serverSideEmit(FILE_DOC_PRESENCE_CHANGED, name, room.fileId) void refreshFileDocPresence(io, name, room.fileId) } @@ -746,7 +710,7 @@ async function runServerSeed( const deadline = Date.now() + FILE_DOC_TIMEOUTS.seedRequestMs while (fileDocRooms.get(name) === room && !isDocSeeded(room.doc)) { // Exactly one task across the cluster builds the seed; the others receive it via the stream (the - // fix for split-brain seeding). Returns a lock token here (single-pod: a sentinel token). + // fix for split-brain seeding). const token = await store.shouldSeed(name) if (token) { await seedUnderLock(name, room, token, actor) @@ -944,13 +908,13 @@ async function mergeMarkdownIntoRoom( // in Redis for multi-task, plus this task's room) so the persist If-Match guard treats this write as // synced rather than an out-of-band conflict. AWAITED so the version is durable before the merge lock // releases, so the next lock holder's staleness check (below) reads a consistent value. - const recordVersion = async (generation?: string) => { + const recordVersion = async (generation: string) => { if (version === undefined) return const room = fileDocRooms.get(name) // Never regress the token: merges/seeds/persists all write it, so a lower value arriving out of // order must not shadow a higher one the doc already incorporates (the Redis side is guarded // identically by SET_VERSION_IF_NEWER_SCRIPT). - if (room && (generation === undefined || docIdOf(room.doc) === generation)) { + if (room && docIdOf(room.doc) === generation) { room.syncedVersion = Math.max(room.syncedVersion ?? 0, version) } await store.setSyncedVersion(name, version, generation) @@ -967,67 +931,47 @@ async function mergeMarkdownIntoRoom( return false } - if (store.enabled) { - // Serialize merges to this file ACROSS tasks — the per-file chain above only covers this process. - // Two copilot edits to the same file landing on different tasks must not diff the SAME shared base - // and publish conflicting full-document rewrites. Retry LONGER than the lock TTL, so a live holder - // always releases (or its lock expires) first and we acquire — never merging against a shared base - // while a peer holds the lock. If somehow still unavailable, skip the live merge (copilot's durable - // file write stands) rather than race. - const token = await acquireFileDocMergeSlot(name) - if (!token) { - logger.warn(`Merge lock unavailable for file ${fileId}; skipping live merge`) - return 'merge-unavailable' - } - try { - // Staleness is checked under the lock against the cluster-wide synced version, so a durable merge - // that lost the race to a newer one (on any process) is dropped rather than regressing the doc. - const shared = await store.getSyncedVersion(name) - const current = Math.max(shared ?? 0, fileDocRooms.get(name)?.syncedVersion ?? 0) - if (isStale(current)) return 'stale' - const generation = await store.getDocumentGeneration(name) - // Defer to an actively-streaming client: it is applying this SAME agent edit into the shared doc - // frame-by-frame, so also publishing a whole-document merge here would double-write the content (the - // client's private shadow never observes this merge, so it re-inserts what we added → duplication). - // Still record the durable version so the persist If-Match stays correct; the client owns the bytes, - // and once streaming stops the flag clears and the final durable merge lands as a near-noop. - if (await store.isAgentStreaming(name)) { - await recordVersion(generation) - return 'applied' - } - // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc - // live (including this one, via its own tailer) applies it and fans it out to its clients, so the - // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream - // means no doc is (or was recently) live → nothing to merge into. AWAIT the publish so the diff is - // durably in the stream before we release the lock (else the next task would diff a stale base). - const base = await store.getStreamState(name, generation) - if (!base) return 'no-live-room' - const diff = await fetchFileDocMerge(fileId, base, markdown) - await store.publishAndWait(name, diff, generation) + // Serialize merges to this file ACROSS tasks — the per-file chain above only covers this process. + // Two copilot edits to the same file landing on different tasks must not diff the SAME shared base + // and publish conflicting full-document rewrites. Retry LONGER than the lock TTL, so a live holder + // always releases (or its lock expires) first and we acquire — never merging against a shared base + // while a peer holds the lock. If somehow still unavailable, skip the live merge (copilot's durable + // file write stands) rather than race. + const token = await acquireFileDocMergeSlot(name) + if (!token) { + logger.warn(`Merge lock unavailable for file ${fileId}; skipping live merge`) + return 'merge-unavailable' + } + try { + // Staleness is checked under the lock against the cluster-wide synced version, so a durable merge + // that lost the race to a newer one (on any process) is dropped rather than regressing the doc. + const shared = await store.getSyncedVersion(name) + const current = Math.max(shared ?? 0, fileDocRooms.get(name)?.syncedVersion ?? 0) + if (isStale(current)) return 'stale' + const generation = await store.getDocumentGeneration(name) + // Defer to an actively-streaming client: it is applying this SAME agent edit into the shared doc + // frame-by-frame, so also publishing a whole-document merge here would double-write the content (the + // client's private shadow never observes this merge, so it re-inserts what we added → duplication). + // Still record the durable version so the persist If-Match stays correct; the client owns the bytes, + // and once streaming stops the flag clears and the final durable merge lands as a near-noop. + if (await store.isAgentStreaming(name)) { await recordVersion(generation) return 'applied' - } finally { - await store.releaseMergeSlot(name, token) } - } - - // Single-replica fallback: apply straight to the local authoritative doc. - const room = fileDocRooms.get(name) - if (!room || room.owners.size === 0 || !isDocSeeded(room.doc)) return 'no-live-room' - if (isStale(room.syncedVersion ?? 0)) return 'stale' - // Defer to an actively-streaming client (see the multi-replica branch above) — it applies this agent - // edit itself, so merging it here too would double-write. Record the version; skip the content merge. - if (room.agentStreamingUntil > Date.now()) { - await recordVersion() + // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc + // live (including this one, via its own tailer) applies it and fans it out to its clients, so the + // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream + // means no doc is (or was recently) live → nothing to merge into. AWAIT the publish so the diff is + // durably in the stream before we release the lock (else the next task would diff a stale base). + const base = await store.getStreamState(name, generation) + if (!base) return 'no-live-room' + const diff = await fetchFileDocMerge(fileId, base, markdown) + await store.publishAndWait(name, diff, generation) + await recordVersion(generation) return 'applied' + } finally { + await store.releaseMergeSlot(name, token) } - const update = await fetchFileDocMerge(fileId, Y.encodeStateAsUpdate(room.doc), markdown) - // The room may have been dropped while the diff was being built; never touch a destroyed doc. - if (fileDocRooms.get(name) !== room) return 'no-live-room' - // No transaction origin → `doc.on('update')` relays to the WHOLE room (every editor sees copilot). - Y.applyUpdate(room.doc, update) - await recordVersion() - return 'applied' } /** @@ -1061,7 +1005,6 @@ function getOrCreateRoom(io: Server, target: FileDocTarget): FileDocRoom { persistTimer: null, persistDeadline: null, syncedVersion: null, - agentStreamingUntil: 0, hydrated, pendingJoins: 0, pendingUpdates: 0, @@ -1094,14 +1037,12 @@ function getOrCreateRoom(io: Server, target: FileDocTarget): FileDocRoom { ) getFileDocStore().publish(name, update, origin === AGENT_SYNC_ORIGIN) // A locally-originated agent frame (this task's stream leader) means a client is applying this agent - // edit itself. Refresh the "actively streaming" markers so a durable `applyMarkdownToLiveFileDoc` - // merge defers to that client instead of double-writing the same content (the two-writer - // duplication). Both are TTL'd, so they self-clear once frames stop and the final durable merge then - // lands as a near-noop. Only AGENT_SYNC_ORIGIN is handled: the in-memory marker is read solely by the - // single-replica merge branch (where every agent frame is AGENT_SYNC_ORIGIN), and the cluster flag is - // read cluster-wide, so a peer task tailing REDIS_AGENT_ORIGIN never needs to set either. + // edit itself. Refresh the cluster-wide "actively streaming" marker so a durable + // `applyMarkdownToLiveFileDoc` merge defers to that client instead of double-writing the same content + // (the two-writer duplication). It is TTL'd, so it self-clears once frames stop and the final durable + // merge then lands as a near-noop. Only AGENT_SYNC_ORIGIN is handled: the flag is read cluster-wide, + // so a peer task tailing REDIS_AGENT_ORIGIN never needs to set it. if (origin === AGENT_SYNC_ORIGIN) { - room.agentStreamingUntil = Date.now() + AGENT_STREAM_FLAG_TTL_MS void getFileDocStore().markAgentStreaming(name, AGENT_STREAM_FLAG_TTL_MS) } // Edit tracking for persistence. Mark the doc dirty on any update applied AFTER it was seeded — a diff --git a/apps/realtime/src/handlers/fixtures/redis.fixture.ts b/apps/realtime/src/handlers/fixtures/redis.fixture.ts new file mode 100644 index 00000000000..1afaf60885a --- /dev/null +++ b/apps/realtime/src/handlers/fixtures/redis.fixture.ts @@ -0,0 +1,344 @@ +/** + * An in-memory stand-in for the node-redis client surface {@link FileDocStore} uses (streams, locks, + * and its Lua scripts, matched by shape), so unit tests can run the store and the relay without a Redis + * server. Tests install it with `vi.mock('redis', () => ({ createClient: () => makeClient() }))` and + * reset {@link fakeRedis}.backing with {@link createBacking} before each test; every client shares that + * backing, so several stores model several tasks on one Redis. + * + * @module + */ +import { sleep } from '@sim/utils/helpers' + +interface TestStreamEntry { + id: string + message: Record +} + +export interface TestRedisClient { + isOpen: boolean + connect(): Promise + quit(): Promise + on(): TestRedisClient + duplicate(): TestRedisClient + xAdd(key: string, id: string, fields: Record): Promise + xRange( + key: string, + start: string, + end: string, + options?: { COUNT?: number } + ): Promise + xRevRange( + key: string, + start: string, + end: string, + options?: { COUNT?: number } + ): Promise + xLen(key: string): Promise + xTrim(key: string, strategy: string, minId: string): Promise + xRead( + streams: { key: string; id: string }[], + options?: { BLOCK?: number; COUNT?: number } + ): Promise<{ name: string; messages: TestStreamEntry[] }[] | null> + set(key: string, value: string, options?: { NX?: boolean }): Promise + get(key: string): Promise + del(keys: string | string[]): Promise + eval( + script: string, + options: { keys: string[]; arguments: string[] } + ): Promise + expire(): Promise +} + +/** + * One shared in-memory Redis backing per test, so several {@link FileDocStore} instances (modelling + * several ECS tasks) all talk to the "same Redis". A minimal fake of just the stream/lock ops the + * store uses. + */ +export interface Backing { + streams: Map + kv: Map + dedupe: Map + seq: number + /** Override generated IDs to model a recreated stream restarting its same-millisecond sequence. */ + nextIds?: string[] + /** Number of upcoming xAdd calls to fail with a transient error (to exercise publish retry). */ + failXAdd: number + /** Set to fail every xRead the way node-redis does once a client has been closed. */ + readerClosed: boolean + /** Failed reads served, so a test can prove the loop is not spinning at the read cadence. */ + reads: number + /** When each FAILED read was attempted, so a test can measure one backoff interval exactly. */ + failedReadTimes: number[] + /** Reads that returned (the idle steady state) — the event that ends a failure streak. */ + idleReads: number + /** `connect()` calls, so a test can prove a closed reader is re-opened rather than abandoned. */ + connects: number + /** Largest stream range response requested, proving replay is paginated. */ + maxRangeCount: number + /** Optional deterministic compaction hook invoked before each range page is read. */ + onRange?: (call: number, key: string, start: string) => void + rangeCalls: number + /** Largest multiplexed XREAD request and COUNT observed. */ + maxReadStreams: number + maxReadCount: number + onSnapshot?: () => Promise + failSnapshotTrim?: boolean + onLength?: () => Promise +} + +function compareStreamIds(left: string, right: string): bigint { + const [leftMs, leftSequence] = left.split('-').map(BigInt) + const [rightMs, rightSequence] = right.split('-').map(BigInt) + return leftMs === rightMs ? leftSequence - rightSequence : leftMs - rightMs +} + +/** A client over the current {@link fakeRedis} backing; `duplicate()` shares it. */ +export function makeClient(): TestRedisClient { + const b = () => { + if (!fakeRedis.backing) throw new Error('backing not initialized') + return fakeRedis.backing + } + const nextId = () => b().nextIds?.shift() ?? `${++b().seq}-0` + const client: TestRedisClient = { + isOpen: true, + connect: async () => { + client.isOpen = true + b().connects++ + }, + quit: async () => {}, + on: () => client, + duplicate: () => makeClient(), + xAdd: async (key: string, _star: string, fields: Record) => { + if (b().failXAdd > 0) { + b().failXAdd-- + throw new Error('transient xAdd failure') + } + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { ...fields } }) + b().streams.set(key, arr) + return id + }, + xRange: async (key: string, start: string, end: string, options?: { COUNT?: number }) => { + b().rangeCalls++ + b().onRange?.(b().rangeCalls, key, start) + const startId = start.startsWith('(') ? start.slice(1) : start + const entries = (b().streams.get(key) ?? []).filter( + (entry) => + (start === '-' || compareStreamIds(entry.id, startId) > 0n) && + (end === '+' || compareStreamIds(entry.id, end) <= 0n) + ) + const count = options?.COUNT ?? entries.length + b().maxRangeCount = Math.max(b().maxRangeCount, count) + return entries.slice(0, count).map((entry) => ({ ...entry })) + }, + xRevRange: async (key: string, _start: string, _end: string, options?: { COUNT?: number }) => + [...(b().streams.get(key) ?? [])] + .reverse() + .slice(0, options?.COUNT) + .map((entry) => ({ ...entry })), + xLen: async (key: string) => { + await b().onLength?.() + return (b().streams.get(key) ?? []).length + }, + xTrim: async (key: string, _strategy: string, minid: string) => { + const arr = b().streams.get(key) ?? [] + b().streams.set( + key, + arr.filter((e) => compareStreamIds(e.id, minid) >= 0n) + ) + }, + xRead: async ( + streams: { key: string; id: string }[], + options?: { BLOCK?: number; COUNT?: number } + ) => { + b().reads++ + b().maxReadStreams = Math.max(b().maxReadStreams, streams.length) + b().maxReadCount = Math.max(b().maxReadCount, options?.COUNT ?? 0) + if (b().readerClosed) { + b().failedReadTimes.push(Date.now()) + client.isOpen = false + throw new Error('The client is closed') + } + const res: { name: string; messages: TestStreamEntry[] }[] = [] + for (const { key, id } of streams) { + const after = (b().streams.get(key) ?? []) + .filter((e) => compareStreamIds(e.id, id) > 0n) + .slice(0, options?.COUNT) + if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) + } + if (res.length) { + b().idleReads++ + return res + } + await sleep(5) + b().idleReads++ + return null + }, + set: async (key: string, val: string, opts?: { NX?: boolean }) => { + if (opts?.NX && b().kv.has(key)) return null + b().kv.set(key, val) + return 'OK' + }, + get: async (key: string) => b().kv.get(key) ?? null, + del: async (keys: string | string[]) => { + const targets = Array.isArray(keys) ? keys : [keys] + for (const key of targets) { + b().kv.delete(key) + b().streams.delete(key) + b().dedupe.delete(key) + } + return targets.length + }, + eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { + const [key] = opts.keys + if (script.startsWith('for _, key in ipairs(KEYS)')) return 1 + if (script.includes("redis.call('exists', KEYS[1])") && !b().streams.has(key)) { + return script.includes('zscore') ? -1 : false + } + if (script.includes('return ARGV[1]')) { + const generation = b().kv.get(opts.keys[1]) + if (generation !== undefined) return generation + if (!b().streams.get(key)?.length) return false + b().kv.set(opts.keys[1], opts.arguments[0]) + return opts.arguments[0] + } + if (script.includes("redis.call('del', KEYS[1], KEYS[4], KEYS[5])")) { + const [, generationKey, versionKey, dedupeKey, agentKey, invalidationKey] = opts.keys + const [version, , marker] = opts.arguments + const current = b().kv.get(versionKey) + const invalidated = b().kv.get(invalidationKey) + if (invalidated && Number(invalidated) >= Number(version)) return null + if (current && Number(current) > Number(version)) return null + if (current === version && b().kv.get(generationKey) === marker) return null + const generation = b().kv.get(generationKey) ?? '' + b().kv.set(generationKey, marker) + b().kv.set(versionKey, version) + b().kv.set(invalidationKey, version) + b().streams.delete(key) + b().dedupe.delete(dedupeKey) + b().kv.delete(agentKey) + return generation + } + if (script.includes('zscore')) { + const [, dedupeKey, generationKey] = opts.keys + const [member, field, value, capacityText, , expectedGeneration] = opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return -1 + const members = b().dedupe.get(dedupeKey) ?? [] + if (members.includes(member)) return 0 + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value, eu: opts.arguments[6], ec: opts.arguments[7] } }) + b().streams.set(key, arr) + members.push(member) + const capacity = Number(capacityText) + if (members.length > capacity) members.splice(0, members.length - capacity) + b().dedupe.set(dedupeKey, members) + return 1 + } + // Atomic seed-if-empty (SEED_IF_EMPTY_SCRIPT): append the entry iff the stream is empty, in one + // synchronous step — mirroring Redis's atomic Lua execution, so two concurrent evals can never both + // append (the second sees a non-empty stream). + if (script.includes('xlen')) { + const [, generationKey, versionKey] = opts.keys + const [field, value, generation, , generationField, version] = opts.arguments + if (Number(b().kv.get(versionKey) ?? 0) > Number(version)) return 0 + const arr = b().streams.get(key) ?? [] + if (arr.length > 0) return 0 + b().kv.set(generationKey, generation) + if (version !== '0') b().kv.set(versionKey, version) + const id = nextId() + arr.push({ id, message: { [field]: value, [generationField]: generation } }) + b().streams.set(key, arr) + return 1 + } + if (script.includes('ARGV[5], ARGV[4]')) { + const [, generationKey] = opts.keys + const [field, value, marker, expectedGeneration, generationField, upTo, compactionField] = + opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return false + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ + id, + message: { + [field]: value, + [marker]: '1', + [generationField]: expectedGeneration, + ...(compactionField ? { [compactionField]: '1' } : {}), + eu: opts.arguments[7], + ec: opts.arguments[8], + ei: opts.arguments[9], + }, + }) + b().streams.set(key, arr) + if (script.includes("redis.call('xtrim'")) { + if (b().failSnapshotTrim) throw new Error('snapshot trim failed') + b().streams.set( + key, + arr.filter((entry) => compareStreamIds(entry.id, upTo) >= 0n) + ) + } + await b().onSnapshot?.() + return id + } + if (script.includes("ARGV[3] ~= ''")) { + const [, generationKey] = opts.keys + const generation = b().kv.get(generationKey) + const expectedGeneration = opts.arguments[3] + if ((generation ?? '') !== expectedGeneration) return false + if (b().failXAdd > 0) { + b().failXAdd-- + throw new Error('transient xAdd failure') + } + const [field, value, marker] = opts.arguments + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value, ...(marker ? { [marker]: '1' } : {}) } }) + b().streams.set(key, arr) + return id + } + if (script.includes('tonumber(c)')) { + const [value, , expectedGeneration] = opts.arguments + const generation = b().kv.get(opts.keys[1]) + if ((generation ?? '') !== expectedGeneration) return 0 + const current = b().kv.get(key) + if (current === undefined || Number(current) < Number(value)) b().kv.set(key, value) + return 1 + } + // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. + const [token] = opts.arguments + if (b().kv.get(key) === token) { + b().kv.delete(key) + return 1 + } + return 0 + }, + expire: async () => 1, + } + return client +} + +/** The backing every fake client reads and writes; reset per test with {@link createBacking}. */ +export const fakeRedis: { backing: Backing | null } = { backing: null } + +export function createBacking(): Backing { + return { + streams: new Map(), + kv: new Map(), + dedupe: new Map(), + seq: 0, + failXAdd: 0, + readerClosed: false, + reads: 0, + failedReadTimes: [], + idleReads: 0, + connects: 0, + maxRangeCount: 0, + rangeCalls: 0, + maxReadStreams: 0, + maxReadCount: 0, + } +} diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index c7b568152b7..7f867964f9a 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -10,7 +10,7 @@ import { waitForConnectionCleanup } from '@/handlers/connection' import { flushAllFileDocRooms, registerFileDocPresenceRelay } from '@/handlers/file-doc' import { getFileDocStore, initFileDocStore } from '@/handlers/file-doc-store' import { type AuthenticatedSocket, authenticateSocket } from '@/middleware/auth' -import { type IRoomManager, MemoryRoomManager, RedisRoomManager } from '@/rooms' +import { type IRoomManager, RedisRoomManager } from '@/rooms' import { createHttpHandler } from '@/routes/http' const logger = createLogger('CollaborativeSocketServer') @@ -19,15 +19,8 @@ const logger = createLogger('CollaborativeSocketServer') const SHUTDOWN_TIMEOUT_MS = 10000 async function createRoomManager(io: SocketIOServer): Promise { - if (env.REDIS_URL) { - logger.info('Initializing Redis-backed RoomManager for multi-pod support') - const manager = new RedisRoomManager(io, env.REDIS_URL) - await manager.initialize() - return manager - } - - logger.warn('No REDIS_URL configured - using in-memory RoomManager (single-pod only)') - const manager = new MemoryRoomManager(io) + logger.info('Initializing Redis-backed RoomManager') + const manager = new RedisRoomManager(io, env.REDIS_URL) await manager.initialize() return manager } @@ -41,7 +34,6 @@ async function main() { nodeEnv: env.NODE_ENV, hasDatabase: !!env.DATABASE_URL, hasAuth: !!env.BETTER_AUTH_SECRET, - hasRedis: !!env.REDIS_URL, }) // Register the HTTP handler before Socket.IO attaches: engine.io captures @@ -52,14 +44,13 @@ async function main() { let httpHandler: ReturnType | undefined httpServer.on('request', (req, res) => httpHandler?.(req, res)) - // Create Socket.IO server with Redis adapter if configured + // Create Socket.IO server with the Redis adapter for cross-pod broadcasting const io = await createSocketIOServer(httpServer) - // Initialize room manager (Redis or in-memory based on config) + // Initialize the Redis-backed room manager const roomManager = await createRoomManager(io) - // Initialize the shared Yjs backend for collaborative file docs (Redis Streams). Enabled only when - // REDIS_URL is set; otherwise the relay runs its original single-replica in-memory doc path. + // Initialize the shared Yjs backend for collaborative file docs (Redis Streams). await initFileDocStore(env.REDIS_URL) registerFileDocPresenceRelay(io) diff --git a/apps/realtime/src/rooms/index.ts b/apps/realtime/src/rooms/index.ts index 29157aa4581..b83628299b3 100644 --- a/apps/realtime/src/rooms/index.ts +++ b/apps/realtime/src/rooms/index.ts @@ -1,4 +1,3 @@ -export { MemoryRoomManager } from '@/rooms/memory-manager' export { RedisRoomManager } from '@/rooms/redis-manager' -export type { IRoomManager, RoomState, UserPresence, UserSession } from '@/rooms/types' +export type { IRoomManager, UserPresence, UserSession } from '@/rooms/types' export { WorkflowRoomService, workflowRoom } from '@/rooms/workflow-room-service' diff --git a/apps/realtime/src/rooms/memory-manager.test.ts b/apps/realtime/src/rooms/memory-manager.test.ts deleted file mode 100644 index 4cb68fc4e53..00000000000 --- a/apps/realtime/src/rooms/memory-manager.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Multi-room semantics for the room manager. Exercises the invariants the - * single-room → multi-room migration must preserve: a socket in two rooms, - * refcounted session cleanup, presence isolation, and full-disconnect cleanup. - */ -import { ROOM_TYPES, type RoomRef } from '@sim/realtime-protocol/rooms' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { MemoryRoomManager } from '@/rooms/memory-manager' -import { sweepStalePresence } from '@/rooms/presence-visibility' -import type { UserPresence } from '@/rooms/types' - -function fakeIo(liveSocketIds: string[] = []) { - const emit = vi.fn() - return { - emit, - io: { - to: vi.fn().mockReturnValue({ emit }), - in: vi.fn().mockReturnValue({ - fetchSockets: vi.fn().mockResolvedValue(liveSocketIds.map((id) => ({ id }))), - }), - } as never, - } -} - -function presence(room: RoomRef, socketId: string, userId: string): UserPresence { - return { - userId, - room, - userName: `user-${userId}`, - socketId, - joinedAt: Date.now(), - lastActivity: Date.now(), - role: 'admin', - } -} - -const WORKFLOW: RoomRef = { type: ROOM_TYPES.WORKFLOW, id: 'wf-1' } -const FILES: RoomRef = { type: ROOM_TYPES.WORKSPACE_FILES, id: 'ws-1' } - -describe('MemoryRoomManager multi-room', () => { - let manager: MemoryRoomManager - - beforeEach(async () => { - manager = new MemoryRoomManager(fakeIo().io) - await manager.initialize() - }) - - it('keeps the shared session alive when leaving one of two rooms (refcount)', async () => { - await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) - await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) - - const removed = await manager.removeUserFromRoom(WORKFLOW, 'socket-1') - expect(removed).toBe(true) - - // The files room and the shared session must survive. - expect(await manager.hasRoom(WORKFLOW)).toBe(false) - expect(await manager.hasRoom(FILES)).toBe(true) - expect(await manager.getUserSession('socket-1')).not.toBeNull() - expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKFLOW)).toBeNull() - expect(await manager.getRoomForSocket('socket-1', ROOM_TYPES.WORKSPACE_FILES)).toEqual(FILES) - }) - - it('isolates presence between rooms of different types', async () => { - await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) - await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) - await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) - - expect(await manager.getRoomUsers(WORKFLOW)).toHaveLength(1) - expect(await manager.getRoomUsers(FILES)).toHaveLength(2) - }) - - it('removes a socket from every room on disconnect and reports them', async () => { - await manager.addUserToRoom(WORKFLOW, 'socket-1', presence(WORKFLOW, 'socket-1', 'user-1')) - await manager.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) - await manager.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) - - const removed = await manager.removeSocketFromAllRooms('socket-1') - expect(removed).toHaveLength(2) - expect(removed).toContainEqual(WORKFLOW) - expect(removed).toContainEqual(FILES) - - expect(await manager.hasRoom(WORKFLOW)).toBe(false) - // The files room still has socket-2. - expect(await manager.getRoomUsers(FILES)).toHaveLength(1) - expect(await manager.getUserSession('socket-1')).toBeNull() - }) - - it('sweepStalePresence reclaims not-live stale entries but keeps live and fresh ones', async () => { - const { io } = fakeIo(['socket-live']) - const m = new MemoryRoomManager(io) - await m.initialize() - - const staleMs = 76 * 60 * 1000 - await m.addUserToRoom(FILES, 'socket-live', presence(FILES, 'socket-live', 'u1')) - await m.addUserToRoom(FILES, 'socket-dead', { - ...presence(FILES, 'socket-dead', 'u2'), - joinedAt: Date.now() - staleMs, - lastActivity: Date.now() - staleMs, - }) - await m.addUserToRoom(FILES, 'socket-recent', presence(FILES, 'socket-recent', 'u3')) - - await sweepStalePresence(m, FILES) - - const remaining = (await m.getRoomUsers(FILES)).map((u) => u.socketId).sort() - // socket-dead: not live + stale → removed. socket-live: live → kept. - // socket-recent: not live but fresh (transient) → kept. - expect(remaining).toEqual(['socket-live', 'socket-recent']) - }) - - it('omits an excluded socket from the presence broadcast (disconnect ghost guard)', async () => { - const { emit, io } = fakeIo() - const m = new MemoryRoomManager(io) - await m.initialize() - await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) - await m.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) - - // Broadcast as if socket-1 is disconnecting: even though its presence entry is - // still present, it must not appear in the emitted list. - await m.broadcastPresenceUpdate(FILES, 'socket-1') - const emitted = emit.mock.calls.at(-1)?.[1] as Array<{ socketId: string }> - expect(emitted.map((u) => u.socketId)).toEqual(['socket-2']) - }) - - it('never emits a presence entry whose socket is no longer live (ghost guard)', async () => { - // Only socket-2 is a live Socket.IO member; socket-1 is an orphaned entry that - // outlived a failed removal. - const { emit, io } = fakeIo(['socket-2']) - const m = new MemoryRoomManager(io) - await m.initialize() - await m.addUserToRoom(FILES, 'socket-1', presence(FILES, 'socket-1', 'user-1')) - await m.addUserToRoom(FILES, 'socket-2', presence(FILES, 'socket-2', 'user-2')) - - await m.broadcastPresenceUpdate(FILES) - const emitted = emit.mock.calls.at(-1)?.[1] as Array<{ socketId: string }> - expect(emitted.map((u) => u.socketId)).toEqual(['socket-2']) - }) -}) diff --git a/apps/realtime/src/rooms/memory-manager.ts b/apps/realtime/src/rooms/memory-manager.ts deleted file mode 100644 index f7ebed862ac..00000000000 --- a/apps/realtime/src/rooms/memory-manager.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { createLogger } from '@sim/logger' -import { - presenceEventName, - type RoomRef, - type RoomType, - roomName, -} from '@sim/realtime-protocol/rooms' -import type { Server } from 'socket.io' -import { filterVisiblePresence } from '@/rooms/presence-visibility' -import type { IRoomManager, RoomState, UserPresence, UserSession } from '@/rooms/types' - -const logger = createLogger('MemoryRoomManager') - -/** Stable string key for a room in the local maps (distinct from the Socket.IO room name). */ -function roomKey(room: RoomRef): string { - return `${room.type}:${room.id}` -} - -/** - * In-memory room manager for single-pod deployments. Used when REDIS_URL is not - * configured. Domain-neutral: keyed by {@link RoomRef}, supports a socket in - * multiple rooms (one per {@link RoomType}). - */ -export class MemoryRoomManager implements IRoomManager { - private rooms = new Map() - /** socketId -> (roomType -> roomId) */ - private socketRooms = new Map>() - private userSessions = new Map() - private _io: Server - - constructor(io: Server) { - this._io = io - } - - get io(): Server { - return this._io - } - - async initialize(): Promise { - logger.info('MemoryRoomManager initialized (single-pod mode)') - } - - isReady(): boolean { - return true - } - - async shutdown(): Promise { - this.rooms.clear() - this.socketRooms.clear() - this.userSessions.clear() - logger.info('MemoryRoomManager shutdown complete') - } - - async addUserToRoom(room: RoomRef, socketId: string, presence: UserPresence): Promise { - const key = roomKey(room) - let state = this.rooms.get(key) - if (!state) { - state = { room, users: new Map(), lastModified: Date.now(), activeConnections: 0 } - this.rooms.set(key, state) - } - - state.users.set(socketId, presence) - state.activeConnections++ - state.lastModified = Date.now() - - let socketRoomMap = this.socketRooms.get(socketId) - if (!socketRoomMap) { - socketRoomMap = new Map() - this.socketRooms.set(socketId, socketRoomMap) - } - socketRoomMap.set(room.type, room.id) - - this.userSessions.set(socketId, { - userId: presence.userId, - userName: presence.userName, - avatarUrl: presence.avatarUrl, - }) - - logger.debug(`Added user ${presence.userId} to room ${key} (socket: ${socketId})`) - } - - async removeUserFromRoom(room: RoomRef, socketId: string): Promise { - const key = roomKey(room) - const state = this.rooms.get(key) - let existed = false - - if (state?.users.has(socketId)) { - existed = true - state.users.delete(socketId) - state.activeConnections = Math.max(0, state.activeConnections - 1) - if (state.users.size === 0) { - this.rooms.delete(key) - logger.info(`Cleaned up empty room: ${key}`) - } - } - - const socketRoomMap = this.socketRooms.get(socketId) - if (socketRoomMap && socketRoomMap.get(room.type) === room.id) { - socketRoomMap.delete(room.type) - // Drop the shared session only when the socket has left its last room. - if (socketRoomMap.size === 0) { - this.socketRooms.delete(socketId) - this.userSessions.delete(socketId) - } - } - - return existed - } - - async removeSocketFromAllRooms(socketId: string): Promise { - const socketRoomMap = this.socketRooms.get(socketId) - if (!socketRoomMap || socketRoomMap.size === 0) { - this.userSessions.delete(socketId) - return [] - } - - const rooms: RoomRef[] = Array.from(socketRoomMap.entries()).map(([type, id]) => ({ type, id })) - for (const room of rooms) { - await this.removeUserFromRoom(room, socketId) - } - // Belt-and-suspenders: ensure session is gone even if the map drifted. - this.socketRooms.delete(socketId) - this.userSessions.delete(socketId) - return rooms - } - - async getRoomsForSocket(socketId: string): Promise { - const socketRoomMap = this.socketRooms.get(socketId) - if (!socketRoomMap) return [] - return Array.from(socketRoomMap.entries()).map(([type, id]) => ({ type, id })) - } - - async getRoomForSocket(socketId: string, type: RoomType): Promise { - const id = this.socketRooms.get(socketId)?.get(type) - return id ? { type, id } : null - } - - async getUserSession(socketId: string): Promise { - return this.userSessions.get(socketId) ?? null - } - - async getRoomUsers(room: RoomRef): Promise { - const state = this.rooms.get(roomKey(room)) - if (!state) return [] - return Array.from(state.users.values()) - } - - async hasRoom(room: RoomRef): Promise { - return this.rooms.has(roomKey(room)) - } - - async deleteRoom(room: RoomRef): Promise { - this.rooms.delete(roomKey(room)) - } - - async updateUserActivity( - room: RoomRef, - socketId: string, - updates: Partial> - ): Promise { - const presence = this.rooms.get(roomKey(room))?.users.get(socketId) - if (!presence) return - - if (updates.cursor !== undefined) presence.cursor = updates.cursor - if (updates.selection !== undefined) presence.selection = updates.selection - if (updates.cell !== undefined) presence.cell = updates.cell - presence.lastActivity = updates.lastActivity ?? Date.now() - } - - async updateRoomLastModified(room: RoomRef): Promise { - const state = this.rooms.get(roomKey(room)) - if (state) state.lastModified = Date.now() - } - - async broadcastPresenceUpdate(room: RoomRef, excludeSocketId?: string): Promise { - const users = await this.getRoomUsers(room) - const visible = await filterVisiblePresence(this._io, room, users, excludeSocketId) - this._io.to(roomName(room)).emit(presenceEventName(room.type), visible) - } - - emitToRoom(room: RoomRef, event: string, payload: T): void { - this._io.to(roomName(room)).emit(event, payload) - } - - async getUniqueUserCount(room: RoomRef): Promise { - const state = this.rooms.get(roomKey(room)) - if (!state) return 0 - const uniqueUsers = new Set() - state.users.forEach((presence) => uniqueUsers.add(presence.userId)) - return uniqueUsers.size - } - - async getTotalActiveConnections(): Promise { - let total = 0 - for (const state of this.rooms.values()) { - total += state.activeConnections - } - return total - } -} diff --git a/apps/realtime/src/rooms/types.ts b/apps/realtime/src/rooms/types.ts index 9ef4e77fa70..29952a9d6d0 100644 --- a/apps/realtime/src/rooms/types.ts +++ b/apps/realtime/src/rooms/types.ts @@ -41,17 +41,7 @@ export interface UserSession { } /** - * Room presence state. - */ -export interface RoomState { - room: RoomRef - users: Map - lastModified: number - activeConnections: number -} - -/** - * Common interface for room managers (in-memory and Redis). + * Common interface for the room manager (implemented by `RedisRoomManager`). * * The manager is domain-neutral: it tracks room membership and presence keyed by * {@link RoomRef}, and knows nothing about workflows, files, or any specific diff --git a/apps/realtime/vitest.setup.ts b/apps/realtime/vitest.setup.ts index 132ea305c23..c50d763909c 100644 --- a/apps/realtime/vitest.setup.ts +++ b/apps/realtime/vitest.setup.ts @@ -1,4 +1,5 @@ process.env.DATABASE_URL ??= 'postgres://localhost/test' +process.env.REDIS_URL ??= 'redis://localhost:6379' process.env.NODE_ENV ??= 'test' process.env.BETTER_AUTH_URL ??= 'http://localhost:3000' process.env.BETTER_AUTH_SECRET ??= 'test-better-auth-secret-at-least-32-chars' diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 15899f16392..1483b6dea40 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -5,6 +5,11 @@ DATABASE_URL="postgresql://postgres:your_password@localhost:5432/simstudio" # DATABASE_URL when unset. # DATABASE_REPLICA_URL="" +# Redis (Required) - rate limiting, OTPs, pub/sub, and realtime coordination. The app refuses to start without it. +# Local default: `docker run -d -p 127.0.0.1:6379:6379 redis:7-alpine` (or let `npx sim-setup` manage one). +REDIS_URL=redis://localhost:6379 +# REDIS_TLS_SERVERNAME= # Required only for a rediss:// URL that targets a bare IP address (TLS hostname verification) + # Authentication (Required unless DISABLE_AUTH=true) BETTER_AUTH_SECRET=your_secret_key # Use `openssl rand -hex 32` to generate, or visit https://www.better-auth.com/docs/installation BETTER_AUTH_URL=http://localhost:3000 diff --git a/apps/sim/app/api/auth/oauth/utils.test.ts b/apps/sim/app/api/auth/oauth/utils.test.ts index 37b6ee5755f..765c1eeee0a 100644 --- a/apps/sim/app/api/auth/oauth/utils.test.ts +++ b/apps/sim/app/api/auth/oauth/utils.test.ts @@ -2,7 +2,7 @@ * Tests for OAuth utility functions */ -import { redisConfigMockFns } from '@sim/testing' +import { createMockRedis, redisConfigMockFns } from '@sim/testing' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -66,7 +66,7 @@ function mockUpdateChain() { describe('OAuth Utils', () => { beforeEach(() => { __resetCoalesceLocallyForTests() - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(createMockRedis()) redisConfigMockFns.mockAcquireLock.mockResolvedValue(true) redisConfigMockFns.mockReleaseLock.mockResolvedValue(true) }) diff --git a/apps/sim/app/api/copilot/chat/delete/route.ts b/apps/sim/app/api/copilot/chat/delete/route.ts index e4c5abaddc1..535801137d7 100644 --- a/apps/sim/app/api/copilot/chat/delete/route.ts +++ b/apps/sim/app/api/copilot/chat/delete/route.ts @@ -52,7 +52,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest) => { logger.info('Chat deleted', { chatId: parsed.chatId }) if (deleted.workspaceId) { - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ workspaceId: deleted.workspaceId, chatId: parsed.chatId, type: 'deleted', diff --git a/apps/sim/app/api/copilot/chats/route.ts b/apps/sim/app/api/copilot/chats/route.ts index 67af8b2f4b6..324ad413a70 100644 --- a/apps/sim/app/api/copilot/chats/route.ts +++ b/apps/sim/app/api/copilot/chats/route.ts @@ -153,7 +153,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return createInternalServerErrorResponse('Failed to create chat') } - chatPubSub?.publishStatusChanged({ workspaceId, chatId: result.chatId, type: 'created' }) + chatPubSub.publishStatusChanged({ workspaceId, chatId: result.chatId, type: 'created' }) return NextResponse.json({ success: true, id: result.chatId }) } catch (error) { diff --git a/apps/sim/app/api/link-preview/route.ts b/apps/sim/app/api/link-preview/route.ts index a7d165a4f76..2cd6cd93e5c 100644 --- a/apps/sim/app/api/link-preview/route.ts +++ b/apps/sim/app/api/link-preview/route.ts @@ -39,15 +39,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const redis = getRedisClient() const cacheKey = `${CACHE_KEY_PREFIX}${createHash('sha256').update(url).digest('hex')}` - if (redis) { - try { - const cached = await redis.get(cacheKey) - if (cached !== null) { - return NextResponse.json({ preview: JSON.parse(cached) }) - } - } catch (error) { - logger.warn('Link preview cache read failed', { error }) + try { + const cached = await redis.get(cacheKey) + if (cached !== null) { + return NextResponse.json({ preview: JSON.parse(cached) }) } + } catch (error) { + logger.warn('Link preview cache read failed', { error }) } let preview: LinkPreview = null @@ -62,17 +60,15 @@ export const GET = withRouteHandler(async (request: NextRequest) => { }) } - if (redis) { - const ttl = preview?.imageRetryable - ? RETRYABLE_CACHE_TTL_SECONDS - : preview - ? CACHE_TTL_SECONDS - : NEGATIVE_CACHE_TTL_SECONDS - try { - await redis.set(cacheKey, JSON.stringify(preview), 'EX', ttl) - } catch (error) { - logger.warn('Link preview cache write failed', { error }) - } + const ttl = preview?.imageRetryable + ? RETRYABLE_CACHE_TTL_SECONDS + : preview + ? CACHE_TTL_SECONDS + : NEGATIVE_CACHE_TTL_SECONDS + try { + await redis.set(cacheKey, JSON.stringify(preview), 'EX', ttl) + } catch (error) { + logger.warn('Link preview cache write failed', { error }) } return NextResponse.json({ preview }) diff --git a/apps/sim/app/api/mcp/events/route.ts b/apps/sim/app/api/mcp/events/route.ts index b294e878957..5f7cf24e07f 100644 --- a/apps/sim/app/api/mcp/events/route.ts +++ b/apps/sim/app/api/mcp/events/route.ts @@ -32,11 +32,10 @@ const mcpEventsHandler = createWorkspaceSSE({ }) }) }, - ready: async () => mcpPubSub?.ready(), + ready: () => mcpPubSub.ready(), }, { subscribe: (workspaceId, send) => { - if (!mcpPubSub) return () => {} return mcpPubSub.onWorkflowToolsChanged((event) => { if (event.workspaceId !== workspaceId) return send('tools_changed', { @@ -46,7 +45,7 @@ const mcpEventsHandler = createWorkspaceSSE({ }) }) }, - ready: async () => mcpPubSub?.ready(), + ready: () => mcpPubSub.ready(), }, ], }) diff --git a/apps/sim/app/api/mothership/events/route.ts b/apps/sim/app/api/mothership/events/route.ts index deb78a641b3..f47ee1f5cf0 100644 --- a/apps/sim/app/api/mothership/events/route.ts +++ b/apps/sim/app/api/mothership/events/route.ts @@ -31,7 +31,6 @@ const mothershipEventsHandler = createWorkspaceSSE({ subscriptions: [ { subscribe: (workspaceId, send, viewerUserId) => { - if (!chatPubSub) return () => {} return chatPubSub.onStatusChanged((event) => { if (event.workspaceId !== workspaceId) return send('task_status', { @@ -44,7 +43,7 @@ const mothershipEventsHandler = createWorkspaceSSE({ }) }) }, - ready: async () => chatPubSub?.ready(), + ready: () => chatPubSub.ready(), }, ], }) @@ -73,7 +72,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { subscriptions: [ { subscribe: (send) => - chatPubSub?.onStatusChanged((event) => { + chatPubSub.onStatusChanged((event) => { if (event.organizationId !== organizationId || event.userId !== principal.userId) return send('task_status', { @@ -82,8 +81,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ...(event.streamId ? { streamId: event.streamId } : {}), timestamp: Date.now(), }) - }) ?? (() => {}), - ready: async () => chatPubSub?.ready(), + }), + ready: () => chatPubSub.ready(), }, ], }) diff --git a/apps/sim/app/workspace/[workspaceId]/layout.test.tsx b/apps/sim/app/workspace/[workspaceId]/layout.test.tsx index b7a28d312af..1cd472e54ee 100644 --- a/apps/sim/app/workspace/[workspaceId]/layout.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/layout.test.tsx @@ -8,7 +8,6 @@ import { renderToReadableStream } from 'react-dom/server' import { beforeEach, describe, expect, it, vi } from 'vitest' const { - mockIsDesktopPresenceAvailable, mockHasSignedInDesktopExecutor, mockWorkspaceChrome, mockPrefetchWorkspaceHostContext, @@ -16,7 +15,6 @@ const { mockPrefetchWorkspaceAccess, mockPrefetchWorkspaceForkAvailability, } = vi.hoisted(() => ({ - mockIsDesktopPresenceAvailable: vi.fn(() => false), mockHasSignedInDesktopExecutor: vi.fn(async () => false), mockWorkspaceChrome: vi.fn( ({ children }: { children: ReactNode; sidebar: ReactNode }) => children @@ -68,10 +66,6 @@ vi.mock('@/app/workspace/[workspaceId]/components/workspace-chrome', () => ({ WorkspaceViewport: ({ children }: { children: ReactNode }) => children, })) -vi.mock('@/lib/desktop/executor/presence', () => ({ - isDesktopPresenceAvailable: mockIsDesktopPresenceAvailable, -})) - vi.mock('@/lib/desktop/executor/repository', () => ({ hasSignedInDesktopExecutor: mockHasSignedInDesktopExecutor, })) @@ -210,24 +204,13 @@ describe('WorkspaceLayout host context', () => { return mockWorkspaceChrome.mock.calls[0][0].sidebar } - it('tells the sidebar the desktop executor runs only where presence is tracked', async () => { - mockIsDesktopPresenceAvailable.mockReturnValue(false) - mockHasSignedInDesktopExecutor.mockResolvedValue(true) - - expect(await sidebarProps()).toMatchObject({ - props: { desktopExecutor: { available: false, registered: false } }, - }) - mockHasSignedInDesktopExecutor.mockResolvedValue(false) - }) - it.each([true, false])( 'tells the sidebar whether the viewer has a desktop that runs their turns (%s)', async (registered) => { - mockIsDesktopPresenceAvailable.mockReturnValue(true) mockHasSignedInDesktopExecutor.mockResolvedValueOnce(registered) expect(await sidebarProps()).toMatchObject({ - props: { desktopExecutor: { available: true, registered } }, + props: { desktopExecutor: { registered } }, }) } ) diff --git a/apps/sim/app/workspace/[workspaceId]/layout.tsx b/apps/sim/app/workspace/[workspaceId]/layout.tsx index ec8bbe988c8..b34a36d9c53 100644 --- a/apps/sim/app/workspace/[workspaceId]/layout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/layout.tsx @@ -8,10 +8,7 @@ import { getSession } from '@/lib/auth' import { getActiveOrganizationId } from '@/lib/auth/session-response' import { isChangelogEnabled } from '@/lib/changelog/feature-flag' import { isDashboardsEnabled } from '@/lib/dashboards/feature-flag' -import { - hasDesktopBackgroundExecutor, - isDesktopBackgroundExecutorAvailable, -} from '@/lib/desktop/executor/availability' +import { hasDesktopBackgroundExecutor } from '@/lib/desktop/executor/availability' import { isMothershipModelSelectorEnabled, isPlanModeEnabled } from '@/lib/mothership/feature-flags' import { resolveOrganizationEntryPath } from '@/lib/navigation/resolve-app-entry' import { isProjectsEnabled } from '@/lib/projects/rollout.server' @@ -173,10 +170,7 @@ async function WorkspaceContent({ sidebar={ } initialSidebarCollapsed={initialSidebarCollapsed} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx index 339422d1931..6504d9bd516 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx @@ -375,11 +375,8 @@ const DRAG_EXEMPT_CLASS = '[-webkit-app-region:no-drag]' interface SidebarProps { organizationHref: string | null - /** - * Whether this install runs the desktop background executor, and whether the user has a desktop - * registered to run their turns, so chats show desktop activity. - */ - desktopExecutor: { available: boolean; registered: boolean } + /** Whether the user has a desktop registered to run their turns, so chats show desktop activity. */ + desktopExecutor: { registered: boolean } } /** diff --git a/apps/sim/ee/scim/TESTING.md b/apps/sim/ee/scim/TESTING.md index 5ef8cdac379..903b15a56ec 100644 --- a/apps/sim/ee/scim/TESTING.md +++ b/apps/sim/ee/scim/TESTING.md @@ -1,7 +1,7 @@ # SCIM integration verification The local integration harness sends real HTTP requests to a running Next.js app -and inspects the resulting PostgreSQL records. It does not mock route handlers, +and inspects the resulting PostgreSQL records and Redis rate-limit buckets. It does not mock route handlers, authentication, application operations, or persistence. Okta and Microsoft Entra request shapes are emulated; this is not certification against either provider's provisioning service. @@ -10,20 +10,21 @@ provisioning service. - Bun and the repository dependencies installed. - A running local app configured for hosted Enterprise SCIM with billing enabled, using a dedicated local - PostgreSQL database with all migrations applied (including `0325`). Leave - Redis unconfigured so the app uses PostgreSQL for rate-limit storage. + PostgreSQL database with all migrations applied (including `0325`), and a + local Redis (required by the app; it holds the rate-limit buckets). Enable People on the app with `PEOPLE=true` to exercise SCIM profile imports. For `next dev`, set `NEXT_PUBLIC_FORCE_HOSTED=true` and `BILLING_ENABLED=true`. - The app's local `BETTER_AUTH_SECRET`, at least 32 characters long. - An app URL using HTTP on `localhost`, `127.0.0.1`, or `::1`. The database must - also use a loopback host, and its name must include `test` or `scim`. + also use a loopback host, and its name must include `test` or `scim`. Redis + must use a loopback host too. The harness creates two isolated synthetic organizations, owner accounts, active Enterprise subscriptions, workspaces, verified `.test` domains, and signed owner sessions. It obtains bearer tokens through the real administration API. Database setup also supplies scenarios unavailable through the SCIM API, such as manual suspension, pre-existing workspace access, and credential expiry. -The rate-limit scenario exhausts only its own connection's database bucket and +The rate-limit scenario exhausts only its own connection's Redis bucket and verifies that overlapping credentials share it while another tenant remains unaffected. @@ -39,6 +40,7 @@ From `apps/sim`, set these values for the local app and database, then run: ```sh export SCIM_E2E_BASE_URL=http://localhost:3000 export SCIM_E2E_DATABASE_URL='postgresql://:@127.0.0.1:/' +export SCIM_E2E_REDIS_URL=redis://127.0.0.1:6379 export SCIM_E2E_AUTH_SECRET='' export SCIM_E2E_REPORT_PATH=/tmp/sim-scim-e2e-report.json bun run test:scim:e2e @@ -46,8 +48,8 @@ bun run test:scim:e2e Set `PEOPLE_E2E_SCREENSHOT_DIR` to run the signed-in People import flow in Chromium and save its screenshots. Install Chromium with `bunx --no-install playwright install chromium` when using that option. -The first three variables are required; the report path is optional. The app and -the harness must use the same database and authentication secret. The report +The first four variables are required; the report path is optional. The app and +the harness must use the same database, Redis, and authentication secret. The report contains check names, outcomes, timing, and HTTP request count; it excludes bearer credentials, session cookies, and secrets. A failed check exits nonzero after cleanup. Request redirects are rejected and each request has a 60-second diff --git a/apps/sim/hooks/queries/desktop-activity.test.tsx b/apps/sim/hooks/queries/desktop-activity.test.tsx index ca000a505bc..e2505ca62b4 100644 --- a/apps/sim/hooks/queries/desktop-activity.test.tsx +++ b/apps/sim/hooks/queries/desktop-activity.test.tsx @@ -26,7 +26,7 @@ interface ActivityWatcherProps { let shown: unknown function ActivityWatcher({ registered }: ActivityWatcherProps) { - shown = useDesktopActivity('ws-1', watchesDesktopActivity({ available: true, registered })) + shown = useDesktopActivity('ws-1', watchesDesktopActivity({ registered })) return null } diff --git a/apps/sim/hooks/queries/desktop-activity.ts b/apps/sim/hooks/queries/desktop-activity.ts index 4143a5602fe..c0f355853f7 100644 --- a/apps/sim/hooks/queries/desktop-activity.ts +++ b/apps/sim/hooks/queries/desktop-activity.ts @@ -32,11 +32,8 @@ async function fetchDesktopActivity( * since its first registration can land after the page rendered; a browser tab open across that * first registration shows activity once reloaded. */ -export function watchesDesktopActivity(desktopExecutor: { - available: boolean - registered: boolean -}): boolean { - return desktopExecutor.available && (desktopExecutor.registered || isDesktopApp()) +export function watchesDesktopActivity(desktopExecutor: { registered: boolean }): boolean { + return desktopExecutor.registered || isDesktopApp() } /** diff --git a/apps/sim/lib/api/contracts/desktop-executor.ts b/apps/sim/lib/api/contracts/desktop-executor.ts index deba49ea953..eda710f4a1f 100644 --- a/apps/sim/lib/api/contracts/desktop-executor.ts +++ b/apps/sim/lib/api/contracts/desktop-executor.ts @@ -30,8 +30,8 @@ const registerDesktopDeviceBodySchema = z.object({ export type RegisterDesktopDeviceBody = z.input /** - * `enabled: false` means no new turn binds to the device, as on an install that cannot track - * desktop presence; the device still finishes calls on runs already bound to it. + * `enabled: false` means no new turn binds to the device; the device still finishes calls on runs + * already bound to it. Current servers always send `true`; older ones without Redis sent `false`. */ export const registerDesktopDeviceResponseSchema = z.object({ enabled: z.boolean(), diff --git a/apps/sim/lib/api/contracts/v2/workflows.ts b/apps/sim/lib/api/contracts/v2/workflows.ts index 3cab387eba2..b44070b22f3 100644 --- a/apps/sim/lib/api/contracts/v2/workflows.ts +++ b/apps/sim/lib/api/contracts/v2/workflows.ts @@ -2539,6 +2539,7 @@ export const v2CancelWorkflowRunDataSchema = z 'Whether this request cancelled anything. `false` with an `already_*` reason means the run had already reached that terminal state, so there was nothing to cancel — still `200`, because a no-op is not an error. `false` with any other reason identifies a degraded or incomplete cancellation step.' ), runId: v2WorkflowRunIdSchema, + /** Kept for wire compatibility; always true since Redis is required, except on a paused-run cancellation whose event failed to publish. */ redisAvailable: z .boolean() .describe('Whether the distributed cancellation channel was available.'), diff --git a/apps/sim/lib/api/contracts/workflows.ts b/apps/sim/lib/api/contracts/workflows.ts index 326ee0b79a1..98adae7c549 100644 --- a/apps/sim/lib/api/contracts/workflows.ts +++ b/apps/sim/lib/api/contracts/workflows.ts @@ -661,6 +661,7 @@ export const cancelWorkflowExecutionReasonSchema = z.enum([ 'already_cancelled', 'already_completed', 'already_failed', + /** Kept for wire compatibility; no longer produced now that Redis is required. */ 'redis_unavailable', 'redis_write_failed', 'paused_event_publish_failed', @@ -673,6 +674,7 @@ export const cancelWorkflowExecutionReasonSchema = z.enum([ const cancelWorkflowExecutionResponseSchema = z.object({ success: z.boolean(), executionId: z.string(), + /** Kept for wire compatibility; always true since Redis is required, except on a paused-run cancellation whose event failed to publish. */ redisAvailable: z.boolean(), durablyRecorded: z.boolean(), locallyAborted: z.boolean(), diff --git a/apps/sim/lib/billing/calculations/usage-reservation.test.ts b/apps/sim/lib/billing/calculations/usage-reservation.test.ts index 058dbfdc153..955731f2efb 100644 --- a/apps/sim/lib/billing/calculations/usage-reservation.test.ts +++ b/apps/sim/lib/billing/calculations/usage-reservation.test.ts @@ -206,14 +206,6 @@ describe('usage-reservation', () => { expect(evalMock).not.toHaveBeenCalled() }) - it('fails closed when hosted Redis is unavailable', async () => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) - await expect(reserveExecutionSlot(baseParams)).rejects.toBeInstanceOf( - UsageReservationUnavailableError - ) - expect(evalMock).not.toHaveBeenCalled() - }) - it('fails closed when the atomic reservation result is unknown', async () => { evalMock.mockRejectedValueOnce(new Error('connection lost')) await expect(reserveExecutionSlot(baseParams)).rejects.toMatchObject({ diff --git a/apps/sim/lib/billing/calculations/usage-reservation.ts b/apps/sim/lib/billing/calculations/usage-reservation.ts index f7a818b3955..f1a6cddcd70 100644 --- a/apps/sim/lib/billing/calculations/usage-reservation.ts +++ b/apps/sim/lib/billing/calculations/usage-reservation.ts @@ -366,7 +366,7 @@ function localKeyArguments(keys: LocalReservationKeys): string[] { } async function rollbackCreatedReservation(params: { - redis: NonNullable> + redis: ReturnType keys: LocalReservationKeys reservationId: string descriptorValue: string @@ -480,11 +480,6 @@ export async function reserveExecutionSlot( } const redis = getRedisClient() - if (!redis) { - throw new UsageReservationUnavailableError( - 'Usage admission is temporarily unavailable. Please retry.' - ) - } const { billingEntity, plan, member } = params const reservationId = requireBoundedIdentifier( @@ -649,11 +644,6 @@ export async function refreshExecutionSlotExpiry( if (!isHosted || !isBillingEnabled) return true const redis = getRedisClient() - if (!redis) { - throw new UsageReservationUnavailableError( - 'Usage admission is temporarily unavailable. Please retry.' - ) - } const now = Date.now() if (!Number.isSafeInteger(expiresAt) || expiresAt <= now) { @@ -720,9 +710,6 @@ export async function releaseExecutionSlot(reservationId: string): Promise } const redis = getRedisClient() - if (!redis) { - return - } try { const boundedReservationId = requireBoundedIdentifier(reservationId, 'reservation id') diff --git a/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts b/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts index c34b1f761a9..e2a48123357 100644 --- a/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts +++ b/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts @@ -289,16 +289,6 @@ describe.runIf(Boolean(redisUrl))('shared reporting usage read', () => { expect(await redis.get(sharedKey(payer))).toBe('9.99') }) - it('sums the whole window in one query when Redis is absent', async () => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) - const period = openReportingWindow() - await insertUsage(payer, 2, new Date(period.start.getTime() + DAY_MS)) - await insertUsage(payer, 3, new Date(Date.now() - MINUTE_MS)) - - await expect(readInNewProcess(payer, period)).resolves.toBe(5) - expect(summedPeriods).toEqual([period]) - }) - it('leases a due refresh to one of the processes missing together', async () => { const period = openReportingWindow() await insertUsage(payer, 2, new Date(period.start.getTime() + DAY_MS)) diff --git a/apps/sim/lib/billing/core/reporting-usage-cache.ts b/apps/sim/lib/billing/core/reporting-usage-cache.ts index 507cb78066f..42f4d4c42ef 100644 --- a/apps/sim/lib/billing/core/reporting-usage-cache.ts +++ b/apps/sim/lib/billing/core/reporting-usage-cache.ts @@ -131,8 +131,8 @@ function parseStoredCost(text: string): number | undefined { } /** - * A sum another process stored, or `undefined` when there is none to use. Redis being absent, - * not ready, slow, or failing, and a value that is not a non-negative number, are all misses: + * A sum another process stored, or `undefined` when there is none to use. Redis being not + * ready, slow, or failing, and a value that is not a non-negative number, are all misses: * the caller sums the ledger, so the cache can cost a read its latency but never its answer. */ async function readSharedReportingUsageCost(key: string): Promise { diff --git a/apps/sim/lib/billing/core/reporting-usage-epoch.ts b/apps/sim/lib/billing/core/reporting-usage-epoch.ts index 9e52b8dc2c4..be813803ce3 100644 --- a/apps/sim/lib/billing/core/reporting-usage-epoch.ts +++ b/apps/sim/lib/billing/core/reporting-usage-epoch.ts @@ -60,16 +60,15 @@ export function payerPrefixEpochKey(entity: BillingEntity): string { */ export function readyRedisClient(): Redis | null { const redis = getRedisClient() - return redis?.status === 'ready' ? redis : null + return redis.status === 'ready' ? redis : null } /** - * Throws when a configured Redis is not ready, since its stored prefixes then outlive the bump: - * each caller logs that like a failed or timed-out SET. Without Redis there is nothing to discard. + * Throws when Redis is not ready, since its stored prefixes then outlive the bump: each caller + * logs that like a failed or timed-out SET. */ async function replaceEpoch(epochKey: string): Promise { const redis = getRedisClient() - if (!redis) return if (redis.status !== 'ready') throw new Error(`Redis not ready (${redis.status})`) await withinDeadline( () => redis.set(epochKey, generateShortId(), 'PX', PREFIX_EPOCH_RETAIN_MS), diff --git a/apps/sim/lib/billing/core/usage-segment-cache.test.ts b/apps/sim/lib/billing/core/usage-segment-cache.test.ts index be6fa422606..71404cd68e5 100644 --- a/apps/sim/lib/billing/core/usage-segment-cache.test.ts +++ b/apps/sim/lib/billing/core/usage-segment-cache.test.ts @@ -1,3 +1,4 @@ +import { createMockRedis } from '@sim/testing/mocks/redis.mock' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { generateShortId } from '@sim/utils/id' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -52,7 +53,25 @@ describe('readThroughSegments', () => { }, {}) beforeEach(() => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + const stored = new Map() + const redis = Object.assign(createMockRedis(), { + mget: async (...keys: string[]) => keys.map((key) => stored.get(key) ?? null), + pipeline: () => { + const writes: [string, string][] = [] + const pipeline = { + set: (key: string, value: string) => { + writes.push([key, value]) + return pipeline + }, + exec: async () => { + for (const [key, value] of writes) stored.set(key, value) + return [] + }, + } + return pipeline + }, + }) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) namespace = `test-${generateShortId()}` }) diff --git a/apps/sim/lib/billing/core/usage-segment-cache.ts b/apps/sim/lib/billing/core/usage-segment-cache.ts index 74aa862f02e..16fe3e7e84f 100644 --- a/apps/sim/lib/billing/core/usage-segment-cache.ts +++ b/apps/sim/lib/billing/core/usage-segment-cache.ts @@ -1,7 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { randomInt } from '@sim/utils/random' -import { LRUCache } from 'lru-cache' import { type UsageSegment, usageHourKey } from '@/lib/billing/core/usage-analytics' import { getRedisClient } from '@/lib/core/config/redis' @@ -36,20 +35,9 @@ const MERGE_GAP_SEGMENTS = 2 /** Bump when a cached value's shape or meaning changes; old entries are then ignored. */ const KEY_VERSION = 'v1' -/** - * The in-process fallback for deployments without Redis, bounded by bytes: a - * member or model entry can run to tens of kilobytes. - */ -const localSegments = new LRUCache({ - max: 50_000, - maxSize: 64 * 1024 * 1024, - sizeCalculation: (value) => value.length, -}) - async function readStored(keys: string[]): Promise<(string | null)[]> { if (keys.length === 0) return [] const redis = getRedisClient() - if (!redis) return keys.map((key) => localSegments.get(key) ?? null) try { return await redis.mget(...keys) } catch (error) { @@ -80,11 +68,6 @@ function writeStored(entries: { key: string; value: string; ttlMs: number }[]): if (entries.length === 0) return const jittered = (ttlMs: number) => ttlMs + randomInt(0, Math.floor(ttlMs * TTL_JITTER_RATIO)) const redis = getRedisClient() - if (!redis) { - for (const { key, value, ttlMs } of entries) - localSegments.set(key, value, { ttl: jittered(ttlMs) }) - return - } const pipeline = redis.pipeline() for (const { key, value, ttlMs } of entries) pipeline.set(key, value, 'PX', jittered(ttlMs)) pipeline.exec().catch((error: unknown) => { diff --git a/apps/sim/lib/cli-auth/approval-store.test.ts b/apps/sim/lib/cli-auth/approval-store.test.ts index d365c0f8fde..05c9ca7d033 100644 --- a/apps/sim/lib/cli-auth/approval-store.test.ts +++ b/apps/sim/lib/cli-auth/approval-store.test.ts @@ -88,12 +88,4 @@ describe('cli-auth approval store', () => { await expect(pollApproval(REQUEST, SECRET)).resolves.toEqual({ status: 'pending' }) }) }) - - describe('without Redis', () => { - beforeEach(() => mockGetRedisClient.mockReturnValue(null)) - - it('fails fast on write', async () => { - await expect(createApproval('user-1', REQUEST, CHALLENGE)).rejects.toThrow('REDIS_URL') - }) - }) }) diff --git a/apps/sim/lib/cli-auth/approval-store.ts b/apps/sim/lib/cli-auth/approval-store.ts index 6b95f3d42d0..560b704c5ca 100644 --- a/apps/sim/lib/cli-auth/approval-store.ts +++ b/apps/sim/lib/cli-auth/approval-store.ts @@ -61,14 +61,6 @@ export interface ApprovalGrant { export type PollResult = { status: 'pending' } | ({ status: 'approved' } & ApprovalGrant) -function requireRedis() { - const redis = getRedisClient() - if (!redis) { - throw new Error('CLI authentication requires Redis. Set REDIS_URL to enable it.') - } - return redis -} - /** * `requestId` is the rendezvous handle the CLI puts in the browser URL, so it is * semi-public (OAuth's `user_code`, not its `device_code`). Hashing it as the @@ -99,7 +91,7 @@ export async function createApproval( scope: 'copilot', } ): Promise { - const redis = requireRedis() + const redis = getRedisClient() const record: ApprovalRecord = { challenge, userId, @@ -132,7 +124,7 @@ export async function createApproval( * {@link completeApproval} on success or {@link releaseMint} on failure. */ export async function pollApproval(requestId: string, pollSecret: string): Promise { - const redis = requireRedis() + const redis = getRedisClient() const raw = await redis.get(approvalKey(requestId)) if (!raw) return { status: 'pending' } @@ -156,12 +148,12 @@ export async function pollApproval(requestId: string, pollSecret: string): Promi /** Consumes the approval after a successful mint — single-use from here on. */ export async function completeApproval(requestId: string): Promise { - const redis = requireRedis() + const redis = getRedisClient() await redis.del(approvalKey(requestId), mintLockKey(requestId)) } /** Releases the mint reservation after a failed mint so a later poll can retry. */ export async function releaseMint(requestId: string): Promise { - const redis = requireRedis() + const redis = getRedisClient() await redis.del(mintLockKey(requestId)) } diff --git a/apps/sim/lib/core/config/appconfig.ts b/apps/sim/lib/core/config/appconfig.ts index d885f7a7d9b..ce01e54a03e 100644 --- a/apps/sim/lib/core/config/appconfig.ts +++ b/apps/sim/lib/core/config/appconfig.ts @@ -222,7 +222,6 @@ async function adoptSharedSnapshot( if (!isInsideTriggerRun()) return false const { getRedisClient } = await import('@/lib/core/config/redis') const redis = getRedisClient() - if (!redis) return false const stored = await redis.get(sharedSnapshotKey(context.ids)) if (!stored) return false const snapshot = toRecordOrNull(JSON.parse(stored)) @@ -256,7 +255,6 @@ function publishSharedSnapshot(ids: AppConfigProfileIdentifiers, entry: CacheEnt if (isInsideTriggerRun()) return const { getRedisClient } = await import('@/lib/core/config/redis') const redis = getRedisClient() - if (!redis) return await redis.eval( PUBLISH_SNAPSHOT_SCRIPT, 1, diff --git a/apps/sim/lib/core/outbox/retention.integration.ts b/apps/sim/lib/core/outbox/retention.integration.ts index 7df03b3c014..b017e70122d 100644 --- a/apps/sim/lib/core/outbox/retention.integration.ts +++ b/apps/sim/lib/core/outbox/retention.integration.ts @@ -3,6 +3,7 @@ import { outboxEvent } from '@sim/db/schema' import { readTestDatabaseUrl, readTestRedisUrl } from '@sim/db/testing/test-infrastructure' import { withUtcTimestamps } from '@sim/db/timestamps' +import { createMockRedis } from '@sim/testing/mocks/redis.mock' import { redisConfigMock, redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { generateId } from '@sim/utils/id' @@ -71,7 +72,7 @@ describe('completed outbox retention in PostgreSQL', () => { }) afterEach(async () => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(createMockRedis()) await redis?.del(...cursorKeys) await connection`TRUNCATE outbox_event` }) diff --git a/apps/sim/lib/core/outbox/retention.ts b/apps/sim/lib/core/outbox/retention.ts index 1de774fcd50..4814c2ee9f3 100644 --- a/apps/sim/lib/core/outbox/retention.ts +++ b/apps/sim/lib/core/outbox/retention.ts @@ -91,12 +91,11 @@ type PruneCursor = z.infer const pruneCursorKey = (eventType: string) => `outbox-prune-cursor:v1:${eventType}` /** - * Each type's cursor from the pass in progress, or null when Redis is unavailable. A missing, + * Each type's cursor from the pass in progress, or null when they cannot be read. A missing, * unreadable or expired cursor starts that type's pass from the oldest row. */ async function readPruneCursors(now: Date): Promise | null> { const redis = getRedisClient() - if (!redis) return null try { const stored = await redis.mget(...PRUNABLE_OUTBOX_EVENT_TYPES.map(pruneCursorKey)) const cursors = new Map() @@ -127,8 +126,8 @@ function parsePruneCursor(stored: string | null): PruneCursor | null { /** Stores each type's next cursor, or clears it when that type's pass is over. */ async function savePruneCursors(next: ReadonlyMap): Promise { + if (next.size === 0) return const redis = getRedisClient() - if (!redis || next.size === 0) return try { const pipeline = redis.pipeline() for (const [eventType, cursor] of next) { @@ -200,8 +199,8 @@ async function pruneCompletedBatch( * Each batch resumes after the last row the previous run's batch deleted, kept in Redis. Starting * again from the oldest row would walk every index entry the earlier batches deleted that vacuum * has not yet removed, which grows with the backlog being drained. A pass starts over from the - * oldest row once a batch comes up short of the cutoff or the pass is older than an hour; without - * Redis every batch starts from the oldest row. + * oldest row once a batch comes up short of the cutoff or the pass is older than an hour; when + * the cursors cannot be read, the batch starts from the oldest row. */ export async function pruneCompletedOutboxEvents(now = new Date()): Promise { const cutoff = new Date(now.getTime() - COMPLETED_OUTBOX_RETENTION_MS) diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts b/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts index 409733a0739..d1cf401689d 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts @@ -85,13 +85,6 @@ describe('HostedKeyQueue', () => { }) describe('dequeue', () => { - it('is a no-op when Redis is unavailable', async () => { - redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null) - - await expect(queue.dequeue(provider, workspaceId, ticketId)).resolves.toBeUndefined() - expect(mockRedis.multi).not.toHaveBeenCalled() - }) - it('swallows errors so callers do not throw on cleanup', async () => { mockRedis.pipeline.exec.mockRejectedValueOnce(new Error('connection lost')) diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts b/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts index 27d3802e8b4..1ef1b665c34 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts @@ -62,7 +62,7 @@ return "waiting" export interface EnqueueResult { /** Position at the moment of enqueue (0 = head, you go next). */ position: number - /** Whether Redis was available — false means we're in no-op mode. */ + /** Whether the ticket was enqueued — false means the enqueue failed and the queue is skipped. */ enabled: boolean } @@ -73,7 +73,7 @@ export interface EnqueueResult { * the head, then attempt to consume from the token bucket. On success or cap * exceeded, they `dequeue` to make room for the next caller. * - * No-op when Redis is unavailable: every method returns "you're the head / + * Fails open when a Redis call fails: the method returns "you're the head / * empty / etc." so the rate limiter falls back to plain bucket racing. */ export class HostedKeyQueue { @@ -87,9 +87,6 @@ export class HostedKeyQueue { ticketId: string ): Promise { const redis = getRedisClient() - if (!redis) { - return { position: 0, enabled: false } - } const listKey = queueListKey(provider, billingActorId) const hbKey = heartbeatKey(provider, billingActorId, ticketId) @@ -127,9 +124,6 @@ export class HostedKeyQueue { ticketId: string ): Promise<'head' | 'waiting' | 'missing'> { const redis = getRedisClient() - if (!redis) { - return 'head' - } const listKey = queueListKey(provider, billingActorId) const hbPrefix = `hosted-queue-tkt:${provider}:${billingActorId}:` @@ -158,7 +152,6 @@ export class HostedKeyQueue { ticketId: string ): Promise { const redis = getRedisClient() - if (!redis) return const listKey = queueListKey(provider, billingActorId) const hbKey = heartbeatKey(provider, billingActorId, ticketId) @@ -180,7 +173,6 @@ export class HostedKeyQueue { */ async dequeue(provider: string, billingActorId: string, ticketId: string): Promise { const redis = getRedisClient() - if (!redis) return const listKey = queueListKey(provider, billingActorId) const hbKey = heartbeatKey(provider, billingActorId, ticketId) diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts index 6f718a1ae28..56188cc6ef5 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts @@ -32,7 +32,7 @@ interface MockQueue { } /** Stub queue that defaults to "you're at the head, no waiting" — i.e. acts as if the - * queue is empty or Redis is unavailable. Tests override per-call to simulate ordering. */ + * queue is empty. Tests override per-call to simulate ordering. */ const createMockQueue = (): MockQueue => { const queue: MockQueue = { enqueue: vi.fn().mockResolvedValue({ position: 0, enabled: true }), diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts index 9289e2bd155..9e46db6aa01 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts @@ -448,8 +448,8 @@ export class HostedKeyRateLimiter { * regular cadence so we don't get reaped as dead. Returns `timedOut: true` once the wait * budget is exhausted before reaching the head (see {@link remainingWaitBudgetMs}). * - * No-op when Redis is unavailable (queue.enqueue returns enabled=false and checkHead - * always returns 'head'). + * Fails open when a queue Redis call fails (queue.enqueue returns enabled=false and checkHead + * returns 'head'). */ private async waitForQueueHead( provider: string, diff --git a/apps/sim/lib/credential-groups/mcp-oauth-state.test.ts b/apps/sim/lib/credential-groups/mcp-oauth-state.test.ts index 406602722c4..96499c6a490 100644 --- a/apps/sim/lib/credential-groups/mcp-oauth-state.test.ts +++ b/apps/sim/lib/credential-groups/mcp-oauth-state.test.ts @@ -107,14 +107,6 @@ describe('Credential Group MCP OAuth state', () => { await expect(consumeCredentialGroupMcpOAuthAttempt('mcp_cg_direct')).resolves.toBeNull() }) - it('fails closed when Redis is unavailable', async () => { - vi.mocked(getRedisClient).mockReturnValue(null) - - await expect( - createCredentialGroupMcpOAuthAttempt({ ...ATTEMPT, state: 'mcp_cg_state-1' }) - ).rejects.toThrow('Credential Group MCP OAuth requires Redis') - }) - it('rejects state outside the managed MCP namespace', async () => { await expect( createCredentialGroupMcpOAuthAttempt({ ...ATTEMPT, state: 'ordinary-state' }) diff --git a/apps/sim/lib/credential-groups/mcp-oauth-state.ts b/apps/sim/lib/credential-groups/mcp-oauth-state.ts index 0c83fcc4f31..63e36d9ccea 100644 --- a/apps/sim/lib/credential-groups/mcp-oauth-state.ts +++ b/apps/sim/lib/credential-groups/mcp-oauth-state.ts @@ -69,12 +69,6 @@ export interface CredentialGroupMcpOAuthAttempt { createdAt: number } -function requireRedis() { - const redis = getRedisClient() - if (!redis) throw new Error('Credential Group MCP OAuth requires Redis') - return redis -} - function attemptKey(state: string): string { return `credential-group:mcp-oauth-attempt:${sha256Hex(state)}` } @@ -146,7 +140,7 @@ export async function createCredentialGroupMcpOAuthAttempt(params: { if (!isCredentialGroupMcpOAuthState(params.state)) { throw new Error('Managed MCP OAuth state has an invalid prefix') } - const redis = requireRedis() + const redis = getRedisClient() const [codeVerifier, invitationToken] = await Promise.all([ encryptSecret(params.codeVerifier), encryptSecret(params.invitationToken), @@ -184,13 +178,13 @@ export async function consumeCredentialGroupMcpOAuthAttempt( state: string ): Promise { if (!isCredentialGroupMcpOAuthState(state)) return null - const raw = await requireRedis().eval(CONSUME_SCRIPT, 1, attemptKey(state)) + const raw = await getRedisClient().eval(CONSUME_SCRIPT, 1, attemptKey(state)) if (raw === null) return null if (typeof raw !== 'string') throw new Error('Credential Group MCP OAuth state is malformed') const parsed: unknown = JSON.parse(raw) assertCredentialGroupOAuthAttemptVersion(parsed, MCP_OAUTH_ATTEMPT_VERSION) if (!isStoredAttempt(parsed)) throw new Error('Credential Group MCP OAuth state is malformed') - await requireRedis().srem(serverAttemptsKey(parsed.mcpServerId), attemptKey(state)) + await getRedisClient().srem(serverAttemptsKey(parsed.mcpServerId), attemptKey(state)) if (Date.now() - parsed.createdAt > MCP_OAUTH_ATTEMPT_TTL_MS) return null const [codeVerifier, invitationToken] = await Promise.all([ decryptSecret(parsed.encryptedCodeVerifier), @@ -217,7 +211,7 @@ export async function consumeCredentialGroupMcpOAuthAttempt( export async function clearCredentialGroupMcpOAuthAttempts(mcpServerIds: string[]): Promise { if (mcpServerIds.length === 0) return - const redis = requireRedis() + const redis = getRedisClient() await Promise.all( mcpServerIds.map((mcpServerId) => redis.eval(CLEAR_SERVER_ATTEMPTS_SCRIPT, 1, serverAttemptsKey(mcpServerId)) diff --git a/apps/sim/lib/credential-groups/oauth-state.test.ts b/apps/sim/lib/credential-groups/oauth-state.test.ts index 7a914e20f38..40f3077c6f0 100644 --- a/apps/sim/lib/credential-groups/oauth-state.test.ts +++ b/apps/sim/lib/credential-groups/oauth-state.test.ts @@ -89,28 +89,6 @@ describe('credential group OAuth state', () => { await expect(consumeCredentialGroupOAuthAttempt(created.state)).resolves.toBeNull() }) - it('fails closed when Redis is unavailable', async () => { - vi.mocked(getRedisClient).mockReturnValue(null) - - await expect( - createCredentialGroupOAuthAttempt({ - provider: 'gmail', - workspaceId: 'workspace-1', - userId: 'user-1', - email: 'person@example.com', - enrollmentId: 'enrollment-1', - credentialGroupId: 'group-1', - optionId: 'option-1', - authorizationAppId: 'google:app', - scopeVersion: 1, - requiredScopes: ['openid'], - redirectUri: 'https://sim.ai/callback', - codeVerifier: 'code-verifier', - invitationToken: 'invitation-token', - }) - ).rejects.toThrow('Credential group OAuth requires Redis') - }) - it('rejects an arbitrary stored return URL while consuming the state only once', async () => { const created = await createCredentialGroupOAuthAttempt({ provider: 'gmail', diff --git a/apps/sim/lib/credential-groups/oauth-state.ts b/apps/sim/lib/credential-groups/oauth-state.ts index c99fc24d93b..3aaa95094d6 100644 --- a/apps/sim/lib/credential-groups/oauth-state.ts +++ b/apps/sim/lib/credential-groups/oauth-state.ts @@ -96,14 +96,6 @@ interface CreateCredentialGroupOAuthAttemptParams { invitationToken: string } -function requireRedis() { - const redis = getRedisClient() - if (!redis) { - throw new Error('Credential group OAuth requires Redis') - } - return redis -} - function attemptKey(state: string): string { return `credential-group:oauth-attempt:${sha256Hex(state)}` } @@ -178,7 +170,7 @@ export async function createCredentialGroupOAuthAttempt( throw new Error('GitHub installation OAuth requires an organization-bound setup attempt') } if (params.connectionIntent) credentialGroupConnectionIntentSchema.parse(params.connectionIntent) - const redis = requireRedis() + const redis = getRedisClient() const state = `${OAUTH_ATTEMPT_STATE_PREFIX}${generateId()}` const nonce = generateId() const [encryptedCodeVerifier, encryptedInvitationToken] = await Promise.all([ @@ -227,7 +219,7 @@ export function isCredentialGroupOAuthState(state: string): boolean { export async function consumeCredentialGroupOAuthAttempt( state: string ): Promise { - const redis = requireRedis() + const redis = getRedisClient() const raw = await redis.eval(CONSUME_SCRIPT, 1, attemptKey(state)) if (raw === null) return null if (typeof raw !== 'string') throw new Error('Credential group OAuth state is malformed') diff --git a/apps/sim/lib/credential-groups/search-connection-completion.ts b/apps/sim/lib/credential-groups/search-connection-completion.ts index 7812ec247d3..00f4d941c3e 100644 --- a/apps/sim/lib/credential-groups/search-connection-completion.ts +++ b/apps/sim/lib/credential-groups/search-connection-completion.ts @@ -18,7 +18,6 @@ export async function recordSearchConnectionCompletion( input: SearchConnectionCompletionScope & { credentialId: string } ) { const redis = getRedisClient() - if (!redis) throw new Error('Search connection completion requires Redis') const stored = await redis.set( completionKey(input), JSON.stringify(receiptSchema.parse({ credentialId: input.credentialId })), @@ -34,7 +33,6 @@ export async function readSearchConnectionCompletion( scope: SearchConnectionCompletionScope ): Promise { const redis = getRedisClient() - if (!redis) throw new Error('Search connection completion requires Redis') const value = await redis.get(completionKey(scope)) return value ? receiptSchema.parse(JSON.parse(value)).credentialId : null } diff --git a/apps/sim/lib/credential-groups/slack-managed-users.ts b/apps/sim/lib/credential-groups/slack-managed-users.ts index 235409f5118..4c6c2597efd 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.ts @@ -163,12 +163,6 @@ export class SlackManagedUsersError extends Error { } } -function requireRedis() { - const redis = getRedisClient() - if (!redis) throw new Error('Slack managed-user setup requires Redis') - return redis -} - function attemptKey(state: string): string { return `credential-group:slack-managed-users:${sha256Hex(state)}` } @@ -725,7 +719,7 @@ export async function createSlackManagedUsersAttempt(params: { 'The combined Slack request has too many permissions.', 'invalid_response' ) - const redis = requireRedis() + const redis = getRedisClient() const state = generateId() const redirectUri = getSlackManagedUsersRedirectUri() const sharedApp = @@ -773,7 +767,7 @@ export async function createSlackManagedUsersAttempt(params: { export async function consumeSlackManagedUsersAttempt( state: string ): Promise { - const redis = requireRedis() + const redis = getRedisClient() const raw = await redis.eval(CONSUME_SCRIPT, 1, attemptKey(state)) return parseSlackManagedUsersAttempt(raw) } @@ -781,7 +775,7 @@ export async function consumeSlackManagedUsersAttempt( export async function loadSlackManagedUsersAttempt( state: string ): Promise { - const redis = requireRedis() + const redis = getRedisClient() const raw = await redis.get(attemptKey(state)) return parseSlackManagedUsersAttempt(raw) } diff --git a/apps/sim/lib/demo-booking/bookers.ts b/apps/sim/lib/demo-booking/bookers.ts index 835546ed704..7b531cad5be 100644 --- a/apps/sim/lib/demo-booking/bookers.ts +++ b/apps/sim/lib/demo-booking/bookers.ts @@ -16,11 +16,10 @@ const CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60 export type DemoBookerStatus = 'verified' | 'unverified' /** - * What a booking request finds for its attendee: the form's verdict, `unseen` - * when no form submission vouches for the email, or `unavailable` when there is - * no Redis to ask. + * What a booking request finds for its attendee: the form's verdict, or `unseen` + * when no form submission vouches for the email. */ -export type DemoBookerLookup = DemoBookerStatus | 'unseen' | 'unavailable' +export type DemoBookerLookup = DemoBookerStatus | 'unseen' /** Keys on a hash, so Redis never holds a visitor's email address. */ function bookerKey(email: string): string { @@ -34,7 +33,6 @@ function bookerKey(email: string): string { */ export async function recordDemoBooker(email: string, status: DemoBookerStatus): Promise { const redis = getRedisClient() - if (!redis) return if (status === 'verified') { await redis.set(bookerKey(email), status, 'EX', BOOKER_TTL_SECONDS) } else { @@ -67,8 +65,6 @@ export async function claimDemoBooker( bookingUid: string ): Promise { const redis = getRedisClient() - if (!redis) return 'unavailable' - const lookup = await redis.eval( CLAIM_SCRIPT, 2, diff --git a/apps/sim/lib/desktop/application/activity.test.ts b/apps/sim/lib/desktop/application/activity.test.ts index 15eb8fec70f..d0a1f31210d 100644 --- a/apps/sim/lib/desktop/application/activity.test.ts +++ b/apps/sim/lib/desktop/application/activity.test.ts @@ -2,12 +2,11 @@ import { createSessionPrincipal } from '@sim/testing/factories/principal.factory import { beforeEach, describe, expect, it, vi } from 'vitest' const { presence, rows } = vi.hoisted(() => ({ - presence: { available: vi.fn(), present: vi.fn() }, + presence: { present: vi.fn() }, rows: vi.fn(), })) vi.mock('@/lib/desktop/executor/presence', () => ({ - isDesktopPresenceAvailable: presence.available, isDesktopPresent: presence.present, })) vi.mock('@/lib/desktop/executor/repository', () => ({ listDesktopActivityRows: rows })) @@ -29,22 +28,7 @@ describe('desktop activity presence', () => { ]) }) - it('does not call a desktop blocked when Sim cannot track presence at all', async () => { - presence.available.mockReturnValue(false) - presence.present.mockResolvedValue(false) - - const { chats } = await listDesktopActivity.execute({ - principal, - input: { workspaceId: 'ws-1' }, - }) - - expect(chats).toEqual([ - { chatId: 'chat-1', streamId: 's-1', state: 'running', deviceName: 'Studio Mac' }, - ]) - }) - - it('calls a desktop blocked when presence is tracked and it is gone', async () => { - presence.available.mockReturnValue(true) + it('calls a desktop blocked when its presence is gone', async () => { presence.present.mockResolvedValue(false) const { chats } = await listDesktopActivity.execute({ diff --git a/apps/sim/lib/desktop/application/activity.ts b/apps/sim/lib/desktop/application/activity.ts index 6fe7439878a..56db41063d7 100644 --- a/apps/sim/lib/desktop/application/activity.ts +++ b/apps/sim/lib/desktop/application/activity.ts @@ -3,7 +3,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { defineOperation } from '@/lib/core/application' import { defineAuthorizedCredentialUserUseCase } from '@/lib/credentials/application/authorized-user-use-case' -import { isDesktopPresenceAvailable, isDesktopPresent } from '@/lib/desktop/executor/presence' +import { isDesktopPresent } from '@/lib/desktop/executor/presence' import { listDesktopActivityRows } from '@/lib/desktop/executor/repository' const logger = createLogger('DesktopActivity') @@ -24,9 +24,8 @@ export interface DesktopChatActivityEntry { deviceName: string } -/** Presence Sim cannot read, or cannot track at all, is not evidence of an offline desktop. */ +/** Presence Sim cannot read is not evidence of an offline desktop. */ async function readPresence(deviceId: string): Promise { - if (!isDesktopPresenceAvailable()) return true try { return await isDesktopPresent(deviceId) } catch (error) { @@ -38,8 +37,8 @@ async function readPresence(deviceId: string): Promise { /** * Which of the caller's chats in a workspace are running on one of their desktops, and in what * state. Lists only the caller's own runs, never their content, so it needs no workspace role. - * Runs already bound are listed until they end, whatever the install's executor availability; the - * sidebar asks only for a user with a desktop that runs their turns. + * Runs already bound are listed until they end; the sidebar asks only for a user with a desktop + * that runs their turns. */ export const listDesktopActivity = defineAuthorizedCredentialUserUseCase({ // permission-group-exempt: reports only the caller's own runs, with no content. diff --git a/apps/sim/lib/desktop/application/executor.integration.ts b/apps/sim/lib/desktop/application/executor.integration.ts index d49700d1068..5a3c4899e4a 100644 --- a/apps/sim/lib/desktop/application/executor.integration.ts +++ b/apps/sim/lib/desktop/application/executor.integration.ts @@ -5,22 +5,16 @@ * Stop runs through the real `requestRunStop`. */ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, presence } = await vi.hoisted(async () => { +const { redisUrl } = await vi.hoisted(async () => { const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') const url = readTestRedisUrl() /** The real Redis module reads this at import. */ if (url) process.env.REDIS_URL = url - return { redisUrl: url, presence: { available: true } } + return { redisUrl: url } }) -/** Redis is always configured here; `presence.available` stands in for an install without it. */ -vi.mock('@/lib/desktop/executor/presence', async (importOriginal) => ({ - ...(await importOriginal()), - isDesktopPresenceAvailable: () => presence.available, -})) - import type { SessionPrincipal } from '@sim/auth/principal' import { db } from '@sim/db' import { @@ -236,10 +230,6 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => return settled } - beforeEach(() => { - presence.available = true - }) - afterAll(async () => { if (chatIds.length) await db.delete(copilotChats).where(inArray(copilotChats.id, chatIds)) if (deviceIds.length) { @@ -252,24 +242,12 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => }) describe('registration', () => { - it('reports the executor on wherever Sim can track presence', async () => { + it('reports the executor on', async () => { const desktop = await signedInDesktop() expect(desktop.enabled).toBe(true) await expect(inbox(desktop)).resolves.toEqual({ items: [] }) }) - it('writes nothing and reports the executor off where Sim cannot track presence', async () => { - presence.available = false - const desktop = await signedInDesktop() - expect(desktop.enabled).toBe(false) - const [stored] = await db - .select() - .from(desktopDevices) - .where(eq(desktopDevices.id, desktop.deviceId)) - expect(stored).toBeUndefined() - await expect(inbox(desktop)).rejects.toBeInstanceOf(DesktopDeviceUnrecognizedError) - }) - it('refuses an install id that belongs to another account', async () => { const owner = await signedInDesktop() const intruder = await signedInDesktop() @@ -845,7 +823,6 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => const run = await boundRun(desktop) const toolCallId = await pendingCall(run.runId) const redis = getRedisClient() - if (!redis) throw new Error('Redis is required') const writes = vi.spyOn(redis, 'set').mockRejectedValue(new Error('Redis unavailable')) try { expect((await inbox(desktop)).items).toMatchObject([{ kind: 'call', toolCallId }]) diff --git a/apps/sim/lib/desktop/application/executor.ts b/apps/sim/lib/desktop/application/executor.ts index 74c0edf6444..8cff3615344 100644 --- a/apps/sim/lib/desktop/application/executor.ts +++ b/apps/sim/lib/desktop/application/executor.ts @@ -10,7 +10,6 @@ import { type CredentialUserAuditEntry, defineAuthorizedCredentialUserUseCase, } from '@/lib/credentials/application/authorized-user-use-case' -import { isDesktopBackgroundExecutorAvailable } from '@/lib/desktop/executor/availability' import { DESKTOP_EXECUTOR_PROTOCOL_VERSION, DESKTOP_INBOX_RECONCILE_MS, @@ -83,15 +82,14 @@ async function requireBoundDevice(principal: SessionPrincipal, deviceId: string) } /** - * The device a new turn binds to: the composer's own, but only while this install can run the - * executor and the device is registered to this very session as an executor. Anything else leaves - * the turn to the chat view, as before the executor existed. + * The device a new turn binds to: the composer's own, but only while the device is registered to + * this very session as an executor. Anything else leaves the turn to the chat view, as before the + * executor existed. */ export async function resolveTurnDesktopDevice( principal: SessionPrincipal, deviceId: string ): Promise { - if (!isDesktopBackgroundExecutorAvailable()) return null const device = await getBoundDesktopDevice( { deviceId, userId: principal.userId, sessionId: principal.sessionId }, { executor: true } @@ -114,8 +112,7 @@ interface RegisterDesktopDeviceInput extends DeviceInput { } /** - * Binds the install to this user and session. Where the executor is unavailable nothing is - * written, and the device stays dormant. + * Binds the install to this user and session. */ export const registerDesktopDevice = defineAuthorizedCredentialUserUseCase({ // permission-group-exempt: registering grants no access; each call's run was admitted under the Chat capability. @@ -131,31 +128,28 @@ export const registerDesktopDevice = defineAuthorizedCredentialUserUseCase({ principal: SessionPrincipal input: RegisterDesktopDeviceInput }) { - const enabled = isDesktopBackgroundExecutorAvailable() - if (enabled) { - const registered = await upsertDesktopDevice({ - id: input.deviceId, - userId: principal.userId, - sessionId: principal.sessionId, - name: input.name, - appVersion: input.appVersion, - platform: input.platform, - capabilities: input.capabilities, - }) - if (!registered) - throw new OrchestrationError( - 'conflict', - 'This device ID belongs to another account or was revoked. Generate a new device ID and register again.' - ) - logger.info('Desktop device registered', { - userId: principal.userId, - deviceId: input.deviceId, - appVersion: input.appVersion, - platform: input.platform, - }) - } + const registered = await upsertDesktopDevice({ + id: input.deviceId, + userId: principal.userId, + sessionId: principal.sessionId, + name: input.name, + appVersion: input.appVersion, + platform: input.platform, + capabilities: input.capabilities, + }) + if (!registered) + throw new OrchestrationError( + 'conflict', + 'This device ID belongs to another account or was revoked. Generate a new device ID and register again.' + ) + logger.info('Desktop device registered', { + userId: principal.userId, + deviceId: input.deviceId, + appVersion: input.appVersion, + platform: input.platform, + }) return { - enabled, + enabled: true, protocolVersion: DESKTOP_EXECUTOR_PROTOCOL_VERSION, leaseMs: SIM_TOOL_EXECUTION_LEASE_SECONDS * 1000, leaseRenewMs: SIM_TOOL_EXECUTION_HEARTBEAT_MS, diff --git a/apps/sim/lib/desktop/application/source-requests.ts b/apps/sim/lib/desktop/application/source-requests.ts index 42f5e6709eb..accdf889ed4 100644 --- a/apps/sim/lib/desktop/application/source-requests.ts +++ b/apps/sim/lib/desktop/application/source-requests.ts @@ -16,12 +16,6 @@ redis.call('DEL', KEYS[1]) return request.encrypted ` -function redis() { - const client = getRedisClient() - if (!client) throw new Error('Desktop connections require Redis') - return client -} - function requestKey(requestId: string) { return `desktop:source-request:${sha256Hex(requestId)}` } @@ -47,7 +41,7 @@ export const createDesktopSourceRequest = defineAuthorizedCredentialUserUseCase( if (!/^[A-Za-z0-9_-]{32}$/.test(requestId)) throw new OrchestrationError('validation', 'Invalid connection request') const { encrypted } = await encryptSecret(input.payload) - const saved = await redis().set( + const saved = await getRedisClient().set( requestKey(requestId), JSON.stringify({ userId: principal.userId, encrypted }), 'EX', @@ -74,7 +68,12 @@ export const consumeDesktopSourceRequest = defineAuthorizedCredentialUserUseCase principal: { userId: string } input: { requestId: string } }) { - const encrypted = await redis().eval(CONSUME, 1, requestKey(input.requestId), principal.userId) + const encrypted = await getRedisClient().eval( + CONSUME, + 1, + requestKey(input.requestId), + principal.userId + ) if (typeof encrypted !== 'string') throw new OrchestrationError( 'not_found', diff --git a/apps/sim/lib/desktop/executor/availability.ts b/apps/sim/lib/desktop/executor/availability.ts index b49f852d8ed..34210e56ff7 100644 --- a/apps/sim/lib/desktop/executor/availability.ts +++ b/apps/sim/lib/desktop/executor/availability.ts @@ -1,20 +1,9 @@ -import { isDesktopPresenceAvailable } from '@/lib/desktop/executor/presence' import { hasSignedInDesktopExecutor } from '@/lib/desktop/executor/repository' -/** - * Whether new turns may bind to a desktop, and whether the workspace shows background desktop - * activity. The executor needs presence tracking, so an install without Redis keeps desktop tools - * in the chat window. Never consulted for a run already bound: it keeps running on its desktop. - */ -export function isDesktopBackgroundExecutorAvailable(): boolean { - return isDesktopPresenceAvailable() -} - /** * Whether the user's turns can run on one of their desktops, so their workspace has background * desktop activity to show. Most users never install the app, and their pages never ask. */ export async function hasDesktopBackgroundExecutor(userId: string): Promise { - if (!isDesktopBackgroundExecutorAvailable()) return false return hasSignedInDesktopExecutor(userId) } diff --git a/apps/sim/lib/desktop/executor/bound-turn.integration.ts b/apps/sim/lib/desktop/executor/bound-turn.integration.ts index 9366ee9fae8..f3889fbf328 100644 --- a/apps/sim/lib/desktop/executor/bound-turn.integration.ts +++ b/apps/sim/lib/desktop/executor/bound-turn.integration.ts @@ -7,9 +7,9 @@ */ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker, presence } = await vi.hoisted(async () => { +const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') const { createServer } = await import('node:http') /** Stands in for the agent worker, which Stop also tells to end the stream. */ @@ -31,15 +31,10 @@ const { redisUrl, inheritedEnv, worker, presence } = await vi.hoisted(async () = if (url) process.env.REDIS_URL = url process.env.COPILOT_TOOL_PERMISSIONS_ENABLED = 'true' process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: { server }, presence: { available: true } } + return { redisUrl: url, inheritedEnv, worker: { server } } }) vi.mock('@/lib/auth', () => authMock) -/** Redis is always configured here; `presence.available` stands in for an install without it. */ -vi.mock('@/lib/desktop/executor/presence', async (importOriginal) => ({ - ...(await importOriginal()), - isDesktopPresenceAvailable: () => presence.available, -})) import type { SessionPrincipal } from '@sim/auth/principal' import { db } from '@sim/db' @@ -360,10 +355,6 @@ describe.runIf(Boolean(redisUrl))("a turn bound to a desktop's background execut }) }) - beforeEach(() => { - presence.available = true - }) - afterAll(async () => { if (chatIds.length > 0) await db.delete(copilotChats).where(inArray(copilotChats.id, chatIds)) if (deviceIds.length > 0) { @@ -374,7 +365,7 @@ describe.runIf(Boolean(redisUrl))("a turn bound to a desktop's background execut await db.delete(user).where(eq(user.id, userId)) }) - it('binds a turn only to an executor registered to this very session, where Sim tracks presence', async () => { + it('binds a turn only to an executor registered to this very session', async () => { const desktop = await signedInDesktop() const viewer = await signedInDesktop({ executor: 0 }) const otherSession: SessionPrincipal = { ...desktop.principal, sessionId: generateId() } @@ -384,8 +375,6 @@ describe.runIf(Boolean(redisUrl))("a turn bound to a desktop's background execut ) expect(await resolveTurnDesktopDevice(otherSession, desktop.deviceId)).toBeNull() expect(await resolveTurnDesktopDevice(viewer.principal, viewer.deviceId)).toBeNull() - presence.available = false - expect(await resolveTurnDesktopDevice(desktop.principal, desktop.deviceId)).toBeNull() }) it( @@ -682,7 +671,6 @@ describe.runIf(Boolean(redisUrl))("a turn bound to a desktop's background execut const desktop = await signedInDesktop() const run = await boundRun(desktop) const redis = getRedisClient() - if (!redis) throw new Error('Redis is required') const writes = vi.spyOn(redis, 'set').mockRejectedValue(new Error('Redis unavailable')) try { /** The pull succeeds, but its presence write fails: the key reads as absent. */ @@ -710,7 +698,6 @@ describe.runIf(Boolean(redisUrl))("a turn bound to a desktop's background execut const desktop = await signedInDesktop() const run = await boundRun(desktop) const redis = getRedisClient() - if (!redis) throw new Error('Redis is required') const reads = vi.spyOn(redis, 'exists').mockRejectedValue(new Error('Redis unavailable')) try { const { toolCallId, answer, context } = await agentCalls(run, 'browser_click', { diff --git a/apps/sim/lib/desktop/executor/presence.ts b/apps/sim/lib/desktop/executor/presence.ts index 407b98f8b1c..28d0d895a49 100644 --- a/apps/sim/lib/desktop/executor/presence.ts +++ b/apps/sim/lib/desktop/executor/presence.ts @@ -14,18 +14,12 @@ function presenceKey(deviceId: string): string { return `desktop:presence:${deviceId}` } -/** Whether presence can be tracked at all, which needs Redis. */ -export function isDesktopPresenceAvailable(): boolean { - return getRedisClient() !== null -} - /** * Records that the device just made a request. Best effort: presence only decides how fast an * unclaimed call fails, so a Redis error must not fail the request that proves the device is awake. */ export async function markDesktopPresent(deviceId: string): Promise { const redis = getRedisClient() - if (!redis) return try { await redis.set(presenceKey(deviceId), '1', 'EX', DESKTOP_PRESENCE_TTL_SECONDS) } catch (error) { @@ -33,9 +27,8 @@ export async function markDesktopPresent(deviceId: string): Promise { } } -/** Whether the device made a request within the presence TTL; a missing Redis reads as absent. */ +/** Whether the device made a request within the presence TTL. */ export async function isDesktopPresent(deviceId: string): Promise { const redis = getRedisClient() - if (!redis) return false return (await redis.exists(presenceKey(deviceId))) === 1 } diff --git a/apps/sim/lib/events/pubsub.test.ts b/apps/sim/lib/events/pubsub.test.ts index 06dd0922dcc..9449381d219 100644 --- a/apps/sim/lib/events/pubsub.test.ts +++ b/apps/sim/lib/events/pubsub.test.ts @@ -32,6 +32,13 @@ async function settled(promise: Promise): Promise { return Promise.race([promise.then(() => true), Promise.resolve().then(() => false)]) } +/** A channel whose connections are open: they open on first use, and `ready` is a use. */ +function openChannel() { + const channel = createPubSubChannel({ channel: 'test', label: 'Test' }) + void channel.ready() + return channel +} + /** The subscriber connection: the channel opens its publisher first. */ function subscriber() { const client = clients.at(-1) @@ -42,7 +49,31 @@ function subscriber() { describe('createPubSubChannel over Redis', () => { beforeEach(() => { clients.length = 0 - redisConfigMockFns.mockGetConfiguredRedisUrl.mockReturnValue('redis://localhost:6379') + redisConfigMockFns.mockGetConfiguredRedisUrl + .mockReset() + .mockReturnValue('redis://localhost:6379') + }) + + it('opens no connection until it is first used', () => { + const channel = createPubSubChannel({ channel: 'test', label: 'Test' }) + expect(clients).toHaveLength(0) + + channel.subscribe(() => {}) + expect(clients).toHaveLength(2) + + channel.publish({ hello: 'world' }) + expect(clients).toHaveLength(2) + }) + + it('reports a missing REDIS_URL on first use, not when the channel is built', () => { + redisConfigMockFns.mockGetConfiguredRedisUrl.mockImplementation(() => { + throw new Error('REDIS_URL is required') + }) + + const channel = createPubSubChannel({ channel: 'test', label: 'Test' }) + + expect(() => channel.publish({})).toThrow('REDIS_URL is required') + expect(clients).toHaveLength(0) }) it('is ready only once its connection has subscribed', async () => { @@ -57,7 +88,7 @@ describe('createPubSubChannel over Redis', () => { }) it('is not ready again until a dropped connection has resubscribed', async () => { - const channel = createPubSubChannel({ channel: 'test', label: 'Test' }) + const channel = openChannel() subscriber().emit('ready') subscriber().subscribed[0](null) @@ -70,7 +101,7 @@ describe('createPubSubChannel over Redis', () => { }) it('is not ready while its subscribe fails, and retries on the next connection', async () => { - const channel = createPubSubChannel({ channel: 'test', label: 'Test' }) + const channel = openChannel() subscriber().emit('ready') subscriber().subscribed[0](new Error('NOPERM')) @@ -83,7 +114,7 @@ describe('createPubSubChannel over Redis', () => { }) it('ignores a subscribe answered after its connection closed', async () => { - const channel = createPubSubChannel({ channel: 'test', label: 'Test' }) + const channel = openChannel() subscriber().emit('ready') subscriber().emit('close') diff --git a/apps/sim/lib/events/pubsub.ts b/apps/sim/lib/events/pubsub.ts index 345a082106a..0be76ba2292 100644 --- a/apps/sim/lib/events/pubsub.ts +++ b/apps/sim/lib/events/pubsub.ts @@ -1,11 +1,11 @@ /** * Generic Pub/Sub Channel Factory * - * Creates a single-channel pub/sub adapter backed by Redis (with EventEmitter fallback). - * Each call creates its own Redis connections — use multiple instances for multiple channels. + * Creates a single-channel pub/sub adapter backed by Redis. Each channel opens its own pair of + * Redis connections, and only on first use, so a module may build its channel at import time + * (including during `next build`, where no REDIS_URL is set) without connecting or throwing. */ -import { EventEmitter } from 'events' import { createLogger } from '@sim/logger' import { noop } from '@sim/utils/helpers' import Redis, { type RedisOptions } from 'ioredis' @@ -149,51 +149,39 @@ class RedisPubSubChannel implements PubSubChannel { } } -class LocalPubSubChannel implements PubSubChannel { - private emitter = new EventEmitter() - - constructor(private config: PubSubChannelConfig) { - this.emitter.setMaxListeners(100) - logger.info(`${config.label}: Using process-local EventEmitter (Redis not configured)`) - } - - publish(event: T): void { - this.emitter.emit(this.config.channel, event) - } - - subscribe(handler: (event: T) => void): () => void { - const isolatedHandler = (event: T) => { - try { - handler(event) - } catch (error) { - logger.error(`Error in ${this.config.label} handler:`, error) - } - } - this.emitter.on(this.config.channel, isolatedHandler) - return () => { - this.emitter.off(this.config.channel, isolatedHandler) - } - } - - ready(): Promise { - return Promise.resolve() +/** + * Builds a channel whose Redis connections open on its first `publish`, `subscribe`, or `ready`. + * A missing or invalid REDIS_URL surfaces from that first call, never from construction. A + * disposed channel publishes nothing, adds no handlers, and never becomes ready. + */ +export function createPubSubChannel(config: PubSubChannelConfig): PubSubChannel { + let connection: RedisPubSubChannel | null = null + let disposed = false + + const open = (): RedisPubSubChannel => { + if (connection) return connection + const redisUrl = getConfiguredRedisUrl() + connection = new RedisPubSubChannel(redisUrl, getRedisConnectionDefaults(redisUrl), config) + return connection } - dispose(): void { - this.emitter.removeAllListeners() - logger.info(`${this.config.label} local pub/sub disposed`) + return { + publish(event) { + if (disposed) return + open().publish(event) + }, + subscribe(handler) { + if (disposed) return noop + return open().subscribe(handler) + }, + ready() { + if (disposed) return new Promise(noop) + return open().ready() + }, + dispose() { + disposed = true + connection?.dispose() + connection = null + }, } } - -export function createPubSubChannel(config: PubSubChannelConfig): PubSubChannel { - const redisUrl = getConfiguredRedisUrl() - if (!redisUrl) return new LocalPubSubChannel(config) - - // Resolve config-derived defaults outside the try so a missing - // REDIS_TLS_SERVERNAME (config error) surfaces instead of silently degrading - // to the in-process EventEmitter — that would break cross-replica pub/sub. - const connectionDefaults = getRedisConnectionDefaults(redisUrl) - - logger.info(`${config.label}: Using Redis`) - return new RedisPubSubChannel(redisUrl, connectionDefaults, config) -} diff --git a/apps/sim/lib/execution/blocked-run-log.ts b/apps/sim/lib/execution/blocked-run-log.ts index 2c531a6b9b3..5093a5874ab 100644 --- a/apps/sim/lib/execution/blocked-run-log.ts +++ b/apps/sim/lib/execution/blocked-run-log.ts @@ -14,17 +14,13 @@ export const BLOCKED_RUN_LOG_WINDOW_SECONDS = 15 * 60 /** * Claims the right to record this window's blocked-run log row for a workflow - * and gate. Returns false when another refusal already recorded one. Without - * Redis, or when Redis fails, it returns true: a duplicate row is better than - * hiding that runs are blocked. Usage-limit refusals only occur on hosted - * billing deployments, which always run Redis. + * and gate. Returns false when another refusal already recorded one. When + * Redis fails it returns true: a duplicate row is better than hiding that runs + * are blocked. */ export async function claimBlockedRunLog(workflowId: string, gate: string): Promise { - const redis = getRedisClient() - if (!redis) return true - try { - const claimed = await redis.set( + const claimed = await getRedisClient().set( `blocked-run-log:v1:${workflowId}:${gate}`, '1', 'EX', diff --git a/apps/sim/lib/execution/cancel-workflow-execution.test.ts b/apps/sim/lib/execution/cancel-workflow-execution.test.ts index b5defe5270a..438a33d71a4 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.test.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.test.ts @@ -158,7 +158,7 @@ describe('cancelWorkflowExecution', () => { mockClearExecutionCancellation.mockReset().mockResolvedValue(undefined) mockMarkExecutionCancelled .mockReset() - .mockResolvedValue({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValue({ durablyRecorded: false, reason: 'redis_write_failed' }) mockAbortManualExecution.mockReset().mockReturnValue(false) mockBeginPausedCancellation.mockReset().mockResolvedValue(false) mockStagePausedCancellation.mockReset().mockResolvedValue({ kind: 'not_paused' }) @@ -271,7 +271,7 @@ describe('cancelWorkflowExecution', () => { }) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) const response = await POST(makeRequest(), makeParams()) @@ -280,7 +280,7 @@ describe('cancelWorkflowExecution', () => { await expect(response.json()).resolves.toMatchObject({ success: false, durablyRecorded: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) expect(mockMarkExecutionCancelled).toHaveBeenCalledOnce() expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() @@ -330,7 +330,7 @@ describe('cancelWorkflowExecution', () => { ]) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) mockAbortManualExecution.mockReturnValue(true) mockCancelWorkflowGroupExecution.mockResolvedValue({ @@ -348,7 +348,7 @@ describe('cancelWorkflowExecution', () => { success: true, durablyRecorded: false, locallyAborted: true, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() expect(mockCancelByExecution).not.toHaveBeenCalled() @@ -443,7 +443,7 @@ describe('cancelWorkflowExecution', () => { mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) mockCancelByExecution.mockResolvedValue(1) mockCancelWorkflowGroupExecution.mockResolvedValue({ @@ -672,26 +672,6 @@ describe('cancelWorkflowExecution', () => { expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() }) - it('returns unsuccessful response when Redis is unavailable', async () => { - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - - const response = await POST(makeRequest(), makeParams()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: false, - executionId: 'ex-1', - redisAvailable: false, - durablyRecorded: false, - locallyAborted: false, - pausedCancelled: false, - reason: 'redis_unavailable', - }) - }) - it('returns unsuccessful response when Redis persistence fails', async () => { mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, @@ -712,10 +692,10 @@ describe('cancelWorkflowExecution', () => { }) }) - it('returns success when local fallback aborts execution without Redis durability', async () => { + it('returns success when an in-process abort lands although the Redis write failed', async () => { mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) mockAbortManualExecution.mockReturnValue(true) @@ -725,18 +705,18 @@ describe('cancelWorkflowExecution', () => { await expect(response.json()).resolves.toEqual({ success: true, executionId: 'ex-1', - redisAvailable: false, + redisAvailable: true, durablyRecorded: false, locallyAborted: true, pausedCancelled: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) }) it('returns success when the queue backend cancels the active job', async () => { mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) mockCancelByExecution.mockResolvedValue(1) @@ -817,7 +797,7 @@ describe('cancelWorkflowExecution', () => { mockGetActiveResumeCancellationTarget.mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) mockMarkExecutionCancelled - .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_write_failed' }) .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) mockCompletePausedCancellation.mockResolvedValueOnce(true) @@ -859,8 +839,8 @@ describe('cancelWorkflowExecution', () => { mockGetActiveResumeCancellationTarget.mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) mockMarkExecutionCancelled - .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) - .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_write_failed' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_write_failed' }) .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) mockCompletePausedCancellation.mockResolvedValueOnce(true) @@ -907,8 +887,8 @@ describe('cancelWorkflowExecution', () => { mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) mockCancelByExecution.mockResolvedValue(1) mockMarkExecutionCancelled - .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) - .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_write_failed' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_write_failed' }) .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) const response = await POST(makeRequest(), makeParams()) @@ -1386,7 +1366,7 @@ describe('cancelWorkflowExecution', () => { await expect(response.json()).resolves.toMatchObject({ success: false, redisAvailable: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() expect(mockWriteTerminalEvent).not.toHaveBeenCalled() @@ -1765,7 +1745,7 @@ describe('cancelWorkflowExecution', () => { databaseMock.db.update.mockReturnValueOnce({ set: mockSet }) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) mockAbortManualExecution.mockReturnValue(true) @@ -1774,7 +1754,7 @@ describe('cancelWorkflowExecution', () => { expect(response.status).toBe(200) await expect(response.json()).resolves.toMatchObject({ success: true, - reason: 'redis_unavailable', + reason: 'redis_write_failed', }) expect(databaseMock.db.update).toHaveBeenCalled() expect(mockSet).toHaveBeenCalledWith({ diff --git a/apps/sim/lib/execution/cancel-workflow-execution.ts b/apps/sim/lib/execution/cancel-workflow-execution.ts index c0dcf084764..ed3a22dbae2 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.ts @@ -40,7 +40,6 @@ export type CancelWorkflowExecutionReason = | 'already_cancelled' | 'already_completed' | 'already_failed' - | 'redis_unavailable' | 'redis_write_failed' | 'paused_event_publish_failed' | 'paused_database_cancel_failed' @@ -51,6 +50,10 @@ export type CancelWorkflowExecutionReason = export interface CancelWorkflowExecutionResult { success: boolean executionId: string + /** + * Kept on the wire for compatibility. Redis is required, so this is true except on a paused-run + * cancellation whose cancellation event failed to publish. + */ redisAvailable: boolean durablyRecorded: boolean locallyAborted: boolean @@ -112,9 +115,10 @@ type ActiveResumeCancellationTarget = NonNullable< Awaited> > +/** Starts unrecorded; the first merged stop signal replaces `cancellation` before anything reads it. */ function createExecutionStopSummary(): ExecutionStopSummary { return { - cancellation: { durablyRecorded: false, reason: 'redis_unavailable' }, + cancellation: { durablyRecorded: false, reason: 'redis_write_failed' }, locallyAborted: false, queueJobsCancelled: 0, accepted: false, @@ -570,7 +574,7 @@ function activeResumeSignalFailureResult( return { success: false, executionId, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + redisAvailable: true, durablyRecorded: stopSummary.cancellation.durablyRecorded, locallyAborted: stopSummary.locallyAborted, pausedCancelled: false, @@ -669,7 +673,7 @@ export async function cancelWorkflowExecution({ return { success: true, executionId, - redisAvailable: cancellation.reason !== 'redis_unavailable', + redisAvailable: true, durablyRecorded: cancellation.durablyRecorded, locallyAborted, pausedCancelled: false, @@ -938,7 +942,7 @@ export async function cancelWorkflowExecution({ return { success: false, executionId, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + redisAvailable: true, durablyRecorded: stopSummary.cancellation.durablyRecorded, locallyAborted: stopSummary.locallyAborted, pausedCancelled: false, @@ -1034,7 +1038,7 @@ export async function cancelWorkflowExecution({ throw new WorkflowRunAlreadyTerminalError({ executionId, executionStatus: competingTerminalStatus, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + redisAvailable: true, locallyAborted: stopSummary.locallyAborted, }) } @@ -1061,7 +1065,7 @@ export async function cancelWorkflowExecution({ return { success: false, executionId, - redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + redisAvailable: true, durablyRecorded: stopSummary.cancellation.durablyRecorded, locallyAborted: stopSummary.locallyAborted, pausedCancelled: false, @@ -1191,9 +1195,7 @@ export async function cancelWorkflowExecution({ success, executionId, redisAvailable: - effectivePausedCancellationPath || pausedCancelled - ? cancellationEventPublished - : stopSummary.cancellation.reason !== 'redis_unavailable', + effectivePausedCancellationPath || pausedCancelled ? cancellationEventPublished : true, durablyRecorded, locallyAborted: stopSummary.locallyAborted, pausedCancelled, diff --git a/apps/sim/lib/execution/cancellation.test.ts b/apps/sim/lib/execution/cancellation.test.ts index c07c1beb3ae..291e077853b 100644 --- a/apps/sim/lib/execution/cancellation.test.ts +++ b/apps/sim/lib/execution/cancellation.test.ts @@ -7,7 +7,6 @@ const { mockRedisPublish, mockSignalSubscribe, mockSignalUnsubscribe, - mockPublishLocalSignal, mockRecordCancellationResult, } = vi.hoisted(() => ({ mockRedisEval: vi.fn(), @@ -15,7 +14,6 @@ const { mockRedisPublish: vi.fn(), mockSignalSubscribe: vi.fn(), mockSignalUnsubscribe: vi.fn(), - mockPublishLocalSignal: vi.fn(), mockRecordCancellationResult: vi.fn(), })) @@ -25,7 +23,6 @@ vi.mock('@/lib/execution/signal', () => ({ getExecutionSignalChannel: (executionId: string) => `execution:signal:${executionId}`, getExecutionSignalHub: () => ({ subscribe: mockSignalSubscribe }), LEGACY_EXECUTION_CANCEL_CHANNEL: 'execution:cancel', - publishLocalExecutionSignal: mockPublishLocalSignal, })) vi.mock('@/lib/core/execution-limits/metrics', () => ({ @@ -50,16 +47,6 @@ describe('markExecutionCancelled', () => { mockRedisPublish.mockResolvedValue(1) }) - it('returns redis_unavailable when no Redis client exists', async () => { - mockGetRedisClient.mockReturnValue(null) - - await expect(markExecutionCancelled('execution-1')).resolves.toEqual({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - expect(mockPublishLocalSignal).toHaveBeenCalledWith('execution-1', 'cancelled') - }) - it('returns recorded when Redis write succeeds', async () => { mockRedisEval.mockResolvedValue(1) mockGetRedisClient.mockReturnValue({ eval: mockRedisEval }) @@ -171,43 +158,6 @@ describe('markExecutionCancelled', () => { resolveExactPublish(1) await vi.waitFor(() => expect(mockRedisPublish).toHaveBeenCalledTimes(2)) }) - - it('publishes through the configured process-local provider when Redis is unavailable', async () => { - mockGetRedisClient.mockReturnValue(null) - - await markExecutionCancelled('execution-3') - - expect(mockRedisEval).not.toHaveBeenCalled() - expect(mockPublishLocalSignal).toHaveBeenCalledWith('execution-3', 'cancelled') - }) - - it('still reports Redis unavailable when the local signal provider rejects the notification', async () => { - mockGetRedisClient.mockReturnValue(null) - mockPublishLocalSignal.mockImplementationOnce(() => { - throw new Error('local signal unavailable') - }) - - await expect(markExecutionCancelled('execution-local-signal-error')).resolves.toEqual({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - }) - - it('expires process-local cancellation records after their retention window', async () => { - vi.useFakeTimers() - try { - mockGetRedisClient.mockReturnValue(null) - const onCancelled = vi.fn() - - await markExecutionCancelled('execution-expiring-local-record') - await vi.advanceTimersByTimeAsync(EXECUTION_CANCEL_MIN_RETENTION_MS) - await subscribeToExecutionCancellation('execution-expiring-local-record', onCancelled) - - expect(onCancelled).not.toHaveBeenCalled() - } finally { - vi.useRealTimers() - } - }) }) describe('subscribeToExecutionCancellation', () => { diff --git a/apps/sim/lib/execution/cancellation.ts b/apps/sim/lib/execution/cancellation.ts index bf41a79b2a6..29cfa8bcf2d 100644 --- a/apps/sim/lib/execution/cancellation.ts +++ b/apps/sim/lib/execution/cancellation.ts @@ -1,21 +1,16 @@ import { createLogger } from '@sim/logger' +import type Redis from 'ioredis' import { getRedisClient } from '@/lib/core/config/redis' import { getExecutionSignalChannel, getExecutionSignalHub, LEGACY_EXECUTION_CANCEL_CHANNEL, - publishLocalExecutionSignal, } from '@/lib/execution/signal' const logger = createLogger('ExecutionCancellation') const EXECUTION_CANCEL_PREFIX = 'execution:cancel:' export const EXECUTION_CANCEL_MIN_RETENTION_MS = 14 * 24 * 60 * 60 * 1000 -const MAX_TIMER_DELAY_MS = 2_147_483_647 -const MAX_LOCAL_CANCELLATION_RECORDS = 50_000 -const localCancelledExecutions = new Map() -let localCancellationCleanupTimer: NodeJS.Timeout | undefined -let localCancellationCleanupAt: number | undefined const MARK_EXECUTION_CANCELLED_SCRIPT = ` redis.call('SET', KEYS[1], '1', 'PXAT', ARGV[1]) redis.call('PUBLISH', ARGV[2], 'cancelled') @@ -29,10 +24,7 @@ export interface MarkExecutionCancelledOptions { export type ExecutionCancellationRecordResult = | { durablyRecorded: true; reason: 'recorded' } - | { - durablyRecorded: false - reason: 'redis_unavailable' | 'redis_write_failed' - } + | { durablyRecorded: false; reason: 'redis_write_failed' } function getCancellationExpiryAt(options: MarkExecutionCancelledOptions): number { const minimumExpiryAt = Date.now() + EXECUTION_CANCEL_MIN_RETENTION_MS @@ -52,7 +44,7 @@ function serializeLegacyCancellationSignal( } async function publishCancellationSignalBestEffort( - redis: NonNullable>, + redis: Redis, executionId: string ): Promise { let executionSignalPublished = false @@ -79,42 +71,6 @@ async function publishCancellationSignalBestEffort( } } -function scheduleLocalCancellationCleanup(candidateExpiryAt?: number): void { - if ( - candidateExpiryAt !== undefined && - localCancellationCleanupTimer && - localCancellationCleanupAt !== undefined && - localCancellationCleanupAt <= candidateExpiryAt - ) { - return - } - if (localCancellationCleanupTimer) clearTimeout(localCancellationCleanupTimer) - localCancellationCleanupTimer = undefined - localCancellationCleanupAt = undefined - if (localCancelledExecutions.size === 0) return - - const now = Date.now() - let nextExpiryAt = candidateExpiryAt - if (nextExpiryAt === undefined) { - nextExpiryAt = Number.POSITIVE_INFINITY - for (const expiryAt of localCancelledExecutions.values()) { - nextExpiryAt = Math.min(nextExpiryAt, expiryAt) - } - } - const delayMs = Math.min(Math.max(0, nextExpiryAt - now), MAX_TIMER_DELAY_MS) - localCancellationCleanupAt = now + delayMs - localCancellationCleanupTimer = setTimeout(() => { - localCancellationCleanupTimer = undefined - localCancellationCleanupAt = undefined - const cleanupAt = Date.now() - for (const [executionId, expiryAt] of localCancelledExecutions) { - if (expiryAt <= cleanupAt) localCancelledExecutions.delete(executionId) - } - scheduleLocalCancellationCleanup() - }, delayMs) - localCancellationCleanupTimer.unref?.() -} - /** Writes the durable key first, then publishes — so a late subscriber still sees the flag on backstop check. */ export async function markExecutionCancelled( executionId: string, @@ -122,31 +78,6 @@ export async function markExecutionCancelled( ): Promise { const redis = getRedisClient() const expiryAt = getCancellationExpiryAt(options) - if (!redis) { - if (!localCancelledExecutions.has(executionId)) { - const oldestExecutionId = localCancelledExecutions.keys().next().value - if ( - localCancelledExecutions.size >= MAX_LOCAL_CANCELLATION_RECORDS && - oldestExecutionId !== undefined - ) { - localCancelledExecutions.delete(oldestExecutionId) - logger.warn('Evicted oldest process-local cancellation record at capacity', { - maxRecords: MAX_LOCAL_CANCELLATION_RECORDS, - }) - } - } else { - localCancelledExecutions.delete(executionId) - } - localCancelledExecutions.set(executionId, expiryAt) - scheduleLocalCancellationCleanup(expiryAt) - try { - publishLocalExecutionSignal(executionId, 'cancelled') - } catch (error) { - logger.warn('Failed to publish process-local cancellation signal', { executionId, error }) - } - return { durablyRecorded: false, reason: 'redis_unavailable' } - } - try { await redis.eval( MARK_EXECUTION_CANCELLED_SCRIPT, @@ -171,18 +102,6 @@ export async function markExecutionCancelled( async function readExecutionCancelledStrict(executionId: string): Promise { const redis = getRedisClient() - if (!redis) { - const expiryAt = localCancelledExecutions.get(executionId) - if (expiryAt === undefined) return false - if (expiryAt <= Date.now()) { - localCancelledExecutions.delete(executionId) - if (expiryAt <= (localCancellationCleanupAt ?? Number.POSITIVE_INFINITY)) { - scheduleLocalCancellationCleanup() - } - return false - } - return true - } return (await redis.exists(`${EXECUTION_CANCEL_PREFIX}${executionId}`)) === 1 } @@ -249,17 +168,6 @@ export async function subscribeToExecutionCancellation( export async function clearExecutionCancellation(executionId: string): Promise { const redis = getRedisClient() - if (!redis) { - const expiryAt = localCancelledExecutions.get(executionId) - localCancelledExecutions.delete(executionId) - if ( - expiryAt !== undefined && - expiryAt <= (localCancellationCleanupAt ?? Number.POSITIVE_INFINITY) - ) { - scheduleLocalCancellationCleanup() - } - return - } try { await redis.del(`${EXECUTION_CANCEL_PREFIX}${executionId}`) diff --git a/apps/sim/lib/execution/event-buffer.integration.ts b/apps/sim/lib/execution/event-buffer.integration.ts new file mode 100644 index 00000000000..c2ade7b7a02 --- /dev/null +++ b/apps/sim/lib/execution/event-buffer.integration.ts @@ -0,0 +1,60 @@ +/** + * The execution stream's active-block snapshot script against a real Redis. A block callback can + * land after the run has ended, and a reconnecting client renders whatever the snapshot holds. + * Failure modes: + * - a late block start overwrites the snapshot of a run that is already terminal + * - a block start on a live run is not recorded + * Each test uses a fresh execution id; the integration setup flushes Redis per file. + */ +import { generateId } from '@sim/utils/id' +import { afterAll, describe, expect, it } from 'vitest' +import { closeRedisConnection } from '@/lib/core/config/redis' +import { + type ActiveBlockStartSnapshot, + initializeExecutionStreamMeta, + markExecutionStreamTerminal, + readExecutionMetaState, + setExecutionActiveBlockStarts, +} from '@/lib/execution/event-buffer' + +const lateBlock: ActiveBlockStartSnapshot = { + eventId: 9, + data: { + blockId: 'function-late', + blockName: 'Late', + blockType: 'function', + executionOrder: 3, + }, +} + +afterAll(async () => { + await closeRedisConnection() +}) + +describe('execution stream active-block snapshot', () => { + it('does not replace a terminal snapshot with a late block start', async () => { + const executionId = `exec-${generateId()}` + await expect(markExecutionStreamTerminal(executionId, 'cancelled')).resolves.toBe(true) + + await expect(setExecutionActiveBlockStarts(executionId, [lateBlock])).resolves.toBe(true) + + await expect(readExecutionMetaState(executionId)).resolves.toMatchObject({ + status: 'found', + meta: { status: 'cancelled', activeBlockStarts: [] }, + }) + }) + + it('records block starts while the run is active', async () => { + const executionId = `exec-${generateId()}` + await expect( + initializeExecutionStreamMeta(executionId, { workflowId: 'wf-1', userId: 'user-1' }) + ).resolves.toBe(true) + + await expect(setExecutionActiveBlockStarts(executionId, [lateBlock])).resolves.toBe(true) + + await expect(readExecutionMetaState(executionId)).resolves.toMatchObject({ + status: 'found', + meta: { status: 'active', activeBlockStarts: [lateBlock] }, + }) + }) +}) diff --git a/apps/sim/lib/execution/event-buffer.ts b/apps/sim/lib/execution/event-buffer.ts index 2bb06bf238a..eb54269cfd6 100644 --- a/apps/sim/lib/execution/event-buffer.ts +++ b/apps/sim/lib/execution/event-buffer.ts @@ -11,7 +11,7 @@ import { ExecutionResourceLimitError, isExecutionResourceLimitError, } from '@/lib/execution/resource-errors' -import { getExecutionSignalChannel, publishLocalExecutionSignal } from '@/lib/execution/signal' +import { getExecutionSignalChannel } from '@/lib/execution/signal' import type { BlockStartedData, ExecutionEvent } from '@/lib/workflows/executor/execution-events' const logger = createLogger('ExecutionEventBuffer') @@ -301,13 +301,6 @@ export interface ExecutionEventEntry { event: ExecutionEvent } -interface MemoryExecutionStream { - events: ExecutionEventEntry[] - meta: ExecutionStreamMeta | null - nextEventId: number - expiresAt: number -} - export interface ExecutionEventWriter { write: (event: ExecutionEvent) => Promise writeTerminal: ( @@ -385,114 +378,6 @@ async function compactEventForBuffer( return { ...event, data: eventData } as ExecutionEvent } -const memoryExecutionStreams = new Map() - -/** - * Redis is required, so the in-memory buffer is never selected; its branches are - * unreachable behind `getRedisClient()`, which throws instead of returning null. - */ -function canUseMemoryEventBuffer(): boolean { - return false -} - -function pruneExpiredMemoryStreams(now = Date.now()): void { - for (const [executionId, stream] of memoryExecutionStreams) { - if (stream.expiresAt <= now) memoryExecutionStreams.delete(executionId) - } -} - -function getMemoryStream(executionId: string): MemoryExecutionStream { - pruneExpiredMemoryStreams() - let stream = memoryExecutionStreams.get(executionId) - if (!stream) { - stream = { - events: [], - meta: null, - nextEventId: 1, - expiresAt: Date.now() + ACTIVE_TTL_SECONDS * 1000, - } - memoryExecutionStreams.set(executionId, stream) - } - return stream -} - -function touchMemoryStream(stream: MemoryExecutionStream): void { - const terminal = stream.meta?.status && stream.meta.status !== 'active' - stream.expiresAt = Date.now() + (terminal ? TERMINAL_TTL_SECONDS : ACTIVE_TTL_SECONDS) * 1000 -} - -function readMemoryMeta(executionId: string): ExecutionMetaReadResult { - pruneExpiredMemoryStreams() - const stream = memoryExecutionStreams.get(executionId) - if (!stream?.meta) return { status: 'missing' } - return { status: 'found', meta: stream.meta } -} - -function readMemoryEvents(executionId: string, afterEventId: number): ExecutionEventsReadResult { - pruneExpiredMemoryStreams() - const stream = memoryExecutionStreams.get(executionId) - if (!stream) return { status: 'ok', events: [] } - const earliestEventId = stream.meta?.earliestEventId - if ( - isReplayBeforeAvailableEvents(afterEventId, earliestEventId, stream.meta?.replayStartEventId) - ) { - return { status: 'pruned', earliestEventId } - } - return { - status: 'ok', - events: stream.events.filter((entry) => entry.eventId > afterEventId), - } -} - -function createMemoryExecutionEventWriter( - executionId: string, - context: ExecutionEventWriterContext = {} -): ExecutionEventWriter { - const writeMemoryEvent = async (event: ExecutionEvent, publish: boolean) => { - const stream = getMemoryStream(executionId) - const compactEvent = await compactEventForBuffer(event, context) - const entry = { - eventId: stream.nextEventId++, - executionId, - event: compactEvent, - } - stream.events.push(entry) - if (stream.events.length > EVENT_LIMIT) { - stream.events = stream.events.slice(-EVENT_LIMIT) - const earliestEventId = stream.events[0]?.eventId - if (earliestEventId !== undefined && stream.meta) { - stream.meta = { - ...stream.meta, - earliestEventId, - updatedAt: new Date().toISOString(), - } - } - } - touchMemoryStream(stream) - if (publish) publishLocalExecutionSignal(executionId, 'event') - return entry - } - - return { - write: (event) => writeMemoryEvent(event, true), - writeTerminal: async (event, status) => { - const entry = await writeMemoryEvent(event, false) - const stream = getMemoryStream(executionId) - stream.meta = { - ...stream.meta, - status, - updatedAt: new Date().toISOString(), - activeBlockStarts: [], - } - touchMemoryStream(stream) - publishLocalExecutionSignal(executionId, 'event') - return entry - }, - flush: async () => {}, - close: async () => {}, - } -} - function isReplayBeforeAvailableEvents( afterEventId: number, earliestEventId?: number, @@ -534,24 +419,6 @@ export async function flushExecutionStreamReplayBuffer( export async function resetExecutionStreamBuffer(executionId: string): Promise { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryEventBuffer()) { - const stream = getMemoryStream(executionId) - stream.events = [] - stream.meta = { - status: 'active', - protocolVersion: EXECUTION_STREAM_PROTOCOL_VERSION, - replayStartEventId: stream.nextEventId, - updatedAt: new Date().toISOString(), - activeBlockStarts: [], - } - touchMemoryStream(stream) - return true - } - logger.warn('resetExecutionStreamBuffer: Redis client unavailable', { executionId }) - return false - } - try { const currentSequence = Number(await redis.get(getSeqKey(executionId)).catch(() => 0)) const replayStartEventId = Number.isFinite(currentSequence) ? currentSequence + 1 : 1 @@ -585,32 +452,6 @@ export async function setExecutionMeta( meta: Partial ): Promise { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryEventBuffer()) { - const stream = getMemoryStream(executionId) - const status = meta.status ?? stream.meta?.status - if (!status) return false - if (meta.activeBlockStarts !== undefined) { - const activeBlockStarts = JSON.stringify(meta.activeBlockStarts) - if ( - meta.activeBlockStarts.length > EVENT_LIMIT || - Buffer.byteLength(activeBlockStarts, 'utf8') > MAX_ACTIVE_BLOCK_SNAPSHOT_BYTES - ) { - return false - } - } - stream.meta = { - ...stream.meta, - ...meta, - status, - updatedAt: new Date().toISOString(), - } - touchMemoryStream(stream) - return true - } - logger.warn('setExecutionMeta: Redis client unavailable', { executionId }) - return false - } try { const key = getMetaKey(executionId) const payload: Record = { @@ -675,24 +516,6 @@ export async function setExecutionActiveBlockStarts( } const redis = getRedisClient() - if (!redis) { - if (canUseMemoryEventBuffer()) { - const stream = getMemoryStream(executionId) - if (!stream.meta?.status) return false - if (stream.meta.status !== 'active') return true - stream.meta = { - ...stream.meta, - activeBlockStarts, - updatedAt: new Date().toISOString(), - } - touchMemoryStream(stream) - publishLocalExecutionSignal(executionId, 'event') - return true - } - logger.warn('setExecutionActiveBlockStarts: Redis client unavailable', { executionId }) - return false - } - try { await redis.eval( SET_ACTIVE_BLOCK_STARTS_SCRIPT, @@ -738,23 +561,6 @@ export async function markExecutionStreamTerminal( status: TerminalExecutionStreamStatus ): Promise { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryEventBuffer()) { - const stream = getMemoryStream(executionId) - stream.meta = { - ...stream.meta, - status, - protocolVersion: EXECUTION_STREAM_PROTOCOL_VERSION, - updatedAt: new Date().toISOString(), - activeBlockStarts: [], - } - touchMemoryStream(stream) - publishLocalExecutionSignal(executionId, 'event') - return true - } - logger.warn('markExecutionStreamTerminal: Redis client unavailable', { executionId }) - return false - } try { await redis.eval( MARK_TERMINAL_META_SCRIPT, @@ -780,11 +586,6 @@ export async function readExecutionMetaState( executionId: string ): Promise { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryEventBuffer()) return readMemoryMeta(executionId) - logger.warn('readExecutionMetaState: Redis client unavailable', { executionId }) - return { status: 'unavailable', error: 'Redis client unavailable' } - } try { const key = getMetaKey(executionId) const meta = await redis.hgetall(key) @@ -838,10 +639,6 @@ export async function readExecutionEventsState( afterEventId: number ): Promise { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryEventBuffer()) return readMemoryEvents(executionId, afterEventId) - return { status: 'unavailable', error: 'Redis client unavailable' } - } try { const meta = await redis.hgetall(getMetaKey(executionId)) const earliestEventId = @@ -893,14 +690,6 @@ export function createExecutionEventWriter( context: ExecutionEventWriterContext = {} ): ExecutionEventWriter { const redis = getRedisClient() - if (!redis) { - if (canUseMemoryEventBuffer()) { - logger.info('createExecutionEventWriter: using in-memory event buffer', { executionId }) - return createMemoryExecutionEventWriter(executionId, context) - } - throw new Error(`Redis execution event buffer unavailable for ${executionId}`) - } - let pending: ExecutionEventEntry[] = [] let nextEventId = 0 let maxReservedId = 0 diff --git a/apps/sim/lib/execution/isolated-vm.test.ts b/apps/sim/lib/execution/isolated-vm.test.ts index cdac6b125d6..4ffcaa97e44 100644 --- a/apps/sim/lib/execution/isolated-vm.test.ts +++ b/apps/sim/lib/execution/isolated-vm.test.ts @@ -1,5 +1,6 @@ import { EventEmitter } from 'node:events' import { + createMockRedis, getMockLogger, inputValidationMock, inputValidationMockFns, @@ -246,14 +247,10 @@ async function loadExecutionModule(options: { ...(options.envOverrides ?? {}), }) - const redisEval = options.redisEvalImpl ? vi.fn(options.redisEvalImpl) : undefined - mockGetRedisClient.mockImplementation(() => - redisEval - ? ({ - eval: redisEval, - } as unknown) - : null - ) + const redis = createMockRedis() + if (options.redisEvalImpl) redis.eval.mockImplementation(options.redisEvalImpl) + else redis.eval.mockResolvedValue(1) + mockGetRedisClient.mockReturnValue(redis) vi.resetModules() @@ -683,28 +680,6 @@ describe('isolated-vm scheduler', () => { expect(result.result).toBeNull() }) - it('falls back to local limits when no Redis client is available', async () => { - const { executeInIsolatedVM } = await loadExecutionModule({ - envOverrides: { - REDIS_URL: 'redis://localhost:6379', - }, - spawns: [() => createReadyProc('ok')], - }) - - const result = await executeInIsolatedVM({ - code: 'return "ok"', - params: {}, - envVars: {}, - contextVariables: {}, - timeoutMs: 100, - requestId: 'req-7', - ownerKey: 'user:redis-down', - }) - - expect(result.error).toBeUndefined() - expect(result.result).toBe('ok') - }) - it('falls back to local limits when the lease evaluation errors', async () => { const { executeInIsolatedVM } = await loadExecutionModule({ envOverrides: { diff --git a/apps/sim/lib/execution/isolated-vm.ts b/apps/sim/lib/execution/isolated-vm.ts index b05e9d9320b..8c7a1f7025d 100644 --- a/apps/sim/lib/execution/isolated-vm.ts +++ b/apps/sim/lib/execution/isolated-vm.ts @@ -459,13 +459,6 @@ async function tryAcquireDistributedLease( timeoutMs: number ): Promise { const redis = getRedisClient() - if (!redis) { - logger.warn('No Redis client for distributed lease acquisition; using local limits', { - ownerKey, - }) - return 'undetermined' - } - const now = Date.now() const leaseTtlMs = Math.max( timeoutMs + QUEUE_TIMEOUT_MS + DISTRIBUTED_LEASE_GRACE_MS, @@ -522,8 +515,6 @@ async function tryAcquireDistributedLease( async function releaseDistributedLease(ownerKey: string, leaseId: string): Promise { const redis = getRedisClient() - if (!redis) return - const key = ownerRedisKey(ownerKey) const script = ` redis.call('ZREM', KEYS[1], ARGV[1]) diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts index ca71441e108..4fd00c7d33c 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts @@ -33,7 +33,7 @@ function fixture() { return { session, machine, key } } afterAll(async () => { - if (keys.length) await getRedisClient()?.del(...keys) + if (keys.length) await getRedisClient().del(...keys) await closeRedisConnection() }) diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts index e55f10fd094..2223af5accc 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.test.ts @@ -1,3 +1,4 @@ +import { createMockRedis } from '@sim/testing/mocks/redis.mock' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { beforeEach, describe, expect, it } from 'vitest' import { @@ -7,18 +8,26 @@ import { } from '@/lib/execution/remote-sandbox/session-file-provenance' const machine = { providerId: 'e2b', sandboxId: 'machine' } as const -beforeEach(() => redisConfigMockFns.mockGetRedisClient.mockReturnValue(null)) -describe('unavailable workbench evidence storage', () => { - it('refuses allocation, input receipt and output classification when Redis is unavailable', async () => { +beforeEach(() => { + const redis = createMockRedis() + const failure = new Error('Connection is closed.') + redis.set.mockRejectedValue(failure) + redis.eval.mockRejectedValue(failure) + redis.get.mockRejectedValue(failure) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) +}) + +describe('failing workbench evidence storage', () => { + it('refuses allocation, input receipt and output classification when Redis fails', async () => { await expect(initializeSessionFileProvenance('chat', machine)).rejects.toThrow( - 'storage is unavailable' + 'Connection is closed.' ) await expect(recordSessionFileInput('chat', machine, true)).rejects.toThrow( - 'storage is unavailable' + 'Connection is closed.' ) await expect(readSessionSecretProvenance('chat', machine)).rejects.toThrow( - 'storage is unavailable' + 'Connection is closed.' ) }) }) diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts index 3234861477e..b81c07cc0b5 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.ts @@ -76,18 +76,13 @@ function key(sessionKey: string, machine: SessionFileIdentity) { .digest('hex') return `mothership:workbench-provenance:v2:${digest}` } -function redis() { - const client = getRedisClient() - if (!client) throw new Error('Workbench provenance storage is unavailable') - return client -} /** Only a newly created physical machine starts with no prior input exposure. */ export async function initializeSessionFileProvenance( sessionKey: string, machine: SessionFileIdentity ) { - await redis().set(key(sessionKey, machine), EMPTY, 'EX', TTL_SECONDS, 'NX') + await getRedisClient().set(key(sessionKey, machine), EMPTY, 'EX', TTL_SECONDS, 'NX') } /** @@ -115,7 +110,7 @@ export async function recordSessionFileInput( normalized.map((entry) => omit(entry, ['sourceValueHash'])) ) const encoded = entries ? JSON.stringify({ status: 'exact', entries }) : UNKNOWN - await redis().eval( + await getRedisClient().eval( RECORD_INPUT, 1, key(sessionKey, machine), @@ -133,7 +128,7 @@ export async function readSessionSecretProvenance( sessionKey: string, machine: SessionFileIdentity ): Promise { - const value = await redis().get(key(sessionKey, machine)) + const value = await getRedisClient().get(key(sessionKey, machine)) if (!value || Buffer.byteLength(value, 'utf8') > PROVENANCE_MAX_SERIALIZED_BYTES) return { status: 'unknown' } try { diff --git a/apps/sim/lib/execution/removal-stop-records.ts b/apps/sim/lib/execution/removal-stop-records.ts index aa96ac0dfea..e5346933f87 100644 --- a/apps/sim/lib/execution/removal-stop-records.ts +++ b/apps/sim/lib/execution/removal-stop-records.ts @@ -214,9 +214,6 @@ export async function recordExecutionStops( * worker. The deadline (`signal`, else {@link SIGNAL_DEADLINE_MS}) bounds only the Redis writes; * every execution this process runs is aborted first. Returns how many flags Redis refused or * the deadline left unsent; never throws. - * - * Without Redis at all the flag is process-local by design (`markExecutionCancelled`) and a - * retry from any process could reach no further, so that outcome does not count as refused. */ export async function signalExecutionStops( executions: readonly Pick[], @@ -243,7 +240,7 @@ export async function signalExecutionStops( const result = await markExecutionCancelled(execution.executionId, { executionDeadlineAt: execution.executionDeadlineAt, }) - if (!result.durablyRecorded && result.reason !== 'redis_unavailable') refused += 1 + if (!result.durablyRecorded) refused += 1 } catch (error) { refused += 1 logger.warn('Failed to signal an execution being stopped', { diff --git a/apps/sim/lib/execution/removal-stops.integration.ts b/apps/sim/lib/execution/removal-stops.integration.ts index c4a7e4e57ac..9874e8d7824 100644 --- a/apps/sim/lib/execution/removal-stops.integration.ts +++ b/apps/sim/lib/execution/removal-stops.integration.ts @@ -175,7 +175,7 @@ async function seedQueuedJob(workspaceId: string, type = 'webhook-execution') { } async function isCancelled(executionId: string) { - return (await getRedisClient()?.exists(`execution:cancel:${executionId}`)) === 1 + return (await getRedisClient().exists(`execution:cancel:${executionId}`)) === 1 } async function jobStatus(jobId: string) { @@ -209,9 +209,8 @@ afterAll(async () => { describe.runIf(Boolean(redisUrl))('removals stop the workflow runs they orphan', () => { afterAll(async () => { - const redis = getRedisClient() - if (redis && executionIds.length > 0) { - await redis.del(...executionIds.map((id) => `execution:cancel:${id}`)) + if (executionIds.length > 0) { + await getRedisClient().del(...executionIds.map((id) => `execution:cancel:${id}`)) } const scoped = [...executionIds, ...workspaceIds] for (const id of scoped) { diff --git a/apps/sim/lib/execution/removal-stops.test.ts b/apps/sim/lib/execution/removal-stops.test.ts index f33e7dff222..93f9f536bab 100644 --- a/apps/sim/lib/execution/removal-stops.test.ts +++ b/apps/sim/lib/execution/removal-stops.test.ts @@ -2,13 +2,13 @@ * The outbox outcome of a removal stop decides whether a run keeps going: an outcome that * completes too early drops the stop, one that never completes dead-letters it. Failure modes: * - Redis refuses the cancellation write → retry, and the queue sweep still runs - * - no Redis at all → the process-local flag is all there is, so the stop completes * - a queue sweep fails → retry * - one queue fails to load → the other is still swept, then retry * - a sweep stops at its cap having cancelled runs → continue without spending an attempt * - a sweep stops at its cap having cancelled nothing → retry, so it cannot loop forever */ import { asyncJobsMock, asyncJobsMockFns } from '@sim/testing/mocks/async-jobs.mock' +import { createMockRedis } from '@sim/testing/mocks/redis.mock' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -39,17 +39,13 @@ const payload = { describe('execution removal stop outcome', () => { beforeEach(() => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(createMockRedis()) mockJobQueue.cancelByWorkspace.mockReset().mockResolvedValue({ cancelledJobs: 0, complete: true, }) }) - it('completes when there is no Redis to record the cancellation in', async () => { - await expect(stopRemovedExecutions(payload, context())).resolves.toBeUndefined() - }) - it('retries when Redis refuses the cancellation, after still sweeping the queue', async () => { redisConfigMockFns.mockGetRedisClient.mockReturnValue({ eval: vi.fn().mockRejectedValue(new Error('READONLY')), diff --git a/apps/sim/lib/execution/signal.test.ts b/apps/sim/lib/execution/signal.test.ts index c9fbd7006fc..a14265fb133 100644 --- a/apps/sim/lib/execution/signal.test.ts +++ b/apps/sim/lib/execution/signal.test.ts @@ -9,7 +9,7 @@ const { connection, mockRedisUrl, mockSubscribe, mockUnsubscribe } = vi.hoisted( options: undefined as Record | undefined, }, mockRedisUrl: { - value: 'redis://localhost:6379' as string | undefined, + value: 'redis://localhost:6379', error: undefined as Error | undefined, }, mockSubscribe: vi.fn(), @@ -34,11 +34,7 @@ vi.mock('ioredis', () => ({ })) import { coldConnectionBudgetMs } from '@/lib/core/config/redis-budget' -import { - connectExecutionSignalHub, - getExecutionSignalHub, - publishLocalExecutionSignal, -} from '@/lib/execution/signal' +import { connectExecutionSignalHub, getExecutionSignalHub } from '@/lib/execution/signal' redisConfigMockFns.mockGetConfiguredRedisUrl.mockImplementation(() => { if (mockRedisUrl.error) throw mockRedisUrl.error @@ -484,19 +480,9 @@ describe('ExecutionSignalHub', () => { }) it('does not throw when Redis is misconfigured, leaving that to the first subscriber', () => { - mockRedisUrl.error = new Error('Cache capability selected Redis but REDIS_URL is missing') + mockRedisUrl.error = new Error('REDIS_URL is required') expect(() => connectExecutionSignalHub()).not.toThrow() expect(connection.client).toBeUndefined() }) - - it('uses a process-local signal hub when Redis is not configured', async () => { - mockRedisUrl.value = undefined - const handler = vi.fn() - await getExecutionSignalHub().subscribe('execution-local', handler) - - publishLocalExecutionSignal('execution-local', 'event') - - expect(handler).toHaveBeenCalledWith('event') - }) }) diff --git a/apps/sim/lib/execution/signal.ts b/apps/sim/lib/execution/signal.ts index 65554f0f16c..c4328ee23b8 100644 --- a/apps/sim/lib/execution/signal.ts +++ b/apps/sim/lib/execution/signal.ts @@ -273,41 +273,6 @@ class RedisExecutionSignalHub implements ExecutionSignalHub { } } -class LocalExecutionSignalHub implements ExecutionSignalHub { - private readonly handlers = new Map>() - - async subscribe(executionId: string, handler: ExecutionSignalHandler): Promise<() => void> { - const channel = getExecutionSignalChannel(executionId) - let channelHandlers = this.handlers.get(channel) - if (!channelHandlers) { - channelHandlers = new Set() - this.handlers.set(channel, channelHandlers) - } - channelHandlers.add(handler) - return () => { - const current = this.handlers.get(channel) - if (!current) return - current.delete(handler) - if (current.size === 0) this.handlers.delete(channel) - } - } - - publish( - executionId: string, - reason: Extract - ): void { - const channelHandlers = this.handlers.get(getExecutionSignalChannel(executionId)) - if (!channelHandlers) return - for (const handler of channelHandlers) { - try { - handler(reason) - } catch (error) { - logger.error('Local execution signal handler failed', { error: toError(error).message }) - } - } - } -} - type ExecutionSignalGlobal = typeof globalThis & { _executionSignalHub?: ExecutionSignalHub } @@ -315,13 +280,7 @@ type ExecutionSignalGlobal = typeof globalThis & { const executionSignalGlobal = globalThis as ExecutionSignalGlobal export function getExecutionSignalHub(): ExecutionSignalHub { - if (executionSignalGlobal._executionSignalHub) return executionSignalGlobal._executionSignalHub - const redisUrl = getConfiguredRedisUrl() - if (!redisUrl) { - executionSignalGlobal._executionSignalHub = new LocalExecutionSignalHub() - return executionSignalGlobal._executionSignalHub - } - executionSignalGlobal._executionSignalHub = new RedisExecutionSignalHub(redisUrl) + executionSignalGlobal._executionSignalHub ??= new RedisExecutionSignalHub(getConfiguredRedisUrl()) return executionSignalGlobal._executionSignalHub } @@ -342,14 +301,3 @@ export function connectExecutionSignalHub(): void { return } } - -export function publishLocalExecutionSignal( - executionId: string, - reason: Extract -): void { - const hub = getExecutionSignalHub() - if (!(hub instanceof LocalExecutionSignalHub)) { - throw new Error('Local execution signals are unavailable while Redis is configured') - } - hub.publish(executionId, reason) -} diff --git a/apps/sim/lib/favicon/resolve.server.ts b/apps/sim/lib/favicon/resolve.server.ts index 0fdc920d2b7..441a5dd6d8f 100644 --- a/apps/sim/lib/favicon/resolve.server.ts +++ b/apps/sim/lib/favicon/resolve.server.ts @@ -45,7 +45,6 @@ const cacheKey = (host: string) => `favicon:v3:${host}` /** A host's key holds the result of the walk that starts at that host. */ async function readCache(host: string): Promise { const redis = getRedisClient() - if (!redis) return null try { const cached = await redis.getBuffer(cacheKey(host)) if (cached === null) return null @@ -67,7 +66,6 @@ async function readCache(host: string): Promise { async function writeCache(host: string, resolution: FaviconResolution): Promise { const redis = getRedisClient() - if (!redis) return const [value, ttl] = resolution.kind === 'hit' ? resolution.provisional diff --git a/apps/sim/lib/knowledge/__integration__/workspace-lifecycle.integration.ts b/apps/sim/lib/knowledge/__integration__/workspace-lifecycle.integration.ts index 73b9e563dc7..40220327f3e 100644 --- a/apps/sim/lib/knowledge/__integration__/workspace-lifecycle.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/workspace-lifecycle.integration.ts @@ -9,11 +9,12 @@ import { workspace, } from '@sim/db/schema' import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { mcpPubsubMock } from '@sim/testing/mocks/mcp-pubsub.mock' import { generateId } from '@sim/utils/id' import { eq, inArray } from 'drizzle-orm' import { afterEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/mcp/pubsub', () => ({ mcpPubSub: null })) +vi.mock('@/lib/mcp/pubsub', () => mcpPubsubMock) vi.mock('@/lib/mcp/service', () => ({ mcpService: { clearCache: vi.fn().mockResolvedValue(undefined) }, })) diff --git a/apps/sim/lib/knowledge/github-setup-state.ts b/apps/sim/lib/knowledge/github-setup-state.ts index deeff458dbd..198f4ad5d00 100644 --- a/apps/sim/lib/knowledge/github-setup-state.ts +++ b/apps/sim/lib/knowledge/github-setup-state.ts @@ -63,12 +63,6 @@ const attemptSchema = z.discriminatedUnion('phase', [ export type GitHubSetupAttempt = z.infer export type GitHubSetupPhase = GitHubSetupAttempt['phase'] -function redis() { - const client = getRedisClient() - if (!client) throw new Error('GitHub setup requires Redis') - return client -} - function attemptKey(scope: GitHubSetupScope) { return `github-search:setup:${sha256Hex(JSON.stringify([scope.organizationId, scope.userId, scope.setupId]))}` } @@ -117,8 +111,8 @@ export async function saveGitHubSetupAttempt( const ttl = remainingTtl(attempt.createdAt) const saved = expected === 'new' - ? (await redis().set(key, value, 'PX', ttl, 'NX')) === 'OK' - : (await redis().eval(TRANSITION, 1, key, value, expected, ttl)) === 1 + ? (await getRedisClient().set(key, value, 'PX', ttl, 'NX')) === 'OK' + : (await getRedisClient().eval(TRANSITION, 1, key, value, expected, ttl)) === 1 if (!saved) throw new OrchestrationError( 'conflict', @@ -129,7 +123,7 @@ export async function saveGitHubSetupAttempt( export async function readGitHubSetupAttempt( scope: GitHubSetupScope ): Promise { - const raw = await redis().get(attemptKey(scope)) + const raw = await getRedisClient().get(attemptKey(scope)) if (!raw) return null const attempt = attemptSchema.parse(JSON.parse(raw)) if ( @@ -147,7 +141,7 @@ export async function readGitHubSetupAttempt( /** Only a random, server-issued nonce is sent to GitHub; the owner remains in Redis. */ export async function issueGitHubSetupCallback(attempt: GitHubSetupAttempt) { const state = generateId() - const stored = await redis().set( + const stored = await getRedisClient().set( callbackKey(state), JSON.stringify(githubSetupScope(attempt)), 'PX', @@ -160,7 +154,7 @@ export async function issueGitHubSetupCallback(attempt: GitHubSetupAttempt) { /** A different browser session cannot consume or discover an attempt through its callback. */ export async function resolveGitHubSetupCallback(state: string, principal: SessionPrincipal) { - const raw = await redis().get(callbackKey(state)) + const raw = await getRedisClient().get(callbackKey(state)) if (!raw) throw expiredGitHubSetupError() const scope = scopeSchema.parse(JSON.parse(raw)) if (scope.userId !== principal.userId || scope.sessionId !== principal.sessionId) diff --git a/apps/sim/lib/logs/execution/progress-markers.ts b/apps/sim/lib/logs/execution/progress-markers.ts index 32dcb11547e..0855fec1786 100644 --- a/apps/sim/lib/logs/execution/progress-markers.ts +++ b/apps/sim/lib/logs/execution/progress-markers.ts @@ -63,7 +63,7 @@ return 1 * terminal/pause boundary (deterministic cleanup is {@link clearProgressMarkers}); * it mirrors the admission reservation so a crashed run's marker key and slot * expire together. Returns `true` only when the marker was durably written to Redis, - * so callers can fall back to the SQL path on a missing client or a failure. + * so callers can fall back to the SQL path when the write fails. */ async function setMarker( executionId: string, @@ -74,8 +74,6 @@ async function setMarker( expiresAt?: number ): Promise { const redis = getMarkerClient() - if (!redis) return false - try { await redis.eval( SET_MARKER_SCRIPT, @@ -201,9 +199,8 @@ function parseCompletedMarker(raw: string | undefined): ExecutionLastCompletedBl } /** - * Read both markers for an execution. Returns an empty object when Redis is - * unavailable (markers were never stored here) or the key holds nothing, and - * `null` when the Redis read itself failed — callers must treat `null` as + * Read both markers for an execution. Returns an empty object when the key + * holds nothing, and `null` when the Redis read itself failed — callers must treat `null` as * "unknown" and skip {@link clearProgressMarkers}, so a transient read error * never wipes the only copy of markers that are still in Redis. */ @@ -211,8 +208,6 @@ export async function getProgressMarkers( executionId: string ): Promise { const redis = getMarkerClient() - if (!redis) return {} - try { const fields = await redis.hgetall(markerKey(executionId)) if (!fields || Object.keys(fields).length === 0) return {} @@ -234,12 +229,10 @@ export async function getProgressMarkers( /** * Delete the markers for an execution. Called at every terminal/pause boundary * after the durable record has been written, so paused executions (which can - * live indefinitely) hold no Redis keys. Fire-and-forget; no-op without Redis. + * live indefinitely) hold no Redis keys. Fire-and-forget: a failed delete is logged. */ export async function clearProgressMarkers(executionId: string): Promise { const redis = getMarkerClient() - if (!redis) return - try { await redis.del(markerKey(executionId)) } catch (error) { diff --git a/apps/sim/lib/mcp/application/workflow-deployments.ts b/apps/sim/lib/mcp/application/workflow-deployments.ts index d0125df9b8d..60b89aae6b3 100644 --- a/apps/sim/lib/mcp/application/workflow-deployments.ts +++ b/apps/sim/lib/mcp/application/workflow-deployments.ts @@ -239,7 +239,7 @@ export const createWorkflowMcpDeploymentServer = defineAuthorizedWorkspaceUseCas }), afterSuccess: ({ context, result }) => result.addedTools.length > 0 - ? mcpPubSub?.publishWorkflowToolsChanged({ + ? mcpPubSub.publishWorkflowToolsChanged({ serverId: result.server.id, workspaceId: context.workspaceId, }) @@ -318,7 +318,7 @@ export const deleteWorkflowMcpDeploymentServer = defineAuthorizedWorkspaceUseCas metadata: { serverName: result.server.name }, }), afterSuccess: ({ context, result }) => - mcpPubSub?.publishWorkflowToolsChanged({ + mcpPubSub.publishWorkflowToolsChanged({ serverId: result.server.id, workspaceId: context.workspaceId, }), @@ -417,7 +417,7 @@ export const deployWorkflowMcpTool = defineAuthorizedWorkspaceUseCase({ }, }), afterSuccess: ({ context, result }) => - mcpPubSub?.publishWorkflowToolsChanged({ + mcpPubSub.publishWorkflowToolsChanged({ serverId: result.server.id, workspaceId: context.workspaceId, }), @@ -464,7 +464,7 @@ export const undeployWorkflowMcpTool = defineAuthorizedWorkspaceUseCase({ }, }), afterSuccess: ({ context, result }) => - mcpPubSub?.publishWorkflowToolsChanged({ + mcpPubSub.publishWorkflowToolsChanged({ serverId: result.server.id, workspaceId: context.workspaceId, }), diff --git a/apps/sim/lib/mcp/connection-manager.ts b/apps/sim/lib/mcp/connection-manager.ts index a47d10f3967..8f420aae107 100644 --- a/apps/sim/lib/mcp/connection-manager.ts +++ b/apps/sim/lib/mcp/connection-manager.ts @@ -12,6 +12,7 @@ import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.ser */ import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' import { backoffWithJitter } from '@sim/utils/retry' import { isTest } from '@/lib/core/config/env-flags' import { McpClient } from '@/lib/mcp/client' @@ -82,25 +83,37 @@ export class McpConnectionManager { private disposed = false private unsubscribePubSub?: () => void - constructor() { - if (mcpPubSub) { - this.unsubscribePubSub = mcpPubSub.onToolsChanged((event) => { - this.notifyLocalListeners(event) - }) - } - } - /** * Subscribe to tools-changed events from any managed connection. * Returns an unsubscribe function. */ subscribe(listener: ToolsChangedListener): () => void { + this.joinCrossProcessEvents() this.listeners.add(listener) return () => { this.listeners.delete(listener) } } + /** + * Joins the cross-process tools-changed channel on the first subscriber rather than at + * construction: the manager is built at import, which `next build` runs without REDIS_URL. + * A subscriber that arrives before Redis is reachable is still served locally, and the + * next subscriber tries again. + */ + private joinCrossProcessEvents(): void { + if (this.disposed || this.unsubscribePubSub) return + try { + this.unsubscribePubSub = mcpPubSub.onToolsChanged((event) => { + this.notifyLocalListeners(event) + }) + } catch (error) { + logger.warn('Could not join the MCP tools-changed channel; retrying on the next subscriber', { + error: toError(error).message, + }) + } + } + /** * Establish a persistent connection to an MCP server. * If the server supports `listChanged`, the connection is kept alive @@ -368,7 +381,7 @@ export class McpConnectionManager { logger.info('Tools changed — publishing to pub/sub', { serverName: state.serverName }) - mcpPubSub?.publishToolsChanged(event) + mcpPubSub.publishToolsChanged(event) } private handleDisconnect(config: McpServerConfig, userId: string, workspaceId: string): void { diff --git a/apps/sim/lib/mcp/orchestration/workflow-mcp-lifecycle.ts b/apps/sim/lib/mcp/orchestration/workflow-mcp-lifecycle.ts index 74c189035b2..7a6ff392b1d 100644 --- a/apps/sim/lib/mcp/orchestration/workflow-mcp-lifecycle.ts +++ b/apps/sim/lib/mcp/orchestration/workflow-mcp-lifecycle.ts @@ -528,7 +528,7 @@ export async function performCreateWorkflowMcpServer( }) if (addedTools.length > 0 && params.publishEffects !== false) { - mcpPubSub?.publishWorkflowToolsChanged({ serverId, workspaceId: params.workspaceId }) + mcpPubSub.publishWorkflowToolsChanged({ serverId, workspaceId: params.workspaceId }) } if (params.projectLegacyAudit !== false) @@ -669,7 +669,7 @@ export async function performDeleteWorkflowMcpServer( } if (params.publishEffects !== false) - mcpPubSub?.publishWorkflowToolsChanged({ + mcpPubSub.publishWorkflowToolsChanged({ serverId: params.serverId, workspaceId: params.workspaceId, }) @@ -894,7 +894,7 @@ export async function performCreateWorkflowMcpTool( } if (params.publishEffects !== false) - mcpPubSub?.publishWorkflowToolsChanged({ + mcpPubSub.publishWorkflowToolsChanged({ serverId: params.serverId, workspaceId: params.workspaceId, }) @@ -1105,7 +1105,7 @@ export async function performUpdateWorkflowMcpTool( if (!tool) return { success: false, error: 'Tool not found', errorCode: 'not_found' } if (params.publishEffects !== false) - mcpPubSub?.publishWorkflowToolsChanged({ + mcpPubSub.publishWorkflowToolsChanged({ serverId: params.serverId, workspaceId: params.workspaceId, }) @@ -1179,7 +1179,7 @@ export async function performDeleteWorkflowMcpTool( if (!tool) return { success: false, error: 'Tool not found', errorCode: 'not_found' } if (params.publishEffects !== false) - mcpPubSub?.publishWorkflowToolsChanged({ + mcpPubSub.publishWorkflowToolsChanged({ serverId: params.serverId, workspaceId: params.workspaceId, }) diff --git a/apps/sim/lib/mcp/pubsub.ts b/apps/sim/lib/mcp/pubsub.ts index 26a8064dcc0..db3d77b298c 100644 --- a/apps/sim/lib/mcp/pubsub.ts +++ b/apps/sim/lib/mcp/pubsub.ts @@ -1,8 +1,8 @@ /** * MCP Pub/Sub Adapter * - * Broadcasts MCP notification events across processes using Redis Pub/Sub. - * Gracefully falls back to process-local EventEmitter when Redis is unavailable. + * Broadcasts MCP notification events across processes using Redis Pub/Sub. The channels + * connect on first use, so importing this module opens nothing. * * Two channels: * - `mcp:tools_changed` — external MCP server sent a listChanged notification @@ -25,48 +25,34 @@ interface McpPubSubAdapter { } type McpPubSubGlobal = typeof globalThis & { - _mcpToolsChannel?: PubSubChannel | null - _mcpWorkflowToolsChannel?: PubSubChannel | null + _mcpToolsChannel?: PubSubChannel + _mcpWorkflowToolsChannel?: PubSubChannel } const g = globalThis as McpPubSubGlobal -if (!('_mcpToolsChannel' in g)) { - g._mcpToolsChannel = - typeof window !== 'undefined' - ? null - : createPubSubChannel({ - channel: 'mcp:tools_changed', - label: 'mcp-tools', - }) -} - -if (!('_mcpWorkflowToolsChannel' in g)) { - g._mcpWorkflowToolsChannel = - typeof window !== 'undefined' - ? null - : createPubSubChannel({ - channel: 'mcp:workflow_tools_changed', - label: 'mcp-workflow-tools', - }) -} +g._mcpToolsChannel ??= createPubSubChannel({ + channel: 'mcp:tools_changed', + label: 'mcp-tools', +}) +g._mcpWorkflowToolsChannel ??= createPubSubChannel({ + channel: 'mcp:workflow_tools_changed', + label: 'mcp-workflow-tools', +}) const toolsChannel = g._mcpToolsChannel const workflowToolsChannel = g._mcpWorkflowToolsChannel -export const mcpPubSub: McpPubSubAdapter | null = - typeof window !== 'undefined' || !toolsChannel || !workflowToolsChannel - ? null - : { - publishToolsChanged: (event) => toolsChannel.publish(event), - publishWorkflowToolsChanged: (event) => workflowToolsChannel.publish(event), - onToolsChanged: (handler) => toolsChannel.subscribe(handler), - onWorkflowToolsChanged: (handler) => workflowToolsChannel.subscribe(handler), - ready: async () => { - await Promise.all([toolsChannel.ready(), workflowToolsChannel.ready()]) - }, - dispose: () => { - toolsChannel.dispose() - workflowToolsChannel.dispose() - }, - } +export const mcpPubSub: McpPubSubAdapter = { + publishToolsChanged: (event) => toolsChannel.publish(event), + publishWorkflowToolsChanged: (event) => workflowToolsChannel.publish(event), + onToolsChanged: (handler) => toolsChannel.subscribe(handler), + onWorkflowToolsChanged: (handler) => workflowToolsChannel.subscribe(handler), + ready: async () => { + await Promise.all([toolsChannel.ready(), workflowToolsChannel.ready()]) + }, + dispose: () => { + toolsChannel.dispose() + workflowToolsChannel.dispose() + }, +} diff --git a/apps/sim/lib/mcp/service-pool.test.ts b/apps/sim/lib/mcp/service-pool.test.ts index 43fdc5bb32c..ed75eeb7af6 100644 --- a/apps/sim/lib/mcp/service-pool.test.ts +++ b/apps/sim/lib/mcp/service-pool.test.ts @@ -113,7 +113,6 @@ vi.mock('@/lib/mcp/resolve-config', () => ({ })) vi.mock('@/lib/mcp/storage', () => ({ createMcpCacheAdapter: () => mockCacheAdapter, - getMcpCacheType: () => 'memory', })) import { withMcpOauthRefreshLock } from '@/lib/mcp/oauth' diff --git a/apps/sim/lib/mcp/service.test.ts b/apps/sim/lib/mcp/service.test.ts index b5839680ff7..25dbd999137 100644 --- a/apps/sim/lib/mcp/service.test.ts +++ b/apps/sim/lib/mcp/service.test.ts @@ -119,7 +119,6 @@ vi.mock('@/lib/mcp/resolve-config', () => ({ vi.mock('@/lib/mcp/storage', () => ({ createMcpCacheAdapter: () => mockCacheAdapter, - getMcpCacheType: () => 'memory', })) import { MAX_MCP_LAST_ERROR_LENGTH } from '@/lib/mcp/constants' diff --git a/apps/sim/lib/mcp/service.ts b/apps/sim/lib/mcp/service.ts index 093073cbe7d..468f0a6bbd4 100644 --- a/apps/sim/lib/mcp/service.ts +++ b/apps/sim/lib/mcp/service.ts @@ -33,11 +33,7 @@ import { } from '@/lib/mcp/oauth' import type { McpOauthCredentials } from '@/lib/mcp/oauth/coordinated-fetch' import { findMissingMcpConfigEnvVars, resolveMcpConfigEnvVars } from '@/lib/mcp/resolve-config' -import { - createMcpCacheAdapter, - getMcpCacheType, - type McpCacheStorageAdapter, -} from '@/lib/mcp/storage' +import { createMcpCacheAdapter, type McpCacheStorageAdapter } from '@/lib/mcp/storage' import { McpConnectionError, McpOauthAuthorizationRequiredError, @@ -296,7 +292,6 @@ class McpService { constructor() { this.cacheAdapter = createMcpCacheAdapter() - logger.info(`MCP Service initialized with ${getMcpCacheType()} cache`) if (mcpConnectionManager) { this.unsubscribeConnectionManager = mcpConnectionManager.subscribe((event) => { diff --git a/apps/sim/lib/mcp/storage/factory.ts b/apps/sim/lib/mcp/storage/factory.ts index cdbb51df8d1..77078da9637 100644 --- a/apps/sim/lib/mcp/storage/factory.ts +++ b/apps/sim/lib/mcp/storage/factory.ts @@ -1,42 +1,15 @@ -import { createLogger } from '@sim/logger' -import { getRedisClient } from '@/lib/core/config/redis' import type { McpCacheStorageAdapter } from './adapter' -import { MemoryMcpCache } from './memory-cache' import { RedisMcpCache } from './redis-cache' -const logger = createLogger('McpCacheFactory') - let cachedAdapter: McpCacheStorageAdapter | null = null /** - * Create MCP cache storage adapter. - * Uses Redis if available, falls back to in-memory cache. - * - * Unlike rate-limiting (which fails if Redis is configured but unavailable), - * MCP caching gracefully falls back to memory since it's an optimization. + * The MCP cache storage adapter, backed by the shared Redis client. */ export function createMcpCacheAdapter(): McpCacheStorageAdapter { if (cachedAdapter) { return cachedAdapter } - - const redis = getRedisClient() - - if (redis) { - logger.info('MCP cache: Using Redis') - cachedAdapter = new RedisMcpCache(redis) - } else { - logger.info('MCP cache: Using in-memory (Redis not configured)') - cachedAdapter = new MemoryMcpCache() - } - + cachedAdapter = new RedisMcpCache() return cachedAdapter } - -/** - * Get the current adapter type for logging/debugging - */ -export function getMcpCacheType(): 'redis' | 'memory' { - const redis = getRedisClient() - return redis ? 'redis' : 'memory' -} diff --git a/apps/sim/lib/mcp/storage/index.ts b/apps/sim/lib/mcp/storage/index.ts index 0990173c82d..2abf5df734c 100644 --- a/apps/sim/lib/mcp/storage/index.ts +++ b/apps/sim/lib/mcp/storage/index.ts @@ -1,2 +1,2 @@ export type { McpCacheStorageAdapter } from './adapter' -export { createMcpCacheAdapter, getMcpCacheType } from './factory' +export { createMcpCacheAdapter } from './factory' diff --git a/apps/sim/lib/mcp/storage/memory-cache.test.ts b/apps/sim/lib/mcp/storage/memory-cache.test.ts deleted file mode 100644 index ed87c9b4e4c..00000000000 --- a/apps/sim/lib/mcp/storage/memory-cache.test.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { sleep } from '@sim/utils/helpers' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import type { McpTool } from '@/lib/mcp/types' -import { MemoryMcpCache } from './memory-cache' - -describe('MemoryMcpCache', () => { - let cache: MemoryMcpCache - - const createTool = (name: string): McpTool => ({ - name, - description: `Test tool: ${name}`, - inputSchema: { type: 'object' }, - serverId: 'server-1', - serverName: 'Test Server', - }) - - beforeEach(() => { - cache = new MemoryMcpCache() - }) - - afterEach(() => { - cache.dispose() - }) - - describe('get', () => { - it('returns cached entry when valid', async () => { - const tools = [createTool('tool-1')] - await cache.set('key-1', tools, 60000) - - const result = await cache.get('key-1') - - expect(result).not.toBeNull() - expect(result?.tools).toEqual(tools) - }) - - it('returns null for expired entry', async () => { - const tools = [createTool('tool-1')] - // Set with 0 TTL so it expires immediately - await cache.set('key-1', tools, 0) - - // Wait a tiny bit to ensure expiry - await sleep(5) - - const result = await cache.get('key-1') - expect(result).toBeNull() - }) - - it('returns a copy of tools to prevent mutation', async () => { - const tools = [createTool('tool-1')] - await cache.set('key-1', tools, 60000) - - const result1 = await cache.get('key-1') - const result2 = await cache.get('key-1') - - expect(result1).not.toBe(result2) - expect(result1?.tools).toEqual(result2?.tools) - }) - }) - - describe('eviction policy', () => { - it('evicts oldest entries when max size is exceeded', async () => { - // Create a cache and add more entries than MAX_CACHE_SIZE (1000) - const tools = [createTool('tool')] - - // Add 1005 entries (5 over the limit of 1000) - for (let i = 0; i < 1005; i++) { - await cache.set(`key-${i}`, tools, 60000) - } - - // The oldest entries (first 5) should be evicted - expect(await cache.get('key-0')).toBeNull() - expect(await cache.get('key-1')).toBeNull() - expect(await cache.get('key-2')).toBeNull() - expect(await cache.get('key-3')).toBeNull() - expect(await cache.get('key-4')).toBeNull() - - // Newer entries should still exist - expect(await cache.get('key-1004')).not.toBeNull() - expect(await cache.get('key-1000')).not.toBeNull() - }) - }) -}) diff --git a/apps/sim/lib/mcp/storage/memory-cache.ts b/apps/sim/lib/mcp/storage/memory-cache.ts deleted file mode 100644 index b9d54194827..00000000000 --- a/apps/sim/lib/mcp/storage/memory-cache.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { McpTool } from '@/lib/mcp/types' -import { MCP_CONSTANTS } from '@/lib/mcp/utils' -import type { McpCacheEntry, McpCacheStorageAdapter } from './adapter' - -const logger = createLogger('McpMemoryCache') - -export class MemoryMcpCache implements McpCacheStorageAdapter { - private cache = new Map() - private readonly maxCacheSize = MCP_CONSTANTS.MAX_CACHE_SIZE - private cleanupInterval: NodeJS.Timeout | null = null - - constructor() { - this.startPeriodicCleanup() - } - - private startPeriodicCleanup(): void { - this.cleanupInterval = setInterval( - () => { - this.cleanupExpiredEntries() - }, - 5 * 60 * 1000 // 5 minutes - ) - // Don't keep Node process alive just for cache cleanup - this.cleanupInterval.unref() - } - - private cleanupExpiredEntries(): void { - const now = Date.now() - const expiredKeys: string[] = [] - - this.cache.forEach((entry, key) => { - if (entry.expiry <= now) { - expiredKeys.push(key) - } - }) - - expiredKeys.forEach((key) => this.cache.delete(key)) - - if (expiredKeys.length > 0) { - logger.debug(`Cleaned up ${expiredKeys.length} expired cache entries`) - } - } - - private evictIfNeeded(): void { - if (this.cache.size <= this.maxCacheSize) { - return - } - - // Evict oldest entries (by insertion order - Map maintains order) - const entriesToRemove = this.cache.size - this.maxCacheSize - const keys = Array.from(this.cache.keys()).slice(0, entriesToRemove) - keys.forEach((key) => this.cache.delete(key)) - - logger.debug(`Evicted ${entriesToRemove} cache entries`) - } - - async get(key: string): Promise { - const entry = this.cache.get(key) - const now = Date.now() - - if (!entry || entry.expiry <= now) { - if (entry) { - this.cache.delete(key) - } - return null - } - - // Return copy to prevent caller from mutating cache - return { - tools: entry.tools, - expiry: entry.expiry, - } - } - - async set(key: string, tools: McpTool[], ttlMs: number): Promise { - const now = Date.now() - const entry: McpCacheEntry = { - tools, - expiry: now + ttlMs, - } - - this.cache.set(key, entry) - this.evictIfNeeded() - } - - async delete(key: string): Promise { - this.cache.delete(key) - } - - async clear(): Promise { - this.cache.clear() - } - - dispose(): void { - if (this.cleanupInterval) { - clearInterval(this.cleanupInterval) - this.cleanupInterval = null - } - this.cache.clear() - logger.info('Memory cache disposed') - } -} diff --git a/apps/sim/lib/mcp/storage/redis-cache.ts b/apps/sim/lib/mcp/storage/redis-cache.ts index f04b9fea119..5a43699d4be 100644 --- a/apps/sim/lib/mcp/storage/redis-cache.ts +++ b/apps/sim/lib/mcp/storage/redis-cache.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import type Redis from 'ioredis' +import { getRedisClient } from '@/lib/core/config/redis' import type { McpTool } from '@/lib/mcp/types' import type { McpCacheEntry, McpCacheStorageAdapter } from './adapter' @@ -8,7 +9,14 @@ const logger = createLogger('McpRedisCache') const REDIS_KEY_PREFIX = 'mcp:tools:' export class RedisMcpCache implements McpCacheStorageAdapter { - constructor(private redis: Redis) {} + /** + * Resolved per command rather than captured, so the module-scope MCP service + * imports without Redis (as `next build` does) and follows the shared client + * across a health-check reconnect. + */ + private get redis(): Redis { + return getRedisClient() + } private getKey(key: string): string { return `${REDIS_KEY_PREFIX}${key}` diff --git a/apps/sim/lib/mcp/workflow-mcp-sync.test.ts b/apps/sim/lib/mcp/workflow-mcp-sync.test.ts index f5bbb7f6533..f96902d23f0 100644 --- a/apps/sim/lib/mcp/workflow-mcp-sync.test.ts +++ b/apps/sim/lib/mcp/workflow-mcp-sync.test.ts @@ -1,4 +1,5 @@ import { flattenMockConditions, hasMockCondition } from '@sim/testing' +import { mcpPubsubMock } from '@sim/testing/mocks/mcp-pubsub.mock' import { workflowsPersistenceUtilsMock } from '@sim/testing/mocks/workflows-persistence-utils.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -18,7 +19,7 @@ vi.mock('@/lib/workflows/triggers/utils', () => ({ hasValidStartBlockInState: mocks.hasValidStartBlock, })) vi.mock('@/lib/workflows/persistence/utils', () => workflowsPersistenceUtilsMock) -vi.mock('@/lib/mcp/pubsub', () => ({ mcpPubSub: null })) +vi.mock('@/lib/mcp/pubsub', () => mcpPubsubMock) vi.mock('@/lib/mcp/workflow-tool-schema', () => ({ applyDescriptionOverrides: (schema: unknown) => schema, extractInputFormatFromBlocks: () => [], diff --git a/apps/sim/lib/mcp/workflow-mcp-sync.ts b/apps/sim/lib/mcp/workflow-mcp-sync.ts index 100ce90c6a2..47442e8291c 100644 --- a/apps/sim/lib/mcp/workflow-mcp-sync.ts +++ b/apps/sim/lib/mcp/workflow-mcp-sync.ts @@ -650,7 +650,7 @@ export async function removeMcpToolsForWorkflow( */ /** Publishes affected servers synchronously so durable callers can observe failures. */ export async function publishMcpToolServerChanges(serverIds: string[]): Promise { - if (!mcpPubSub || !serverIds.length) return + if (!serverIds.length) return const servers = await db .select({ id: workflowMcpServer.id, workspaceId: workflowMcpServer.workspaceId }) .from(workflowMcpServer) diff --git a/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts b/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts index 04161091426..9bc9467ccea 100644 --- a/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts +++ b/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts @@ -217,18 +217,13 @@ function noteExercised(method: string, route: string) { } describe('content reaching Chat is admitted per item', () => { - let redis: NonNullable> + let redis: ReturnType let fixture: ContentFixture let connector: { connectorId: string; documentId: string } beforeAll(async () => { fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-content-delivery-')) - const client = getRedisClient() - if (!client) - throw new Error( - 'Set TEST_REDIS_URL to a disposable local Redis or use bun run test:integration' - ) - redis = client + redis = getRedisClient() // Compile the lazy tool graph before request deadlines and per-case provider fixtures exist. await matchV2Route('/api/v2/tools/file_get_content/execute')?.load() fixture = await seedContentFixture() diff --git a/apps/sim/lib/mothership/agent-cli/project-file-write-transport.integration.ts b/apps/sim/lib/mothership/agent-cli/project-file-write-transport.integration.ts index 24fd0a67ac5..e939299975b 100644 --- a/apps/sim/lib/mothership/agent-cli/project-file-write-transport.integration.ts +++ b/apps/sim/lib/mothership/agent-cli/project-file-write-transport.integration.ts @@ -608,7 +608,6 @@ describe.skipIf(!redisUrl)('Project workbench callback across requests', () => { executionLeaseExpiresAt: new Date(Date.now() + 60_000), }) const redis = getRedisClient() - if (!redis) throw new Error('Expected integration Redis') for (const leaseToken of [token, otherToken]) { const prefix = `mothership:sandbox-resources:${leaseToken}` callbackRedisKeys.push(`${prefix}:context`, `${prefix}:inbox`, `${prefix}:seen`) @@ -780,7 +779,7 @@ describe.skipIf(!redisUrl)('Project workbench callback across requests', () => { }) afterAll(async () => { - if (callbackRedisKeys.length) await getRedisClient()?.del(...callbackRedisKeys) + if (callbackRedisKeys.length) await getRedisClient().del(...callbackRedisKeys) for (const f of fixtures) { await db.delete(uploadSession).where(eq(uploadSession.userId, f.userId)) await db diff --git a/apps/sim/lib/mothership/agent-cli/saved-run-read.live.test.ts b/apps/sim/lib/mothership/agent-cli/saved-run-read.live.test.ts index 71801fffbca..dc84ea845bb 100644 --- a/apps/sim/lib/mothership/agent-cli/saved-run-read.live.test.ts +++ b/apps/sim/lib/mothership/agent-cli/saved-run-read.live.test.ts @@ -227,7 +227,12 @@ vi.mock('@/lib/mothership/request/session/controller-lease', async (original) => ...(await original()), assertChatStreamLease: vi.fn().mockResolvedValue(undefined), })) -vi.mock('@/lib/mothership/chat-status', () => ({ chatPubSub: null })) +vi.mock('@/lib/mothership/chat-status', async () => { + const { mothershipChatStatusMock } = await import( + '@sim/testing/mocks/mothership-chat-status.mock' + ) + return mothershipChatStatusMock +}) vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: () => {} })) vi.mock('@/lib/mothership/inbox/agentmail-client', () => ({ getAttachment: async (_inbox: string, _message: string, id: string) => { diff --git a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts index 3b382e2bd02..1fab43088b0 100644 --- a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts +++ b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts @@ -80,9 +80,7 @@ export function createWorkbenchFileProvenance(scope: WorkbenchFileScope) { stream, scope.signal, async (digest) => { - const redis = getRedisClient() - if (!redis) throw new Error('Workbench file classification storage is unavailable') - await redis.eval( + await getRedisClient().eval( RECORD_RECEIPT, 1, key(machine, digest), @@ -113,9 +111,7 @@ export function createWorkbenchFileProvenance(scope: WorkbenchFileScope) { const observeUpload: SessionFileObserver = (machine, stream) => { uploaded = undefined return hashStream(stream, scope.signal, async (digest) => { - const redis = getRedisClient() - if (!redis) throw new Error('Workbench file classification storage is unavailable') - const value = await redis.get(key(machine, digest)) + const value = await getRedisClient().get(key(machine, digest)) if (value === null) { /** Newly generated bytes inherit settled machine history; an existing bad receipt never falls back. */ const history = await readSessionSecretProvenance(scope.sessionKey, machine) diff --git a/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts b/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts index 5db03635d7f..5c4c573619f 100644 --- a/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts +++ b/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts @@ -75,19 +75,18 @@ const FIRST_RECOVERY = { attempts: 1, claimedAt: 0, notBefore: 0 } function redis() { const client = getRedisClient() - if (!client) throw new Error('The integration suite requires TEST_REDIS_URL') return client } /** The sweep's resume point lives in shared Redis; each test and the next suite start fresh. */ async function resetSweepCursor() { - await getRedisClient()?.del('copilot:orphaned-runs:sweep-cursor') + await getRedisClient().del('copilot:orphaned-runs:sweep-cursor') } beforeEach(resetSweepCursor) afterAll(async () => { - chatPubSub?.dispose() + chatPubSub.dispose() await resetSweepCursor() await closeRedisConnection() await new Promise((resolve) => worker.server.close(() => resolve())) @@ -725,7 +724,7 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { legacy: true, }) const announced: string[] = [] - const unsubscribe = chatPubSub!.onStatusChanged((event) => { + const unsubscribe = chatPubSub.onStatusChanged((event) => { if (event.type === 'completed') announced.push(event.chatId) }) diff --git a/apps/sim/lib/mothership/async-runs/orphaned-runs.ts b/apps/sim/lib/mothership/async-runs/orphaned-runs.ts index e3ceb6ee2fb..4c340109207 100644 --- a/apps/sim/lib/mothership/async-runs/orphaned-runs.ts +++ b/apps/sim/lib/mothership/async-runs/orphaned-runs.ts @@ -349,7 +349,7 @@ async function releaseChatLocks(fences: ChatLockFence[]): Promise { async function readSweepCursor(key = SWEEP_CURSOR_KEY): Promise { try { - return (await getRedisClient()?.get(key)) ?? undefined + return (await getRedisClient().get(key)) ?? undefined } catch (error) { logger.warn('Orphaned-run sweep cursor is unreadable; starting from the first run', { error: getErrorMessage(error), @@ -361,7 +361,6 @@ async function readSweepCursor(key = SWEEP_CURSOR_KEY): Promise { try { const redis = getRedisClient() - if (!redis) return if (cursor) await redis.set(key, cursor, 'EX', SWEEP_CURSOR_TTL_SECONDS) else await redis.del(key) } catch (error) { diff --git a/apps/sim/lib/mothership/chat-status.test.ts b/apps/sim/lib/mothership/chat-status.test.ts index a59c9a6f053..024ea2eaafe 100644 --- a/apps/sim/lib/mothership/chat-status.test.ts +++ b/apps/sim/lib/mothership/chat-status.test.ts @@ -1,21 +1,20 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { type ChatStatusEvent, chatPubSub, publishChatStatusChanged, } from '@/lib/mothership/chat-status' -/** Events as a subscriber receives them from the channel (process-local without Redis). */ +/** Events as they are handed to the Redis status channel. */ let received: ChatStatusEvent[] = [] -let unsubscribe: () => void = () => {} beforeEach(() => { received = [] - unsubscribe = chatPubSub?.onStatusChanged((event) => received.push(event)) ?? (() => {}) + vi.spyOn(chatPubSub, 'publishStatusChanged').mockImplementation((event) => { + received.push(event) + }) }) -afterEach(() => unsubscribe()) - describe('chat status ownership', () => { it('names the chat owner on workspace events, so each member can tell their own chats', () => { publishChatStatusChanged( diff --git a/apps/sim/lib/mothership/chat-status.ts b/apps/sim/lib/mothership/chat-status.ts index 94e830ca214..fe6e38f7977 100644 --- a/apps/sim/lib/mothership/chat-status.ts +++ b/apps/sim/lib/mothership/chat-status.ts @@ -1,8 +1,8 @@ /** * Chat Status Pub/Sub Adapter * - * Broadcasts chat status events across processes using Redis Pub/Sub. - * Gracefully falls back to process-local EventEmitter when Redis is unavailable. + * Broadcasts chat status events across processes using Redis Pub/Sub. The channel connects on + * first use, so importing this module opens nothing. * * The Redis channel and SSE label retain the legacy `task:status_changed` * identifier so live status updates keep flowing across pods during a rolling @@ -26,28 +26,24 @@ export type ChatStatusEvent = ChatStatusOwner & { } type ChatPubSubGlobal = typeof globalThis & { - _chatStatusChannel?: PubSubChannel | null + _chatStatusChannel?: PubSubChannel } const g = globalThis as ChatPubSubGlobal -if (!('_chatStatusChannel' in g)) { - g._chatStatusChannel = - typeof window !== 'undefined' - ? null - : createPubSubChannel({ channel: 'task:status_changed', label: 'task' }) -} +g._chatStatusChannel ??= createPubSubChannel({ + channel: 'task:status_changed', + label: 'task', +}) const channel = g._chatStatusChannel -export const chatPubSub = channel - ? { - publishStatusChanged: (event: ChatStatusEvent) => channel.publish(event), - onStatusChanged: (handler: (event: ChatStatusEvent) => void) => channel.subscribe(handler), - ready: () => channel.ready(), - dispose: () => channel.dispose(), - } - : null +export const chatPubSub = { + publishStatusChanged: (event: ChatStatusEvent) => channel.publish(event), + onStatusChanged: (handler: (event: ChatStatusEvent) => void) => channel.subscribe(handler), + ready: () => channel.ready(), + dispose: () => channel.dispose(), +} /** Projects canonical chat ownership into the same status channel for both surfaces. */ export function publishChatStatusChanged( @@ -56,13 +52,13 @@ export function publishChatStatusChanged( ): void { if (chat.organizationId) { if (!chat.userId || chat.workspaceId) throw new Error('Invalid organization chat owner') - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ organizationId: chat.organizationId, userId: chat.userId, ...event, }) } else if (chat.workspaceId) { - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ workspaceId: chat.workspaceId, ...(chat.userId ? { userId: chat.userId } : {}), ...event, diff --git a/apps/sim/lib/mothership/inbox/executor.ts b/apps/sim/lib/mothership/inbox/executor.ts index 65f960a88d5..34a7dd4cb6c 100644 --- a/apps/sim/lib/mothership/inbox/executor.ts +++ b/apps/sim/lib/mothership/inbox/executor.ts @@ -176,7 +176,7 @@ export async function executeInboxTask(taskId: string): Promise { .returning({ id: copilotChats.id }) // The rename won — do not announce a title the row no longer holds. if (stamped.length === 0) return - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ workspaceId: ws.id, chatId, type: 'renamed', @@ -187,7 +187,7 @@ export async function executeInboxTask(taskId: string): Promise { logger.warn('Failed to generate chat title', { chatId, err }) }) - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ workspaceId: ws.id, chatId, type: 'created', @@ -197,7 +197,7 @@ export async function executeInboxTask(taskId: string): Promise { const userMessageId = generateId() if (chatId) { - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ workspaceId: ws.id, chatId, type: 'started', @@ -315,7 +315,7 @@ export async function executeInboxTask(taskId: string): Promise { .where(eq(mothershipInboxTask.id, taskId)) if (chatId) { - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ workspaceId: ws.id, chatId, type: 'completed', diff --git a/apps/sim/lib/mothership/jobs/lane.integration.ts b/apps/sim/lib/mothership/jobs/lane.integration.ts index 3f48510b313..8fd6ee7189a 100644 --- a/apps/sim/lib/mothership/jobs/lane.integration.ts +++ b/apps/sim/lib/mothership/jobs/lane.integration.ts @@ -194,7 +194,6 @@ const scheduleIds: string[] = [] function redis() { const client = getRedisClient() - if (!client) throw new Error('The integration suite requires TEST_REDIS_URL') return client } @@ -371,7 +370,7 @@ describe.runIf(Boolean(redisUrl))('the scheduled job lane', () => { await db.delete(permissions).where(eq(permissions.userId, userId)) await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) await db.delete(user).where(eq(user.id, userId)) - chatPubSub?.dispose() + chatPubSub.dispose() await closeRedisConnection() }) diff --git a/apps/sim/lib/mothership/jobs/lane.test.ts b/apps/sim/lib/mothership/jobs/lane.test.ts index 496c6ed0890..61a277c709e 100644 --- a/apps/sim/lib/mothership/jobs/lane.test.ts +++ b/apps/sim/lib/mothership/jobs/lane.test.ts @@ -12,11 +12,8 @@ describe('runJobLane', () => { resetDbChainMock() }) - it.each([ - { COPILOT_API_KEY: undefined, REDIS_URL: 'redis://localhost:6379' }, - { COPILOT_API_KEY: 'key', REDIS_URL: undefined }, - ])('touches nothing where turns cannot be durable (%o)', async (env) => { - setEnv(env) + it('touches nothing where the worker is not configured', async () => { + setEnv({ COPILOT_API_KEY: undefined }) expect(await runJobLane()).toEqual({ claimed: 0 }) expect(dbChainMockFns.execute).not.toHaveBeenCalled() diff --git a/apps/sim/lib/mothership/jobs/lane.ts b/apps/sim/lib/mothership/jobs/lane.ts index da3aef937ca..cf38e011d28 100644 --- a/apps/sim/lib/mothership/jobs/lane.ts +++ b/apps/sim/lib/mothership/jobs/lane.ts @@ -32,12 +32,12 @@ export function jobLaneWorkQuery(now: Date) { * first means a run that met its condition, or a third failure that turned its job off, stops * the job before its next occurrence is claimed. * - * Jobs run only where their turns are durable: the worker is configured, and Redis holds the - * chat locks that let Sim take a turn over after its process dies. Elsewhere the lane does - * nothing, not even a query. + * Jobs run only where the worker is configured, so their turns are durable (Redis holds the chat + * locks that let Sim take a turn over after its process dies). Elsewhere the lane does nothing, + * not even a query. */ export async function runJobLane(now: Date = new Date()): Promise<{ claimed: number }> { - if (!env.COPILOT_API_KEY || !env.REDIS_URL) return { claimed: 0 } + if (!env.COPILOT_API_KEY) return { claimed: 0 } const [work] = await db.execute<{ due: boolean; open: boolean }>(jobLaneWorkQuery(now)) if (work?.open) await reconcileJobRuns(now) const claimed = work?.due ? await claimDueJobOccurrences(now, JOB_CLAIM_LIMIT) : [] diff --git a/apps/sim/lib/mothership/memory/application/spaces.integration.ts b/apps/sim/lib/mothership/memory/application/spaces.integration.ts index 40ba34f2532..bcc433955b6 100644 --- a/apps/sim/lib/mothership/memory/application/spaces.integration.ts +++ b/apps/sim/lib/mothership/memory/application/spaces.integration.ts @@ -226,7 +226,6 @@ describe('private KG selection through authorized application boundaries', () => }) it('publishes the creating owner with a private workspace chat event', async () => { - if (!chatPubSub) throw new Error('Server chat status channel unavailable') await chatPubSub.ready() const event = createDeferred<{ chatId: string; userId?: string }>() const unsubscribe = chatPubSub.onStatusChanged((value) => { diff --git a/apps/sim/lib/mothership/persistence/tool-confirm/index.ts b/apps/sim/lib/mothership/persistence/tool-confirm/index.ts index dcabe6e2771..059d5033d26 100644 --- a/apps/sim/lib/mothership/persistence/tool-confirm/index.ts +++ b/apps/sim/lib/mothership/persistence/tool-confirm/index.ts @@ -79,34 +79,26 @@ export function publishToolConfirmation(event: AsyncCompletionEnvelope): void { toolCallId: event.toolCallId, status: event.status, }) - const redis = getRedisClient() - if (redis) { - void redis - .set( - toolConfirmationKey(event.toolCallId), - JSON.stringify(event), - 'EX', - TOOL_CONFIRMATION_TTL_SECONDS - ) - .then(() => { - logger.info('Persisted tool confirmation in Redis', { - toolCallId: event.toolCallId, - status: event.status, - redisKey: toolConfirmationKey(event.toolCallId), - }) + void getRedisClient() + .set( + toolConfirmationKey(event.toolCallId), + JSON.stringify(event), + 'EX', + TOOL_CONFIRMATION_TTL_SECONDS + ) + .then(() => { + logger.info('Persisted tool confirmation in Redis', { + toolCallId: event.toolCallId, + status: event.status, + redisKey: toolConfirmationKey(event.toolCallId), }) - .catch((error) => { - logger.warn('Failed to persist tool confirmation in Redis', { - toolCallId: event.toolCallId, - error: toError(error).message, - }) + }) + .catch((error) => { + logger.warn('Failed to persist tool confirmation in Redis', { + toolCallId: event.toolCallId, + error: toError(error).message, }) - } else { - logger.warn('Redis unavailable while publishing tool confirmation', { - toolCallId: event.toolCallId, - status: event.status, }) - } toolConfirmationChannel.publish(event) } diff --git a/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts b/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts index cd59f1c6831..f5055a5ce81 100644 --- a/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts +++ b/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts @@ -143,7 +143,6 @@ const TOOL_ARGS = {} function redis() { const client = getRedisClient() - if (!client) throw new Error('The integration suite requires TEST_REDIS_URL') return client } @@ -396,7 +395,7 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () await db.delete(organization).where(eq(organization.id, organizationId)) await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) for (const id of [ownerId, userId]) await db.delete(user).where(eq(user.id, id)) - chatPubSub?.dispose() + chatPubSub.dispose() await closeRedisConnection() }) diff --git a/apps/sim/lib/mothership/request/application/abandoned-runs.ts b/apps/sim/lib/mothership/request/application/abandoned-runs.ts index 5b0dd9dcc34..c121b092de3 100644 --- a/apps/sim/lib/mothership/request/application/abandoned-runs.ts +++ b/apps/sim/lib/mothership/request/application/abandoned-runs.ts @@ -115,11 +115,11 @@ export async function takeOverAbandonedRuns(): Promise { } /** - * Needs Redis, whose chat locks are the only proof a run is abandoned. Stops on shutdown, so a - * draining process adopts no run it is about to abandon. + * Redis chat locks are the only proof a run is abandoned. Stops on shutdown, so a draining + * process adopts no run it is about to abandon. */ export function startAbandonedRunTakeover(): void { - if (!env.COPILOT_API_KEY || !env.REDIS_URL || timer) return + if (!env.COPILOT_API_KEY || timer) return timer = setInterval(() => { void takeOverAbandonedRuns().catch((error) => logger.warn('Abandoned run scan failed', { error: getErrorMessage(error) }) diff --git a/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts b/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts index 63cce9648e7..376a0786b86 100644 --- a/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts +++ b/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts @@ -502,7 +502,7 @@ describe.runIf(Boolean(redisUrl))('removing a run’s chat, owner or access stop if (organizationIds.length) await db.delete(organization).where(inArray(organization.id, organizationIds)) if (userIds.length) await db.delete(user).where(inArray(user.id, userIds)) - chatPubSub?.dispose() + chatPubSub.dispose() await closeRedisConnection() }) @@ -564,7 +564,7 @@ describe.runIf(Boolean(redisUrl))('removing a run’s chat, owner or access stop expect(await deleteCopilotChat({ chatId: run.chatId, userId: ownerId })).not.toBeNull() - await expect.poll(() => getRedisClient()?.exists(`execution:cancel:${executionId}`)).toBe(1) + await expect.poll(() => getRedisClient().exists(`execution:cancel:${executionId}`)).toBe(1) }) it('still stops the run when the worker cannot take the pushed Stop', async () => { @@ -1177,7 +1177,7 @@ describe.runIf(Boolean(redisUrl))('removing a run’s chat, owner or access stop const workspaceRun = await seedRun({ userId: memberId, workspaceId }) const assistantRun = await seedRun({ userId: memberId, organizationId }) /** A live controller keeps the run unsettled, as it would be while both removals commit. */ - await getRedisClient()?.set( + await getRedisClient().set( chatStreamLockKey(workspaceRun.chatId), workspaceRun.streamId, 'EX', @@ -1190,7 +1190,7 @@ describe.runIf(Boolean(redisUrl))('removing a run’s chat, owner or access stop await expect.poll(pushedStops).toEqual(expected) await sleep(300) expect(pushedStops()).toEqual(expected) - await getRedisClient()?.del(chatStreamLockKey(workspaceRun.chatId)) + await getRedisClient().del(chatStreamLockKey(workspaceRun.chatId)) }) it('answers Stop for a suspended owner’s run', async () => { diff --git a/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts b/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts index 090ad17dcf3..0aefa8fe558 100644 --- a/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts +++ b/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts @@ -1,3 +1,4 @@ +import { createMockRedis, redisConfigMockFns } from '@sim/testing' import { flushMicrotasks } from '@sim/testing/helpers/async' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { @@ -36,6 +37,19 @@ import type { StreamEvent, } from '@/lib/mothership/request/types' +beforeEach(() => { + // Preview sessions persist through a pipeline; a rejected write would retry on `sleep`, which + // the byte-rate cases' fake timers never advance. + redisConfigMockFns.mockGetRedisClient.mockReturnValue({ + ...createMockRedis(), + pipeline: vi.fn(() => ({ + hset: vi.fn(), + expire: vi.fn(), + exec: vi.fn().mockResolvedValue([]), + })), + }) +}) + const STREAM_ID = 'stream-1' const EDIT_TOOL_CALL_ID = 'edit-content-1' const WORKSPACE_FILE_TOOL_CALL_ID = 'workspace-file-1' diff --git a/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts b/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts index c1254b1c987..8fbbe7a48a7 100644 --- a/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts +++ b/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts @@ -155,7 +155,6 @@ const JOB_SECRETS = { secretScope: 'selected' as const, mountedSecrets: ['REPORT function redis() { const client = getRedisClient() - if (!client) throw new Error('The integration suite requires TEST_REDIS_URL') return client } @@ -285,7 +284,7 @@ describe.runIf(Boolean(redisUrl))('an unattended turn with no browser attached', await db.delete(permissions).where(eq(permissions.userId, userId)) await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) await db.delete(user).where(eq(user.id, userId)) - chatPubSub?.dispose() + chatPubSub.dispose() await closeRedisConnection() }) diff --git a/apps/sim/lib/mothership/request/session/abort.test.ts b/apps/sim/lib/mothership/request/session/abort.test.ts index 31a797c5b29..c73f5bd41c2 100644 --- a/apps/sim/lib/mothership/request/session/abort.test.ts +++ b/apps/sim/lib/mothership/request/session/abort.test.ts @@ -237,10 +237,6 @@ describe('startAbortPoller heartbeat', () => { }) describe('getChatStreamLockOwners', () => { - beforeEach(() => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) - }) - it('returns a verified empty owner map when no chat ids are provided', async () => { const result = await getChatStreamLockOwners([]) expect(result.status).toBe('verified') @@ -279,7 +275,8 @@ describe('getChatStreamLockOwners', () => { it('trusts verified Redis null over a process-local pending stream', async () => { const mget = vi.fn().mockResolvedValue([null]) - redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never) + const get = vi.fn().mockResolvedValue(null) + redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget, get } as never) await acquirePendingChatStream('chat-local', 'stream-local') try { @@ -292,16 +289,10 @@ describe('getChatStreamLockOwners', () => { } }) - it('returns unknown status when Redis is unavailable', async () => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) - - const result = await getChatStreamLockOwners(['chat-1', 'chat-2']) - - expect(result.status).toBe('unknown') - expect(result.ownersByChatId.size).toBe(0) - }) - - it('preserves local pending stream owners when Redis is unavailable', async () => { + it('preserves local pending stream owners when mget rejects', async () => { + const mget = vi.fn().mockRejectedValue(new Error('redis down')) + const get = vi.fn().mockResolvedValue(null) + redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget, get } as never) await acquirePendingChatStream('chat-local', 'stream-local') try { diff --git a/apps/sim/lib/mothership/request/session/abort.ts b/apps/sim/lib/mothership/request/session/abort.ts index 840b41198d8..434256962e1 100644 --- a/apps/sim/lib/mothership/request/session/abort.ts +++ b/apps/sim/lib/mothership/request/session/abort.ts @@ -19,10 +19,7 @@ import { const logger = createLogger('SessionAbort') const activeStreams = new Map() -const pendingChatStreams = new Map< - string, - { promise: Promise; resolve: () => void; streamId: string; lease: ChatStreamLease } ->() +const pendingChatStreams = new Map() const DEFAULT_ABORT_POLL_MS = 250 @@ -48,17 +45,12 @@ export interface ChatStreamLockOwnersResult { } function registerPendingChatStream(chatId: string, streamId: string, lease: ChatStreamLease): void { - let resolve!: () => void - const promise = new Promise((r) => { - resolve = r - }) - pendingChatStreams.set(chatId, { promise, resolve, streamId, lease }) + pendingChatStreams.set(chatId, { streamId, lease }) } function resolvePendingChatStream(chatId: string, streamId: string, lease?: ChatStreamLease): void { const entry = pendingChatStreams.get(chatId) if (entry && entry.streamId === streamId && (!lease || entry.lease.value === lease.value)) { - entry.resolve() pendingChatStreams.delete(chatId) } } @@ -91,24 +83,20 @@ export async function waitForPendingChatStream( const entry = pendingChatStreams.get(chatId) const localPending = !!entry && (!expectedStreamId || entry.streamId === expectedStreamId) - if (redis) { - try { - const ownerStreamId = await redis.get(chatStreamLockKey(chatId)) - const lockReleased = - !ownerStreamId || - (expectedStreamId !== undefined && streamIdFromLock(ownerStreamId) !== expectedStreamId) - if (!localPending && lockReleased) { - return true - } - } catch (error) { - logger.warn('Failed to inspect chat stream lock while waiting', { - chatId, - expectedStreamId, - error: toError(error).message, - }) + try { + const ownerStreamId = await redis.get(chatStreamLockKey(chatId)) + const lockReleased = + !ownerStreamId || + (expectedStreamId !== undefined && streamIdFromLock(ownerStreamId) !== expectedStreamId) + if (!localPending && lockReleased) { + return true } - } else if (!localPending) { - return true + } catch (error) { + logger.warn('Failed to inspect chat stream lock while waiting', { + chatId, + expectedStreamId, + error: toError(error).message, + }) } if (Date.now() >= deadline) { @@ -125,10 +113,6 @@ export async function getPendingChatStreamId(chatId: string): Promise= deadline) { - span.setAttribute(TraceAttr.LockAcquired, false) - span.setAttribute(TraceAttr.LockTimedOut, true) - return false } - await sleep(200) - } - } - - for (;;) { - const existing = pendingChatStreams.get(chatId) - if (!existing) { - registerPendingChatStream(chatId, streamId, lease) - span.setAttribute(TraceAttr.LockAcquired, true) - return true + } catch (error) { + logger.warn('Failed to acquire chat stream lock', { + chatId, + streamId, + error: toError(error).message, + }) } - const settled = await Promise.race([ - existing.promise.then(() => true), - sleep(timeoutMs).then(() => false as const), - ]) - if (!settled) { + if (Date.now() >= deadline) { span.setAttribute(TraceAttr.LockAcquired, false) span.setAttribute(TraceAttr.LockTimedOut, true) return false } + await sleep(200) } } ) diff --git a/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts b/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts index a84bc2bf7c0..40f31f877cd 100644 --- a/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts +++ b/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts @@ -61,7 +61,7 @@ describe.runIf(Boolean(redisUrl))('replay buffer lifetime', () => { expect(await acquirePendingChatStream(chatId, streamId, 0)).toBe(true) await appendText(streamId, 'before the park') const [ownerBudgetKey] = getRedisBudgetKeys({ kind: 'copilot_stream', id: streamId }) - const redis = getRedisClient()! + const redis = getRedisClient() const chargedBytes = await redis.get(ownerBudgetKey) /** Shortening every TTL to 5 s stands in for a park longer than each; the park outlasts it twice. */ for (const key of [ @@ -96,7 +96,7 @@ describe.runIf(Boolean(redisUrl))('replay buffer lifetime', () => { const streamId = generateId() await appendText(streamId, 'live') - const redis = getRedisClient()! + const redis = getRedisClient() expect(await redis.ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThanOrEqual(55) expect(await redis.ttl(`mothership_stream:${streamId}:seq`)).toBeGreaterThanOrEqual(55) }) @@ -108,7 +108,7 @@ describe.runIf(Boolean(redisUrl))('replay buffer lifetime', () => { await refreshBufferTtl(streamId) - const redis = getRedisClient()! + const redis = getRedisClient() expect(await redis.ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThan(250) expect(await redis.ttl(`mothership_stream:${streamId}:seq`)).toBeGreaterThan(250) }) @@ -116,7 +116,7 @@ describe.runIf(Boolean(redisUrl))('replay buffer lifetime', () => { it('reports a gap to a cursor ahead of a buffer whose numbering restarted', async () => { const streamId = generateId() for (let index = 0; index < 5; index++) await appendText(streamId, `part ${index}`) - const redis = getRedisClient()! + const redis = getRedisClient() await redis.del(`mothership_stream:${streamId}:events`, `mothership_stream:${streamId}:seq`) await appendText(streamId, 'after expiry') diff --git a/apps/sim/lib/mothership/request/session/buffer.ts b/apps/sim/lib/mothership/request/session/buffer.ts index bd89f4c3680..33293c3d664 100644 --- a/apps/sim/lib/mothership/request/session/buffer.ts +++ b/apps/sim/lib/mothership/request/session/buffer.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import type Redis from 'ioredis' import { env, envNumber } from '@/lib/core/config/env' import { getRedisClient } from '@/lib/core/config/redis' import { @@ -82,12 +83,9 @@ export function getStreamConfig(): StreamConfig { async function withRedisRetry( metadata: RedisOperationMetadata, - operation: (redis: NonNullable>) => Promise + operation: (redis: Redis) => Promise ): Promise { const redis = getRedisClient() - if (!redis) { - throw new Error('Redis is required for mothership stream durability') - } let lastError: unknown @@ -535,7 +533,6 @@ export async function getLatestSeq(streamId: string): Promise { /** The streams among these whose replay buffer has not yet expired. */ export async function findStreamsWithReplay(streamIds: string[]): Promise> { const redis = getRedisClient() - if (!redis) throw new Error('Redis is required for mothership stream durability') if (streamIds.length === 0) return new Set() const pipeline = redis.pipeline() for (const streamId of streamIds) pipeline.exists(getSeqKey(streamId)) diff --git a/apps/sim/lib/mothership/request/session/controller-lease.ts b/apps/sim/lib/mothership/request/session/controller-lease.ts index cc5e51f4fab..ad1880e7004 100644 --- a/apps/sim/lib/mothership/request/session/controller-lease.ts +++ b/apps/sim/lib/mothership/request/session/controller-lease.ts @@ -24,8 +24,7 @@ export function streamIdFromLock(value: string): string { } export async function assertChatStreamLease(lease: ChatStreamLease): Promise { - const redis = getRedisClient() - if (!redis || (await redis.get(lease.key)) !== lease.value) { + if ((await getRedisClient().get(lease.key)) !== lease.value) { throw new StreamControllerSupersededError() } } diff --git a/apps/sim/lib/mothership/request/session/file-preview-session.ts b/apps/sim/lib/mothership/request/session/file-preview-session.ts index df93b3a03ac..1d342191827 100644 --- a/apps/sim/lib/mothership/request/session/file-preview-session.ts +++ b/apps/sim/lib/mothership/request/session/file-preview-session.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import type Redis from 'ioredis' import { getRedisClient } from '@/lib/core/config/redis' import { getStreamConfig } from './buffer' import { @@ -36,12 +37,9 @@ type RedisOperationMetadata = { async function withRedisRetry( metadata: RedisOperationMetadata, - operation: (redis: NonNullable>) => Promise + operation: (redis: Redis) => Promise ): Promise { const redis = getRedisClient() - if (!redis) { - throw new Error('Redis is required for mothership preview durability') - } let lastError: unknown diff --git a/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts b/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts index 30446bad33b..f619af25365 100644 --- a/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts +++ b/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts @@ -134,7 +134,7 @@ vi.mock('@/lib/mothership/request/session/buffer', async (importOriginal) => { throw new Error('simulated transient Redis write failure') } /** The lock expires under a live controller, and nobody else holds it. */ - if (hit && fault.effect === 'lose_lease') await getRedisClient()!.del(lease.key) + if (hit && fault.effect === 'lose_lease') await getRedisClient().del(lease.key) return actual.appendEvents(...args) }, readEvents: async (...args: Parameters) => { @@ -182,7 +182,6 @@ const chatIds: string[] = [] function redis() { const client = getRedisClient() - if (!client) throw new Error('The integration suite requires TEST_REDIS_URL') return client } diff --git a/apps/sim/lib/mothership/request/session/replay-budget.integration.ts b/apps/sim/lib/mothership/request/session/replay-budget.integration.ts index c3e5f49e109..3b2714ee464 100644 --- a/apps/sim/lib/mothership/request/session/replay-budget.integration.ts +++ b/apps/sim/lib/mothership/request/session/replay-budget.integration.ts @@ -179,7 +179,6 @@ const MB = 1024 * 1024 function redis() { const client = getRedisClient() - if (!client) throw new Error('The integration suite requires TEST_REDIS_URL') return client } diff --git a/apps/sim/lib/mothership/request/session/replay-gap.integration.ts b/apps/sim/lib/mothership/request/session/replay-gap.integration.ts index 6adf931fe58..d2e72d9cb41 100644 --- a/apps/sim/lib/mothership/request/session/replay-gap.integration.ts +++ b/apps/sim/lib/mothership/request/session/replay-gap.integration.ts @@ -214,7 +214,7 @@ describe.runIf(Boolean(redisUrl))('reconnects past the replay ring', () => { it('re-syncs a cursor ahead of a ring whose numbering restarted', async () => { const { streamId } = await liveRunWithTrimmedRing() - await getRedisClient()!.del( + await getRedisClient().del( `mothership_stream:${streamId}:events`, `mothership_stream:${streamId}:seq` ) @@ -285,7 +285,7 @@ describe.runIf(Boolean(redisUrl))('reconnects past the replay ring', () => { const body = response.text() // Let the tail reach its poll loop, past the reconnect-time gap check. await sleep(500) - await getRedisClient()!.del( + await getRedisClient().del( `mothership_stream:${streamId}:events`, `mothership_stream:${streamId}:seq` ) diff --git a/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts b/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts index 96983f03d13..e2860ab6b32 100644 --- a/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts +++ b/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts @@ -136,8 +136,8 @@ async function orphanedRunWithTrimmedRing() { ), { streamId } ) - await getRedisClient()!.set(`mothership_stream:${streamId}:seq`, '4') - await getRedisClient()!.zremrangebyrank(`mothership_stream:${streamId}:events`, 0, 1) + await getRedisClient().set(`mothership_stream:${streamId}:seq`, '4') + await getRedisClient().zremrangebyrank(`mothership_stream:${streamId}:events`, 0, 1) const frame = (seq: number, type: string, payload: unknown) => ({ v: 1, type, @@ -254,7 +254,7 @@ describe.runIf(Boolean(redisUrl))('recovering a run whose ring lost its head', ( // the response rather than running, or billing, the turn again. expect(worker.requests.map((request) => request.path)).toEqual(['/api/mothership']) expect(worker.requests[0].body).toMatchObject({ messageId: streamId, receivedTextChars: 0 }) - expect(await getRedisClient()!.get(chatStreamLockKey(chatId))).toBeNull() + expect(await getRedisClient().get(chatStreamLockKey(chatId))).toBeNull() }) it('shows replayed tools once and never re-runs a re-handed call the dead controller ran', async () => { @@ -352,7 +352,7 @@ describe.runIf(Boolean(redisUrl))('recovering a run whose ring lost its head', ( it('numbers a recovered turn past a ring whose events are all unreadable', async () => { const { streamId, runId, frame } = await orphanedRunWithTrimmedRing() - const redis = getRedisClient()! + const redis = getRedisClient() const eventsKey = `mothership_stream:${streamId}:events` await redis.del(eventsKey) await redis.zadd(eventsKey, 1, 'corrupt-1', 2, 'corrupt-2', 3, 'corrupt-3', 4, 'corrupt-4') diff --git a/apps/sim/lib/mothership/request/session/viewer.ts b/apps/sim/lib/mothership/request/session/viewer.ts index 66842900e63..11263585849 100644 --- a/apps/sim/lib/mothership/request/session/viewer.ts +++ b/apps/sim/lib/mothership/request/session/viewer.ts @@ -25,7 +25,6 @@ function getViewersKey(streamId: string) { async function markStreamViewer(streamId: string, attachmentId: string): Promise { const redis = getRedisClient() - if (!redis) return const key = getViewersKey(streamId) const now = Date.now() // Drops readers that died without leaving, which a live reader's refresh keeps the key past. @@ -38,9 +37,7 @@ async function markStreamViewer(streamId: string, attachmentId: string): Promise } async function unmarkStreamViewer(streamId: string, attachmentId: string): Promise { - const redis = getRedisClient() - if (!redis) return - await redis.zrem(getViewersKey(streamId), attachmentId) + await getRedisClient().zrem(getViewersKey(streamId), attachmentId) } function warnMarkFailed(streamId: string, error: unknown) { @@ -68,13 +65,11 @@ export async function holdStreamViewer(streamId: string): Promise<() => void> { /** * Whether a reconnected reader is attached to the stream. Answers true whenever it cannot - * tell (no Redis, a failed read), so an unknown never cuts a present view's pickup window. + * tell (a failed read), so an unknown never cuts a present view's pickup window. */ export async function hasStreamViewer(streamId: string): Promise { - const redis = getRedisClient() - if (!redis) return true try { - return (await redis.zcount(getViewersKey(streamId), Date.now(), '+inf')) > 0 + return (await getRedisClient().zcount(getViewersKey(streamId), Date.now(), '+inf')) > 0 } catch (error) { logger.warn('Could not read whether a chat stream reader is attached', { streamId, diff --git a/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts b/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts index 1f675be2a67..c2d0c531d23 100644 --- a/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts +++ b/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts @@ -73,7 +73,6 @@ interface ResumeEngine { */ async function publishedConfirmation(toolCallId: string) { const client = getRedisClient() - if (!client) throw new Error('The integration suite requires TEST_REDIS_URL') const value = await client.get(`copilot:tool-confirmation:${toolCallId}`) return value === null ? null : JSON.parse(value) } diff --git a/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts b/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts index d86a9232014..cd30a830fb1 100644 --- a/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts +++ b/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts @@ -47,7 +47,7 @@ import { import { POST as wakeRoute } from '@/app/api/mothership/wake/route' afterAll(async () => { - chatPubSub?.dispose() + chatPubSub.dispose() await closeRedisConnection() if (inheritedRedisUrl === undefined) Reflect.deleteProperty(process.env, 'REDIS_URL') else process.env.REDIS_URL = inheritedRedisUrl @@ -198,7 +198,7 @@ describe.runIf(Boolean(redisUrl))('task wake retries', () => { await started /** The first wake's lock outlives its TTL while the lookup hangs. */ const firstLease = getLocalChatStreamLease(chatId, runId) - await getRedisClient()?.del(firstLease?.key ?? '') + await getRedisClient().del(firstLease?.key ?? '') expect((await wake(chatId, runId)).status).toBe(202) failLookup() diff --git a/apps/sim/lib/mothership/tasks/wake.ts b/apps/sim/lib/mothership/tasks/wake.ts index 23ef4c22d9e..44dbd50cf56 100644 --- a/apps/sim/lib/mothership/tasks/wake.ts +++ b/apps/sim/lib/mothership/tasks/wake.ts @@ -37,7 +37,7 @@ export async function runWakeTurn(input: WakeRequest): Promise { const { taskId, chatId, workspaceId, organizationId, userId, message, runId } = input const userMessageId = runId const owner = organizationId ? { organizationId, userId } : { workspaceId: workspaceId!, userId } - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ ...owner, chatId, type: 'started', @@ -100,7 +100,7 @@ export async function runWakeTurn(input: WakeRequest): Promise { logger.error('Wake turn failed', { taskId, chatId, error: getErrorMessage(error) }) } finally { await releasePendingChatStream(chatId, userMessageId) - chatPubSub?.publishStatusChanged({ + chatPubSub.publishStatusChanged({ ...owner, chatId, type: 'completed', diff --git a/apps/sim/lib/mothership/tools/hosted-workbench.live.test.ts b/apps/sim/lib/mothership/tools/hosted-workbench.live.test.ts index fdc2394abde..ac41aad4c4a 100644 --- a/apps/sim/lib/mothership/tools/hosted-workbench.live.test.ts +++ b/apps/sim/lib/mothership/tools/hosted-workbench.live.test.ts @@ -73,7 +73,7 @@ describe.skipIf(!enabled)('configured hosted Mship workbench', () => { SANDBOX_PROVIDER: 'e2b', E2B_ENABLED: 'true', }) - expect(await getRedisClient()?.ping()).toBe('PONG') + expect(await getRedisClient().ping()).toBe('PONG') }) afterEach(async () => { diff --git a/apps/sim/lib/mothership/tools/sandbox-project-files.ts b/apps/sim/lib/mothership/tools/sandbox-project-files.ts index a6c0fd83016..54827fb6100 100644 --- a/apps/sim/lib/mothership/tools/sandbox-project-files.ts +++ b/apps/sim/lib/mothership/tools/sandbox-project-files.ts @@ -54,11 +54,6 @@ export async function proxySandboxProjectFileRequest( userId: scope.userId, })) ?? new ResolvedSecretTraceRegistry([], { userId: scope.userId }) const bindingKey = `mothership:sandbox-resources:${token}:project-uploads:${encodeURIComponent(projectId)}` - const redis = () => { - const client = getRedisClient() - if (!client) throw new Error('Workbench upload binding is unavailable') - return client - } const transport = createProjectFileUploadTransport({ endpoint, projectId, @@ -66,7 +61,7 @@ export async function proxySandboxProjectFileRequest( resolveSecretTraceRegistry, uploadBinding: { async record(uploadId) { - await redis().eval( + await getRedisClient().eval( "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], ARGV[2]); return 1", 1, bindingKey, @@ -75,7 +70,7 @@ export async function proxySandboxProjectFileRequest( ) }, async contains(uploadId) { - return (await redis().sismember(bindingKey, uploadId)) === 1 + return (await getRedisClient().sismember(bindingKey, uploadId)) === 1 }, }, /** Direct uploads conservatively retain the physical machine's complete source history. */ diff --git a/apps/sim/lib/mothership/tools/sandbox-resources.ts b/apps/sim/lib/mothership/tools/sandbox-resources.ts index 5590f81255e..6e034dcdda2 100644 --- a/apps/sim/lib/mothership/tools/sandbox-resources.ts +++ b/apps/sim/lib/mothership/tools/sandbox-resources.ts @@ -78,12 +78,6 @@ const contextKey = (token: string) => `mothership:sandbox-resources:${token}:con const inboxKey = (token: string) => `mothership:sandbox-resources:${token}:inbox` const seenKey = (token: string) => `mothership:sandbox-resources:${token}:seen` -function redisClient() { - const redis = getRedisClient() - if (!redis) throw new Error('Sandbox resource tracking requires Redis') - return redis -} - /** Only the admitted in-process tool can bind an endpoint; arguments cannot select a chat. */ export async function sandboxResourceEndpoint( endpoint: string, @@ -125,10 +119,10 @@ export async function sandboxResourceEndpoint( } active.fileOwnerProtocolVersion = fileOwnerProtocolVersion active.workspaceFileCopy = workspaceFileCopy - await redisClient().set(contextKey(token), JSON.stringify(scope), 'EX', CONTEXT_TTL_SECONDS) + await getRedisClient().set(contextKey(token), JSON.stringify(scope), 'EX', CONTEXT_TTL_SECONDS) active.token = token if (active.signal.aborted) { - await redisClient().del(contextKey(token)) + await getRedisClient().del(contextKey(token)) active.signal.throwIfAborted() } return `${endpoint.replace(/\/$/, '')}/api/mothership/sandbox/${token}` @@ -140,7 +134,7 @@ export async function readSandboxResourceScope( apiKey: string | null ): Promise { if (!/^[0-9a-f-]{36}$/.test(token) || !apiKey) return null - const raw = await redisClient().get(contextKey(token)) + const raw = await getRedisClient().get(contextKey(token)) if (!raw) return null const scope = scopeSchema.parse(JSON.parse(raw)) if (scope.apiKeyHash !== sha256Hex(apiKey) || !(await isActiveSandboxResourceOwner(scope))) @@ -169,9 +163,9 @@ export async function recordSandboxResourceEffects( } for (const effect of effects) { try { - if (!(await redisClient().exists(contextKey(token)))) return + if (!(await getRedisClient().exists(contextKey(token)))) return await persistResourceEffect(scope.chatId, effect) - await redisClient().eval( + await getRedisClient().eval( "if redis.call('exists', KEYS[1]) == 0 then return 0 end if redis.call('sadd', KEYS[3], ARGV[3]) == 0 then return 0 end redis.call('rpush', KEYS[2], ARGV[1]); redis.call('expire', KEYS[2], ARGV[2]); redis.call('expire', KEYS[3], ARGV[2]); return 1", 3, contextKey(token), @@ -199,7 +193,7 @@ export async function withSandboxResourceScope( if (active.token) { void (async () => { try { - await redisClient().del(contextKey(active.token!)) + await getRedisClient().del(contextKey(active.token!)) } catch { logger.warn('Sandbox resource scope revocation deferred to lease fencing', { toolCallId: identity.toolCallId, @@ -212,7 +206,7 @@ export async function withSandboxResourceScope( const drain = async (all = false): Promise => { if (!active.token || signal.aborted) return true try { - const redis = redisClient() + const redis = getRedisClient() do { const entries = await redis.lrange(inboxKey(active.token), 0, 63) if (entries.length && !(await isActiveSandboxResourceOwner(identity))) return true @@ -243,7 +237,7 @@ export async function withSandboxResourceScope( const renew = async () => { if (!active.token || signal.aborted) return try { - await redisClient().expire(contextKey(active.token), CONTEXT_TTL_SECONDS) + await getRedisClient().expire(contextKey(active.token), CONTEXT_TTL_SECONDS) } catch { logger.warn('Sandbox resource scope renewal deferred', { toolCallId: identity.toolCallId }) } @@ -263,9 +257,9 @@ export async function withSandboxResourceScope( signal.removeEventListener('abort', revoke) if (active.token) { try { - await redisClient().del(contextKey(active.token)) - if (await drain(true)) await redisClient().del(inboxKey(active.token)) - await redisClient().del(seenKey(active.token)) + await getRedisClient().del(contextKey(active.token)) + if (await drain(true)) await getRedisClient().del(inboxKey(active.token)) + await getRedisClient().del(seenKey(active.token)) } catch { logger.warn('Sandbox resource scope cleanup deferred to expiry', { toolCallId: identity.toolCallId, diff --git a/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts b/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts index 13ffcf24854..37b643eb925 100644 --- a/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts +++ b/apps/sim/lib/mothership/tools/server/files/edit-roundtrip.test.ts @@ -1,6 +1,8 @@ import { readWorkspaceFileMetadataMock } from '@sim/testing/mocks/read-workspace-file-metadata.mock' +import { createMockRedis } from '@sim/testing/mocks/redis.mock' +import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { generateId } from '@sim/utils/id' -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ServerToolContext } from '@/lib/mothership/tools/server/base-tool' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/file-manager' @@ -84,6 +86,36 @@ function context(workspaceId = generateId()): ServerToolContext { } } +/** + * The prepare and apply tools hand the intent over through Redis hashes, so the client keeps the + * hashes it is given; everything else stays the shared mock's default. + */ +function createHashBackedRedis() { + const hashes = new Map>() + const redis = createMockRedis() + redis.hset.mockImplementation(async (key: string, field: string, value: string) => { + const hash = hashes.get(key) ?? new Map() + hashes.set(key, hash) + hash.set(field, value) + return 1 + }) + redis.hget.mockImplementation( + async (key: string, field: string) => hashes.get(key)?.get(field) ?? null + ) + redis.hgetall.mockImplementation(async (key: string) => + Object.fromEntries(hashes.get(key) ?? new Map()) + ) + redis.hdel.mockImplementation(async (key: string, ...fields: string[]) => { + const hash = hashes.get(key) + return fields.filter((field) => hash?.delete(field)).length + }) + return redis +} + +beforeEach(() => { + redisConfigMockFns.mockGetRedisClient.mockReturnValue(createHashBackedRedis()) +}) + describe('prepared file write across tool invocations', () => { it('creates then applies content in a new tool context without another prepare', async () => { const prepareContext = context() diff --git a/apps/sim/lib/mothership/tools/server/files/intent-store.test.ts b/apps/sim/lib/mothership/tools/server/files/intent-store.integration.ts similarity index 68% rename from apps/sim/lib/mothership/tools/server/files/intent-store.test.ts rename to apps/sim/lib/mothership/tools/server/files/intent-store.integration.ts index 83665802d28..2b49a25c6f0 100644 --- a/apps/sim/lib/mothership/tools/server/files/intent-store.test.ts +++ b/apps/sim/lib/mothership/tools/server/files/intent-store.integration.ts @@ -1,12 +1,29 @@ +/** + * The file intent store against a real Redis: prepare_file_edit stages an intent that its paired + * apply_file_edit must consume, even when sibling file subagents in the same message stage theirs + * concurrently. Failure modes: + * - a concurrent sibling's later intent is consumed by the wrong subagent + * - consuming one channel's intent removes its sibling's + * - a caller with no channel cannot reach the message's latest intent + * - an intent older than its TTL is still handed out, and lingers in the hash + * - apply_file_edit arrives before its prepare and fails instead of waiting + * - the wait outlives its deadline when no prepare ever lands + * Each test uses a fresh workspace id; the integration setup flushes Redis per file. + */ import { sleep } from '@sim/utils/helpers' import { generateShortId } from '@sim/utils/id' -import { describe, expect, it } from 'vitest' +import { afterAll, describe, expect, it } from 'vitest' +import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' import { consumeLatestFileIntent, type PendingFileIntent, storeFileIntent, waitForLatestFileIntent, -} from './intent-store' +} from '@/lib/mothership/tools/server/files/intent-store' + +afterAll(async () => { + await closeRedisConnection() +}) function makeIntent(overrides: Partial): PendingFileIntent { return { @@ -16,7 +33,18 @@ function makeIntent(overrides: Partial): PendingFileIntent { userId: 'user-1', chatId: 'chat-1', messageId: 'msg-1', - fileRecord: { id: overrides.fileId ?? 'file-x' } as unknown as PendingFileIntent['fileRecord'], + fileRecord: { + id: overrides.fileId ?? 'file-x', + workspaceId: 'ws-1', + name: 'notes.md', + key: 'workspace/ws-1/notes.md', + path: '/api/files/serve/workspace/ws-1/notes.md', + size: 0, + type: 'text/markdown', + uploadedBy: 'user-1', + uploadedAt: new Date(0), + updatedAt: new Date(0), + }, createdAt: Date.now(), ...overrides, } @@ -104,6 +132,24 @@ describe('file intent store channel scoping', () => { const latest = await consumeLatestFileIntent(ws, scope) expect(latest?.fileId).toBe('fileB') }) + + it('never hands out an intent older than its TTL, and removes it from the hash', async () => { + const ws = uniqueWorkspace() + const scope = { chatId: 'chat-1', messageId: 'msg-1', channelId: 'F1' } + await storeFileIntent( + ws, + 'fileA', + makeIntent({ + workspaceId: ws, + fileId: 'fileA', + channelId: 'F1', + createdAt: Date.now() - 2 * 60 * 60 * 1000, + }) + ) + + expect(await consumeLatestFileIntent(ws, scope)).toBeUndefined() + expect(await getRedisClient().hlen(`mothership_file_intent:${ws}`)).toBe(0) + }) }) describe('waitForLatestFileIntent', () => { diff --git a/apps/sim/lib/mothership/tools/server/files/intent-store.ts b/apps/sim/lib/mothership/tools/server/files/intent-store.ts index 7fdfdb1c833..d0772967400 100644 --- a/apps/sim/lib/mothership/tools/server/files/intent-store.ts +++ b/apps/sim/lib/mothership/tools/server/files/intent-store.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import type Redis from 'ioredis' import { getRedisClient } from '@/lib/core/config/redis' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/file-manager' @@ -50,11 +51,6 @@ const INTENT_TTL_MS = 60 * 60 * 1000 const INTENT_TTL_SECONDS = INTENT_TTL_MS / 1000 const REDIS_KEY_PREFIX = 'mothership_file_intent:' const RETRY_DELAYS_MS = [0, 50, 150] as const -const memoryStore = new Map() - -function buildKey(workspaceId: string, fileId: string): string { - return `${workspaceId}:${fileId}` -} function getWorkspaceRedisKey(workspaceId: string): string { return `${REDIS_KEY_PREFIX}${workspaceId}` @@ -76,24 +72,12 @@ function buildScopedField(fileId: string, scope?: FileIntentScope): string { return `${scope?.chatId ?? ''}:${scope?.messageId ?? ''}:${fileId}` } -function cleanupStale(): void { - const now = Date.now() - for (const [key, intent] of memoryStore) { - if (now - intent.createdAt > INTENT_TTL_MS) { - memoryStore.delete(key) - } - } -} - async function withRedisRetry( operation: string, workspaceId: string, - work: (redis: NonNullable>) => Promise + work: (redis: Redis) => Promise ): Promise { const redis = getRedisClient() - if (!redis) { - throw new Error('Redis client unavailable') - } let lastError: unknown for (let attempt = 0; attempt < RETRY_DELAYS_MS.length; attempt++) { @@ -140,13 +124,6 @@ export async function storeFileIntent( fileId: string, intent: PendingFileIntent ): Promise { - const redis = getRedisClient() - if (!redis) { - cleanupStale() - memoryStore.set(buildKey(workspaceId, buildScopedField(fileId, intent)), intent) - return - } - await withRedisRetry('store_file_intent', workspaceId, async (client) => { const key = getWorkspaceRedisKey(workspaceId) const pipeline = client.pipeline() @@ -161,12 +138,6 @@ export async function peekFileIntent( fileId: string, scope?: FileIntentScope ): Promise { - const redis = getRedisClient() - if (!redis) { - cleanupStale() - return memoryStore.get(buildKey(workspaceId, buildScopedField(fileId, scope))) - } - const raw = await withRedisRetry('peek_file_intent', workspaceId, async (client) => { const key = getWorkspaceRedisKey(workspaceId) return client.hget(key, buildScopedField(fileId, scope)) @@ -214,29 +185,6 @@ export async function consumeLatestFileIntent( workspaceId: string, scope?: FileIntentScope ): Promise { - const redis = getRedisClient() - if (!redis) { - cleanupStale() - let latest: PendingFileIntent | undefined - let latestKey: string | undefined - for (const [key, intent] of memoryStore) { - if ( - intent.workspaceId === workspaceId && - scopeMatches(intent, scope) && - channelMatches(intent, scope) - ) { - if (!latest || intent.createdAt > latest.createdAt) { - latest = intent - latestKey = key - } - } - } - if (latestKey) { - memoryStore.delete(latestKey) - } - return latest - } - const entries = await withRedisRetry('read_workspace_file_intents', workspaceId, async (client) => client.hgetall(getWorkspaceRedisKey(workspaceId)) ) diff --git a/apps/sim/lib/oauth/terminal-errors.test.ts b/apps/sim/lib/oauth/terminal-errors.test.ts index 57e9c65cd18..3da9cfed950 100644 --- a/apps/sim/lib/oauth/terminal-errors.test.ts +++ b/apps/sim/lib/oauth/terminal-errors.test.ts @@ -1,5 +1,5 @@ import { redisConfigMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { clearOAuthRefreshDeadFlag, getOAuthRefreshCoordinationIdentity, @@ -34,10 +34,6 @@ function createFakeRedis(): FakeRedis { } } -beforeEach(() => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) -}) - describe('isTerminalRefreshError', () => { it('treats a revoked refresh token as terminal but not a transient failure', () => { expect(isTerminalRefreshError('invalid_grant')).toBe(true) @@ -108,12 +104,6 @@ describe('markCredentialDead / getRecentTerminalError / clearDeadFlag', () => { } ) - it('all functions are no-ops when Redis is unavailable', async () => { - await expect(markCredentialDead('acc-1', 'code')).resolves.toBeUndefined() - await expect(getRecentTerminalError('acc-1')).resolves.toBeNull() - await expect(clearDeadFlag('acc-1')).resolves.toBeUndefined() - }) - it('absorbs Redis errors without throwing', async () => { const redis = createFakeRedis() redis.set.mockRejectedValueOnce(new Error('boom')) diff --git a/apps/sim/lib/oauth/terminal-errors.ts b/apps/sim/lib/oauth/terminal-errors.ts index 17174f9b477..1ecc17377df 100644 --- a/apps/sim/lib/oauth/terminal-errors.ts +++ b/apps/sim/lib/oauth/terminal-errors.ts @@ -12,7 +12,6 @@ function deadKey(accountId: string): string { export async function markCredentialDead(accountId: string, code: string): Promise { const redis = getRedisClient() - if (!redis) return try { await redis.set(deadKey(accountId), code, 'EX', DEAD_CACHE_TTL_SEC) } catch (error) { @@ -26,7 +25,6 @@ export async function markCredentialDead(accountId: string, code: string): Promi export async function getRecentTerminalError(accountId: string): Promise { const redis = getRedisClient() - if (!redis) return null try { return await redis.get(deadKey(accountId)) } catch (error) { @@ -40,7 +38,6 @@ export async function getRecentTerminalError(accountId: string): Promise { const redis = getRedisClient() - if (!redis) return try { await redis.del(deadKey(accountId)) } catch (error) { diff --git a/apps/sim/lib/realtime/event-log.integration.ts b/apps/sim/lib/realtime/event-log.integration.ts new file mode 100644 index 00000000000..228ea3d5150 --- /dev/null +++ b/apps/sim/lib/realtime/event-log.integration.ts @@ -0,0 +1,150 @@ +/** + * The durable event log's Lua append against a real Redis. Readers replay from a cursor and + * refetch when they fall off the back of the buffer, so each bound the script enforces has to + * move `earliestEventId` in step with what it drops. Failure modes: + * - appends mint ids out of order, or a replay returns events at or before the cursor + * - the entry cap trims without telling a reader behind it that it was pruned + * - the byte ceiling lets a buffer grow past it while under the entry cap + * - the byte ceiling drops the newest entry when it alone exceeds the ceiling + * - a reader with a cursor on an expired or never-seen stream replays nothing instead of refetching + * - a client tailing from the latest id replays the backlog, or misses the next append + * Each test uses a fresh stream id; the integration setup flushes Redis per file. + */ +import { generateShortId } from '@sim/utils/id' +import { afterAll, describe, expect, it } from 'vitest' +import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { + appendEvent, + type EventLogConfig, + type EventLogEntry, + getLatestEventId, + readEventsSince, +} from '@/lib/realtime/event-log' + +interface TestEntry extends EventLogEntry { + streamId: string + value: string +} + +const baseConfig: EventLogConfig = { + prefix: 'test:event-log:', + ttlSeconds: 3600, + cap: 3, + maxBytes: 0, + readChunk: 500, +} + +function serializerFor(streamId: string, value: string) { + return { + entryPrefix: '{"eventId":', + entrySuffix: `,"streamId":${JSON.stringify(streamId)},"value":${JSON.stringify(value)}}`, + buildEntry: (eventId: number): TestEntry => ({ eventId, streamId, value }), + } +} + +function freshStream(): string { + return `stream-${generateShortId()}` +} + +async function appendAll(config: EventLogConfig, streamId: string, values: string[]) { + const appended: Array = [] + for (const value of values) { + appended.push(await appendEvent(config, streamId, serializerFor(streamId, value))) + } + return appended +} + +afterAll(async () => { + await closeRedisConnection() +}) + +describe('event log over Redis', () => { + it('mints increasing ids and replays only the events after the cursor, in order', async () => { + const streamId = freshStream() + + const appended = await appendAll(baseConfig, streamId, ['a', 'b', 'c']) + + expect(appended.map((entry) => entry?.eventId)).toEqual([1, 2, 3]) + await expect(readEventsSince(baseConfig, streamId, 1)).resolves.toEqual({ + status: 'ok', + events: [ + { eventId: 2, streamId, value: 'b' }, + { eventId: 3, streamId, value: 'c' }, + ], + }) + }) + + it('trims to the entry cap and reports a cursor behind the trim as pruned', async () => { + const streamId = freshStream() + + await appendAll(baseConfig, streamId, ['a', 'b', 'c', 'd', 'e']) + + await expect(readEventsSince(baseConfig, streamId, 0)).resolves.toEqual({ + status: 'pruned', + earliestEventId: 3, + }) + await expect(readEventsSince(baseConfig, streamId, 2)).resolves.toMatchObject({ + status: 'ok', + events: [{ eventId: 3 }, { eventId: 4 }, { eventId: 5 }], + }) + expect(await getRedisClient().zcard(`${baseConfig.prefix}${streamId}:events`)).toBe(3) + }) + + it('drops the oldest entries once the buffer exceeds its byte ceiling, under the entry cap', async () => { + const streamId = freshStream() + const value = 'x'.repeat(100) + const entryBytes = Buffer.byteLength( + `{"eventId":1,"streamId":${JSON.stringify(streamId)},"value":${JSON.stringify(value)}}` + ) + const config = { ...baseConfig, cap: 100, maxBytes: entryBytes * 2 + 10 } + + await appendAll(config, streamId, [value, value, value, value]) + + const result = await readEventsSince(config, streamId, 2) + expect(result).toMatchObject({ status: 'ok', events: [{ eventId: 3 }, { eventId: 4 }] }) + await expect(readEventsSince(config, streamId, 1)).resolves.toEqual({ + status: 'pruned', + earliestEventId: 3, + }) + const meta = await getRedisClient().hgetall(`${config.prefix}${streamId}:meta`) + expect(Number(meta.bytes)).toBeLessThanOrEqual(config.maxBytes) + }) + + it('keeps the newest entry even when it alone exceeds the byte ceiling', async () => { + const streamId = freshStream() + const config = { ...baseConfig, cap: 100, maxBytes: 16 } + + await appendAll(config, streamId, ['first', 'x'.repeat(200)]) + + await expect(readEventsSince(config, streamId, 1)).resolves.toMatchObject({ + status: 'ok', + events: [{ eventId: 2, value: 'x'.repeat(200) }], + }) + }) + + it('reports a non-zero cursor against a stream with no sequence as pruned', async () => { + await expect(readEventsSince(baseConfig, freshStream(), 5)).resolves.toEqual({ + status: 'pruned', + earliestEventId: undefined, + }) + }) + + it('tails from the latest id without replaying the backlog and sees the next append', async () => { + const streamId = freshStream() + expect(await getLatestEventId(baseConfig, streamId)).toBe(0) + await appendAll(baseConfig, streamId, ['a', 'b']) + + const latest = await getLatestEventId(baseConfig, streamId) + expect(latest).toBe(2) + await expect(readEventsSince(baseConfig, streamId, latest)).resolves.toEqual({ + status: 'ok', + events: [], + }) + + await appendAll(baseConfig, streamId, ['c']) + await expect(readEventsSince(baseConfig, streamId, latest)).resolves.toEqual({ + status: 'ok', + events: [{ eventId: 3, streamId, value: 'c' }], + }) + }) +}) diff --git a/apps/sim/lib/slack-search/oauth-state.ts b/apps/sim/lib/slack-search/oauth-state.ts index 27836bc061e..ea7d8b37697 100644 --- a/apps/sim/lib/slack-search/oauth-state.ts +++ b/apps/sim/lib/slack-search/oauth-state.ts @@ -48,19 +48,13 @@ redis.call('DEL', KEYS[1]) return value ` -function requireRedis() { - const redis = getRedisClient() - if (!redis) throw new Error('Slack OAuth setup requires Redis') - return redis -} - function stateKey(state: string) { return `slack-search:oauth:${sha256Hex(state)}` } export async function storeSlackSearchOAuthAttempt(attempt: SlackSearchOAuthAttempt) { const state = generateId() - const saved = await requireRedis().set( + const saved = await getRedisClient().set( stateKey(state), JSON.stringify(attemptSchema.parse(attempt)), 'EX', @@ -73,7 +67,7 @@ export async function storeSlackSearchOAuthAttempt(attempt: SlackSearchOAuthAtte /** Consumes only the initiating browser session's attempt; another session cannot invalidate it. */ export async function consumeSlackSearchOAuthAttempt(state: string, principal: SessionPrincipal) { - const value = await requireRedis().eval( + const value = await getRedisClient().eval( CONSUME, 1, stateKey(state), diff --git a/apps/sim/lib/slack-search/onboarding-state.ts b/apps/sim/lib/slack-search/onboarding-state.ts index 847b310e113..006e87734ac 100644 --- a/apps/sim/lib/slack-search/onboarding-state.ts +++ b/apps/sim/lib/slack-search/onboarding-state.ts @@ -13,12 +13,6 @@ const stateSchema = z.object({ }) export type SlackSearchOnboardingState = z.infer -function requireRedis() { - const redis = getRedisClient() - if (!redis) throw new Error('Slack onboarding requires Redis') - return redis -} - function stateKey(token: string) { return `slack-search:onboarding:${sha256Hex(token)}` } @@ -26,7 +20,7 @@ function stateKey(token: string) { /** Navigation proof only: every read and retry separately verifies the signed-in Slack sender. */ export async function storeSlackSearchOnboardingState(state: SlackSearchOnboardingState) { const token = generateId() - const result = await requireRedis().set( + const result = await getRedisClient().set( stateKey(token), JSON.stringify(stateSchema.parse(state)), 'EX', @@ -38,7 +32,7 @@ export async function storeSlackSearchOnboardingState(state: SlackSearchOnboardi } export async function readSlackSearchOnboardingState(token: string) { - const value = await requireRedis().get(stateKey(token)) + const value = await getRedisClient().get(stateKey(token)) if (!value) throw new OrchestrationError( 'validation', diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts index 516836c8bcf..8fccb04b209 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts @@ -1,4 +1,4 @@ -import { redisConfigMockFns, resetRedisConfigMock } from '@sim/testing' +import { createMockRedis, redisConfigMockFns, resetRedisConfigMock } from '@sim/testing' import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' import { fileUtilsServerMock, @@ -25,22 +25,7 @@ import type { UserFile } from '@/executor/types' const mockCreateKnowledgeAccessProvider = knowledgeAccessScopeMockFns.mockCreateKnowledgeAccessProvider -const { mockRedis } = vi.hoisted(() => { - const mockRedis = { - get: vi.fn(), - set: vi.fn(), - hget: vi.fn(), - hset: vi.fn(), - hgetall: vi.fn(), - expire: vi.fn(), - scan: vi.fn(), - del: vi.fn(), - eval: vi.fn(), - } - return { - mockRedis, - } -}) +const mockRedis = createMockRedis() const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient const mockDownloadFile = storageServiceMockFns.mockDownloadFile @@ -68,14 +53,13 @@ describe('hydrateUserFilesWithBase64', () => { beforeEach(() => { mockDownloadFile.mockReset() mockDownloadServableFileFromStorage.mockReset() - mockGetRedisClient.mockReturnValue(null) + mockGetRedisClient.mockReturnValue(mockRedis) mockRedis.get.mockResolvedValue(null) mockRedis.set.mockResolvedValue('OK') mockRedis.hget.mockResolvedValue(null) mockRedis.hset.mockResolvedValue(1) mockRedis.hgetall.mockResolvedValue({}) mockRedis.expire.mockResolvedValue(1) - mockRedis.scan.mockResolvedValue(['0', []]) mockRedis.del.mockResolvedValue(1) mockRedis.eval.mockResolvedValue([1, 'ok', 0, 0]) mockVerifyFileAccess.mockResolvedValue(true) @@ -144,7 +128,6 @@ describe('hydrateUserFilesWithBase64', () => { }) it('bypasses a byte-only cache when model hydration must verify document contributors', async () => { - mockGetRedisClient.mockReturnValue(mockRedis) mockRedis.get.mockResolvedValue(Buffer.from('%PDF-cached').toString('base64')) const contentUpdatedAt = new Date('2026-08-06T00:00:00.000Z') const contributor = { @@ -322,7 +305,6 @@ describe('hydrateUserFilesWithBase64', () => { }) it('releases reserved Redis budget when cleaning up execution cache entries', async () => { - mockGetRedisClient.mockReturnValue(mockRedis) const rawEntry = JSON.stringify({ bytes: 12, userId: 'user-1' }) mockRedis.hgetall.mockResolvedValueOnce({ 'key:file-1': rawEntry, @@ -351,7 +333,6 @@ describe('hydrateUserFilesWithBase64', () => { }) it('releases indexed budget entries even when cache keys already expired', async () => { - mockGetRedisClient.mockReturnValue(mockRedis) mockRedis.hgetall.mockResolvedValueOnce({ 'key:file-1': JSON.stringify({ bytes: 7, userId: 'user-1' }), }) diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.ts index f38a9391a23..15a4059a17e 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.ts @@ -2,6 +2,7 @@ import type { Principal } from '@sim/auth/principal' import type { Logger } from '@sim/logger' import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' +import type Redis from 'ioredis' import { getRedisClient } from '@/lib/core/config/redis' import { getRedisBudgetKeys, getRedisBudgetLimits } from '@/lib/core/redis/byte-budget.server' import { generateRequestId } from '@/lib/core/utils/request' @@ -180,29 +181,6 @@ export interface Base64HydrationOptions { ) => Promise } -class InMemoryBase64Cache implements Base64Cache { - private entries = new Map() - - async get(file: UserFile): Promise { - const key = getFileCacheKey(file) - const entry = this.entries.get(key) - if (!entry) { - return null - } - if (entry.expiresAt <= Date.now()) { - this.entries.delete(key) - return null - } - return entry.value - } - - async set(file: UserFile, value: string, ttlSeconds: number): Promise { - const key = getFileCacheKey(file) - const expiresAt = Date.now() + ttlSeconds * 1000 - this.entries.set(key, { value, expiresAt }) - } -} - /** * The base64 cache only saves a repeat read from storage — the bytes it would have stored are * already in hand by the time it is written. Exceeding a Redis budget therefore means "do not @@ -226,13 +204,6 @@ function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Bas const redis = getRedisClient() const { executionId } = options - if (!redis) { - logger.warn('Redis unavailable for base64 cache, using in-memory fallback', { - requestId: options.requestId, - }) - return new InMemoryBase64Cache() - } - return { async get(file: UserFile) { try { @@ -367,7 +338,7 @@ function parseBudgetEntry(value: unknown): Base64BudgetEntry | null { } async function cleanupBudgetEntry( - redis: NonNullable>, + redis: Redis, executionId: string, fileKey: string, rawEntry: string, @@ -659,10 +630,6 @@ export function containsUserFileWithMetadata(value: unknown): boolean { */ export async function cleanupExecutionBase64Cache(executionId: string): Promise { const redis = getRedisClient() - if (!redis) { - return - } - const logger = createLogger('UserFileBase64') try { diff --git a/apps/sim/lib/webhooks/pending-verification.test.ts b/apps/sim/lib/webhooks/pending-verification.test.ts index 8e113befecb..c0d6d8aacbe 100644 --- a/apps/sim/lib/webhooks/pending-verification.test.ts +++ b/apps/sim/lib/webhooks/pending-verification.test.ts @@ -1,4 +1,4 @@ -import { redisConfigMockFns } from '@sim/testing' +import { createMockRedis, redisConfigMockFns } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { clearPendingWebhookVerification, @@ -10,7 +10,15 @@ import { describe('pending webhook verification', () => { beforeEach(() => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + const stored = new Map() + const redis = createMockRedis() + redis.set.mockImplementation(async (key: string, value: string) => { + stored.set(key, value) + return 'OK' + }) + redis.get.mockImplementation(async (key: string) => stored.get(key) ?? null) + redis.del.mockImplementation(async (key: string) => (stored.delete(key) ? 1 : 0)) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) }) afterEach(async () => { diff --git a/apps/sim/lib/webhooks/pending-verification.ts b/apps/sim/lib/webhooks/pending-verification.ts index d4ac76de444..e875db08bc2 100644 --- a/apps/sim/lib/webhooks/pending-verification.ts +++ b/apps/sim/lib/webhooks/pending-verification.ts @@ -7,8 +7,6 @@ const logger = createLogger('WebhookPendingVerification') const DEFAULT_TTL_SECONDS = 120 const REDIS_KEY_PREFIX = 'webhook:pending-verification:' -const inMemoryPendingVerificationStore = new Map() - export interface PendingWebhookVerification { path: string provider: string @@ -89,20 +87,6 @@ function isExpired(entry: PendingWebhookVerification): boolean { return entry.expiresAt <= Date.now() } -function getInMemoryPendingWebhookVerification(path: string): PendingWebhookVerification | null { - const entry = inMemoryPendingVerificationStore.get(path) - if (!entry) { - return null - } - - if (isExpired(entry)) { - inMemoryPendingVerificationStore.delete(path) - return null - } - - return entry -} - export function requiresPendingWebhookVerification( provider: string, metadata?: Record @@ -137,12 +121,12 @@ export async function registerPendingWebhookVerification( expiresAt: Date.now() + ttlSeconds * 1000, } - const redis = getRedisClient() - if (redis) { - await redis.set(getRedisKey(registration.path), JSON.stringify(entry), 'EX', ttlSeconds) - } else { - inMemoryPendingVerificationStore.set(registration.path, entry) - } + await getRedisClient().set( + getRedisKey(registration.path), + JSON.stringify(entry), + 'EX', + ttlSeconds + ) logger.info('Registered pending webhook verification', { provider: registration.provider, @@ -155,39 +139,30 @@ export async function getPendingWebhookVerification( path: string ): Promise { const redis = getRedisClient() - if (redis) { - const value = await redis.get(getRedisKey(path)) - if (!value) { - return null - } + const value = await redis.get(getRedisKey(path)) + if (!value) { + return null + } - try { - const entry = JSON.parse(value) as PendingWebhookVerification - if (isExpired(entry)) { - await redis.del(getRedisKey(path)) - return null - } - return entry - } catch (error) { - logger.warn('Failed to parse pending webhook verification entry', { - path, - error: toError(error).message, - }) + try { + const entry = JSON.parse(value) as PendingWebhookVerification + if (isExpired(entry)) { await redis.del(getRedisKey(path)) return null } + return entry + } catch (error) { + logger.warn('Failed to parse pending webhook verification entry', { + path, + error: toError(error).message, + }) + await redis.del(getRedisKey(path)) + return null } - - return getInMemoryPendingWebhookVerification(path) } export async function clearPendingWebhookVerification(path: string): Promise { - const redis = getRedisClient() - if (redis) { - await redis.del(getRedisKey(path)) - } else { - inMemoryPendingVerificationStore.delete(path) - } + await getRedisClient().del(getRedisKey(path)) logger.info('Cleared pending webhook verification', { path }) } diff --git a/apps/sim/lib/webhooks/polling/admission-refusals.ts b/apps/sim/lib/webhooks/polling/admission-refusals.ts index c57de7e2380..4baee5250ef 100644 --- a/apps/sim/lib/webhooks/polling/admission-refusals.ts +++ b/apps/sim/lib/webhooks/polling/admission-refusals.ts @@ -23,7 +23,6 @@ const refusalKey = (workspaceId: string) => `poll-admission-refused:v1:${workspa export async function recordPollAdmissionRefusal(workspaceId: string): Promise { if (!isBillingEnabled) return const redis = getRedisClient() - if (!redis) return try { await redis.set(refusalKey(workspaceId), '1', 'EX', POLL_ADMISSION_REFUSAL_TTL_SECONDS) } catch (error) { @@ -44,7 +43,6 @@ export async function findRecentlyRefusedWorkspaces( ): Promise> { if (!isBillingEnabled || workspaceIds.length === 0) return new Set() const redis = getRedisClient() - if (!redis) return new Set() try { const flags = await redis.mget(...workspaceIds.map(refusalKey)) return new Set(workspaceIds.filter((_, index) => flags[index] !== null)) diff --git a/apps/sim/lib/webhooks/slack-stream-sessions.ts b/apps/sim/lib/webhooks/slack-stream-sessions.ts index 1bf09e6d41e..62ee29c2a24 100644 --- a/apps/sim/lib/webhooks/slack-stream-sessions.ts +++ b/apps/sim/lib/webhooks/slack-stream-sessions.ts @@ -19,14 +19,6 @@ function getSessionKey(credentialId: string, target: SlackStreamSessionTarget): return `slack:agent-session:${credentialId}:${target.channel}:${target.threadTs}` } -function requireRedis() { - const redis = getRedisClient() - if (!redis) { - throw new Error('Redis is required for Slack agent session streaming') - } - return redis -} - function parseExecution(value: string): SlackStreamSessionExecution { const parsed = JSON.parse(value) as unknown if ( @@ -51,7 +43,7 @@ export async function registerSlackStreamSession( target: SlackStreamSessionTarget, execution: SlackStreamSessionExecution ): Promise { - const redis = requireRedis() + const redis = getRedisClient() const key = getSessionKey(credentialId, target) await redis .multi() @@ -65,7 +57,7 @@ export async function unregisterSlackStreamSession( target: SlackStreamSessionTarget, executionId: string ): Promise { - const redis = requireRedis() + const redis = getRedisClient() const key = getSessionKey(credentialId, target) await redis.eval( ` @@ -85,7 +77,7 @@ export async function listSlackStreamSessions( credentialId: string, target: SlackStreamSessionTarget ): Promise { - const redis = requireRedis() + const redis = getRedisClient() const values = await redis.hvals(getSessionKey(credentialId, target)) if (!Array.isArray(values)) { throw new Error('Invalid Redis response for Slack agent session lookup') diff --git a/apps/sim/lib/workflows/lifecycle.ts b/apps/sim/lib/workflows/lifecycle.ts index 971186c58f6..cf374718fb5 100644 --- a/apps/sim/lib/workflows/lifecycle.ts +++ b/apps/sim/lib/workflows/lifecycle.ts @@ -490,7 +490,7 @@ export async function finishWorkflowArchive( options.strictExternalCleanup ) - if (workspaceId && mcpPubSub) { + if (workspaceId) { for (const serverId of new Set(serverIds)) { try { mcpPubSub.publishWorkflowToolsChanged({ serverId, workspaceId }) diff --git a/apps/sim/lib/workspaces/lifecycle.ts b/apps/sim/lib/workspaces/lifecycle.ts index 4b7b79e2b60..1cb608592b2 100644 --- a/apps/sim/lib/workspaces/lifecycle.ts +++ b/apps/sim/lib/workspaces/lifecycle.ts @@ -178,16 +178,14 @@ export async function finishEnvironmentArchive( if (cleanupErrors.length > 0) throw cleanupErrors[0] } finally { await mcpService.clearCache(workspaceId).catch(() => undefined) - if (mcpPubSub) { - for (const serverId of effects.serverIds) { - try { - mcpPubSub.publishWorkflowToolsChanged({ serverId, workspaceId }) - } catch (error) { - logger.warn(`MCP tools-changed publish failed for server ${serverId}`, { - requestId, - error, - }) - } + for (const serverId of effects.serverIds) { + try { + mcpPubSub.publishWorkflowToolsChanged({ serverId, workspaceId }) + } catch (error) { + logger.warn(`MCP tools-changed publish failed for server ${serverId}`, { + requestId, + error, + }) } } } diff --git a/apps/sim/scripts/backfill-projects.ts b/apps/sim/scripts/backfill-projects.ts index ca924b17f93..96b9c0a499e 100644 --- a/apps/sim/scripts/backfill-projects.ts +++ b/apps/sim/scripts/backfill-projects.ts @@ -556,7 +556,7 @@ The registered migration first commits the durable column-authority switch after const { mcpPubSub } = await import('@/lib/mcp/pubsub') const { closeRedisConnection } = await import('@/lib/core/config/redis') const { db, dbReplica } = await import('@sim/db') - mcpPubSub?.dispose() + mcpPubSub.dispose() await closeRedisConnection() await Promise.all( [...new Set([db.$client, dbReplica.$client])].map((client) => client.end({ timeout: 5 })) diff --git a/apps/sim/scripts/backfill-table-workflow-deployments.test.ts b/apps/sim/scripts/backfill-table-workflow-deployments.test.ts deleted file mode 100644 index 54545d3f8c3..00000000000 --- a/apps/sim/scripts/backfill-table-workflow-deployments.test.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { resetDbChainMock } from '@sim/testing' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockLoadRuntimeSecrets, mockPerformFullDeploy } = vi.hoisted(() => ({ - mockLoadRuntimeSecrets: vi.fn(), - mockPerformFullDeploy: vi.fn(), -})) - -vi.mock('@sim/runtime-secrets', () => ({ - loadRuntimeSecrets: mockLoadRuntimeSecrets, -})) - -vi.mock('@/lib/workflows/orchestration/deploy', () => ({ - performFullDeploy: mockPerformFullDeploy, -})) - -import { - backfillTableWorkflowDeployments, - type TableWorkflowDeploymentCandidate, - type TableWorkflowDeploymentStore, -} from '@/scripts/backfill-table-workflow-deployments' - -const ORIGINAL_ENV = { - DATABASE_URL: process.env.DATABASE_URL, - DATABASE_URL_WEB: process.env.DATABASE_URL_WEB, - REDIS_TLS_SERVERNAME: process.env.REDIS_TLS_SERVERNAME, - REDIS_URL: process.env.REDIS_URL, - SIM_ENV_SECRET_ID: process.env.SIM_ENV_SECRET_ID, -} - -function restoreEnvironmentVariable(key: keyof typeof ORIGINAL_ENV): void { - const value = ORIGINAL_ENV[key] - if (value === undefined) { - Reflect.deleteProperty(process.env, key) - } else { - process.env[key] = value - } -} - -function candidate(workflowId: string): TableWorkflowDeploymentCandidate { - return { - workflowId, - workspaceId: 'workspace-1', - userId: 'user-1', - } -} - -function store( - overrides: Partial = {} -): TableWorkflowDeploymentStore { - return { - assertIntegrity: vi.fn().mockResolvedValue(undefined), - listCandidates: vi.fn().mockResolvedValue([]), - isDeployed: vi.fn().mockResolvedValue(false), - ...overrides, - } -} - -describe('backfillTableWorkflowDeployments', () => { - beforeEach(() => { - resetDbChainMock() - }) - - afterEach(() => { - restoreEnvironmentVariable('DATABASE_URL') - restoreEnvironmentVariable('DATABASE_URL_WEB') - restoreEnvironmentVariable('REDIS_TLS_SERVERNAME') - restoreEnvironmentVariable('REDIS_URL') - restoreEnvironmentVariable('SIM_ENV_SECRET_ID') - }) - - it('deploys bounded keyset pages and verifies the final desired state', async () => { - const listCandidates = vi - .fn() - .mockResolvedValueOnce([candidate('workflow-a'), candidate('workflow-b')]) - .mockResolvedValueOnce([candidate('workflow-c')]) - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([]) - const deploymentState = new Map() - const isDeployed = vi - .fn() - .mockImplementation(async (workflowId) => deploymentState.get(workflowId) ?? false) - const deploy = vi.fn(async (workflow: TableWorkflowDeploymentCandidate) => { - deploymentState.set(workflow.workflowId, true) - return { - success: true, - activeDeployment: { - deploymentVersionId: `version-${workflow.workflowId}`, - version: 1, - deployedAt: new Date().toISOString(), - }, - } - }) - const backfillStore = store({ listCandidates, isDeployed }) - - await expect( - backfillTableWorkflowDeployments(backfillStore, deploy, { batchSize: 2 }) - ).resolves.toEqual({ - scanned: 3, - deployed: 3, - alreadyDeployed: 0, - skippedLocked: 0, - skippedUndeployable: 0, - }) - expect(listCandidates.mock.calls).toEqual([ - ['', 2], - ['workflow-b', 2], - ['workflow-c', 2], - ['', 1], - ]) - expect(backfillStore.assertIntegrity).toHaveBeenCalledTimes(2) - expect(deploy.mock.calls.map(([workflow]) => workflow.workflowId)).toEqual([ - 'workflow-a', - 'workflow-b', - 'workflow-c', - ]) - }) - - it('reports and skips locked workflows while continuing the backfill', async () => { - const listCandidates = vi - .fn() - .mockResolvedValueOnce([candidate('workflow-a'), candidate('workflow-b')]) - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([candidate('workflow-a')]) - .mockResolvedValueOnce([]) - const deploymentState = new Map() - const isDeployed = vi - .fn() - .mockImplementation(async (workflowId) => deploymentState.get(workflowId) ?? false) - const deploy = vi.fn(async (workflow: TableWorkflowDeploymentCandidate) => { - if (workflow.workflowId === 'workflow-a') { - return { - success: false, - error: 'Workflow is locked by its containing folder', - errorCode: 'locked' as const, - } - } - deploymentState.set(workflow.workflowId, true) - return { - success: true, - activeDeployment: { - deploymentVersionId: `version-${workflow.workflowId}`, - version: 1, - deployedAt: new Date().toISOString(), - }, - } - }) - - await expect( - backfillTableWorkflowDeployments(store({ listCandidates, isDeployed }), deploy) - ).resolves.toEqual({ - scanned: 2, - deployed: 1, - alreadyDeployed: 0, - skippedLocked: 1, - skippedUndeployable: 0, - }) - expect(deploy.mock.calls.map(([workflow]) => workflow.workflowId)).toEqual([ - 'workflow-a', - 'workflow-b', - ]) - expect(listCandidates.mock.calls.slice(-2)).toEqual([ - ['', 1], - ['workflow-a', 1], - ]) - }) - - it('fails fast when canonical deployment fails', async () => { - const listCandidates = vi - .fn() - .mockResolvedValueOnce([candidate('workflow-a'), candidate('workflow-b')]) - const deploy = vi.fn().mockResolvedValue({ - success: false, - error: 'invalid trigger configuration', - }) - - await expect( - backfillTableWorkflowDeployments(store({ listCandidates }), deploy) - ).rejects.toThrow('Failed to deploy table workflow workflow-a: invalid trigger configuration') - expect(deploy).toHaveBeenCalledTimes(1) - }) - - it('fails when deployment does not activate or persist a valid active version', async () => { - const firstList = vi - .fn() - .mockResolvedValueOnce([candidate('workflow-a')]) - await expect( - backfillTableWorkflowDeployments(store({ listCandidates: firstList }), async () => ({ - success: true, - activeDeployment: null, - })) - ).rejects.toThrow('did not reach an active deployment state') - - const secondList = vi - .fn() - .mockResolvedValueOnce([candidate('workflow-b')]) - await expect( - backfillTableWorkflowDeployments(store({ listCandidates: secondList }), async () => ({ - success: true, - activeDeployment: { - deploymentVersionId: 'version-b', - version: 1, - deployedAt: new Date().toISOString(), - }, - })) - ).rejects.toThrow('completed without a valid active version') - }) - - it('rejects invalid batch and page behavior before it can loop or skip data', async () => { - const invalidBatchStore = store() - await expect( - backfillTableWorkflowDeployments(invalidBatchStore, vi.fn(), { batchSize: 0 }) - ).rejects.toThrow('positive integer') - expect(invalidBatchStore.assertIntegrity).not.toHaveBeenCalled() - - const oversizedStore = store({ - listCandidates: vi.fn().mockResolvedValue([candidate('workflow-a'), candidate('workflow-b')]), - }) - await expect( - backfillTableWorkflowDeployments(oversizedStore, vi.fn(), { batchSize: 1 }) - ).rejects.toThrow('oversized page') - - const duplicateStore = store({ - listCandidates: vi.fn().mockResolvedValue([candidate('workflow-a'), candidate('workflow-a')]), - }) - await expect( - backfillTableWorkflowDeployments(duplicateStore, vi.fn(), { batchSize: 2 }) - ).rejects.toThrow('duplicate workflow ids') - }) -}) diff --git a/apps/sim/scripts/backfill-table-workflow-deployments.ts b/apps/sim/scripts/backfill-table-workflow-deployments.ts deleted file mode 100644 index 5c8ddd77556..00000000000 --- a/apps/sim/scripts/backfill-table-workflow-deployments.ts +++ /dev/null @@ -1,515 +0,0 @@ -#!/usr/bin/env bun - -/** - * Deploys every mutable workflow referenced by a table workflow group. - * - * The script is idempotent and resumable: it pages over only workflows that do - * not have the full desired state, and each deployment uses a stable idempotency - * key. It runs the canonical deployment orchestration so webhook, schedule, - * MCP, audit, and notification side effects stay consistent with a - * user-initiated deployment. - * - * Usage: - * DATABASE_URL=... bun run apps/sim/scripts/backfill-table-workflow-deployments.ts - * AWS_PROFILE=sim-admin bun --no-env-file apps/sim/scripts/backfill-table-workflow-deployments.ts --environment=staging - * AWS_PROFILE=sim-admin bun --no-env-file apps/sim/scripts/backfill-table-workflow-deployments.ts --environment=production - */ - -import { createLogger } from '@sim/logger' -import { loadRuntimeSecrets } from '@sim/runtime-secrets' -import { getErrorMessage } from '@sim/utils/errors' -import { sql } from 'drizzle-orm' -import type { PerformFullDeployResult } from '@/lib/workflows/orchestration/deploy' - -const logger = createLogger('BackfillTableWorkflowDeployments') - -export const TABLE_WORKFLOW_DEPLOYMENT_BATCH_SIZE = 25 -const BACKFILL_ACTOR_ID = 'table-workflow-deployment-backfill' -const BACKFILL_OPERATION_VERSION = 'v2' -const RUNTIME_SECRET_IDS = { - production: '/production/sim/env-vars', - staging: '/staging/sim/env-vars', -} as const -/** Container-private services that a locally executed hosted backfill must not initialize. */ -const LOCAL_HOSTED_OMITTED_VARIABLES = ['REDIS_URL', 'REDIS_TLS_SERVERNAME'] as const - -type TableWorkflowDeploymentBackfillEnvironment = keyof typeof RUNTIME_SECRET_IDS - -interface TableWorkflowDeploymentBackfillCliOptions { - environment?: TableWorkflowDeploymentBackfillEnvironment -} - -export interface TableWorkflowDeploymentCandidate { - workflowId: string - workspaceId: string - userId: string -} - -export interface TableWorkflowDeploymentStore { - assertIntegrity(): Promise - listCandidates( - afterWorkflowId: string, - limit: number - ): Promise - isDeployed(workflowId: string): Promise -} - -export interface TableWorkflowDeploymentSummary { - scanned: number - deployed: number - alreadyDeployed: number - skippedLocked: number - skippedUndeployable: number -} - -interface TableWorkflowDeploymentBackfillOptions { - batchSize?: number -} - -export type DeployTableWorkflow = ( - candidate: TableWorkflowDeploymentCandidate -) => Promise - -interface InvalidTableSchemaRow extends Record { - table_id: string -} - -interface InvalidWorkflowGroupRow extends Record { - group_index: string - table_id: string -} - -interface InvalidWorkflowReferenceRow extends Record { - table_id: string - table_workspace_id: string - workflow_id: string - workflow_workspace_id: string -} - -interface MultipleActiveVersionsRow extends Record { - active_version_count: number - workflow_id: string -} - -interface CandidateRow extends Record { - user_id: string - workflow_id: string - workspace_id: string -} - -interface DeploymentStateRow extends Record { - active_version_count: number - is_deployed: boolean -} - -function isTableWorkflowDeploymentBackfillEnvironment( - value: string -): value is TableWorkflowDeploymentBackfillEnvironment { - return Object.hasOwn(RUNTIME_SECRET_IDS, value) -} - -/** Parses the deliberately small CLI surface for the backfill. */ -export function parseTableWorkflowDeploymentBackfillArgs( - args: readonly string[] -): TableWorkflowDeploymentBackfillCliOptions { - let environment: TableWorkflowDeploymentBackfillCliOptions['environment'] - - for (const arg of args) { - if (!arg.startsWith('--environment=')) { - throw new Error(`Unknown argument: ${arg}`) - } - if (environment) { - throw new Error('The --environment argument can only be provided once') - } - - const requestedEnvironment = arg.slice('--environment='.length) - if (!isTableWorkflowDeploymentBackfillEnvironment(requestedEnvironment)) { - throw new Error(`Unsupported backfill environment: ${requestedEnvironment || '(empty)'}`) - } - environment = requestedEnvironment - } - - return { environment } -} - -/** Loads staging configuration before modules that read database settings are imported. */ -export async function prepareTableWorkflowDeploymentBackfillEnvironment( - args: readonly string[] -): Promise { - const { environment } = parseTableWorkflowDeploymentBackfillArgs(args) - if (!environment) return - - const runtimeSecretId = RUNTIME_SECRET_IDS[environment] - const configuredSecretId = process.env.SIM_ENV_SECRET_ID - if (configuredSecretId && configuredSecretId !== runtimeSecretId) { - throw new Error( - `SIM_ENV_SECRET_ID is already set to ${configuredSecretId}; expected ${runtimeSecretId}` - ) - } - - const configuredDatabaseVariables = ['DATABASE_URL', 'DATABASE_URL_WEB'].filter( - (key) => key in process.env - ) - if (configuredDatabaseVariables.length > 0) { - throw new Error( - `Unset ${configuredDatabaseVariables.join(', ')} before using --environment=${environment} so local configuration cannot override ${environment}` - ) - } - - process.env.SIM_ENV_SECRET_ID = runtimeSecretId - await loadRuntimeSecrets() - - if (!process.env.DATABASE_URL && !process.env.DATABASE_URL_WEB) { - throw new Error(`${runtimeSecretId} did not provide a database URL`) - } - - for (const key of LOCAL_HOSTED_OMITTED_VARIABLES) { - Reflect.deleteProperty(process.env, key) - } -} - -async function getDatabase() { - const { db } = await import('@sim/db') - return db -} - -function validateCandidatePage( - candidates: TableWorkflowDeploymentCandidate[], - afterWorkflowId: string, - limit: number -): string | null { - if (candidates.length === 0) return null - if (candidates.length > limit) { - throw new Error('Table workflow deployment store returned an oversized page') - } - - const pageIds = new Set(candidates.map((candidate) => candidate.workflowId)) - if (pageIds.size !== candidates.length) { - throw new Error('Table workflow deployment store returned duplicate workflow ids') - } - - const lastWorkflowId = candidates.at(-1)?.workflowId - if (!lastWorkflowId || lastWorkflowId === afterWorkflowId) { - throw new Error('Table workflow deployment store returned a non-advancing page') - } - return lastWorkflowId -} - -async function assertOnlySkippedCandidatesRemain( - store: TableWorkflowDeploymentStore, - skippedWorkflowIds: ReadonlySet -): Promise { - let afterWorkflowId = '' - for (;;) { - const candidates = await store.listCandidates(afterWorkflowId, 1) - const lastWorkflowId = validateCandidatePage(candidates, afterWorkflowId, 1) - if (!lastWorkflowId) return - - const unexpectedCandidate = candidates.find( - (candidate) => !skippedWorkflowIds.has(candidate.workflowId) - ) - if (unexpectedCandidate) { - throw new Error( - `Table workflow deployment backfill left workflow ${unexpectedCandidate.workflowId} undeployed` - ) - } - afterWorkflowId = lastWorkflowId - } -} - -/** Ensures the table group references can be traversed without silently dropping corrupt data. */ -async function assertTableWorkflowIntegrity(): Promise { - const db = await getDatabase() - const [invalidSchema] = await db.execute(sql` - SELECT id AS table_id - FROM user_table_definitions - WHERE schema ? 'workflowGroups' - AND jsonb_typeof(schema->'workflowGroups') IS DISTINCT FROM 'array' - LIMIT 1 - `) - if (invalidSchema) { - throw new Error( - `Table ${invalidSchema.table_id} has a workflowGroups value that is not an array` - ) - } - - const [invalidGroup] = await db.execute(sql` - SELECT - table_definition.id AS table_id, - workflow_group.ordinality::text AS group_index - FROM user_table_definitions AS table_definition - CROSS JOIN LATERAL jsonb_array_elements( - COALESCE(table_definition.schema->'workflowGroups', '[]'::jsonb) - ) WITH ORDINALITY AS workflow_group(value, ordinality) - WHERE jsonb_typeof(workflow_group.value) IS DISTINCT FROM 'object' - OR NOT (workflow_group.value ? 'workflowId') - OR jsonb_typeof(workflow_group.value->'workflowId') IS DISTINCT FROM 'string' - OR ( - workflow_group.value->>'workflowId' = '' - AND ( - NOT (workflow_group.value ? 'enrichmentId') - OR jsonb_typeof(workflow_group.value->'enrichmentId') IS DISTINCT FROM 'string' - OR workflow_group.value->>'enrichmentId' = '' - ) - ) - LIMIT 1 - `) - if (invalidGroup) { - throw new Error( - `Table ${invalidGroup.table_id} workflow group ${invalidGroup.group_index} has an invalid workflowId` - ) - } - - const [invalidReference] = await db.execute(sql` - WITH table_workflow_groups AS ( - SELECT - table_definition.id AS table_id, - table_definition.workspace_id AS table_workspace_id, - workflow_group.value->>'workflowId' AS workflow_id - FROM user_table_definitions AS table_definition - CROSS JOIN LATERAL jsonb_array_elements( - COALESCE(table_definition.schema->'workflowGroups', '[]'::jsonb) - ) AS workflow_group(value) - ) - SELECT - table_workflow_groups.table_id, - table_workflow_groups.table_workspace_id, - table_workflow_groups.workflow_id, - workflow.workspace_id AS workflow_workspace_id - FROM table_workflow_groups - INNER JOIN workflow ON workflow.id = table_workflow_groups.workflow_id - AND workflow.archived_at IS NULL - WHERE table_workflow_groups.workflow_id <> '' - AND workflow.workspace_id IS DISTINCT FROM table_workflow_groups.table_workspace_id - LIMIT 1 - `) - if (invalidReference) { - throw new Error( - `Table ${invalidReference.table_id} in workspace ${invalidReference.table_workspace_id} ` + - `references workflow ${invalidReference.workflow_id} in workspace ${invalidReference.workflow_workspace_id}` - ) - } - - const [multipleActiveVersions] = await db.execute(sql` - WITH referenced_workflow_ids AS ( - SELECT DISTINCT workflow_group.value->>'workflowId' AS workflow_id - FROM user_table_definitions AS table_definition - CROSS JOIN LATERAL jsonb_array_elements( - COALESCE(table_definition.schema->'workflowGroups', '[]'::jsonb) - ) AS workflow_group(value) - INNER JOIN workflow ON workflow.id = workflow_group.value->>'workflowId' - AND workflow.archived_at IS NULL - WHERE workflow_group.value->>'workflowId' <> '' - ) - SELECT - deployment_version.workflow_id, - COUNT(*)::int AS active_version_count - FROM workflow_deployment_version AS deployment_version - INNER JOIN referenced_workflow_ids - ON referenced_workflow_ids.workflow_id = deployment_version.workflow_id - WHERE deployment_version.is_active = true - GROUP BY deployment_version.workflow_id - HAVING COUNT(*) > 1 - LIMIT 1 - `) - if (multipleActiveVersions) { - throw new Error( - `Workflow ${multipleActiveVersions.workflow_id} has ${multipleActiveVersions.active_version_count} active deployment versions` - ) - } -} - -export const postgresTableWorkflowDeploymentStore: TableWorkflowDeploymentStore = { - assertIntegrity: assertTableWorkflowIntegrity, - - async listCandidates(afterWorkflowId, limit) { - const db = await getDatabase() - const rows = await db.execute(sql` - WITH referenced_workflow_ids AS ( - SELECT DISTINCT workflow_group.value->>'workflowId' AS workflow_id - FROM user_table_definitions AS table_definition - CROSS JOIN LATERAL jsonb_array_elements( - COALESCE(table_definition.schema->'workflowGroups', '[]'::jsonb) - ) AS workflow_group(value) - WHERE workflow_group.value->>'workflowId' <> '' - ) - SELECT - workflow.id AS workflow_id, - workflow.workspace_id AS workspace_id, - workflow.user_id - FROM referenced_workflow_ids - INNER JOIN workflow ON workflow.id = referenced_workflow_ids.workflow_id - AND workflow.archived_at IS NULL - WHERE workflow.id COLLATE "C" > ${afterWorkflowId}::text COLLATE "C" - AND ( - workflow.is_deployed = false - OR NOT EXISTS ( - SELECT 1 - FROM workflow_deployment_version AS active_version - WHERE active_version.workflow_id = workflow.id - AND active_version.is_active = true - ) - ) - ORDER BY workflow.id COLLATE "C" - LIMIT ${limit} - `) - - return rows.map((row) => ({ - workflowId: row.workflow_id, - workspaceId: row.workspace_id, - userId: row.user_id, - })) - }, - - async isDeployed(workflowId) { - const db = await getDatabase() - const [state] = await db.execute(sql` - SELECT - workflow.is_deployed, - (COUNT(deployment_version.id) FILTER (WHERE deployment_version.is_active))::int - AS active_version_count - FROM workflow - LEFT JOIN workflow_deployment_version AS deployment_version - ON deployment_version.workflow_id = workflow.id - WHERE workflow.id = ${workflowId} - GROUP BY workflow.id, workflow.is_deployed - `) - if (!state) { - throw new Error(`Workflow ${workflowId} disappeared during the deployment backfill`) - } - if (state.active_version_count > 1) { - throw new Error( - `Workflow ${workflowId} has ${state.active_version_count} active deployment versions` - ) - } - return state.is_deployed && state.active_version_count === 1 - }, -} - -/** Deploys one table workflow through the same orchestration used by application surfaces. */ -export async function deployTableWorkflow( - candidate: TableWorkflowDeploymentCandidate -): Promise { - const { performFullDeploy } = await import('@/lib/workflows/orchestration/deploy') - return performFullDeploy({ - workflowId: candidate.workflowId, - userId: candidate.userId, - actorId: BACKFILL_ACTOR_ID, - captureAnalytics: false, - requestId: `${BACKFILL_ACTOR_ID}:${BACKFILL_OPERATION_VERSION}:${candidate.workflowId}`, - idempotencyKey: `${BACKFILL_ACTOR_ID}:${BACKFILL_OPERATION_VERSION}:${candidate.workflowId}`, - }) -} - -/** - * Reaches and verifies that every mutable table workflow has an active deployment. - */ -export async function backfillTableWorkflowDeployments( - store: TableWorkflowDeploymentStore, - deploy: DeployTableWorkflow, - options: TableWorkflowDeploymentBackfillOptions = {} -): Promise { - const batchSize = options.batchSize ?? TABLE_WORKFLOW_DEPLOYMENT_BATCH_SIZE - if (!Number.isInteger(batchSize) || batchSize <= 0) { - throw new Error('Table workflow deployment backfill batch size must be a positive integer') - } - - await store.assertIntegrity() - - const summary: TableWorkflowDeploymentSummary = { - scanned: 0, - deployed: 0, - alreadyDeployed: 0, - skippedLocked: 0, - skippedUndeployable: 0, - } - const skippedWorkflowIds = new Set() - let afterWorkflowId = '' - - for (;;) { - const candidates = await store.listCandidates(afterWorkflowId, batchSize) - const lastWorkflowId = validateCandidatePage(candidates, afterWorkflowId, batchSize) - if (!lastWorkflowId) break - - for (const candidate of candidates) { - summary.scanned += 1 - if (await store.isDeployed(candidate.workflowId)) { - summary.alreadyDeployed += 1 - } else { - logger.info('Deploying workflow referenced by a table workflow group', { - workflowId: candidate.workflowId, - workspaceId: candidate.workspaceId, - }) - const result = await deploy(candidate) - if (!result.success) { - if (result.errorCode === 'locked') { - skippedWorkflowIds.add(candidate.workflowId) - summary.skippedLocked += 1 - logger.warn('Skipping locked workflow referenced by a table workflow group', { - workflowId: candidate.workflowId, - workspaceId: candidate.workspaceId, - reason: result.error ?? 'Workflow is locked', - }) - continue - } - if (result.errorCode === 'validation') { - skippedWorkflowIds.add(candidate.workflowId) - summary.skippedUndeployable += 1 - logger.warn('Skipping undeployable workflow referenced by a table workflow group', { - workflowId: candidate.workflowId, - workspaceId: candidate.workspaceId, - reason: result.error ?? 'Workflow deployment validation failed', - }) - continue - } - throw new Error( - `Failed to deploy table workflow ${candidate.workflowId}: ${result.error ?? 'deployment returned no error'}` - ) - } - if (!result.activeDeployment) { - throw new Error( - `Table workflow ${candidate.workflowId} did not reach an active deployment state` - ) - } - if (!(await store.isDeployed(candidate.workflowId))) { - throw new Error( - `Table workflow ${candidate.workflowId} deployment completed without a valid active version` - ) - } - summary.deployed += 1 - } - } - - afterWorkflowId = lastWorkflowId - } - - await store.assertIntegrity() - await assertOnlySkippedCandidatesRemain(store, skippedWorkflowIds) - - return summary -} - -export async function runTableWorkflowDeploymentBackfill(): Promise { - logger.info('Starting table workflow deployment backfill') - const summary = await backfillTableWorkflowDeployments( - postgresTableWorkflowDeploymentStore, - deployTableWorkflow - ) - logger.info('Table workflow deployment backfill completed', summary) -} - -async function main(): Promise { - await prepareTableWorkflowDeploymentBackfillEnvironment(process.argv.slice(2)) - await runTableWorkflowDeploymentBackfill() -} - -if ((import.meta as { main?: boolean }).main) { - main() - .then(() => process.exit(0)) - .catch((error: unknown) => { - logger.error('Table workflow deployment backfill failed', { - error: getErrorMessage(error), - }) - process.exit(1) - }) -} diff --git a/apps/sim/scripts/test-docs-search-provider-e2e.ts b/apps/sim/scripts/test-docs-search-provider-e2e.ts index f7660e0ee5a..58fcb8ab986 100644 --- a/apps/sim/scripts/test-docs-search-provider-e2e.ts +++ b/apps/sim/scripts/test-docs-search-provider-e2e.ts @@ -11,15 +11,17 @@ import { z } from 'zod' const logger = createLogger('DocsProviderE2E') const setting = process.env.DOCS_PROVIDER_E2E_DATABASE_URL +const redisSetting = process.env.DOCS_PROVIDER_E2E_REDIS_URL const reportSetting = process.env.DOCS_PROVIDER_E2E_REPORT_PATH const origin = process.env.DOCS_PROVIDER_E2E_BASE_URL const providerKey = process.env.OPENAI_API_KEY const live = process.env.DOCS_PROVIDER_E2E_ALLOW_LIVE === '1' assert( - setting && reportSetting && origin && (!live || providerKey), - 'Fixture, report, docs origin and provider credential are required' + setting && redisSetting && reportSetting && origin && (!live || providerKey), + 'Fixture, Redis, report, docs origin and provider credential are required' ) const databaseUrl = new URL(setting) +const redisUrl = new URL(redisSetting) const docsOrigin = new URL(origin) assert( ['localhost', '127.0.0.1', '[::1]'].includes(databaseUrl.hostname), @@ -31,6 +33,11 @@ assert( 'Use a dedicated docs_search_provider_test database (or the empty docs_search_test fixture without live providers)' ) assert(['postgres:', 'postgresql:'].includes(databaseUrl.protocol)) +assert( + ['localhost', '127.0.0.1', '[::1]'].includes(redisUrl.hostname) && + ['redis:', 'rediss:'].includes(redisUrl.protocol), + 'Only a loopback Redis is permitted' +) assert( ['localhost', '127.0.0.1', '[::1]'].includes(docsOrigin.hostname), 'Only a loopback docs build may be tested' @@ -39,6 +46,7 @@ const systemVariables = new Set(['PATH', 'HOME', 'TMPDIR', 'TMP', 'TEMP', 'USER' for (const key of Object.keys(process.env)) if (!systemVariables.has(key)) delete process.env[key] Object.assign(process.env, { DATABASE_URL: databaseUrl.href, + REDIS_URL: redisUrl.href, OPENAI_API_KEY: live ? (providerKey ?? '') : '', NEXT_PUBLIC_APP_URL: 'http://localhost:3000', NODE_ENV: 'test', @@ -132,11 +140,6 @@ try { created_at timestamp NOT NULL DEFAULT now(), updated_at timestamp NOT NULL DEFAULT now() )` created = true - await sql`CREATE TABLE rate_limit_bucket ( - key text PRIMARY KEY, tokens decimal NOT NULL, - last_refill_at timestamp NOT NULL, blocked_until timestamp, - capacity_state jsonb, updated_at timestamp NOT NULL DEFAULT now() - )` let expectedChunks = 0 await check( 'Scoped publication fits one bounded provider batch before calling the provider', @@ -417,7 +420,7 @@ try { server.close((error) => (error ? reject(error) : resolve())) }) } - if (created) await sql`DROP TABLE IF EXISTS docs_embeddings, rate_limit_bucket` + if (created) await sql`DROP TABLE IF EXISTS docs_embeddings` await sql.end({ timeout: 1 }) await writeFile( reportPath, diff --git a/apps/sim/scripts/test-scim-e2e.ts b/apps/sim/scripts/test-scim-e2e.ts index 146887ddb52..f21fb3174a5 100644 --- a/apps/sim/scripts/test-scim-e2e.ts +++ b/apps/sim/scripts/test-scim-e2e.ts @@ -8,6 +8,7 @@ import { sleep } from '@sim/utils/helpers' import { generateId, generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import { makeSignature } from 'better-auth/crypto' +import Redis from 'ioredis' import postgres from 'postgres' /** @@ -31,6 +32,7 @@ function requiredEnvironment(name: string): string { const baseUrl = new URL(requiredEnvironment('SCIM_E2E_BASE_URL')) const databaseUrl = new URL(requiredEnvironment('SCIM_E2E_DATABASE_URL')) +const redisUrl = new URL(requiredEnvironment('SCIM_E2E_REDIS_URL')) const authSecret = requiredEnvironment('SCIM_E2E_AUTH_SECRET') const loopbackHosts = new Set(['localhost', '127.0.0.1', '[::1]']) assert(loopbackHosts.has(baseUrl.hostname), 'The app must use a loopback host') @@ -40,9 +42,12 @@ assert.equal(baseUrl.pathname, '/', 'App URL must be an origin without a path') assert(loopbackHosts.has(databaseUrl.hostname), 'The database must use a loopback host') assert(['postgres:', 'postgresql:'].includes(databaseUrl.protocol), 'Expected a PostgreSQL URL') assert(/(?:test|scim)/i.test(databaseUrl.pathname), 'Use a dedicated test or SCIM database') +assert(loopbackHosts.has(redisUrl.hostname), 'Redis must use a loopback host') +assert(['redis:', 'rediss:'].includes(redisUrl.protocol), 'Expected a Redis URL') assert(authSecret.length >= 32, 'Use a local Better Auth secret of at least 32 characters') const sql = postgres(databaseUrl.toString(), { max: 2 }) +const redis = new Redis(redisUrl.toString(), { maxRetriesPerRequest: 2 }) const fixtures: Fixture[] = [] const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = [] @@ -213,6 +218,11 @@ async function permission(userId: string, workspaceId: string) { return rows[0]?.permission_type } +/** The app's Redis token bucket for one SCIM connection's shared rate limit. */ +function connectionBucketKey(connectionId: string): string { + return `ratelimit:tb:route:scim:connection:${connectionId}` +} + async function cleanup() { for (const fixture of [...fixtures].reverse()) { await sql.begin(async (tx) => { @@ -223,9 +233,11 @@ async function cleanup() { await tx`delete from workspace where owner_id in ${tx(userIds)} or billed_account_user_id in ${tx(userIds)}` } await tx`delete from subscription where reference_id = ${fixture.orgId}` - await tx`delete from rate_limit_bucket where key in ( - select 'route:scim:connection:' || id from scim_connection where organization_id = ${fixture.orgId} - )` + const connections = + await tx`select id from scim_connection where organization_id = ${fixture.orgId}` + for (const connection of connections) { + await redis.del(connectionBucketKey(string(connection.id))) + } if (userIds.length) await tx`delete from project where owner_id in ${tx(userIds)}` await tx`delete from project where organization_id = ${fixture.orgId}` await tx`delete from organization where id = ${fixture.orgId}` @@ -1855,10 +1867,13 @@ async function run() { const connectionId = string( (await sql`select id from scim_connection where organization_id = ${primary.orgId}`)[0]?.id ) - const bucketKey = `route:scim:connection:${connectionId}` - const exhausted = await sql`update rate_limit_bucket set tokens = 0, last_refill_at = now() - where key = ${bucketKey} returning key` - assert.equal(exhausted.length, 1, 'Run the local app with PostgreSQL rate-limit storage') + const bucketKey = connectionBucketKey(connectionId) + assert.equal( + await redis.exists(bucketKey), + 1, + 'The connection bucket is missing; point the app and the harness at the same Redis' + ) + await redis.hset(bucketKey, { tokens: 0, lastRefillAt: Date.now() }) try { const limited = await scim(token, '/Users', { expected: 429 }) assert(Number(limited.headers.get('retry-after')) >= 1) @@ -1866,7 +1881,7 @@ async function run() { await scim(newToken, '/Users', { expected: 429 }) await scim(otherToken, '/Users') } finally { - await sql`delete from rate_limit_bucket where key = ${bucketKey}` + await redis.del(bucketKey) } await admin(primary, `/credentials/${credentialId}`, { method: 'DELETE' }) await scim(token, '/Users', { expected: 401 }) @@ -1907,11 +1922,13 @@ try { process.exitCode = 1 } await sql.end() + redis.disconnect() const report = { startedAt, completedAt: new Date().toISOString(), status: process.exitCode ? 'failed' : 'passed', - transport: 'Real HTTP against a local Next.js app and PostgreSQL; provider payload emulation', + transport: + 'Real HTTP against a local Next.js app, PostgreSQL, and Redis; provider payload emulation', requestCount, checks, } diff --git a/docker-compose.local.yml b/docker-compose.local.yml index eceba713eb5..40a1c1801ca 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -101,9 +101,10 @@ services: command: ['bun', 'run', 'db:migrate'] restart: 'no' - # Backs pub/sub (live Chat task-status and table events) and the shared caches. - # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the - # pub/sub channels have no fallback — without Redis, live status never streams. + # Required: rate limiting, OTPs, pub/sub (live Chat task-status and table + # events), realtime coordination, and the shared caches all live here, and the + # app and realtime refuse to start without REDIS_URL. Set REDIS_URL in .env to + # use a managed Redis instead. # Not published to the host: only the app and realtime containers need it, and # binding 6379 would collide with a Redis already running locally. redis: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 226a9a49829..6b9627efea2 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -120,9 +120,10 @@ services: command: ['bun', 'run', 'db:migrate'] restart: 'no' - # Backs pub/sub (live Chat task-status and table events) and the shared caches. - # The app falls back to PostgreSQL for storage when REDIS_URL is unset, but the - # pub/sub channels have no fallback — without Redis, live status never streams. + # Required: rate limiting, OTPs, pub/sub (live Chat task-status and table + # events), realtime coordination, and the shared caches all live here, and the + # app and realtime refuse to start without REDIS_URL. Set REDIS_URL in .env to + # use a managed Redis instead. # Not published to the host: only the app and realtime containers need it, and # binding 6379 would collide with a Redis already running locally. redis: diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index ea1fa023d49..28da45a6661 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.14.0 +version: 1.15.0 appVersion: "v0.8.26" kubeVersion: ">=1.25.0-0" home: https://sim.ai diff --git a/helm/sim/README.md b/helm/sim/README.md index 7c013d3a3b5..c159465bba8 100644 --- a/helm/sim/README.md +++ b/helm/sim/README.md @@ -509,7 +509,7 @@ autoscaling: targetMemoryUtilizationPercentage: 80 ``` -When `autoscaling.enabled=true`, the chart omits `spec.replicas` from the Deployment so the HPA owns replica count. Requires `metrics-server` in the cluster. The realtime Deployment gets the same HPA unless `autoscaling.realtime.enabled=false` — scale realtime past one replica only with `REDIS_URL` set (Socket.IO Redis adapter), or cross-pod collaboration events are dropped. +When `autoscaling.enabled=true`, the chart omits `spec.replicas` from the Deployment so the HPA owns replica count. Requires `metrics-server` in the cluster. The realtime Deployment gets the same HPA unless `autoscaling.realtime.enabled=false` — realtime bridges cross-pod collaboration events through Redis, which every install has (bundled, or your `REDIS_URL`). --- @@ -606,6 +606,12 @@ kubectl --namespace sim logs deploy/sim-app -c migrations --- +## Upgrading to 1.13.0 + +* **Redis is required.** The application no longer falls back to PostgreSQL or in-memory state for rate limiting, OTPs, pub/sub, or realtime coordination, and refuses to start without `REDIS_URL`. Installs that keep the bundled Redis (`redis.enabled: true`, the default) or already supply `REDIS_URL` need no action. + + With `redis.enabled: false`, the chart now fails to render unless `REDIS_URL` comes from `app.env.REDIS_URL`, a pre-created `app.secrets.existingSecret` (which the chart trusts, since it cannot read it), or `externalSecrets.remoteRefs.app.REDIS_URL` when External Secrets is enabled. If you previously ran with Redis disabled, either re-enable the bundled Redis or point `REDIS_URL` at a managed instance before upgrading. + ## Upgrading to 1.6.4 * `appVersion` — the default image tag for every first-party image (`app`, `realtime`, `migrations`, `pii`, `copilot`) when `image.tag` is unset — moves from `v0.7.44` to `v0.8.18`. It had not been bumped since chart 1.2.0, so an unpinned install deployed an application dozens of releases behind the chart it shipped with, and values keys added by newer charts had no effect because the running image did not read them. **An unpinned release rolls every first-party pod on upgrade.** Installs that pin `image.tag` or `image.digest` are unaffected; pinning explicitly remains the recommendation for production. @@ -620,7 +626,7 @@ Two changes alter behavior on an existing release. Neither requires action, but **An existing `REDIS_URL` always wins, wherever it comes from — no action needed on upgrade.** The bundled URL ships as a ConfigMap listed *before* the app Secret in `envFrom`. Kubernetes resolves duplicate keys by letting the last source win, so a `REDIS_URL` in your chart-managed Secret, a pre-created `existingSecret`, or one synced by External Secrets overrides the bundled value — the chart never has to read it. The bundled Redis simply fills the gap when nothing else provides a URL. - Set `app.env.REDIS_URL` to skip the bundled Deployment entirely (no unused pod), or `redis.enabled: false` to opt out. + Set `app.env.REDIS_URL` to skip the bundled Deployment entirely (no unused pod). (Since 1.13.0, `redis.enabled: false` also requires a `REDIS_URL` from one of those sources.) ## Upgrading to 1.2.0 diff --git a/helm/sim/examples/values-aws.yaml b/helm/sim/examples/values-aws.yaml index 150aca75eb9..aed865f7e4e 100644 --- a/helm/sim/examples/values-aws.yaml +++ b/helm/sim/examples/values-aws.yaml @@ -114,8 +114,8 @@ app: realtime: enabled: true # Safe to scale past 1: Socket.IO bridges cross-pod events through its Redis - # adapter, and the chart bundles Redis by default. Only keep this at 1 if you - # set redis.enabled=false without supplying app.env.REDIS_URL. + # adapter. Sim requires Redis — the chart bundles it unless you set + # redis.enabled=false and supply REDIS_URL yourself. replicaCount: 1 # Node selector for realtime pods diff --git a/helm/sim/examples/values-azure.yaml b/helm/sim/examples/values-azure.yaml index 5b99a927cfe..ec2c6b4c941 100644 --- a/helm/sim/examples/values-azure.yaml +++ b/helm/sim/examples/values-azure.yaml @@ -129,8 +129,8 @@ app: realtime: enabled: true # Safe to scale past 1: Socket.IO bridges cross-pod events through its Redis - # adapter, and the chart bundles Redis by default. Only keep this at 1 if you - # set redis.enabled=false without supplying app.env.REDIS_URL. + # adapter. Sim requires Redis — the chart bundles it unless you set + # redis.enabled=false and supply REDIS_URL yourself. replicaCount: 1 # Node selector for realtime pods diff --git a/helm/sim/examples/values-existing-secret.yaml b/helm/sim/examples/values-existing-secret.yaml index d5416a1e920..1b440e661a6 100644 --- a/helm/sim/examples/values-existing-secret.yaml +++ b/helm/sim/examples/values-existing-secret.yaml @@ -35,8 +35,8 @@ app: realtime: enabled: true # Safe to scale past 1: Socket.IO bridges cross-pod events through its Redis - # adapter, and the chart bundles Redis by default. Only keep this at 1 if you - # set redis.enabled=false without supplying app.env.REDIS_URL. + # adapter. Sim requires Redis — the chart bundles it unless you set + # redis.enabled=false and supply REDIS_URL yourself. replicaCount: 1 env: NEXT_PUBLIC_APP_URL: "https://sim.example.com" @@ -70,5 +70,7 @@ postgresql: # --namespace sim \ # --from-literal=POSTGRES_PASSWORD="$(openssl rand -base64 16 | tr -d '/+=')" -# Redis: bundled by default. If the Secret above already contains REDIS_URL, -# it overrides the bundled value automatically — nothing to configure. +# Redis (required): bundled by default. If the Secret above already contains +# REDIS_URL, it overrides the bundled value automatically — nothing to configure. +# To drop the bundled Redis, set redis.enabled=false and make sure the Secret +# holds REDIS_URL. diff --git a/helm/sim/examples/values-external-db.yaml b/helm/sim/examples/values-external-db.yaml index a70f7a19271..0d7250ac1cc 100644 --- a/helm/sim/examples/values-external-db.yaml +++ b/helm/sim/examples/values-external-db.yaml @@ -72,8 +72,8 @@ app: realtime: enabled: true # Safe to scale past 1: Socket.IO bridges cross-pod events through its Redis - # adapter, and the chart bundles Redis by default. Only keep this at 1 if you - # set redis.enabled=false without supplying app.env.REDIS_URL.. + # adapter. Sim requires Redis — the chart bundles it unless you set + # redis.enabled=false and supply REDIS_URL yourself. replicaCount: 1 resources: @@ -153,9 +153,8 @@ ingress: # Production-ready features (autoscaling, disruption budgets, network policy) autoscaling: enabled: true - # Realtime scales with the app because Redis is available by default. Set this - # to false only if you run without Redis (redis.enabled=false and no - # app.env.REDIS_URL) — multi-replica realtime would drop cross-pod events. + # Realtime scales with the app; Redis (always required) bridges its + # cross-pod events. Set false to pin realtime at realtime.replicaCount. realtime: enabled: false minReplicas: 2 diff --git a/helm/sim/examples/values-external-secrets.yaml b/helm/sim/examples/values-external-secrets.yaml index 80ba6e84025..00a2f0bf858 100644 --- a/helm/sim/examples/values-external-secrets.yaml +++ b/helm/sim/examples/values-external-secrets.yaml @@ -65,8 +65,8 @@ app: realtime: enabled: true # Safe to scale past 1: Socket.IO bridges cross-pod events through its Redis - # adapter, and the chart bundles Redis by default. Only keep this at 1 if you - # set redis.enabled=false without supplying app.env.REDIS_URL. + # adapter. Sim requires Redis — the chart bundles it unless you set + # redis.enabled=false and supply REDIS_URL yourself. replicaCount: 1 envDefaults: NEXT_PUBLIC_APP_URL: "https://sim.example.com" @@ -126,5 +126,6 @@ postgresql: # mountPath: "kubernetes" # role: "external-secrets" -# Redis: bundled by default. Map externalSecrets.remoteRefs.app.REDIS_URL to -# use a managed instance instead; the synced value overrides the bundled one. +# Redis (required): bundled by default. Map externalSecrets.remoteRefs.app.REDIS_URL +# to use a managed instance instead; the synced value overrides the bundled one. +# With redis.enabled=false that mapping is required and the chart enforces it. diff --git a/helm/sim/examples/values-gcp.yaml b/helm/sim/examples/values-gcp.yaml index 5fef2e296f2..df51dd6001a 100644 --- a/helm/sim/examples/values-gcp.yaml +++ b/helm/sim/examples/values-gcp.yaml @@ -115,8 +115,8 @@ app: realtime: enabled: true # Safe to scale past 1: Socket.IO bridges cross-pod events through its Redis - # adapter, and the chart bundles Redis by default. Only keep this at 1 if you - # set redis.enabled=false without supplying app.env.REDIS_URL. + # adapter. Sim requires Redis — the chart bundles it unless you set + # redis.enabled=false and supply REDIS_URL yourself. replicaCount: 1 # Node selector for realtime pods diff --git a/helm/sim/examples/values-production.yaml b/helm/sim/examples/values-production.yaml index b8b50570dcb..f98a73ecdaa 100644 --- a/helm/sim/examples/values-production.yaml +++ b/helm/sim/examples/values-production.yaml @@ -73,10 +73,8 @@ app: realtime: enabled: true # Safe to scale: Socket.IO bridges events across pods through its Redis - # adapter, and the chart bundles Redis by default (redis.enabled). If you - # set redis.enabled=false without supplying app.env.REDIS_URL, drop this - # back to 1 and disable autoscaling.realtime below — multi-replica realtime - # without Redis silently drops cross-pod collaboration updates. + # adapter. Sim requires Redis — the chart bundles it (redis.enabled) unless + # you supply REDIS_URL for a managed instance. replicaCount: 2 resources: @@ -182,9 +180,8 @@ certManager: # Horizontal Pod Autoscaler (automatically scales pods based on CPU/memory usage) autoscaling: enabled: true - # Realtime scales with the app because Redis is available by default. Set - # this to false if you run without Redis (redis.enabled=false and no - # app.env.REDIS_URL) — multi-replica realtime would drop cross-pod events. + # Realtime scales with the app; Redis (always required) bridges its + # cross-pod events. Set false to pin realtime at realtime.replicaCount. realtime: enabled: true minReplicas: 2 diff --git a/helm/sim/templates/_helpers.tpl b/helm/sim/templates/_helpers.tpl index 398fb417022..bc45cca79e1 100644 --- a/helm/sim/templates/_helpers.tpl +++ b/helm/sim/templates/_helpers.tpl @@ -321,10 +321,25 @@ Skip validation when using existing secrets or External Secrets Operator {{- fail "externalDatabase.password must only contain alphanumeric characters, hyphens, underscores, or periods to ensure DATABASE_URL compatibility." }} {{- end }} {{- end }} +{{- include "sim.validateRedis" . }} {{- /* ESO coverage validation - every key set in app.env / realtime.env must be mapped in externalSecrets.remoteRefs.app */ -}} {{- include "sim.validateExternalSecretCoverage" . }} {{- end }} +{{/* +Sim requires Redis. With redis.enabled=false the operator must supply +REDIS_URL: inline in app.env, from a pre-created Secret (which the chart cannot +read, so it is trusted), or mapped through External Secrets (enforced in +sim.validateExternalSecretCoverage). +*/}} +{{- define "sim.validateRedis" -}} +{{- $useExistingAppSecret := and .Values.app.secrets .Values.app.secrets.existingSecret .Values.app.secrets.existingSecret.enabled }} +{{- $useExternalSecrets := and .Values.externalSecrets .Values.externalSecrets.enabled }} +{{- if and (or .Values.app.enabled .Values.realtime.enabled) (not .Values.redis.enabled) (not (.Values.app.env.REDIS_URL | default "")) (not (or $useExistingAppSecret $useExternalSecrets)) }} +{{- fail "Sim requires Redis: redis.enabled=false but no REDIS_URL is provided. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." }} +{{- end }} +{{- end }} + {{/* Validate that every key set in app.env / realtime.env is also mapped in externalSecrets.remoteRefs.app when ESO is enabled. When ESO is on, the @@ -356,6 +371,9 @@ than enforced. {{- if .Values.cronjobs.enabled -}} {{- $required = append $required "CRON_SECRET" -}} {{- end -}} +{{- if not .Values.redis.enabled -}} +{{- $required = append $required "REDIS_URL" -}} +{{- end -}} {{- range $key := $required -}} {{- $inEnv := index $appEnv $key -}} {{- $mapped := index $remoteRefs $key -}} diff --git a/helm/sim/tests/redis_test.yaml b/helm/sim/tests/redis_test.yaml index 581f5874edf..26bc10bce87 100644 --- a/helm/sim/tests/redis_test.yaml +++ b/helm/sim/tests/redis_test.yaml @@ -36,6 +36,8 @@ tests: template: deployment-redis.yaml set: redis.enabled: false + app.secrets.existingSecret.enabled: true + app.secrets.existingSecret.name: my-app-secrets asserts: - hasDocuments: count: 0 @@ -115,10 +117,89 @@ tests: template: configmap-redis.yaml set: redis.enabled: false + app.secrets.existingSecret.enabled: true + app.secrets.existingSecret.name: my-app-secrets asserts: - hasDocuments: count: 0 + - it: fails to render when redis.enabled=false and nothing supplies REDIS_URL + template: deployment-app.yaml + set: + redis.enabled: false + realtime.enabled: false + asserts: + - failedTemplate: + errorMessage: "Sim requires Redis: redis.enabled=false but no REDIS_URL is provided. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." + + - it: fails realtime too when redis.enabled=false and nothing supplies REDIS_URL + template: deployment-realtime.yaml + set: + redis.enabled: false + app.enabled: false + asserts: + - failedTemplate: + errorMessage: "Sim requires Redis: redis.enabled=false but no REDIS_URL is provided. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." + + - it: renders with redis.enabled=false when app.env.REDIS_URL is set + template: deployment-app.yaml + set: + redis.enabled: false + app.env.REDIS_URL: "rediss://:pw@managed.example.com:6380" + asserts: + - isKind: + of: Deployment + + - it: trusts a pre-created app Secret to carry REDIS_URL when redis.enabled=false + template: deployment-app.yaml + set: + redis.enabled: false + app.secrets.existingSecret.enabled: true + app.secrets.existingSecret.name: my-app-secrets + asserts: + - isKind: + of: Deployment + + - it: fails under External Secrets when redis.enabled=false and REDIS_URL is not mapped + template: deployment-app.yaml + set: + redis.enabled: false + realtime.enabled: false + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: my-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: sim/app/better-auth-secret + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: sim/app/encryption-key + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: sim/app/internal-api-secret + externalSecrets.remoteRefs.app.CRON_SECRET: sim/app/cron-secret + externalSecrets.remoteRefs.postgresql.password: sim/postgresql/password + app.env.BETTER_AUTH_SECRET: "" + app.env.ENCRYPTION_KEY: "" + app.env.INTERNAL_API_SECRET: "" + app.env.CRON_SECRET: "" + asserts: + - failedTemplate: + errorPattern: "Required key 'REDIS_URL' is missing" + + - it: renders under External Secrets when redis.enabled=false and REDIS_URL is mapped + template: deployment-app.yaml + set: + redis.enabled: false + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: my-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: sim/app/better-auth-secret + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: sim/app/encryption-key + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: sim/app/internal-api-secret + externalSecrets.remoteRefs.app.CRON_SECRET: sim/app/cron-secret + externalSecrets.remoteRefs.app.REDIS_URL: sim/app/redis-url + externalSecrets.remoteRefs.postgresql.password: sim/postgresql/password + app.env.BETTER_AUTH_SECRET: "" + app.env.ENCRYPTION_KEY: "" + app.env.INTERNAL_API_SECRET: "" + app.env.CRON_SECRET: "" + asserts: + - isKind: + of: Deployment + - it: does not preset free-tier plan caps, which are opt-in on a self-host template: deployment-app.yaml asserts: diff --git a/helm/sim/values.schema.json b/helm/sim/values.schema.json index c0b49a44c17..514f8ec2520 100644 --- a/helm/sim/values.schema.json +++ b/helm/sim/values.schema.json @@ -477,11 +477,11 @@ }, "redis": { "type": "object", - "description": "In-cluster Redis for pub/sub, the Socket.IO adapter, and the idempotency/progress stores. Disable and set app.env.REDIS_URL to use a managed instance.", + "description": "In-cluster Redis. Sim requires Redis (rate limiting, pub/sub, the Socket.IO adapter, and the idempotency/progress stores). Disable only when you supply REDIS_URL for a managed instance via app.env, a pre-created app Secret, or externalSecrets.remoteRefs.app.", "properties": { "enabled": { "type": "boolean", - "description": "Deploy the bundled Redis. Skipped automatically when app.env.REDIS_URL is set or a secret-manager mode is active." + "description": "Deploy the bundled Redis. Skipped automatically when app.env.REDIS_URL is set. Setting false requires REDIS_URL from app.env, app.secrets.existingSecret, or externalSecrets.remoteRefs.app; the chart fails to render otherwise." }, "image": { "type": "object", diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index fe3bacd77c3..a08ff73f63f 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -123,7 +123,7 @@ app: # Optional: API Key Encryption (RECOMMENDED for production) # Generate with: openssl rand -hex 32 (produces the required 64-hex-char / 32-byte value). API_ENCRYPTION_KEY: "" # OPTIONAL - encrypts API keys at rest; if unset, keys are stored in plain text - REDIS_URL: "" # OPTIONAL - external Redis connection string. Overrides the bundled Redis (see the `redis:` section) and suppresses its Deployment. A REDIS_URL from a pre-created Secret or External Secrets also wins, without needing to be repeated here. + REDIS_URL: "" # REQUIRED when redis.enabled=false - external Redis connection string (redis:// or rediss://). Sim requires Redis; leave empty to use the bundled one (see the `redis:` section). When set, it overrides the bundled Redis and suppresses its Deployment. A REDIS_URL from a pre-created Secret or External Secrets also wins, without needing to be repeated here. # Email & Communication # Configure one provider — the mailer auto-detects in priority order: @@ -566,12 +566,13 @@ realtime: extraVolumes: [] extraVolumeMounts: [] -# Redis — pub/sub, the Socket.IO adapter, and the idempotency/progress stores. -# Bundled by default so a chart install matches the Docker Compose stack. -# REQUIRED once app.replicaCount or realtime.replicaCount exceeds 1: without it -# cross-pod events are silently dropped. For production prefer a managed -# instance (ElastiCache, Memorystore, Azure Cache) — set enabled: false and put -# its connection string in app.env.REDIS_URL, which then takes over. +# Redis — REQUIRED. Backs rate limiting, pub/sub, the Socket.IO adapter, OTPs, +# and the idempotency/progress stores; the app refuses to start without it. +# Bundled by default so a chart install matches the Docker Compose stack. For +# production prefer a managed instance (ElastiCache, Memorystore, Azure Cache) — +# set enabled: false and supply its connection string as app.env.REDIS_URL, in a +# pre-created app Secret, or via externalSecrets.remoteRefs.app.REDIS_URL. The +# chart fails to render when enabled: false and none of those is configured. redis: enabled: true @@ -1099,12 +1100,10 @@ serviceAccount: # if you use customMetrics) installed in the cluster. autoscaling: enabled: false - # Also scale the realtime Deployment with the same HPA settings. Multi-replica - # realtime REQUIRES Redis past one replica (Socket.IO adapter) — without it, - # cross-pod collaboration events are silently dropped. The chart bundles Redis - # by default, so this is satisfied unless you set redis.enabled=false without - # supplying app.env.REDIS_URL. Set false to - # keep realtime pinned at realtime.replicaCount while the app still scales. + # Also scale the realtime Deployment with the same HPA settings. Realtime + # bridges cross-pod collaboration events through Redis (always present: the + # chart bundles it or you supply REDIS_URL). Set false to keep realtime pinned + # at realtime.replicaCount while the app still scales. realtime: enabled: true minReplicas: 1 @@ -1955,7 +1954,7 @@ externalSecrets: ENCRYPT_ACCOUNT_TOKENS: "" # Path to MSHIP_SYSPROMPT_OVERRIDE in external store (optional) MSHIP_SYSPROMPT_OVERRIDE: "" - # Path to REDIS_URL in external store (optional) + # Path to REDIS_URL in external store (required when redis.enabled=false; overrides the bundled Redis) REDIS_URL: "" # PostgreSQL password (for internal PostgreSQL) diff --git a/packages/deployment-config/src/env-capabilities.ts b/packages/deployment-config/src/env-capabilities.ts index 9d464e1e87b..a5fdf9a5855 100644 --- a/packages/deployment-config/src/env-capabilities.ts +++ b/packages/deployment-config/src/env-capabilities.ts @@ -1171,42 +1171,6 @@ export function getRedisConfigurationError(values: EnvCapabilityValues): string return null } -/** - * The pre-required-Redis cache capability, kept only for the `sim-setup` wizard - * until its setup flow treats REDIS_URL as a core value. It is not part of - * {@link ENV_CAPABILITIES} and the app never reads it: Redis is required. - * - * @deprecated Use {@link getRedisConfigurationError}; REDIS_URL is a core configuration key. - */ -export const CACHE_CAPABILITY = defineCapability({ - strategy: 'selected', - id: 'cache', - label: 'Cache', - whenUnset: 'first-ready', - defaultProvider: { id: 'database', kind: 'built-in', label: 'Postgres' }, - providers: [ - { - id: 'redis', - label: 'Redis', - activation: { mode: 'any-present', keys: ['REDIS_URL'] }, - requires: envField('REDIS_URL', { - validation: { - kind: 'url', - protocols: REDIS_URL_PROTOCOLS, - message: 'must be a valid redis:// or rediss:// URL', - }, - }), - optionalFields: [envField('REDIS_TLS_SERVERNAME')], - validate: (values) => { - const error = getRedisConfigurationError(values) - return error?.startsWith('REDIS_TLS_SERVERNAME') - ? [{ kind: 'missing', fields: ['REDIS_TLS_SERVERNAME'], message: error }] - : [] - }, - }, - ], -} as const) - export const OCR_CAPABILITY = defineCapability({ strategy: 'selected', id: 'knowledge', diff --git a/packages/sim-setup/README.md b/packages/sim-setup/README.md index 97d576bef2b..e9a7192d629 100644 --- a/packages/sim-setup/README.md +++ b/packages/sim-setup/README.md @@ -178,7 +178,6 @@ npx sim-setup add email npx sim-setup add storage npx sim-setup add sandbox npx sim-setup add jobs -npx sim-setup add cache npx sim-setup add knowledge npx sim-setup add knowledge-embeddings npx sim-setup add chat diff --git a/packages/sim-setup/src/capability-config.test.ts b/packages/sim-setup/src/capability-config.test.ts index 92e200774f4..785e407c982 100644 --- a/packages/sim-setup/src/capability-config.test.ts +++ b/packages/sim-setup/src/capability-config.test.ts @@ -8,11 +8,9 @@ import { CAPABILITY_SETUPS, defineCapabilitySetup } from './capability-config' describe('capability setup configuration', () => { it('maps every runtime capability and provider exactly once', () => { - // The cache setup outlives the runtime capability until the wizard prompts for - // REDIS_URL as a required core value. - expect( - CAPABILITY_SETUPS.map((setup) => setup.definition.id).filter((id) => id !== 'cache') - ).toEqual(ENV_CAPABILITIES.map((capability) => capability.id)) + expect(CAPABILITY_SETUPS.map((setup) => setup.definition.id)).toEqual( + ENV_CAPABILITIES.map((capability) => capability.id) + ) for (const setup of CAPABILITY_SETUPS) { expect(Object.keys(setup.providers).sort()).toEqual( setup.definition.providers.map((provider) => provider.id).sort() diff --git a/packages/sim-setup/src/capability-config.ts b/packages/sim-setup/src/capability-config.ts index 578c22a0e90..cd672cfc154 100644 --- a/packages/sim-setup/src/capability-config.ts +++ b/packages/sim-setup/src/capability-config.ts @@ -6,7 +6,6 @@ */ import { ASYNC_JOBS_CAPABILITY, - CACHE_CAPABILITY, type CapabilityDefinition, EMAIL_CAPABILITY, ENV_CAPABILITIES, @@ -653,36 +652,6 @@ export const JOBS_SETUP = defineCapabilitySetup(ASYNC_JOBS_CAPABILITY, { optionOrder: ['database', 'trigger-dev'], }) -export const CACHE_SETUP = defineCapabilitySetup(CACHE_CAPABILITY, { - label: 'Redis cache', - message: 'Cache and realtime coordination?', - actions: {}, - defaultOption: { hint: 'built-in default' }, - providers: { - redis: { - hint: 'recommended for multiple replicas', - prompts: [ - { - type: 'field', - key: 'REDIS_URL', - input: 'text', - required: true, - defaultValue: 'redis://localhost:6379', - validate: true, - }, - { - type: 'field', - key: 'REDIS_TLS_SERVERNAME', - input: 'text', - required: true, - when: { kind: 'provider-missing-field', key: 'REDIS_TLS_SERVERNAME' }, - }, - ], - }, - }, - optionOrder: ['database', 'redis'], -}) - export const KNOWLEDGE_SETUP = defineCapabilitySetup(OCR_CAPABILITY, { label: 'Knowledge and OCR', message: 'PDF OCR provider?', @@ -972,7 +941,6 @@ export const CAPABILITY_SETUPS = [ STORAGE_SETUP, SANDBOX_SETUP, JOBS_SETUP, - CACHE_SETUP, KNOWLEDGE_SETUP, KNOWLEDGE_EMBEDDINGS_SETUP, ] as const diff --git a/packages/sim-setup/src/capability-status.test.ts b/packages/sim-setup/src/capability-status.test.ts index 497a4b7b8a4..2fdc441e452 100644 --- a/packages/sim-setup/src/capability-status.test.ts +++ b/packages/sim-setup/src/capability-status.test.ts @@ -49,7 +49,6 @@ describe('env capability status', () => { SMTP_HOST: 'localhost', TRIGGER_DEV_ENABLED: 'true', TRIGGER_PROJECT_ID: 'project', - REDIS_URL: 'not-a-url', OCR_AZURE_ENDPOINT: 'https://ocr.example.com', }) @@ -61,7 +60,6 @@ describe('env capability status', () => { missingFields: ['SMTP_PORT'], }) expect(status.features.jobs).toMatchObject({ state: 'partial', providerId: 'trigger-dev' }) - expect(status.features.cache).toMatchObject({ state: 'invalid', providerId: 'redis' }) expect(status.features.knowledge).toMatchObject({ state: 'partial', providerId: 'azure-mistral', diff --git a/packages/sim-setup/src/capability-status.ts b/packages/sim-setup/src/capability-status.ts index 44717153041..11583acbf42 100644 --- a/packages/sim-setup/src/capability-status.ts +++ b/packages/sim-setup/src/capability-status.ts @@ -6,7 +6,6 @@ */ import { ASYNC_JOBS_CAPABILITY, - CACHE_CAPABILITY, EMAIL_CAPABILITY, type EnvCapabilityConfigurationError, type EnvCapabilityValues, @@ -72,10 +71,6 @@ export interface JobsCapabilityStatus extends FeatureStatusBase<'jobs'> { providerId: 'database' | 'trigger-dev' } -export interface CacheCapabilityStatus extends FeatureStatusBase<'cache'> { - providerId: 'database' | 'redis' -} - export interface KnowledgeCapabilityStatus extends FeatureStatusBase<'knowledge'> { providerId: 'local' | 'mistral' | 'azure-mistral' | null } @@ -107,7 +102,6 @@ interface FeatureStatusById { storage: StorageCapabilityStatus sandbox: SandboxCapabilityStatus jobs: JobsCapabilityStatus - cache: CacheCapabilityStatus knowledge: KnowledgeCapabilityStatus 'knowledge-embeddings': KnowledgeEmbeddingsCapabilityStatus llm: LlmCapabilityStatus @@ -318,29 +312,6 @@ function inspectJobs(values: EnvCapabilityValues): JobsCapabilityStatus { } } -function inspectCache(values: EnvCapabilityValues): CacheCapabilityStatus { - const inspection = inspectCapability(CACHE_CAPABILITY, values) - if (!inspection.error && inspection.providerId) { - return { - ...featureMetadata('cache'), - state: inspection.providerId === 'database' ? 'default' : 'configured', - providerId: inspection.providerId, - } - } - - if (!inspection.error) { - throw new Error('Cache inspection failed without a configuration error') - } - const redis = inspection.providers.find((provider) => provider.id === 'redis') - const state: CapabilityStatusIssue['state'] = redis?.state === 'invalid' ? 'invalid' : 'partial' - return { - ...featureMetadata('cache'), - state, - providerId: 'redis', - issue: issue(state, inspection.error), - } -} - function inspectKnowledge(values: EnvCapabilityValues): KnowledgeCapabilityStatus { const inspection = inspectCapability(OCR_CAPABILITY, values) if (!inspection.error && inspection.providerId) { @@ -486,7 +457,6 @@ const FEATURE_STATUS_BUILDERS = { storage: inspectStorage, sandbox: inspectSandbox, jobs: inspectJobs, - cache: inspectCache, knowledge: inspectKnowledge, 'knowledge-embeddings': inspectKnowledgeEmbeddings, llm: inspectLlm, @@ -500,7 +470,6 @@ export function buildEnvCapabilityStatus(values: EnvCapabilityValues): EnvCapabi storage: FEATURE_STATUS_BUILDERS.storage(values), sandbox: FEATURE_STATUS_BUILDERS.sandbox(values), jobs: FEATURE_STATUS_BUILDERS.jobs(values), - cache: FEATURE_STATUS_BUILDERS.cache(values), knowledge: FEATURE_STATUS_BUILDERS.knowledge(values), 'knowledge-embeddings': FEATURE_STATUS_BUILDERS['knowledge-embeddings'](values), llm: FEATURE_STATUS_BUILDERS.llm(values), diff --git a/packages/sim-setup/src/checks.ts b/packages/sim-setup/src/checks.ts index 368a8227066..39840a89585 100644 --- a/packages/sim-setup/src/checks.ts +++ b/packages/sim-setup/src/checks.ts @@ -1,9 +1,9 @@ import { ASYNC_JOBS_CAPABILITY, - CACHE_CAPABILITY, CORE_CONFIGURATION_KEYS, EMAIL_CAPABILITY, EnvCapabilityConfigurationError, + getRedisConfigurationError, inspectCapability, inspectOAuthClientCapability, OAUTH_CLIENT_CAPABILITIES, @@ -121,6 +121,7 @@ const REQUIRED_KEYS: Partial> = { 'BETTER_AUTH_SECRET', 'INTERNAL_API_SECRET', 'NEXT_PUBLIC_APP_URL', + 'REDIS_URL', ], db: ['DATABASE_URL'], // Compose's single root .env only carries the secrets that have no safe @@ -264,6 +265,17 @@ function checkSchema(ctx: CheckContext): Finding[] { }) } } + if (key === 'REDIS_URL') { + const redisError = getRedisConfigurationError(file.vars) + if (redisError) { + findings.push({ + group: 'schema', + status: 'fail', + message: `${rel(file)}: ${redisError}`, + fix: `correct the Redis settings in ${rel(file)}`, + }) + } + } } if (missing.length === 0 && findings.every((f) => !f.message.startsWith(rel(file)))) { findings.push({ @@ -347,10 +359,6 @@ function checkCoherence(ctx: CheckContext): Finding[] { command: getSetupCommand(ASYNC_JOBS_CAPABILITY.id), resolve: () => requireCapability(ASYNC_JOBS_CAPABILITY, sim.vars), }, - { - command: getSetupCommand(CACHE_CAPABILITY.id), - resolve: () => requireCapability(CACHE_CAPABILITY, sim.vars), - }, { command: getSetupCommand(SANDBOX_CAPABILITY.id), resolve: () => { @@ -674,10 +682,21 @@ async function checkDatabase(sim: EnvFile): Promise { } async function checkRedis(sim: EnvFile): Promise { - const inspection = inspectCapability(CACHE_CAPABILITY, sim.vars) - if (inspection.error || inspection.providerId !== 'redis') return [] const redisUrl = sim.vars.get('REDIS_URL') - if (!redisUrl) throw new Error('Redis resolved as ready without REDIS_URL') + // A Compose install's REDIS_URL comes from the service definition, not the root .env. + if (!redisUrl && sim.target === 'root') { + return [{ group: 'live', status: 'skip', message: 'redis: REDIS_URL is set by Compose' }] + } + if (!redisUrl || getRedisConfigurationError(sim.vars)) { + return [ + { + group: 'live', + status: 'fail', + message: 'redis: REDIS_URL is missing or invalid', + fix: 'set REDIS_URL to a reachable redis:// or rediss:// URL (npx sim-setup can manage a Redis container)', + }, + ] + } const ping = await redisPing(redisUrl) return [ ping.ok @@ -686,7 +705,7 @@ async function checkRedis(sim: EnvFile): Promise { group: 'live', status: 'fail', message: `redis unreachable: ${ping.error}`, - fix: 'fix REDIS_URL or remove it (optional for single-replica)', + fix: 'start Redis (npx sim-setup can manage a Redis container) or fix REDIS_URL', }, ] } diff --git a/packages/sim-setup/src/configuration-sources.ts b/packages/sim-setup/src/configuration-sources.ts index b0c6aa483be..a07565472b1 100644 --- a/packages/sim-setup/src/configuration-sources.ts +++ b/packages/sim-setup/src/configuration-sources.ts @@ -96,6 +96,7 @@ const SPLIT_REQUIRED_KEYS = { 'BETTER_AUTH_SECRET', 'INTERNAL_API_SECRET', 'NEXT_PUBLIC_APP_URL', + 'REDIS_URL', ], db: ['DATABASE_URL'], } as const diff --git a/packages/sim-setup/src/env-files.ts b/packages/sim-setup/src/env-files.ts index 1cbbe9ed0da..6183c821351 100644 --- a/packages/sim-setup/src/env-files.ts +++ b/packages/sim-setup/src/env-files.ts @@ -28,6 +28,7 @@ export const SHARED_KEYS = [ 'INTERNAL_API_SECRET', 'BETTER_AUTH_URL', 'NEXT_PUBLIC_APP_URL', + 'REDIS_URL', ] as const export const SECRET_KEYS = [ diff --git a/packages/sim-setup/src/index.ts b/packages/sim-setup/src/index.ts index 6177b110e16..73c76f6e328 100644 --- a/packages/sim-setup/src/index.ts +++ b/packages/sim-setup/src/index.ts @@ -7,7 +7,7 @@ import { theme } from './theme' import { SETUP_VERSION } from './version' const SETUP_FEATURES = - 'email | storage | sandbox | jobs | cache | knowledge | knowledge-embeddings | chat | llm | integration ' + 'email | storage | sandbox | jobs | knowledge | knowledge-embeddings | chat | llm | integration ' const USAGE = `Usage: sim-setup [--quick] [--dir ] [--mode compose|dev|k8s] diff --git a/packages/sim-setup/src/modes/dev.ts b/packages/sim-setup/src/modes/dev.ts index 6f0dd0663eb..fdab5cd329f 100644 --- a/packages/sim-setup/src/modes/dev.ts +++ b/packages/sim-setup/src/modes/dev.ts @@ -61,16 +61,6 @@ async function runMigrations(dsn: string): Promise { if (!proceed) throw new Error(`aborted: db:migrate failed:\n${error}`) } -async function promptRedis(detection: Detection, existing?: string): Promise { - const wants = await p.confirm({ - message: - 'Configure Redis? (powers live Chat status and table events; storage falls back to Postgres)', - initialValue: true, - }) - if (!wants) return null - return resolveRedis(detection, existing) -} - export async function runDevMode( detection: Detection, quick: boolean @@ -114,18 +104,13 @@ export async function runDevMode( Object.assign(values, chatFlagValues(copilotKey)) Object.assign(values, await promptLlmKeys(detection, !quick)) - // Redis is set up in every mode, quick included. Storage falls back to - // PostgreSQL without it, but the pub/sub channels (live Chat task-status, - // table events) have no fallback — skipping it silently produces an install - // where live updates never arrive. Quick configures it with no questions at - // all; custom keeps the opt-out and the where-should-it-live ladder. + // Redis is required: quick adopts or starts one with no questions, custom + // walks the where-should-it-live ladder. const redisUrl = quick ? await ensureRedis(detection, simAfter.vars.get('REDIS_URL')) - : await promptRedis(detection, simAfter.vars.get('REDIS_URL')) - if (redisUrl) { - values.REDIS_URL = redisUrl - writeEnvValues('realtime', { REDIS_URL: redisUrl }) - } + : await resolveRedis(detection, simAfter.vars.get('REDIS_URL')) + values.REDIS_URL = redisUrl + writeEnvValues('realtime', { REDIS_URL: redisUrl }) const stagedVars = new Map(simAfter.vars) for (const [key, value] of Object.entries(values)) stagedVars.set(key, value) diff --git a/packages/sim-setup/src/redis.ts b/packages/sim-setup/src/redis.ts index 26bbdc81c1c..ab61608e97f 100644 --- a/packages/sim-setup/src/redis.ts +++ b/packages/sim-setup/src/redis.ts @@ -73,7 +73,7 @@ async function promptRedisUrl(existing?: string): Promise { }) if (!(await pingWithSpinner(url, 'Pinging Redis…'))) { throw new SetupError(`Redis at ${url} is not answering.`, [ - 'fix the URL and re-run, or skip Redis — single-instance runs fine without it', + 'fix the URL and re-run — Sim requires a reachable Redis', ]) } return url @@ -81,13 +81,10 @@ async function promptRedisUrl(existing?: string): Promise { /** * Non-interactive Redis for quick mode: adopt whatever already answers, else - * start the managed container. Quick's contract is sensible defaults with no - * questions, and Redis is not optional enough to skip — pub/sub (live Chat - * status, table events) has no PostgreSQL fallback. Returns null only when - * nothing answers and Docker is unavailable, so the caller can warn instead of - * failing the whole setup. + * start the managed container. Redis is required, so when nothing answers and + * Docker is unavailable setup fails with the steps to provide one. */ -export async function ensureRedis(detection: Detection, existing?: string): Promise { +export async function ensureRedis(detection: Detection, existing?: string): Promise { if (existing && (await redisPing(existing)).ok) return existing if (detection.redisPortOpen && (await redisPing(LOCAL_URL)).ok) { p.log.step(`Using the Redis already on :6379`) @@ -102,10 +99,10 @@ export async function ensureRedis(detection: Detection, existing?: string): Prom } } if (!(await ensureDocker(false))) { - p.log.warn( - 'Docker is unavailable, so Redis was not configured — live Chat status and table events will not stream.' - ) - return null + throw new SetupError('Sim requires Redis, and none is reachable.', [ + 'start Docker so the wizard can run a Redis container, then re-run it', + `or start Redis yourself and set REDIS_URL in apps/sim/.env (e.g. ${LOCAL_URL})`, + ]) } return startManagedRedis(detection) } diff --git a/packages/sim-setup/src/setup-status.test.ts b/packages/sim-setup/src/setup-status.test.ts index 73cacf08eb9..f62a3f3a1ba 100644 --- a/packages/sim-setup/src/setup-status.test.ts +++ b/packages/sim-setup/src/setup-status.test.ts @@ -59,4 +59,22 @@ describe('setup status', () => { expect(output).toContain('SPOTIFY_CLIENT_SECRET') expect(output).not.toContain('partial-client-value') }) + it.each([ + ['missing', { REDIS_URL: '' }, { reason: 'missing' }], + [ + 'not a Redis URL', + { REDIS_URL: 'http://localhost:6379' }, + { detail: expect.stringContaining('must be a valid redis://') }, + ], + [ + 'a rediss:// IP without a TLS server name', + { REDIS_URL: 'rediss://10.0.0.5:6379' }, + { detail: expect.stringContaining('REDIS_TLS_SERVERNAME is required') }, + ], + ])('fails when Redis is %s', (_, overrides, issue) => { + const report = buildSetupStatusReport(source({ ...CORE_VALUES, ...overrides })) + + expect(report.failed).toBe(true) + expect(report.coreIssues).toEqual([expect.objectContaining({ key: 'REDIS_URL', ...issue })]) + }) }) diff --git a/packages/sim-setup/src/setup-status.ts b/packages/sim-setup/src/setup-status.ts index cdc2b6b35bc..451a7b6c949 100644 --- a/packages/sim-setup/src/setup-status.ts +++ b/packages/sim-setup/src/setup-status.ts @@ -1,5 +1,6 @@ import { type EnvCapabilityValues, + getRedisConfigurationError, hasEnvCapabilityValue, } from '@sim/deployment-config/env-capabilities' import { @@ -21,7 +22,13 @@ type FeatureStatus = EnvCapabilityFeatureStatuses[keyof EnvCapabilityFeatureStat interface CoreConfigurationIssue { key: (typeof REQUIRED_APP_KEYS)[number] - reason: 'missing' | 'placeholder' | 'invalid secret' | 'invalid URL' + reason: + | 'missing' + | 'placeholder' + | 'invalid secret' + | 'invalid URL' + | 'invalid Redis configuration' + detail?: string } export type SetupStatusSource = Omit @@ -63,6 +70,11 @@ function inspectCoreConfiguration(values: EnvCapabilityValues): CoreConfiguratio } catch { issues.push({ key, reason: 'invalid URL' }) } + } else if (key === 'REDIS_URL') { + const redisError = getRedisConfigurationError(values) + if (redisError) { + issues.push({ key, reason: 'invalid Redis configuration', detail: redisError }) + } } } return issues @@ -98,8 +110,6 @@ function featureDetail(feature: FeatureStatus): string { return feature.providerId ? titleCase(feature.providerId) : 'Not configured' case 'jobs': return feature.providerId === 'database' ? 'Database queue (default)' : 'Trigger.dev' - case 'cache': - return feature.providerId === 'database' ? 'Postgres (default)' : 'Redis' case 'knowledge': if (feature.providerId === 'local') return 'Local parser (default)' if (feature.providerId === 'azure-mistral') return 'Azure Mistral OCR' @@ -265,7 +275,7 @@ export function renderSetupStatusReport(report: SetupStatusReport): string { lines.push(` ${glyph.pass} All ${REQUIRED_APP_KEYS.length} required app values are present`) } else { for (const coreIssue of report.coreIssues) { - lines.push(` ${glyph.fail} ${coreIssue.key}: ${coreIssue.reason}`) + lines.push(` ${glyph.fail} ${coreIssue.key}: ${coreIssue.detail ?? coreIssue.reason}`) } for (const configurationIssue of source.configurationIssues ?? []) { lines.push(` ${glyph.fail} ${configurationIssue}`) diff --git a/packages/sim-setup/src/wizard.ts b/packages/sim-setup/src/wizard.ts index 05fae742ae8..d2f869fbbba 100644 --- a/packages/sim-setup/src/wizard.ts +++ b/packages/sim-setup/src/wizard.ts @@ -150,7 +150,7 @@ export async function runWizard(flags: WizardFlags): Promise { { value: 'custom', label: 'Custom', - hint: 'every option: redis, trigger.dev, image variants', + hint: 'every option: where Redis lives, trigger.dev, image variants', }, ], initialValue: 'quick', diff --git a/packages/testing/src/mocks/mcp-pubsub.mock.ts b/packages/testing/src/mocks/mcp-pubsub.mock.ts index 8db7097e6e2..24ce0a73ab8 100644 --- a/packages/testing/src/mocks/mcp-pubsub.mock.ts +++ b/packages/testing/src/mocks/mcp-pubsub.mock.ts @@ -26,9 +26,8 @@ export const mcpPubsubMockFns = { } /** - * Static mock module for `@/lib/mcp/pubsub`. `mcpPubSub` is the non-null (server-side) adapter - * whose methods are the fns above. Tests that need the `null` adapter (no Redis, browser) keep a - * local `{ mcpPubSub: null }` factory. + * Static mock module for `@/lib/mcp/pubsub`. `mcpPubSub` is the adapter whose methods are the fns + * above. * * @example * ```ts diff --git a/packages/testing/src/mocks/mothership-chat-status.mock.ts b/packages/testing/src/mocks/mothership-chat-status.mock.ts index 9785be123ed..33524b72b65 100644 --- a/packages/testing/src/mocks/mothership-chat-status.mock.ts +++ b/packages/testing/src/mocks/mothership-chat-status.mock.ts @@ -25,8 +25,8 @@ export const mothershipChatStatusMockFns = { } /** - * Static mock module for `@/lib/mothership/chat-status`. `chatPubSub` is the non-null - * (Redis/EventEmitter available) adapter whose methods are the fns above. + * Static mock module for `@/lib/mothership/chat-status`. `chatPubSub` is the adapter whose + * methods are the fns above. * * @example * ```ts diff --git a/packages/testing/src/mocks/redis.mock.ts b/packages/testing/src/mocks/redis.mock.ts index 947165d11f0..7ea7a66aac4 100644 --- a/packages/testing/src/mocks/redis.mock.ts +++ b/packages/testing/src/mocks/redis.mock.ts @@ -1,5 +1,38 @@ +import { toError } from '@sim/utils/errors' import { vi } from 'vitest' +type MockRedisCommand = (...args: unknown[]) => unknown + +/** + * A pipeline over a mock client: each queued command runs the client's own mock of the same name + * on `exec`, in order, so tests that stub (or make stateful) a client command see pipelined calls + * too. `exec` resolves the ioredis `[error, result]` tuple per command. + */ +function createMockPipeline(client: Record) { + const queued: Array<() => unknown> = [] + const pipeline: Record = { + exec: vi.fn(async () => { + const results: Array<[Error | null, unknown]> = [] + for (const run of queued.splice(0)) { + try { + results.push([null, await run()]) + } catch (error) { + results.push([toError(error), null]) + } + } + return results + }), + } + for (const [name, command] of Object.entries(client)) { + if (typeof command !== 'function' || name in pipeline || name === 'pipeline') continue + pipeline[name] = (...args: unknown[]) => { + queued.push(() => (command as MockRedisCommand)(...args)) + return pipeline + } + } + return pipeline +} + /** * Creates a mock Redis client with common operations. * @@ -19,7 +52,7 @@ export function createMockRedis() { * the handlers the client registered. */ const listeners = new Map void>>() - return { + const client = { // Hash operations hset: vi.fn().mockResolvedValue(1), hget: vi.fn().mockResolvedValue(null), @@ -94,4 +127,9 @@ export function createMockRedis() { // Status status: 'ready', } + + return Object.assign(client, { + /** Queues commands against this client's own mocks; see {@link createMockPipeline}. */ + pipeline: vi.fn(() => createMockPipeline(client)), + }) } diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 4fdbbe06774..d77f15a4452 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -194,7 +194,6 @@ "apps/sim/lib/billing/subscriptions/utils.ts": 7, "apps/sim/lib/content/mdx.tsx": 14, "apps/sim/lib/content/registry-factory.ts": 3, - "apps/sim/lib/core/config/redis.ts": 1, "apps/sim/lib/core/idempotency/service.ts": 6, "apps/sim/lib/core/security/redaction.ts": 6, "apps/sim/lib/core/utils/display-filters.ts": 11, @@ -1313,10 +1312,8 @@ "apps/sim/lib/concurrency/singleflight.test.ts": 3, "apps/sim/lib/core/async-jobs/backends/trigger-dev.integration.ts": 1, "apps/sim/lib/core/network/gateway.server.test.ts": 3, - "apps/sim/lib/core/rate-limiter/provider-admission.test.ts": 1, "apps/sim/lib/core/rate-limiter/provider-capacity-state.test.ts": 1, - "apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts": 9, - "apps/sim/lib/core/rate-limiter/storage/factory.ts": 1, + "apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts": 7, "apps/sim/lib/core/resource-scope.server.ts": 3, "apps/sim/lib/core/security/guarded-request-fetch.server.test.ts": 3, "apps/sim/lib/core/security/input-validation.test.ts": 2, @@ -1400,7 +1397,6 @@ "apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts": 3, "apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts": 22, "apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts": 2, - "apps/sim/lib/knowledge/__integration__/provider-admission.integration.ts": 1, "apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts": 3, "apps/sim/lib/knowledge/__integration__/providers-live.integration.ts": 10, "apps/sim/lib/knowledge/__integration__/reranker-admission.integration.ts": 1, @@ -1494,7 +1490,7 @@ "apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts": 4, "apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts": 2, "apps/sim/lib/mothership/assistant/tool-policy.test.ts": 2, - "apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts": 10, + "apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts": 9, "apps/sim/lib/mothership/auth/internal.ts": 1, "apps/sim/lib/mothership/billing/service-store.integration.ts": 3, "apps/sim/lib/mothership/chat/application/fork.ts": 2, @@ -1514,12 +1510,8 @@ "apps/sim/lib/mothership/request/handlers/tool.ts": 1, "apps/sim/lib/mothership/request/lifecycle/run.test.ts": 1, "apps/sim/lib/mothership/request/lifecycle/start.ts": 1, - "apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts": 4, - "apps/sim/lib/mothership/request/session/recovery-storm.integration.ts": 1, "apps/sim/lib/mothership/request/session/recovery.test.ts": 1, "apps/sim/lib/mothership/request/session/replay-budget.integration.ts": 2, - "apps/sim/lib/mothership/request/session/replay-gap.integration.ts": 2, - "apps/sim/lib/mothership/request/session/stream-recovery.integration.ts": 4, "apps/sim/lib/mothership/request/tools/executor.test.ts": 1, "apps/sim/lib/mothership/resources/client-persistence-queue.ts": 1, "apps/sim/lib/mothership/resources/search.test.ts": 2, diff --git a/scripts/check-source-conventions.baseline.json b/scripts/check-source-conventions.baseline.json index bc0c41b2690..22dd1e4f682 100644 --- a/scripts/check-source-conventions.baseline.json +++ b/scripts/check-source-conventions.baseline.json @@ -179,9 +179,7 @@ "apps/sim/lib/mcp/error-diagnostics.test.ts": 1, "apps/sim/lib/mcp/oauth/revoke.test.ts": 1, "apps/sim/lib/mcp/oauth/storage.test.ts": 1, - "apps/sim/lib/mcp/storage/factory.ts": 3, - "apps/sim/lib/mcp/storage/memory-cache.test.ts": 1, - "apps/sim/lib/mcp/storage/memory-cache.ts": 1, + "apps/sim/lib/mcp/storage/factory.ts": 2, "apps/sim/lib/mcp/storage/redis-cache.ts": 1, "apps/sim/lib/mcp/tool-validation.test.ts": 2, "apps/sim/lib/mcp/workflow-mcp-sync.ts": 2, @@ -233,7 +231,6 @@ "apps/sim/lib/mothership/tool-executor/register-handlers.ts": 6, "apps/sim/lib/mothership/tools/client/run-tool-execution.test.ts": 1, "apps/sim/lib/mothership/tools/server/files/edit-content.ts": 3, - "apps/sim/lib/mothership/tools/server/files/intent-store.test.ts": 1, "apps/sim/lib/mothership/tools/server/files/workspace-file.ts": 3, "apps/sim/lib/mothership/tools/server/user/get-credentials.test.ts": 1, "apps/sim/lib/mothership/tools/workflow-tools.test.ts": 1, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index 0020779f67a..ce22c5a35ba 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -64,7 +64,6 @@ "apps/realtime/src/env.ts#isDev", "apps/realtime/src/env.ts#isHosted", "apps/realtime/src/env.ts#isTest", - "apps/realtime/src/rooms/index.ts#RoomState", "apps/sim/app/(auth)/oauth/consent/consent-view.tsx#OAuthConsentRefusal", "apps/sim/app/(interfaces)/chat/components/message/message.tsx#ChatAttachment", "apps/sim/app/(landing)/comparisons/comparison-sections.ts#ComparisonRowDef", @@ -4158,25 +4157,14 @@ "apps/sim/lib/core/rate-limiter/index.ts#resetHostedKeyRateLimiter", "apps/sim/lib/core/rate-limiter/index.ts#toTokenBucketConfig", "apps/sim/lib/core/rate-limiter/route-helpers.ts#DEFAULT_PUBLIC_IP_ROUTE_LIMIT", - "apps/sim/lib/core/rate-limiter/storage/factory.ts#getAdapterType", - "apps/sim/lib/core/rate-limiter/storage/factory.ts#setStorageAdapter", - "apps/sim/lib/core/rate-limiter/storage/index.ts#DbTokenBucket", "apps/sim/lib/core/rate-limiter/storage/index.ts#RedisTokenBucket", - "apps/sim/lib/core/rate-limiter/storage/index.ts#getAdapterType", "apps/sim/lib/core/rate-limiter/storage/index.ts#resetStorageAdapter", - "apps/sim/lib/core/rate-limiter/storage/index.ts#setStorageAdapter", "apps/sim/lib/core/security/egress/validate.ts#EgressValidationFailure", "apps/sim/lib/core/security/egress/validate.ts#EgressValidationSuccess", "apps/sim/lib/core/security/otp.ts#OTP_EXPIRY_MS", "apps/sim/lib/core/security/otp.ts#OTP_EXPIRY_SECONDS", "apps/sim/lib/core/security/redaction.ts#TRUNCATED_MARKER", "apps/sim/lib/core/security/redaction.ts#isLargeDataKey", - "apps/sim/lib/core/storage/index.ts#isDatabaseStorage", - "apps/sim/lib/core/storage/index.ts#isRedisStorage", - "apps/sim/lib/core/storage/index.ts#requireRedis", - "apps/sim/lib/core/storage/storage.ts#isDatabaseStorage", - "apps/sim/lib/core/storage/storage.ts#isRedisStorage", - "apps/sim/lib/core/storage/storage.ts#requireRedis", "apps/sim/lib/core/utils/multipart.ts#MultipartFilePart", "apps/sim/lib/core/utils/response-format.ts#hasResponseFormatSelection", "apps/sim/lib/core/utils/smooth-bottom-chase.ts#CHASE_REST_GAP", @@ -5862,7 +5850,6 @@ "packages/sim-setup/src/arguments.ts#LIFECYCLE_COMMANDS", "packages/sim-setup/src/arguments.ts#LifecycleCommand", "packages/sim-setup/src/arguments.ts#WizardMode", - "packages/sim-setup/src/capability-config.ts#CACHE_SETUP", "packages/sim-setup/src/capability-config.ts#CapabilitySetupId", "packages/sim-setup/src/capability-config.ts#KNOWLEDGE_SETUP", "packages/sim-setup/src/capability-config.ts#OAUTH_CLIENT_SETUP_FIELDS", @@ -5879,7 +5866,6 @@ "packages/sim-setup/src/capability-setup.ts#getCapabilitySetupProviderMissingFields", "packages/sim-setup/src/capability-setup.ts#markCurrentlyUsed", "packages/sim-setup/src/capability-setup.ts#resolveCurrentCapabilitySetupOptionId", - "packages/sim-setup/src/capability-status.ts#CacheCapabilityStatus", "packages/sim-setup/src/capability-status.ts#CapabilityStatusIssue", "packages/sim-setup/src/capability-status.ts#CapabilityStatusState", "packages/sim-setup/src/capability-status.ts#EmailCapabilityStatus", diff --git a/scripts/test-patterns-baseline.json b/scripts/test-patterns-baseline.json index a21ba5665e6..5856fb99cf6 100644 --- a/scripts/test-patterns-baseline.json +++ b/scripts/test-patterns-baseline.json @@ -47,7 +47,6 @@ "local-factory\tapps/sim/ee/sso/components/sso-settings.test.tsx\t@sim/emcn", "local-factory\tapps/sim/executor/execution/block-executor.test.ts\t@/lib/logs/execution/pii-redaction", "local-factory\tapps/sim/executor/utils/output-filter.test.ts\t@/blocks", - "local-factory\tapps/sim/lib/mcp/workflow-mcp-sync.test.ts\t@/lib/mcp/pubsub", "local-factory\tapps/sim/lib/mothership/chat/application/use-cases.test.ts\t@/lib/core/application/workspace-authorization", "local-factory\tapps/sim/lib/mothership/chat/payload.test.ts\t@/tools/utils", "local-factory\tapps/sim/lib/mothership/request/application/recover-stream.test.ts\t@/lib/core/application/workspace-authorization", @@ -85,7 +84,6 @@ "real-wait\tapps/desktop/src/main/terminal/tmux.test.ts\tsleep() #1", "real-wait\tapps/desktop/src/main/terminal/tmux.test.ts\tsleep() #2", "real-wait\tapps/desktop/src/main/terminal/tmux.test.ts\tsleep() #3", - "real-wait\tapps/realtime/src/handlers/file-doc-store.test.ts\tsleep() #1", "real-wait\tapps/realtime/src/handlers/file-doc.redis-store.test.ts\tsetTimeout promise #1", "real-wait\tapps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.test.tsx\tsetTimeout promise #1", "real-wait\tapps/sim/app/api/v2/chat/route.test.ts\tsleep() #1", @@ -172,7 +170,6 @@ "real-wait\tapps/sim/lib/knowledge/transfer/export-archive.test.ts\tsleep() #1", "real-wait\tapps/sim/lib/knowledge/transfer/export-archive.test.ts\tsleep() #2", "real-wait\tapps/sim/lib/mcp/oauth/storage.test.ts\tsetTimeout promise #1", - "real-wait\tapps/sim/lib/mcp/storage/memory-cache.test.ts\tsleep() #1", "real-wait\tapps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts\tsleep() #1", "real-wait\tapps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts\tsleep() #2", "real-wait\tapps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts\tsleep() #3", @@ -180,7 +177,6 @@ "real-wait\tapps/sim/lib/mothership/tools/client/local-filesystem.test.ts\tsleep() #1", "real-wait\tapps/sim/lib/mothership/tools/client/local-filesystem.test.ts\tsleep() #2", "real-wait\tapps/sim/lib/mothership/tools/client/run-tool-execution.test.ts\tsleep() #1", - "real-wait\tapps/sim/lib/mothership/tools/server/files/intent-store.test.ts\tsleep() #1", "real-wait\tapps/sim/lib/table/import.test.ts\tsleep() #1", "real-wait\tpackages/audit/src/log.test.ts\tsleep() #1", "real-wait\tpackages/sim-cli/src/commands/protocol/files-get.test.ts\tsleep() #1", From bae8785b09738d49b37be517a8fdaba41fe243a0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 18:15:57 -0800 Subject: [PATCH 3/6] fix(redis): resolve rate-limit storage lazily and address review findings --- .claude/rules/sim-testing.md | 5 +- .cursor/rules/sim-testing.mdc | 5 +- .github/workflows/checks.yml | 2 +- .../docs/platform/self-hosting/redis.mdx | 31 +++- .../scripts/fixtures/content-baseline.json | 15 +- .../src/handlers/file-doc-store.test.ts | 28 +++ apps/realtime/src/handlers/file-doc-store.ts | 22 ++- .../src/handlers/file-doc.redis-store.test.ts | 6 +- .../src/handlers/fixtures/redis.fixture.ts | 13 +- .../copilot/api-keys/validate/route.test.ts | 2 +- apps/sim/app/api/link-preview/route.ts | 5 +- .../lib/billing/core/usage-segment-cache.ts | 16 +- .../lib/core/config/env-capabilities.test.ts | 7 - .../lib/core/rate-limiter/hosted-key/queue.ts | 12 +- .../rate-limiter/hosted-key/rate-limiter.ts | 13 +- .../sim/lib/core/rate-limiter/rate-limiter.ts | 13 +- apps/sim/lib/desktop/executor/presence.ts | 2 +- apps/sim/lib/favicon/resolve.server.ts | 5 +- .../lib/logs/execution/progress-markers.ts | 6 +- .../persistence/tool-confirm/index.ts | 24 ++- .../tool-confirm/tool-confirm.test.ts | 45 +++++ apps/sim/lib/oauth/terminal-errors.ts | 6 +- apps/sim/lib/realtime/event-log.ts | 2 +- .../uploads/utils/user-file-base64.server.ts | 6 +- .../webhooks/polling/admission-refusals.ts | 4 +- apps/sim/vitest.integration.setup.ts | 14 +- helm/sim/README.md | 10 +- helm/sim/templates/_helpers.tpl | 68 +++++++- helm/sim/templates/external-secret-app.yaml | 2 +- helm/sim/templates/secrets-app.yaml | 2 +- helm/sim/tests/redis_test.yaml | 159 +++++++++++++++++- helm/sim/values.yaml | 8 +- packages/sim-setup/src/redis.ts | 7 +- scripts/test-integration.ts | 2 +- 34 files changed, 479 insertions(+), 88 deletions(-) diff --git a/.claude/rules/sim-testing.md b/.claude/rules/sim-testing.md index a3a736b0501..f0a8bb1117b 100644 --- a/.claude/rules/sim-testing.md +++ b/.claude/rules/sim-testing.md @@ -46,7 +46,10 @@ contracts, and demonstrated regressions. `vi.mock` in the file. - `TEST_DATABASE_URL` must be loopback and its database name must contain a `test` segment (`sim_test`); `TEST_REDIS_URL` must be loopback. `packages/db/testing/test-infrastructure.ts` owns - those checks. Isolate with a unique schema or generated IDs, and clean up in `afterAll`. + those checks. `apps/sim` integration setup runs `FLUSHDB` before every file, so its + `TEST_REDIS_URL` must also select a non-zero database (`redis://127.0.0.1:6379/15`); never + point it at a Redis holding data you want to keep. Isolate with a unique schema or generated + IDs, and clean up in `afterAll`. - Integration files run one at a time against one shared database. A new `*.integration.ts` is picked up by CI with no workflow change, and the run writes the JSON report configured in `vitest.shared.ts`, which CI uploads. Never exclude a failing suite from the run in diff --git a/.cursor/rules/sim-testing.mdc b/.cursor/rules/sim-testing.mdc index 46778c442a0..9780636a28c 100644 --- a/.cursor/rules/sim-testing.mdc +++ b/.cursor/rules/sim-testing.mdc @@ -43,7 +43,10 @@ contracts, and demonstrated regressions. `vi.mock` in the file. - `TEST_DATABASE_URL` must be loopback and its database name must contain a `test` segment (`sim_test`); `TEST_REDIS_URL` must be loopback. `packages/db/testing/test-infrastructure.ts` owns - those checks. Isolate with a unique schema or generated IDs, and clean up in `afterAll`. + those checks. `apps/sim` integration setup runs `FLUSHDB` before every file, so its + `TEST_REDIS_URL` must also select a non-zero database (`redis://127.0.0.1:6379/15`); never + point it at a Redis holding data you want to keep. Isolate with a unique schema or generated + IDs, and clean up in `afterAll`. - Integration files run one at a time against one shared database. A new `*.integration.ts` is picked up by CI with no workflow change, and the run writes the JSON report configured in `vitest.shared.ts`, which CI uploads. Never exclude a failing suite from the run in diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7254a04149d..0ada1aeb1b1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -122,7 +122,7 @@ jobs: --health-retries 10 env: TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test - TEST_REDIS_URL: redis://127.0.0.1:6379 + TEST_REDIS_URL: redis://127.0.0.1:6379/15 DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test BETTER_AUTH_SECRET: oauth-postgres-ci-secret-at-least-32-characters NEXT_PUBLIC_APP_URL: https://test.sim.ai diff --git a/apps/docs/content/docs/platform/self-hosting/redis.mdx b/apps/docs/content/docs/platform/self-hosting/redis.mdx index a1390af8f2f..90413b10a52 100644 --- a/apps/docs/content/docs/platform/self-hosting/redis.mdx +++ b/apps/docs/content/docs/platform/self-hosting/redis.mdx @@ -22,13 +22,30 @@ There are no PostgreSQL or in-memory fallbacks: each of these runs only on Redis | Execution progress markers | Event buffers and progress for running executions | | Distributed execution limits | Per-deployment concurrency and provider admission limits | | CLI pairing-code approval store | API-key login through pairing; OAuth login uses a separate path | +| OAuth flow state | Short-lived state for in-progress OAuth, MCP OAuth, and Slack OAuth connections | +| Pending webhook verifications | Challenge state while a provider verifies a newly registered webhook | +| Distributed locks | Leases that keep a job, schedule, or stream on one replica at a time | | Collaborative document store (realtime) | Shared state for collaboratively edited documents | The app checks `REDIS_URL` when it starts and refuses to boot when it is missing or is not a `redis://` or `rediss://` URL. `next build` does not need Redis, so an image builds without it; the running server does not. -Everything Sim keeps in Redis is cache, coordination state, or an in-flight event — never committed data, which lives in PostgreSQL and object storage. Persistence is therefore not required. Losing or restarting the instance is not free, though: cancellation markers and the cross-pod half of execution streaming live here, so active runs stop streaming and a cancellation issued across the gap may not land. Completed work is unaffected. +Everything Sim keeps in Redis is cache, coordination state, or an in-flight event — never committed data, which lives in PostgreSQL and object storage. Persistence is therefore not required. Losing or restarting the instance is not free, though: + +- Active runs stop streaming, and a cancellation issued across the gap may not land, because cancellation markers and the cross-pod half of execution streaming live here. +- Pending email OTPs stop working and must be requested again. +- OAuth, MCP OAuth, and Slack OAuth connections in progress fail and must be started again. +- Pending webhook verifications are lost. +- Rate-limit buckets reset. + +Completed work is unaffected. + +### Eviction policy + +Use `maxmemory-policy noeviction`, as the bundled Helm Redis does, and size the instance so it never fills. Alert on memory use well before the limit: with `noeviction`, a full Redis rejects writes instead of silently dropping keys. + +Most managed services default to an evicting policy such as `volatile-lru` or `allkeys-lru`. Under memory pressure, `volatile-*` policies evict only keys with a TTL and `allkeys-*` policies evict any key. Either way, the short-lived keys Sim depends on are at risk: OTPs, OAuth state, locks, and rate-limit buckets. An evicted key looks like an expired code, a failed OAuth connection, a duplicated job, or a reset limit. Change the policy in the service's parameter group or configuration. ## Configuration @@ -45,6 +62,8 @@ Both the app and the realtime service need it and must point at the same instanc `docker-compose.prod.yml` already includes a `redis:7-alpine` service and wires `REDIS_URL=redis://redis:6379` into both the app and realtime containers. No additional Redis URL is needed for that bundled instance. +That service sets no `maxmemory`, so Redis can grow until the host runs out of memory. Watch its memory use on a long-running host. + The port is deliberately not published to the host, so a Redis already running locally will not collide. Override `REDIS_URL` in `.env` to point at an external instance instead. @@ -52,7 +71,15 @@ The port is deliberately not published to the host, so a Redis already running l The chart deploys Redis by default, matching the Compose stack. No additional Redis URL is needed for that bundled instance. -For production, prefer a managed instance — disable the bundled one and supply a URL. With `redis.enabled: false`, the chart refuses to render unless `REDIS_URL` comes from `app.env`, a pre-created `app.secrets.existingSecret`, or `externalSecrets.remoteRefs.app.REDIS_URL`: +For production, prefer a managed instance — disable the bundled one and supply a URL. With `redis.enabled: false`, the chart refuses to render unless `REDIS_URL` reaches every enabled app and realtime pod. Accepted sources: + +- `app.env.REDIS_URL` or `realtime.env.REDIS_URL`, written into the app Secret that both pods load +- A top-level `extraEnvVars` entry named `REDIS_URL`, such as one with `valueFrom.secretKeyRef`, which is set on both pods +- A pre-created `app.secrets.existingSecret`, which the chart trusts because it cannot read it +- `externalSecrets.remoteRefs.app.REDIS_URL` when External Secrets is enabled +- `app.envDefaults.REDIS_URL` covers only the app pod, and `realtime.envDefaults.REDIS_URL` only the realtime pod, so set both when both are enabled + +The simplest form: ```yaml redis: diff --git a/apps/docs/scripts/fixtures/content-baseline.json b/apps/docs/scripts/fixtures/content-baseline.json index 8300736be86..15d2f0af918 100644 --- a/apps/docs/scripts/fixtures/content-baseline.json +++ b/apps/docs/scripts/fixtures/content-baseline.json @@ -174974,7 +174974,7 @@ "bytes": 6715, "frontmatter": { "title": "Redis", - "description": "When Redis is optional, when it is required, and how to configure it" + "description": "Why Sim requires Redis, what it backs, and how to configure it" }, "headings": [ { @@ -174982,6 +174982,11 @@ "url": "#what-it-backs", "depth": 2 }, + { + "title": "Eviction policy", + "url": "#eviction-policy", + "depth": 3 + }, { "title": "Configuration", "url": "#configuration", @@ -175072,28 +175077,28 @@ "meta": null, "sha256": "eb73d81bd9564fdbb8cd1a57ac1eb1bdd906e057d04d5a6414ddbd7b67ff2cc3", "bytes": 103, - "line": 27 + "line": 52 }, { "lang": "yaml", "meta": null, "sha256": "59ac0be78ca58845392d8e9d91445a890fad94ebf9659d1ac2a1c0183e04e0c3", "bytes": 97, - "line": 49 + "line": 84 }, { "lang": "bash", "meta": null, "sha256": "75128b92e81c8670e89b93d25c8a6f695fc5ba9bf9e84c21b3a7a8dd45357014", "bytes": 116, - "line": 87 + "line": 122 }, { "lang": "bash", "meta": null, "sha256": "f81661b02c93f78580f48d5e525a71d31d59cedd7974bcdcfc00588cf2f070aa", "bytes": 312, - "line": 96 + "line": 131 } ], "links": [ diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index c192028aeb6..cee579af064 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -179,6 +179,34 @@ describe('FileDocStore', () => { } }) + it('opens one pair of connections when init is called concurrently', async () => { + const store = new FileDocStore(REDIS_URL) + stores.push(store) + await Promise.all([store.init(), store.init(), store.init()]) + expect(fakeRedis.backing!.connects).toBe(2) + }) + + /** + * A restart that lands while the previous reader loop is between reads must retire that loop: + * otherwise it resumes on the new connection and two loops tail the same rooms forever. + */ + it('runs a single reader loop after a shutdown and restart', async () => { + vi.useFakeTimers() + const doc = new Y.Doc() + try { + const store = await newStore() + await store.shutdown() + await store.init() + await store.attachRoom(NAME, doc) + await vi.advanceTimersByTimeAsync(1000) + expect(fakeRedis.backing!.idleReads).toBeGreaterThan(0) + expect(fakeRedis.backing!.maxInFlightReads).toBe(1) + } finally { + doc.destroy() + vi.useRealTimers() + } + }) + it('elects exactly one seeder across tasks (no split-brain seed)', async () => { const a = await newStore() const b = await newStore() diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 1652b6e59a6..8a927bf4536 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -362,6 +362,13 @@ export class FileDocStore { private read: RedisClientType | null = null private readonly rooms = new Map() private running = false + /** + * Bumped on every start, so a reader loop left over from before a restart exits instead of + * running alongside the new one. + */ + private generation = 0 + /** The in-flight start, shared by concurrent {@link init} calls so they open one pair of connections. */ + private starting: Promise | null = null private heartbeat: ReturnType | null = null constructor(private readonly redisUrl: string) {} @@ -369,6 +376,13 @@ export class FileDocStore { /** Connect the two Redis clients and start the multiplexed reader + TTL heartbeat. Idempotent. */ async init(): Promise { if (this.running) return + this.starting ??= this.start().finally(() => { + this.starting = null + }) + return this.starting + } + + private async start(): Promise { const options = { url: this.redisUrl, socket: { @@ -394,7 +408,7 @@ export class FileDocStore { this.read.on('error', (err) => logger.error('FileDocStore read client error:', err)) await Promise.all([this.write.connect(), this.read.connect()]) this.running = true - void this.runReader() + void this.runReader(++this.generation) this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) logger.info('FileDocStore ready — shared Yjs backend over Redis Streams enabled') } @@ -1089,10 +1103,10 @@ export class FileDocStore { * The single multiplexed tail loop: block-read every locally-open room's stream from its last id and * apply new entries. One blocking connection for the whole process regardless of open-file count. */ - private async runReader(): Promise { + private async runReader(generation: number): Promise { let failures = 0 let blockingBatchIndex = 0 - while (this.running && this.read) { + while (this.running && this.read && generation === this.generation) { const snapshot = new Map(this.rooms) if (snapshot.size === 0) { await sleep(IDLE_POLL_MS) @@ -1134,7 +1148,7 @@ export class FileDocStore { } failures = 0 } catch (error) { - if (!this.running) break + if (!this.running || generation !== this.generation) break await this.recoverReader(++failures, error) } } diff --git a/apps/realtime/src/handlers/file-doc.redis-store.test.ts b/apps/realtime/src/handlers/file-doc.redis-store.test.ts index 4c86bcda1b2..9dd27d6928c 100644 --- a/apps/realtime/src/handlers/file-doc.redis-store.test.ts +++ b/apps/realtime/src/handlers/file-doc.redis-store.test.ts @@ -247,12 +247,12 @@ function requestSyncStep2(handlers: Record): void { handlers[FILE_DOC_EVENTS.MESSAGE](encoding.toUint8Array(encoder)) } -describe('file-doc join readiness (shared store enabled)', () => { +describe('file-doc join readiness against the shared store', () => { /** Every socket the test created, so a room emit can be routed to its members. */ const sockets: FakeSocket[] = [] - // One store for the whole file: `initFileDocStore` is idempotent once enabled, so a per-test - // shutdown would leave every later test running against a store with closed clients. + // One store for the whole file: `beforeEach` resets the fake backing, so the tests share one + // started store instead of restarting it around every case. beforeAll(async () => { await initFileDocStore('redis://fake') }) diff --git a/apps/realtime/src/handlers/fixtures/redis.fixture.ts b/apps/realtime/src/handlers/fixtures/redis.fixture.ts index 1afaf60885a..ff09bab203f 100644 --- a/apps/realtime/src/handlers/fixtures/redis.fixture.ts +++ b/apps/realtime/src/handlers/fixtures/redis.fixture.ts @@ -71,6 +71,9 @@ export interface Backing { failedReadTimes: number[] /** Reads that returned (the idle steady state) — the event that ends a failure streak. */ idleReads: number + /** Reads currently awaiting a reply, and the most ever in flight at once: one per reader loop. */ + inFlightReads: number + maxInFlightReads: number /** `connect()` calls, so a test can prove a closed reader is re-opened rather than abandoned. */ connects: number /** Largest stream range response requested, proving replay is paginated. */ @@ -171,7 +174,13 @@ export function makeClient(): TestRedisClient { b().idleReads++ return res } - await sleep(5) + b().inFlightReads++ + b().maxInFlightReads = Math.max(b().maxInFlightReads, b().inFlightReads) + try { + await sleep(5) + } finally { + b().inFlightReads-- + } b().idleReads++ return null }, @@ -335,6 +344,8 @@ export function createBacking(): Backing { reads: 0, failedReadTimes: [], idleReads: 0, + inFlightReads: 0, + maxInFlightReads: 0, connects: 0, maxRangeCount: 0, rangeCalls: 0, diff --git a/apps/sim/app/api/copilot/api-keys/validate/route.test.ts b/apps/sim/app/api/copilot/api-keys/validate/route.test.ts index c8b4943b687..160e3c756fd 100644 --- a/apps/sim/app/api/copilot/api-keys/validate/route.test.ts +++ b/apps/sim/app/api/copilot/api-keys/validate/route.test.ts @@ -431,7 +431,7 @@ describe('POST /api/copilot/api-keys/validate billing protocols', () => { expect(mockResolveBillingAttribution).not.toHaveBeenCalled() }) - it('admits a direct-v1 key without Redis while ignoring a local workspace ID', async () => { + it('admits a direct-v1 key against the account payer, ignoring the workspace ID it sends', async () => { mockSerializeAccountBillingDecisionHeader.mockImplementation((decision: object) => encodeURIComponent(JSON.stringify(decision)) ) diff --git a/apps/sim/app/api/link-preview/route.ts b/apps/sim/app/api/link-preview/route.ts index 2cd6cd93e5c..b9c922d71dc 100644 --- a/apps/sim/app/api/link-preview/route.ts +++ b/apps/sim/app/api/link-preview/route.ts @@ -37,10 +37,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ preview: null }) } - const redis = getRedisClient() const cacheKey = `${CACHE_KEY_PREFIX}${createHash('sha256').update(url).digest('hex')}` try { - const cached = await redis.get(cacheKey) + const cached = await getRedisClient().get(cacheKey) if (cached !== null) { return NextResponse.json({ preview: JSON.parse(cached) }) } @@ -66,7 +65,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ? CACHE_TTL_SECONDS : NEGATIVE_CACHE_TTL_SECONDS try { - await redis.set(cacheKey, JSON.stringify(preview), 'EX', ttl) + await getRedisClient().set(cacheKey, JSON.stringify(preview), 'EX', ttl) } catch (error) { logger.warn('Link preview cache write failed', { error }) } diff --git a/apps/sim/lib/billing/core/usage-segment-cache.ts b/apps/sim/lib/billing/core/usage-segment-cache.ts index 16fe3e7e84f..adabdb1fe8a 100644 --- a/apps/sim/lib/billing/core/usage-segment-cache.ts +++ b/apps/sim/lib/billing/core/usage-segment-cache.ts @@ -37,8 +37,8 @@ const KEY_VERSION = 'v1' async function readStored(keys: string[]): Promise<(string | null)[]> { if (keys.length === 0) return [] - const redis = getRedisClient() try { + const redis = getRedisClient() return await redis.mget(...keys) } catch (error) { logger.warn('Usage segment cache read failed; reading the ledger', { @@ -67,12 +67,16 @@ function parseStored(key: string, value: string | null | undefined): T | unde function writeStored(entries: { key: string; value: string; ttlMs: number }[]): void { if (entries.length === 0) return const jittered = (ttlMs: number) => ttlMs + randomInt(0, Math.floor(ttlMs * TTL_JITTER_RATIO)) - const redis = getRedisClient() - const pipeline = redis.pipeline() - for (const { key, value, ttlMs } of entries) pipeline.set(key, value, 'PX', jittered(ttlMs)) - pipeline.exec().catch((error: unknown) => { + const logWriteFailure = (error: unknown) => { logger.warn('Usage segment cache write failed', { error: getErrorMessage(error) }) - }) + } + try { + const pipeline = getRedisClient().pipeline() + for (const { key, value, ttlMs } of entries) pipeline.set(key, value, 'PX', jittered(ttlMs)) + pipeline.exec().catch(logWriteFailure) + } catch (error) { + logWriteFailure(error) + } } interface ReadThroughSegmentsArgs { diff --git a/apps/sim/lib/core/config/env-capabilities.test.ts b/apps/sim/lib/core/config/env-capabilities.test.ts index 5f3231f485e..00d7eb5af12 100644 --- a/apps/sim/lib/core/config/env-capabilities.test.ts +++ b/apps/sim/lib/core/config/env-capabilities.test.ts @@ -1,9 +1,7 @@ import { ASYNC_JOBS_CAPABILITY, - CORE_CONFIGURATION_KEYS, defineCapability, EMAIL_CAPABILITY, - ENV_CAPABILITIES, EnvCapabilityConfigurationError, envField, getRedisConfigurationError, @@ -678,11 +676,6 @@ describe('env capabilities', () => { getRedisConfigurationError({ REDIS_URL: 'rediss://cache.example.com:6379' }) ).toBeNull() }) - - it('treats REDIS_URL as core configuration rather than an optional capability', () => { - expect(CORE_CONFIGURATION_KEYS).toContain('REDIS_URL') - expect(ENV_CAPABILITIES.map((capability) => capability.id)).not.toContain('cache') - }) }) describe('OAuth and deployment metadata', () => { diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts b/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts index 1ef1b665c34..fa7a2261119 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts @@ -86,12 +86,11 @@ export class HostedKeyQueue { billingActorId: string, ticketId: string ): Promise { - const redis = getRedisClient() - const listKey = queueListKey(provider, billingActorId) const hbKey = heartbeatKey(provider, billingActorId, ticketId) try { + const redis = getRedisClient() const pipeline = redis.multi() pipeline.rpush(listKey, ticketId) pipeline.expire(listKey, QUEUE_LIST_TTL_SECONDS) @@ -123,12 +122,11 @@ export class HostedKeyQueue { billingActorId: string, ticketId: string ): Promise<'head' | 'waiting' | 'missing'> { - const redis = getRedisClient() - const listKey = queueListKey(provider, billingActorId) const hbPrefix = `hosted-queue-tkt:${provider}:${billingActorId}:` try { + const redis = getRedisClient() const result = (await redis.eval(CHECK_HEAD_SCRIPT, 1, listKey, hbPrefix, ticketId)) as | 'head' | 'waiting' @@ -151,11 +149,10 @@ export class HostedKeyQueue { billingActorId: string, ticketId: string ): Promise { - const redis = getRedisClient() - const listKey = queueListKey(provider, billingActorId) const hbKey = heartbeatKey(provider, billingActorId, ticketId) try { + const redis = getRedisClient() const pipeline = redis.multi() pipeline.set(hbKey, '1', 'EX', TICKET_HEARTBEAT_TTL_SECONDS) pipeline.expire(listKey, QUEUE_LIST_TTL_SECONDS) @@ -172,11 +169,10 @@ export class HostedKeyQueue { * call multiple times. LREM count=1 removes at most one matching entry. */ async dequeue(provider: string, billingActorId: string, ticketId: string): Promise { - const redis = getRedisClient() - const listKey = queueListKey(provider, billingActorId) const hbKey = heartbeatKey(provider, billingActorId, ticketId) try { + const redis = getRedisClient() const pipeline = redis.multi() pipeline.lrem(listKey, 1, ticketId) pipeline.del(hbKey) diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts index 9e46db6aa01..75a7c101306 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts @@ -106,16 +106,25 @@ interface WaitState { * across all users within the same workspace. */ export class HostedKeyRateLimiter { - private storage: RateLimitStorageAdapter + private readonly injectedStorage: RateLimitStorageAdapter | undefined private queue: HostedKeyQueue /** Round-robin counter per provider for even key distribution */ private roundRobinCounters = new Map() constructor(storage?: RateLimitStorageAdapter, queue?: HostedKeyQueue) { - this.storage = storage ?? createStorageAdapter() + this.injectedStorage = storage this.queue = queue ?? getHostedKeyQueue() } + /** + * Resolved on each use so constructing the limiter never touches Redis + * (`next build` evaluates module-scope instances without REDIS_URL) and the + * factory's cached adapter follows Redis reconnects. + */ + private get storage(): RateLimitStorageAdapter { + return this.injectedStorage ?? createStorageAdapter() + } + private buildActorStorageKey(provider: string, billingActorId: string): string { return `hosted:${provider}:actor:${billingActorId}:${ACTOR_REQUESTS_DIMENSION}` } diff --git a/apps/sim/lib/core/rate-limiter/rate-limiter.ts b/apps/sim/lib/core/rate-limiter/rate-limiter.ts index 838ced01b68..07af2087989 100644 --- a/apps/sim/lib/core/rate-limiter/rate-limiter.ts +++ b/apps/sim/lib/core/rate-limiter/rate-limiter.ts @@ -33,10 +33,19 @@ export interface RateLimitStatus { } export class RateLimiter { - private storage: RateLimitStorageAdapter + private readonly injectedStorage: RateLimitStorageAdapter | undefined constructor(storage?: RateLimitStorageAdapter) { - this.storage = storage ?? createStorageAdapter() + this.injectedStorage = storage + } + + /** + * Resolved on each use, never in the constructor: route modules build a + * limiter at module scope, which `next build` evaluates without REDIS_URL, + * and the factory's cached adapter follows Redis reconnects. + */ + private get storage(): RateLimitStorageAdapter { + return this.injectedStorage ?? createStorageAdapter() } private getRateLimitKey(userId: string, subscription: SubscriptionInfo | null): string { diff --git a/apps/sim/lib/desktop/executor/presence.ts b/apps/sim/lib/desktop/executor/presence.ts index 28d0d895a49..8b344edbb03 100644 --- a/apps/sim/lib/desktop/executor/presence.ts +++ b/apps/sim/lib/desktop/executor/presence.ts @@ -19,8 +19,8 @@ function presenceKey(deviceId: string): string { * unclaimed call fails, so a Redis error must not fail the request that proves the device is awake. */ export async function markDesktopPresent(deviceId: string): Promise { - const redis = getRedisClient() try { + const redis = getRedisClient() await redis.set(presenceKey(deviceId), '1', 'EX', DESKTOP_PRESENCE_TTL_SECONDS) } catch (error) { logger.warn('Could not record desktop presence', { deviceId, error: toError(error).message }) diff --git a/apps/sim/lib/favicon/resolve.server.ts b/apps/sim/lib/favicon/resolve.server.ts index 441a5dd6d8f..b46e402a3fd 100644 --- a/apps/sim/lib/favicon/resolve.server.ts +++ b/apps/sim/lib/favicon/resolve.server.ts @@ -44,8 +44,8 @@ const cacheKey = (host: string) => `favicon:v3:${host}` /** A host's key holds the result of the walk that starts at that host. */ async function readCache(host: string): Promise { - const redis = getRedisClient() try { + const redis = getRedisClient() const cached = await redis.getBuffer(cacheKey(host)) if (cached === null) return null if (cached.toString() === UNAVAILABLE_MARKER) return { kind: 'unavailable' } @@ -65,7 +65,6 @@ async function readCache(host: string): Promise { } async function writeCache(host: string, resolution: FaviconResolution): Promise { - const redis = getRedisClient() const [value, ttl] = resolution.kind === 'hit' ? resolution.provisional @@ -81,7 +80,7 @@ async function writeCache(host: string, resolution: FaviconResolution): Promise< ? ([MISS_MARKER, MISS_TTL_SECONDS] as const) : ([UNAVAILABLE_MARKER, UNAVAILABLE_TTL_SECONDS] as const) try { - await redis.set(cacheKey(host), value, 'EX', ttl) + await getRedisClient().set(cacheKey(host), value, 'EX', ttl) } catch (error) { logger.warn('Favicon cache write failed', { error }) } diff --git a/apps/sim/lib/logs/execution/progress-markers.ts b/apps/sim/lib/logs/execution/progress-markers.ts index 0855fec1786..9baa19ced36 100644 --- a/apps/sim/lib/logs/execution/progress-markers.ts +++ b/apps/sim/lib/logs/execution/progress-markers.ts @@ -73,8 +73,8 @@ async function setMarker( marker: ExecutionLastStartedBlock | ExecutionLastCompletedBlock, expiresAt?: number ): Promise { - const redis = getMarkerClient() try { + const redis = getMarkerClient() await redis.eval( SET_MARKER_SCRIPT, 1, @@ -207,8 +207,8 @@ function parseCompletedMarker(raw: string | undefined): ExecutionLastCompletedBl export async function getProgressMarkers( executionId: string ): Promise { - const redis = getMarkerClient() try { + const redis = getMarkerClient() const fields = await redis.hgetall(markerKey(executionId)) if (!fields || Object.keys(fields).length === 0) return {} @@ -232,8 +232,8 @@ export async function getProgressMarkers( * live indefinitely) hold no Redis keys. Fire-and-forget: a failed delete is logged. */ export async function clearProgressMarkers(executionId: string): Promise { - const redis = getMarkerClient() try { + const redis = getMarkerClient() await redis.del(markerKey(executionId)) } catch (error) { logger.error(`Failed to clear progress markers for execution ${executionId}`, { diff --git a/apps/sim/lib/mothership/persistence/tool-confirm/index.ts b/apps/sim/lib/mothership/persistence/tool-confirm/index.ts index 059d5033d26..a22dc03aec1 100644 --- a/apps/sim/lib/mothership/persistence/tool-confirm/index.ts +++ b/apps/sim/lib/mothership/persistence/tool-confirm/index.ts @@ -19,6 +19,11 @@ import { MothershipStreamV1ToolOutcome } from '@/lib/mothership/generated/mother const logger = createLogger('CopilotOrchestratorPersistence') const TOOL_CONFIRMATION_TTL_SECONDS = 60 * 10 const DURABLE_CONFIRMATION_POLL_MS = 5_000 +/** + * How long the first durable check waits for the subscription to land. Past it the wait falls + * back to polling, as it must while Redis cannot subscribe. + */ +const SUBSCRIPTION_READY_WAIT_MS = DURABLE_CONFIRMATION_POLL_MS const toolConfirmationKey = (toolCallId: string) => `copilot:tool-confirmation:${toolCallId}` type ToolConfirmGlobal = typeof globalThis & { @@ -127,11 +132,13 @@ export async function waitForToolConfirmation( let settled = false let timeoutId: ReturnType | null = null let pollId: ReturnType | null = null + let readyWaitId: ReturnType | null = null let unsubscribe: (() => void) | null = null const cleanup = () => { if (timeoutId) clearTimeout(timeoutId) if (pollId) clearTimeout(pollId) + if (readyWaitId) clearTimeout(readyWaitId) if (unsubscribe) unsubscribe() abortSignal?.removeEventListener('abort', onAbort) } @@ -208,6 +215,21 @@ export async function waitForToolConfirmation( } abortSignal?.addEventListener('abort', onAbort, { once: true }) - void checkDurableConfirmation('subscribe').then(scheduleDurablePoll) + // The first durable check runs once the subscription is live. A publisher commits the row + // before it publishes, so a signal sent before then is in the row this check reads, and + // every later one reaches the handler above. + const subscriptionReady = Promise.race([ + toolConfirmationChannel.ready(), + new Promise((resolveWait) => { + readyWaitId = setTimeout(resolveWait, SUBSCRIPTION_READY_WAIT_MS) + }), + ]) + void subscriptionReady.then(async () => { + if (readyWaitId) clearTimeout(readyWaitId) + readyWaitId = null + if (settled) return + await checkDurableConfirmation('subscribe') + scheduleDurablePoll() + }) }) } diff --git a/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts b/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts index e0ea9178dae..8062f59811f 100644 --- a/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts +++ b/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts @@ -5,6 +5,8 @@ import { import { beforeEach, describe, expect, it, vi } from 'vitest' const channelHandlers = new Set<(event: any) => void>() +/** When this process's subscription lands; publishes before then reach no handler. */ +let subscriptionReady: Promise = Promise.resolve() vi.mock('@/lib/mothership/async-runs/repository', () => mothershipAsyncRunsMock) @@ -19,6 +21,9 @@ vi.mock('@/lib/events/pubsub', () => ({ channelHandlers.delete(handler) } }, + ready() { + return subscriptionReady + }, dispose() {}, }), })) @@ -44,6 +49,7 @@ describe('copilot orchestrator persistence', () => { beforeEach(() => { channelHandlers.clear() + subscriptionReady = Promise.resolve() revokeExpiredSimToolExecutions.mockResolvedValue([]) row = null getAsyncToolCalls.mockImplementation(async () => (row ? [row] : [])) @@ -184,6 +190,45 @@ describe('copilot orchestrator persistence', () => { }) }) + it('reads the durable row only once its subscription lands, so a detach in between is not missed', async () => { + vi.useFakeTimers() + try { + let landSubscription: () => void = () => {} + subscriptionReady = new Promise((resolve) => { + landSubscription = resolve + }) + row = { + status: 'pending', + error: null, + result: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + } + let result: unknown + const waitPromise = waitForToolConfirmation('tool-1', 20_000, undefined, { + acceptStatus: (status) => status === 'background', + }).then((value) => { + result = value + }) + await vi.advanceTimersByTimeAsync(0) + + // Another pod detaches the call while this one's SUBSCRIBE is still in flight: the row + // commits and the publish reaches no handler here. + row = { + status: 'delivered', + error: null, + result: null, + updatedAt: new Date('2026-01-01T00:00:01.000Z'), + } + landSubscription() + await vi.advanceTimersByTimeAsync(0) + + expect(result).toEqual({ status: 'background', timestamp: '2026-01-01T00:00:01.000Z' }) + await waitPromise + } finally { + vi.useRealTimers() + } + }) + it('keeps a no-deadline human wait alive until confirmation arrives', async () => { vi.useFakeTimers() try { diff --git a/apps/sim/lib/oauth/terminal-errors.ts b/apps/sim/lib/oauth/terminal-errors.ts index 1ecc17377df..3f235618767 100644 --- a/apps/sim/lib/oauth/terminal-errors.ts +++ b/apps/sim/lib/oauth/terminal-errors.ts @@ -11,8 +11,8 @@ function deadKey(accountId: string): string { } export async function markCredentialDead(accountId: string, code: string): Promise { - const redis = getRedisClient() try { + const redis = getRedisClient() await redis.set(deadKey(accountId), code, 'EX', DEAD_CACHE_TTL_SEC) } catch (error) { logger.warn('Failed to mark credential dead in Redis', { @@ -24,8 +24,8 @@ export async function markCredentialDead(accountId: string, code: string): Promi } export async function getRecentTerminalError(accountId: string): Promise { - const redis = getRedisClient() try { + const redis = getRedisClient() return await redis.get(deadKey(accountId)) } catch (error) { logger.warn('Failed to read terminal error flag from Redis', { @@ -37,8 +37,8 @@ export async function getRecentTerminalError(accountId: string): Promise { - const redis = getRedisClient() try { + const redis = getRedisClient() await redis.del(deadKey(accountId)) } catch (error) { logger.warn('Failed to clear terminal error flag from Redis', { diff --git a/apps/sim/lib/realtime/event-log.ts b/apps/sim/lib/realtime/event-log.ts index d7d32278fc9..46591d179fa 100644 --- a/apps/sim/lib/realtime/event-log.ts +++ b/apps/sim/lib/realtime/event-log.ts @@ -149,8 +149,8 @@ export async function appendEvent( streamId: string, serializer: EntrySerializer ): Promise { - const redis = getRedisClient() try { + const redis = getRedisClient() const result = await redis.eval( APPEND_EVENT_SCRIPT, 3, diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.ts index 15a4059a17e..92b89021922 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.ts @@ -201,14 +201,13 @@ function logSkippedCacheWrite( } function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Base64Cache { - const redis = getRedisClient() const { executionId } = options return { async get(file: UserFile) { try { const key = getFullCacheKey(executionId, file) - return await redis.get(key) + return await getRedisClient().get(key) } catch (error) { logger.warn('Redis get failed, skipping cache', error, { requestId: options.requestId }) return null @@ -218,6 +217,7 @@ function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Bas const key = getFullCacheKey(executionId, file) const valueBytes = Buffer.byteLength(value, 'utf8') try { + const redis = getRedisClient() if (!executionId) { await redis.set(key, value, 'EX', ttlSeconds) return @@ -629,10 +629,10 @@ export function containsUserFileWithMetadata(value: unknown): boolean { * Should be called at the end of workflow execution. */ export async function cleanupExecutionBase64Cache(executionId: string): Promise { - const redis = getRedisClient() const logger = createLogger('UserFileBase64') try { + const redis = getRedisClient() const budgetEntries = await redis.hgetall(getBudgetIndexKey(executionId)) let deletedCount = 0 for (const [fileKey, rawEntry] of Object.entries(budgetEntries ?? {})) { diff --git a/apps/sim/lib/webhooks/polling/admission-refusals.ts b/apps/sim/lib/webhooks/polling/admission-refusals.ts index 4baee5250ef..78a50466221 100644 --- a/apps/sim/lib/webhooks/polling/admission-refusals.ts +++ b/apps/sim/lib/webhooks/polling/admission-refusals.ts @@ -22,8 +22,8 @@ const refusalKey = (workspaceId: string) => `poll-admission-refused:v1:${workspa */ export async function recordPollAdmissionRefusal(workspaceId: string): Promise { if (!isBillingEnabled) return - const redis = getRedisClient() try { + const redis = getRedisClient() await redis.set(refusalKey(workspaceId), '1', 'EX', POLL_ADMISSION_REFUSAL_TTL_SECONDS) } catch (error) { logger.debug('Failed to record poll admission refusal', { @@ -42,8 +42,8 @@ export async function findRecentlyRefusedWorkspaces( workspaceIds: readonly string[] ): Promise> { if (!isBillingEnabled || workspaceIds.length === 0) return new Set() - const redis = getRedisClient() try { + const redis = getRedisClient() const flags = await redis.mget(...workspaceIds.map(refusalKey)) return new Set(workspaceIds.filter((_, index) => flags[index] !== null)) } catch (error) { diff --git a/apps/sim/vitest.integration.setup.ts b/apps/sim/vitest.integration.setup.ts index 9a204df41f2..fe8fa35118a 100644 --- a/apps/sim/vitest.integration.setup.ts +++ b/apps/sim/vitest.integration.setup.ts @@ -57,10 +57,18 @@ Object.assign(process.env, { }) /** - * Every file starts from an empty Redis, so keys one file leaves behind never decide another's - * outcome. Files run one at a time (`fileParallelism: false`), and `readTestRedisUrl` only accepts - * a credential-free loopback URL, which is the disposable instance the harness provisions. + * Every file starts from an empty Redis database, so keys one file leaves behind never decide + * another's outcome. Files run one at a time (`fileParallelism: false`). `readTestRedisUrl` only + * accepts a credential-free loopback URL, and the flush additionally requires the URL to select a + * non-zero database (`redis://127.0.0.1:6379/15`): a local Redis left on its default database 0 is + * someone's dev data, and pointing TEST_REDIS_URL at it must fail here rather than wipe it. */ +const redisDatabase = Number(new URL(redisUrl).pathname.slice(1) || '0') +if (!Number.isInteger(redisDatabase) || redisDatabase <= 0) { + throw new Error( + 'TEST_REDIS_URL must select a non-zero database (e.g. redis://127.0.0.1:6379/15): every integration file flushes it' + ) +} const redisReset = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 1 }) try { await redisReset.connect() diff --git a/helm/sim/README.md b/helm/sim/README.md index c159465bba8..6821976a5ea 100644 --- a/helm/sim/README.md +++ b/helm/sim/README.md @@ -610,7 +610,15 @@ kubectl --namespace sim logs deploy/sim-app -c migrations * **Redis is required.** The application no longer falls back to PostgreSQL or in-memory state for rate limiting, OTPs, pub/sub, or realtime coordination, and refuses to start without `REDIS_URL`. Installs that keep the bundled Redis (`redis.enabled: true`, the default) or already supply `REDIS_URL` need no action. - With `redis.enabled: false`, the chart now fails to render unless `REDIS_URL` comes from `app.env.REDIS_URL`, a pre-created `app.secrets.existingSecret` (which the chart trusts, since it cannot read it), or `externalSecrets.remoteRefs.app.REDIS_URL` when External Secrets is enabled. If you previously ran with Redis disabled, either re-enable the bundled Redis or point `REDIS_URL` at a managed instance before upgrading. + With `redis.enabled: false`, the chart now fails to render unless `REDIS_URL` reaches every enabled app and realtime pod. Accepted sources: + + * `app.env.REDIS_URL` or `realtime.env.REDIS_URL`, written into the app Secret both pods load. + * A top-level `extraEnvVars` entry named `REDIS_URL` (for example with `valueFrom.secretKeyRef`), set inline on both pods. + * A pre-created `app.secrets.existingSecret`, which the chart trusts since it cannot read it. + * `externalSecrets.remoteRefs.app.REDIS_URL` when External Secrets is enabled, now also checked for a realtime-only release. + * `app.envDefaults.REDIS_URL` covers only the app pod and `realtime.envDefaults.REDIS_URL` only the realtime pod, so set both when both components are enabled. + + The app Secret (chart-managed or External Secrets) now also renders for a realtime-only release (`app.enabled: false`), since the realtime pod loads it. If you previously ran with Redis disabled, either re-enable the bundled Redis or point `REDIS_URL` at a managed instance before upgrading. ## Upgrading to 1.6.4 diff --git a/helm/sim/templates/_helpers.tpl b/helm/sim/templates/_helpers.tpl index bc45cca79e1..db1ecfa5aa4 100644 --- a/helm/sim/templates/_helpers.tpl +++ b/helm/sim/templates/_helpers.tpl @@ -327,19 +327,67 @@ Skip validation when using existing secrets or External Secrets Operator {{- end }} {{/* -Sim requires Redis. With redis.enabled=false the operator must supply -REDIS_URL: inline in app.env, from a pre-created Secret (which the chart cannot -read, so it is trusted), or mapped through External Secrets (enforced in -sim.validateExternalSecretCoverage). +Sim requires Redis. With redis.enabled=false every enabled component (app, +realtime) must get REDIS_URL from a source that reaches its pod: + + - both pods: a top-level extraEnvVars entry named REDIS_URL (inlined on app + and realtime), or app.env.REDIS_URL / realtime.env.REDIS_URL (written into + the app Secret both pods envFrom) + - app only: app.envDefaults.REDIS_URL (inlined on the app container) + - realtime only: realtime.envDefaults.REDIS_URL (inlined on realtime) + +A pre-created app Secret cannot be read, so it is trusted. Under External +Secrets the same check runs in sim.validateExternalSecretCoverage, where a +remoteRefs.app.REDIS_URL mapping stands in for the app Secret. */}} {{- define "sim.validateRedis" -}} {{- $useExistingAppSecret := and .Values.app.secrets .Values.app.secrets.existingSecret .Values.app.secrets.existingSecret.enabled }} {{- $useExternalSecrets := and .Values.externalSecrets .Values.externalSecrets.enabled }} -{{- if and (or .Values.app.enabled .Values.realtime.enabled) (not .Values.redis.enabled) (not (.Values.app.env.REDIS_URL | default "")) (not (or $useExistingAppSecret $useExternalSecrets)) }} -{{- fail "Sim requires Redis: redis.enabled=false but no REDIS_URL is provided. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." }} +{{- if not (or $useExistingAppSecret $useExternalSecrets) }} +{{- $inSecret := or (include "sim.nonEmptyEnv" (list .Values.app.env "REDIS_URL")) (include "sim.nonEmptyEnv" (list .Values.realtime.env "REDIS_URL")) }} +{{- $uncovered := include "sim.redisUncoveredComponents" (dict "root" . "shared" $inSecret) }} +{{- if $uncovered }} +{{- fail (printf "Sim requires Redis: redis.enabled=false but no REDIS_URL reaches the %s pod. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), add a REDIS_URL entry to extraEnvVars (e.g. valueFrom.secretKeyRef), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." $uncovered) }} +{{- end }} {{- end }} {{- end }} +{{/* +"true" when the env map (first arg) holds a non-empty value for the key (second arg). +*/}} +{{- define "sim.nonEmptyEnv" -}} +{{- $value := index (index . 0 | default dict) (index . 1) -}} +{{- if and $value (ne (toString $value) "") (ne (toString $value) "") -}} +true +{{- end -}} +{{- end }} + +{{/* +With redis.enabled=false, the enabled components ("app", "realtime", joined by +" and ") that no REDIS_URL source reaches. `shared` is whether the app Secret +carries REDIS_URL, which both pods envFrom. Empty when every enabled component +is covered, or when the chart deploys Redis itself. +*/}} +{{- define "sim.redisUncoveredComponents" -}} +{{- $root := .root -}} +{{- if not $root.Values.redis.enabled -}} +{{- $shared := .shared -}} +{{- range $root.Values.extraEnvVars | default list -}} +{{- if and (eq (toString .name) "REDIS_URL") (or .valueFrom (and .value (ne (toString .value) ""))) -}} +{{- $shared = true -}} +{{- end -}} +{{- end -}} +{{- $uncovered := list -}} +{{- if and $root.Values.app.enabled (not $shared) (not (include "sim.nonEmptyEnv" (list $root.Values.app.envDefaults "REDIS_URL"))) -}} +{{- $uncovered = append $uncovered "app" -}} +{{- end -}} +{{- if and $root.Values.realtime.enabled (not $shared) (not (include "sim.nonEmptyEnv" (list $root.Values.realtime.envDefaults "REDIS_URL"))) -}} +{{- $uncovered = append $uncovered "realtime" -}} +{{- end -}} +{{- join " and " $uncovered -}} +{{- end -}} +{{- end }} + {{/* Validate that every key set in app.env / realtime.env is also mapped in externalSecrets.remoteRefs.app when ESO is enabled. When ESO is on, the @@ -366,14 +414,16 @@ than enforced. runtime CrashLoopBackOff with cryptic env errors. Fail at template time if the key is neither set in app.env nor mapped via remoteRefs.app. */}} +{{- $redisInSecret := or (include "sim.nonEmptyEnv" (list $appEnv "REDIS_URL")) (index $remoteRefs "REDIS_URL") -}} +{{- $redisUncovered := include "sim.redisUncoveredComponents" (dict "root" . "shared" $redisInSecret) -}} +{{- if $redisUncovered -}} +{{- fail (printf "Required key 'REDIS_URL' is missing for the %s pod: redis.enabled=false and externalSecrets.enabled=true but the key is neither set in app.env, mapped in externalSecrets.remoteRefs.app, nor supplied by an extraEnvVars entry. Map it via externalSecrets.remoteRefs.app.REDIS_URL='path/in/store' so it is synced into the app Secret." $redisUncovered) }} +{{- end -}} {{- if .Values.app.enabled -}} {{- $required := list "BETTER_AUTH_SECRET" "ENCRYPTION_KEY" "INTERNAL_API_SECRET" -}} {{- if .Values.cronjobs.enabled -}} {{- $required = append $required "CRON_SECRET" -}} {{- end -}} -{{- if not .Values.redis.enabled -}} -{{- $required = append $required "REDIS_URL" -}} -{{- end -}} {{- range $key := $required -}} {{- $inEnv := index $appEnv $key -}} {{- $mapped := index $remoteRefs $key -}} diff --git a/helm/sim/templates/external-secret-app.yaml b/helm/sim/templates/external-secret-app.yaml index ea82d6a1f6e..cca1dc34c85 100644 --- a/helm/sim/templates/external-secret-app.yaml +++ b/helm/sim/templates/external-secret-app.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.externalSecrets.enabled .Values.app.enabled }} +{{- if and .Values.externalSecrets.enabled (or .Values.app.enabled .Values.realtime.enabled) }} # ExternalSecret for app credentials (syncs from external secret managers) # # The data list is generated by iterating every non-empty entry in diff --git a/helm/sim/templates/secrets-app.yaml b/helm/sim/templates/secrets-app.yaml index cf598096b5f..6af810ca5c1 100644 --- a/helm/sim/templates/secrets-app.yaml +++ b/helm/sim/templates/secrets-app.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.app.enabled (include "sim.createAppSecrets" .) }} +{{- if and (or .Values.app.enabled .Values.realtime.enabled) (include "sim.createAppSecrets" .) }} {{/* Secret for app + realtime env. Every key in .Values.app.env and .Values.realtime.env is written here and mounted via envFrom on the diff --git a/helm/sim/tests/redis_test.yaml b/helm/sim/tests/redis_test.yaml index 26bc10bce87..009e9d7d3ee 100644 --- a/helm/sim/tests/redis_test.yaml +++ b/helm/sim/tests/redis_test.yaml @@ -130,7 +130,7 @@ tests: realtime.enabled: false asserts: - failedTemplate: - errorMessage: "Sim requires Redis: redis.enabled=false but no REDIS_URL is provided. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." + errorMessage: "Sim requires Redis: redis.enabled=false but no REDIS_URL reaches the app pod. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), add a REDIS_URL entry to extraEnvVars (e.g. valueFrom.secretKeyRef), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." - it: fails realtime too when redis.enabled=false and nothing supplies REDIS_URL template: deployment-realtime.yaml @@ -139,7 +139,7 @@ tests: app.enabled: false asserts: - failedTemplate: - errorMessage: "Sim requires Redis: redis.enabled=false but no REDIS_URL is provided. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." + errorMessage: "Sim requires Redis: redis.enabled=false but no REDIS_URL reaches the realtime pod. Set app.env.REDIS_URL to your Redis (redis:// or rediss://), add a REDIS_URL entry to extraEnvVars (e.g. valueFrom.secretKeyRef), supply it via app.secrets.existingSecret or externalSecrets.remoteRefs.app.REDIS_URL, or set redis.enabled=true to deploy the bundled Redis." - it: renders with redis.enabled=false when app.env.REDIS_URL is set template: deployment-app.yaml @@ -150,6 +150,161 @@ tests: - isKind: of: Deployment + - it: accepts an extraEnvVars REDIS_URL, which is inlined on the app pod + template: deployment-app.yaml + set: + redis.enabled: false + extraEnvVars: + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: managed-redis + key: url + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_URL + valueFrom: + secretKeyRef: + name: managed-redis + key: url + + - it: accepts an extraEnvVars REDIS_URL, which is inlined on the realtime pod + template: deployment-realtime.yaml + set: + redis.enabled: false + extraEnvVars: + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: managed-redis + key: url + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_URL + valueFrom: + secretKeyRef: + name: managed-redis + key: url + + - it: rejects an extraEnvVars REDIS_URL entry with an empty value + template: deployment-app.yaml + set: + redis.enabled: false + realtime.enabled: false + extraEnvVars: + - name: REDIS_URL + value: "" + asserts: + - failedTemplate: + errorPattern: "no REDIS_URL reaches the app pod" + + # realtime.env lands in the app Secret, which both the app and realtime pods envFrom. + - it: accepts realtime.env.REDIS_URL, written into the Secret both pods share + template: secrets-app.yaml + set: + redis.enabled: false + realtime.env.REDIS_URL: "rediss://:pw@managed.example.com:6380" + asserts: + - equal: + path: stringData.REDIS_URL + value: "rediss://:pw@managed.example.com:6380" + + - it: renders the shared app Secret for a realtime-only release so realtime.env.REDIS_URL reaches it + template: secrets-app.yaml + set: + redis.enabled: false + app.enabled: false + realtime.env.REDIS_URL: "rediss://:pw@managed.example.com:6380" + asserts: + - equal: + path: stringData.REDIS_URL + value: "rediss://:pw@managed.example.com:6380" + + - it: accepts app.envDefaults.REDIS_URL for an app-only release + template: deployment-app.yaml + set: + redis.enabled: false + realtime.enabled: false + app.envDefaults.REDIS_URL: "rediss://:pw@managed.example.com:6380" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_URL + value: "rediss://:pw@managed.example.com:6380" + + # app.envDefaults is inlined on the app container only, so realtime is still uncovered. + - it: rejects app.envDefaults.REDIS_URL alone when realtime is enabled + template: deployment-realtime.yaml + set: + redis.enabled: false + app.envDefaults.REDIS_URL: "rediss://:pw@managed.example.com:6380" + asserts: + - failedTemplate: + errorPattern: "no REDIS_URL reaches the realtime pod" + + - it: accepts app.envDefaults plus realtime.envDefaults REDIS_URL, one per pod + template: deployment-realtime.yaml + set: + redis.enabled: false + app.envDefaults.REDIS_URL: "rediss://:pw@managed.example.com:6380" + realtime.envDefaults.REDIS_URL: "rediss://:pw@managed.example.com:6380" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_URL + value: "rediss://:pw@managed.example.com:6380" + + - it: fails a realtime-only External Secrets release when REDIS_URL is not mapped + template: deployment-realtime.yaml + set: + redis.enabled: false + app.enabled: false + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: my-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: sim/app/better-auth-secret + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: sim/app/encryption-key + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: sim/app/internal-api-secret + externalSecrets.remoteRefs.app.CRON_SECRET: sim/app/cron-secret + externalSecrets.remoteRefs.postgresql.password: sim/postgresql/password + app.env.BETTER_AUTH_SECRET: "" + app.env.ENCRYPTION_KEY: "" + app.env.INTERNAL_API_SECRET: "" + app.env.CRON_SECRET: "" + asserts: + - failedTemplate: + errorPattern: "Required key 'REDIS_URL' is missing for the realtime pod" + + - it: syncs the mapped REDIS_URL for a realtime-only External Secrets release + template: external-secret-app.yaml + set: + redis.enabled: false + app.enabled: false + externalSecrets.remoteRefs.app.REDIS_URL: sim/app/redis-url + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: my-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: sim/app/better-auth-secret + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: sim/app/encryption-key + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: sim/app/internal-api-secret + externalSecrets.remoteRefs.app.CRON_SECRET: sim/app/cron-secret + externalSecrets.remoteRefs.postgresql.password: sim/postgresql/password + app.env.BETTER_AUTH_SECRET: "" + app.env.ENCRYPTION_KEY: "" + app.env.INTERNAL_API_SECRET: "" + app.env.CRON_SECRET: "" + asserts: + - contains: + path: spec.data + content: + secretKey: REDIS_URL + remoteRef: + key: sim/app/redis-url + - it: trusts a pre-created app Secret to carry REDIS_URL when redis.enabled=false template: deployment-app.yaml set: diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index a08ff73f63f..5d7b45dbac7 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -570,9 +570,11 @@ realtime: # and the idempotency/progress stores; the app refuses to start without it. # Bundled by default so a chart install matches the Docker Compose stack. For # production prefer a managed instance (ElastiCache, Memorystore, Azure Cache) — -# set enabled: false and supply its connection string as app.env.REDIS_URL, in a -# pre-created app Secret, or via externalSecrets.remoteRefs.app.REDIS_URL. The -# chart fails to render when enabled: false and none of those is configured. +# set enabled: false and supply its connection string as app.env.REDIS_URL (or +# realtime.env.REDIS_URL), a REDIS_URL entry in extraEnvVars, a pre-created app +# Secret, or externalSecrets.remoteRefs.app.REDIS_URL. app.envDefaults and +# realtime.envDefaults each reach only their own pod. The chart fails to render +# when enabled: false and no source reaches every enabled app/realtime pod. redis: enabled: true diff --git a/packages/sim-setup/src/redis.ts b/packages/sim-setup/src/redis.ts index ab61608e97f..fdd41c7b907 100644 --- a/packages/sim-setup/src/redis.ts +++ b/packages/sim-setup/src/redis.ts @@ -33,8 +33,9 @@ async function startManagedRedis(detection: Detection): Promise { const hostPort = detection.redisPortOpen ? 6380 : 6379 const url = `redis://${LOOPBACK_HOST}:${hostPort}` // A prior sim-redis (unhealthy, or bound to a stale port) would collide on the - // name. Unlike Postgres this container has no data volume, so removing it - // loses nothing — no prompt needed. `rm -f` is a no-op when none exists. + // name. It has no data volume and holds no durable data, so it is replaced + // without a prompt; only short-lived state (live OTPs, OAuth state, rate-limit + // buckets) is dropped. `rm -f` is a no-op when none exists. spawnSync('docker', ['rm', '-f', REDIS_CONTAINER], { stdio: 'ignore' }) docker([ 'run', @@ -145,7 +146,7 @@ export async function resolveRedis(detection: Detection, existing?: string): Pro return managedUrl } p.log.warn( - `${REDIS_CONTAINER} exists but is not answering — starting a fresh one will replace it (it holds no data).` + `${REDIS_CONTAINER} exists but is not answering — starting a fresh one will replace it. It holds no durable data, but live OTPs, OAuth state, and rate-limit buckets are dropped.` ) } diff --git a/scripts/test-integration.ts b/scripts/test-integration.ts index 095a7854a1f..3ff0669f3d4 100644 --- a/scripts/test-integration.ts +++ b/scripts/test-integration.ts @@ -159,7 +159,7 @@ try { DATABASE_URL: databaseUrl, MIGRATION_DATABASE_URL: databaseUrl, TEST_DATABASE_URL: databaseUrl, - TEST_REDIS_URL: `redis://${redisEndpoint}`, + TEST_REDIS_URL: `redis://${redisEndpoint}/15`, ...(scale ? { KNOWLEDGE_SCALE_REPORT_FILE: scaleReportFile } : {}), } run('bun', ['run', `db:${provision}`], { cwd: path.join(root, 'packages/db'), env: environment }) From a31e812282be36c62e426ccec102de21bc06b05b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 18:52:16 -0800 Subject: [PATCH 4/6] improvement(redis): address review threads and simplify the leftovers of the fallback removal --- .claude/rules/sim-testing.md | 2 +- .cursor/rules/sim-testing.mdc | 2 +- .../src/handlers/file-doc-store.test.ts | 112 +++++++------- apps/realtime/src/handlers/file-doc.ts | 4 +- apps/realtime/src/index.ts | 3 - apps/sim/.env.example | 2 +- .../runs/[runId]/cancel/route.test.ts | 1 - apps/sim/lib/api/contracts/v2/workflows.ts | 7 +- apps/sim/lib/api/contracts/workflows.ts | 7 +- .../core/reporting-usage-cache.integration.ts | 11 +- .../billing/core/usage-segment-cache.test.ts | 28 ++-- apps/sim/lib/core/config/redis.ts | 11 +- .../lib/core/outbox/retention.integration.ts | 17 +-- .../rate-limiter/hosted-key/queue.test.ts | 4 +- .../lib/core/rate-limiter/hosted-key/queue.ts | 8 +- .../hosted-key/rate-limiter.test.ts | 2 +- .../rate-limiter/hosted-key/rate-limiter.ts | 6 +- .../rate-limiter/provider-admission.test.ts | 4 +- .../application/executor.integration.ts | 16 +- apps/sim/lib/desktop/application/executor.ts | 3 +- .../executor/bound-turn.integration.ts | 12 +- apps/sim/lib/events/pubsub.test.ts | 21 +++ apps/sim/lib/events/pubsub.ts | 48 +++++- .../cancel-workflow-execution.test.ts | 2 - .../execution/cancel-workflow-execution.ts | 8 +- .../session-file-provenance.integration.ts | 12 +- .../execution/removal-stops.integration.ts | 15 +- .../workflow-run-already-terminal-error.ts | 3 - .../lib/logs/execution/progress-markers.ts | 4 +- apps/sim/lib/mcp/oauth/storage.ts | 4 +- ...roject-file-write-transport.integration.ts | 8 +- .../async-runs/orphaned-runs.integration.ts | 33 ++-- .../lib/mothership/jobs/lane.integration.ts | 36 ++--- .../persistence/tool-confirm/index.ts | 36 ++--- .../tool-confirm/tool-confirm.test.ts | 73 +++++++-- .../application/abandoned-runs.integration.ts | 27 ++-- .../removal-stops-runs.integration.ts | 10 +- .../request/go/file-preview-adapter.test.ts | 14 -- .../lifecycle/unattended.integration.ts | 28 ++-- .../request/session/buffer-ttl.integration.ts | 8 +- .../session/recovery-storm.integration.ts | 28 ++-- .../session/replay-budget.integration.ts | 142 +++++++++--------- .../request/session/replay-gap.integration.ts | 10 +- .../session/stream-recovery.integration.ts | 10 +- .../workflow-client-settlement.integration.ts | 32 ++-- .../application/prepare-wake.integration.ts | 13 +- .../tools/handlers/workflow/mutations.test.ts | 1 - apps/sim/lib/workflows/api/route-policies.ts | 2 +- helm/sim/templates/_helpers.tpl | 22 ++- helm/sim/tests/redis_test.yaml | 45 ++++++ packages/sim-setup/src/checks.test.ts | 11 ++ packages/sim-setup/src/checks.ts | 4 + packages/sim-setup/src/redis.test.ts | 83 ++++++++++ packages/sim-setup/src/redis.ts | 26 +++- packages/sim-setup/src/setup-status.test.ts | 5 - scripts/check-explicit-any.baseline.json | 3 +- scripts/check-test-types.baseline.json | 8 +- 57 files changed, 615 insertions(+), 482 deletions(-) create mode 100644 packages/sim-setup/src/redis.test.ts diff --git a/.claude/rules/sim-testing.md b/.claude/rules/sim-testing.md index f0a8bb1117b..52f947b8474 100644 --- a/.claude/rules/sim-testing.md +++ b/.claude/rules/sim-testing.md @@ -34,7 +34,7 @@ contracts, and demonstrated regressions. | Suffix | Needs | Run with | In CI | |--------|-------|----------|-------| | `*.test.ts(x)` | nothing; global mocks from `vitest.setup.ts` | `bun run --cwd test` | `test` jobs | -| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`, required) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | +| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`) and Redis (`TEST_REDIS_URL`: required by `apps/sim` and `apps/realtime`; `packages/db` runs without it locally and requires it in CI) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | | `*.live.test.ts` | provider APIs, hosted sandboxes, local runtimes, or sibling checkouts | `bun run --cwd apps/sim test --mode live ` | never | | `apps/desktop/e2e/*.spec.ts` | the packaged Electron app | Playwright | desktop E2E workflow | | `apps/sim/scripts/test-*-e2e.ts` | a running app over HTTP | its `package.json` script when one exists (`bun run test:scim:e2e`; `test:workflow-version-compare:e2e` adds `--no-env-file`), else `bun scripts/test--e2e.ts` from apps/sim | `e2e` jobs (`.github/scripts/http-e2e.sh`) | diff --git a/.cursor/rules/sim-testing.mdc b/.cursor/rules/sim-testing.mdc index 9780636a28c..a49fff9a55b 100644 --- a/.cursor/rules/sim-testing.mdc +++ b/.cursor/rules/sim-testing.mdc @@ -31,7 +31,7 @@ contracts, and demonstrated regressions. | Suffix | Needs | Run with | In CI | |--------|-------|----------|-------| | `*.test.ts(x)` | nothing; global mocks from `vitest.setup.ts` | `bun run --cwd test` | `test` jobs | -| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`, required) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | +| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`) and Redis (`TEST_REDIS_URL`: required by `apps/sim` and `apps/realtime`; `packages/db` runs without it locally and requires it in CI) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | | `*.live.test.ts` | provider APIs, hosted sandboxes, local runtimes, or sibling checkouts | `bun run --cwd apps/sim test --mode live ` | never | | `apps/desktop/e2e/*.spec.ts` | the packaged Electron app | Playwright | desktop E2E workflow | | `apps/sim/scripts/test-*-e2e.ts` | a running app over HTTP | its `package.json` script when one exists (`bun run test:scim:e2e`; `test:workflow-version-compare:e2e` adds `--no-env-file`), else `bun scripts/test--e2e.ts` from apps/sim | `e2e` jobs (`.github/scripts/http-e2e.sh`) | diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index cee579af064..e53f933c971 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -8,7 +8,7 @@ vi.mock('redis', async () => { }) import { FileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN } from '@/handlers/file-doc-store' -import { createBacking, fakeRedis } from '@/handlers/fixtures/redis.fixture' +import { type Backing, createBacking, fakeRedis } from '@/handlers/fixtures/redis.fixture' const REDIS_URL = 'redis://fake' const NAME = 'workspace-file-doc:file-1' @@ -69,9 +69,15 @@ function seedFor(text: string): Uint8Array { let stores: FileDocStore[] = [] +/** The in-memory Redis behind every fake client; each test starts a fresh one. */ +function currentBacking(): Backing { + if (!fakeRedis.backing) throw new Error('backing not initialized') + return fakeRedis.backing +} + /** An existing stream from a relay predating generation markers; modern seeds use seedIfEmpty. */ function seedLegacyStream(update = updateFor('')): void { - const backing = fakeRedis.backing! + const backing = currentBacking() backing.streams.set(`filedoc:stream:${NAME}`, [ { id: `${++backing.seq}-0`, message: { u: Buffer.from(update).toString('base64') } }, ]) @@ -160,19 +166,19 @@ describe('FileDocStore', () => { const store = await newStore() const doc = new Y.Doc() await store.attachRoom(NAME, doc) - fakeRedis.backing!.readerClosed = true + currentBacking().readerClosed = true - fakeRedis.backing!.connects = 0 // ignore the two `init` connects; count only recovery attempts - const before = fakeRedis.backing!.reads + currentBacking().connects = 0 // ignore the two `init` connects; count only recovery attempts + const before = currentBacking().reads await vi.advanceTimersByTimeAsync(3000) - const attempts = fakeRedis.backing!.reads - before + const attempts = currentBacking().reads - before // A fixed 500ms retry manages 6 attempts in this window; backing off (500 → 1s → 2s → …) // manages about 3. The ±20% jitter moves the exact count, so assert that it slowed down. expect(attempts).toBeGreaterThan(0) expect(attempts).toBeLessThanOrEqual(4) // …and it tried to bring the connection back rather than leaving the tailer dead forever. - expect(fakeRedis.backing!.connects).toBeGreaterThan(0) + expect(currentBacking().connects).toBeGreaterThan(0) doc.destroy() } finally { vi.useRealTimers() @@ -183,7 +189,7 @@ describe('FileDocStore', () => { const store = new FileDocStore(REDIS_URL) stores.push(store) await Promise.all([store.init(), store.init(), store.init()]) - expect(fakeRedis.backing!.connects).toBe(2) + expect(currentBacking().connects).toBe(2) }) /** @@ -199,8 +205,8 @@ describe('FileDocStore', () => { await store.init() await store.attachRoom(NAME, doc) await vi.advanceTimersByTimeAsync(1000) - expect(fakeRedis.backing!.idleReads).toBeGreaterThan(0) - expect(fakeRedis.backing!.maxInFlightReads).toBe(1) + expect(currentBacking().idleReads).toBeGreaterThan(0) + expect(currentBacking().maxInFlightReads).toBe(1) } finally { doc.destroy() vi.useRealTimers() @@ -263,7 +269,7 @@ describe('FileDocStore', () => { userId: 'editor', connectionId: 'socket-editor', }) - fakeRedis.backing!.streams.delete(`filedoc:stream:${NAME}`) + currentBacking().streams.delete(`filedoc:stream:${NAME}`) await expect(store.publishAndWait(NAME, delta, generation)).rejects.toThrow('replaced') await expect( @@ -278,14 +284,14 @@ describe('FileDocStore', () => { connectionId: 'socket-editor', }) ).rejects.toThrow('replaced') - expect(fakeRedis.backing!.streams.has(`filedoc:stream:${NAME}`)).toBe(false) + expect(currentBacking().streams.has(`filedoc:stream:${NAME}`)).toBe(false) }) it('rejects a shared replay if the document generation changes between pages', async () => { const store = await newStore() await store.seedIfEmpty(NAME, seedFor('old generation'), 10) - fakeRedis.backing!.onRange = () => { - fakeRedis.backing!.kv.set(`filedoc:generation:${NAME}`, 'new generation') + currentBacking().onRange = () => { + currentBacking().kv.set(`filedoc:generation:${NAME}`, 'new generation') } await expect(store.getStreamState(NAME)).rejects.toThrow('replaced') }) @@ -293,14 +299,14 @@ describe('FileDocStore', () => { it('fails safely when an uncompacted stream exceeds the replay entry budget', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(updateFor('')).toString('base64') - fakeRedis.backing!.streams.set( + currentBacking().streams.set( streamKey, Array.from({ length: 2_001 }, (_, index) => ({ id: `${index + 1}-0`, message: { u: noop }, })) ) - fakeRedis.backing!.seq = 2_001 + currentBacking().seq = 2_001 const store = await newStore() await expect(store.getStreamState(NAME)).rejects.toThrow('replay exceeded its safety limit') @@ -309,14 +315,14 @@ describe('FileDocStore', () => { it('never exposes a partially replayed document when room attachment exceeds its budget', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(updateFor('')).toString('base64') - fakeRedis.backing!.streams.set( + currentBacking().streams.set( streamKey, Array.from({ length: 2_001 }, (_, index) => ({ id: `${index + 1}-0`, message: { u: noop }, })) ) - fakeRedis.backing!.seq = 2_001 + currentBacking().seq = 2_001 const store = await newStore() const doc = new Y.Doc() @@ -330,24 +336,24 @@ describe('FileDocStore', () => { it('recovers from compaction that trims unread pages during replay', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(updateFor('')).toString('base64') - fakeRedis.backing!.streams.set( + currentBacking().streams.set( streamKey, Array.from({ length: 8 }, (_, index) => ({ id: `${index + 1}-0`, message: { u: noop }, })) ) - fakeRedis.backing!.seq = 8 - fakeRedis.backing!.onRange = (call, key) => { + currentBacking().seq = 8 + currentBacking().onRange = (call, key) => { if (call !== 2 || key !== streamKey) return - fakeRedis.backing!.streams.set(streamKey, [ + currentBacking().streams.set(streamKey, [ { id: '9-0', message: { u: Buffer.from(updateFor('compacted')).toString('base64'), s: '1' }, }, ]) - fakeRedis.backing!.seq = 9 - fakeRedis.backing!.onRange = undefined + currentBacking().seq = 9 + currentBacking().onRange = undefined } const store = await newStore() @@ -370,16 +376,16 @@ describe('FileDocStore', () => { for (let i = 1; i <= 8; i++) source.getText('body').insert(source.getText('body').length, String(i)) const snapshot = Y.encodeStateAsUpdate(source) - fakeRedis.backing!.streams.set(streamKey, entries.slice()) + currentBacking().streams.set(streamKey, entries.slice()) for (let i = 9; i <= 11; i++) source.getText('body').insert(source.getText('body').length, String(i)) - fakeRedis.backing!.onRange = (_call, key, start) => { + currentBacking().onRange = (_call, key, start) => { if (key !== streamKey || start !== '(4-0') return - fakeRedis.backing!.streams.set(streamKey, [ + currentBacking().streams.set(streamKey, [ ...entries.slice(7), { id: '12-0', message: { u: Buffer.from(snapshot).toString('base64'), s: '1' } }, ]) - fakeRedis.backing!.onRange = undefined + currentBacking().onRange = undefined } const store = await newStore() const recovered = new Y.Doc() @@ -411,8 +417,8 @@ describe('FileDocStore', () => { })) entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) - fakeRedis.backing!.streams.set(streamKey, entries) - fakeRedis.backing!.seq = 402 + currentBacking().streams.set(streamKey, entries) + currentBacking().seq = 402 const a = await newStore() // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the @@ -438,7 +444,7 @@ describe('FileDocStore', () => { // The appended snapshot entry must carry the snapshot marker, so a fresh catch-up task treats it as // edited content (not a bare seed) and persists on last-disconnect. - const stream = fakeRedis.backing!.streams.get(streamKey)! + const stream = currentBacking().streams.get(streamKey)! expect(stream[stream.length - 1].message.s).toBe('1') }) @@ -461,7 +467,7 @@ describe('FileDocStore', () => { // the peer tailer applies it as REDIS_AGENT_ORIGIN — excluded from the relay's edited/persist gate. a.publish(NAME, updateFor('agent frame'), true) await vi.waitFor(() => expect(origins).toContain(REDIS_AGENT_ORIGIN), { timeout: 2000 }) - const stream = fakeRedis.backing!.streams.get(streamKey)! + const stream = currentBacking().streams.get(streamKey)! expect(stream.some((e) => e.message.a === '1')).toBe(true) // The normal edit's entry carries no agent marker. expect(stream.filter((e) => e.message.a === '1')).toHaveLength(1) @@ -499,7 +505,7 @@ describe('FileDocStore', () => { } await vi.waitFor(() => { - const stream = fakeRedis.backing!.streams.get(`filedoc:stream:${NAME}`)! + const stream = currentBacking().streams.get(`filedoc:stream:${NAME}`)! expect(stream.length).toBeLessThan(400) expect(stream.some((entry) => entry.message.c === '1')).toBe(true) }) @@ -515,7 +521,7 @@ describe('FileDocStore', () => { it('retries a transient append failure so the edit is not lost from the shared log', async () => { seedLegacyStream() const a = await newStore() - fakeRedis.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed + currentBacking().failXAdd = 2 // first two xAdd attempts throw; the third must succeed a.publish(NAME, updateFor('resilient')) await vi.waitFor( async () => { @@ -542,7 +548,7 @@ describe('FileDocStore', () => { connectionId: 'socket-editor', }) - expect(fakeRedis.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + expect(currentBacking().streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) }) it('does not drop different payloads that reuse an acknowledged update id', async () => { @@ -558,7 +564,7 @@ describe('FileDocStore', () => { connectionId: 'socket-editor', }) - expect(fakeRedis.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) + expect(currentBacking().streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) }) it('preserves the inclusive barrier and delta bytes observed during compaction', async () => { @@ -570,7 +576,7 @@ describe('FileDocStore', () => { room.uncompactedDeltaBytes = 12 * 1024 * 1024 const retainedBarrierBytes = room.lastDeltaBytes const lateUpdate = updateFor('concurrent edit') - fakeRedis.backing!.onSnapshot = async () => { + currentBacking().onSnapshot = async () => { await store.publishAndWait(NAME, lateUpdate) await store.catchUp(NAME) } @@ -588,7 +594,7 @@ describe('FileDocStore', () => { const replacer = await newStore() const oldDoc = docWithText('old') oldDoc.getMap('config').set('docId', 'old-generation') - fakeRedis.backing!.nextIds = ['1000-0', '1000-1', '1000-2'] + currentBacking().nextIds = ['1000-0', '1000-1', '1000-2'] await store.seedIfEmpty(NAME, Y.encodeStateAsUpdate(oldDoc), 10) await store.publishAndWait(NAME, updateFor('first edit'), 'old-generation') await store.publishAndWait(NAME, updateFor('second edit'), 'old-generation') @@ -602,31 +608,33 @@ describe('FileDocStore', () => { freshDoc.getText('body').insert(5, ' accepted edit') const freshEdit = Y.encodeStateAsUpdate(freshDoc, beforeEdit) const streamKey = `filedoc:stream:${NAME}` - fakeRedis.backing!.nextIds = ['1000-3', '1000-0', '1000-1'] - fakeRedis.backing!.onSnapshot = async () => { + currentBacking().nextIds = ['1000-3', '1000-0', '1000-1'] + currentBacking().onSnapshot = async () => { await replacer.invalidateDocument(NAME, 20) await replacer.seedIfEmpty(NAME, freshSeed, 20) await replacer.publishClientUpdateAndWait(NAME, 'fresh-edit', freshEdit, 'new-generation', { userId: 'editor', connectionId: 'socket-editor', }) - expect(fakeRedis.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ - '1000-0', - '1000-1', - ]) + expect( + currentBacking() + .streams.get(streamKey) + ?.map((entry) => entry.id) + ).toEqual(['1000-0', '1000-1']) } await storeInternals(store).maybeCompact(NAME) - expect(fakeRedis.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ - '1000-0', - '1000-1', - ]) + expect( + currentBacking() + .streams.get(streamKey) + ?.map((entry) => entry.id) + ).toEqual(['1000-0', '1000-1']) await replacer.publishClientUpdateAndWait(NAME, 'fresh-edit', freshEdit, 'new-generation', { userId: 'editor', connectionId: 'socket-editor', }) - expect(fakeRedis.backing!.streams.get(streamKey)).toHaveLength(2) + expect(currentBacking().streams.get(streamKey)).toHaveLength(2) const persisted = await replacer.getStreamState(NAME) expect(persisted).not.toBeNull() const replayed = new Y.Doc() @@ -682,7 +690,7 @@ describe('FileDocStore', () => { const tokenA = await a.shouldSeed(NAME) expect(tokenA).toBeTruthy() // Simulate A's lock expiring mid-seed so B also wins the freed lock over a still-empty stream. - fakeRedis.backing!.kv.delete(`filedoc:seedlock:${NAME}`) + currentBacking().kv.delete(`filedoc:seedlock:${NAME}`) const tokenB = await b.shouldSeed(NAME) expect(tokenB).toBeTruthy() // Both tasks now race to seed with distinct client ids. @@ -742,8 +750,8 @@ describe('FileDocStore', () => { })) entries.push({ id: '401-0', message: { u: Buffer.from(peerUpdates[0]).toString('base64') } }) entries.push({ id: '402-0', message: { u: Buffer.from(peerUpdates[1]).toString('base64') } }) - fakeRedis.backing!.streams.set(streamKey, entries) - fakeRedis.backing!.seq = 402 + currentBacking().streams.set(streamKey, entries) + currentBacking().seq = 402 // Two tasks whose local docs lag at DIFFERENT points (400 and 401): both cross the threshold and // compact concurrently. Each must only trim what its own snapshot subsumes, so the union of both diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 901d6d5e651..87fa605dd90 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -709,8 +709,8 @@ async function runServerSeed( const store = getFileDocStore() const deadline = Date.now() + FILE_DOC_TIMEOUTS.seedRequestMs while (fileDocRooms.get(name) === room && !isDocSeeded(room.doc)) { - // Exactly one task across the cluster builds the seed; the others receive it via the stream (the - // fix for split-brain seeding). + // Exactly one task across the cluster builds the seed (no split-brain); the others receive it + // via the stream. const token = await store.shouldSeed(name) if (token) { await seedUnderLock(name, room, token, actor) diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index 7f867964f9a..ab3a824386d 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -44,13 +44,10 @@ async function main() { let httpHandler: ReturnType | undefined httpServer.on('request', (req, res) => httpHandler?.(req, res)) - // Create Socket.IO server with the Redis adapter for cross-pod broadcasting const io = await createSocketIOServer(httpServer) - // Initialize the Redis-backed room manager const roomManager = await createRoomManager(io) - // Initialize the shared Yjs backend for collaborative file docs (Redis Streams). await initFileDocStore(env.REDIS_URL) registerFileDocPresenceRelay(io) diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 1483b6dea40..5bc0362cfaf 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -8,7 +8,7 @@ DATABASE_URL="postgresql://postgres:your_password@localhost:5432/simstudio" # Redis (Required) - rate limiting, OTPs, pub/sub, and realtime coordination. The app refuses to start without it. # Local default: `docker run -d -p 127.0.0.1:6379:6379 redis:7-alpine` (or let `npx sim-setup` manage one). REDIS_URL=redis://localhost:6379 -# REDIS_TLS_SERVERNAME= # Required only for a rediss:// URL that targets a bare IP address (TLS hostname verification) +# REDIS_TLS_SERVERNAME= # Required only for a rediss:// URL that targets a bare IPv4 address (TLS hostname verification) # Authentication (Required unless DISABLE_AUTH=true) BETTER_AUTH_SECRET=your_secret_key # Use `openssl rand -hex 32` to generate, or visit https://www.better-auth.com/docs/installation diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts index 3255af4de16..5cf711d03fc 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/runs/[runId]/cancel/route.test.ts @@ -106,7 +106,6 @@ describe('POST /api/v2/workflows/[workflowId]/runs/[runId]/cancel', () => { new WorkflowRunAlreadyTerminalError({ executionId: RUN_ID, executionStatus, - redisAvailable: true, locallyAborted: false, }) ) diff --git a/apps/sim/lib/api/contracts/v2/workflows.ts b/apps/sim/lib/api/contracts/v2/workflows.ts index b44070b22f3..1283b0c0dac 100644 --- a/apps/sim/lib/api/contracts/v2/workflows.ts +++ b/apps/sim/lib/api/contracts/v2/workflows.ts @@ -2539,7 +2539,12 @@ export const v2CancelWorkflowRunDataSchema = z 'Whether this request cancelled anything. `false` with an `already_*` reason means the run had already reached that terminal state, so there was nothing to cancel — still `200`, because a no-op is not an error. `false` with any other reason identifies a degraded or incomplete cancellation step.' ), runId: v2WorkflowRunIdSchema, - /** Kept for wire compatibility; always true since Redis is required, except on a paused-run cancellation whose event failed to publish. */ + /** + * Kept for wire compatibility; it no longer reports Redis availability. False only when a + * cancellation that must publish its cancellation event (a paused run's, or a reconciled + * workflow-group run's) did not: the stop was not confirmed, the pause could not be reconciled, + * or the publish failed. True otherwise. + */ redisAvailable: z .boolean() .describe('Whether the distributed cancellation channel was available.'), diff --git a/apps/sim/lib/api/contracts/workflows.ts b/apps/sim/lib/api/contracts/workflows.ts index 98adae7c549..5886725417c 100644 --- a/apps/sim/lib/api/contracts/workflows.ts +++ b/apps/sim/lib/api/contracts/workflows.ts @@ -674,7 +674,12 @@ export const cancelWorkflowExecutionReasonSchema = z.enum([ const cancelWorkflowExecutionResponseSchema = z.object({ success: z.boolean(), executionId: z.string(), - /** Kept for wire compatibility; always true since Redis is required, except on a paused-run cancellation whose event failed to publish. */ + /** + * Kept for wire compatibility; it no longer reports Redis availability. False only when a + * cancellation that must publish its cancellation event (a paused run's, or a reconciled + * workflow-group run's) did not: the stop was not confirmed, the pause could not be reconciled, + * or the publish failed. True otherwise. + */ redisAvailable: z.boolean(), durablyRecorded: z.boolean(), locallyAborted: z.boolean(), diff --git a/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts b/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts index e2a48123357..86cda785550 100644 --- a/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts +++ b/apps/sim/lib/billing/core/reporting-usage-cache.integration.ts @@ -1,7 +1,7 @@ /** * The shared reporting-usage read against a real ledger in a disposable PostgreSQL schema and a - * real Redis. Skipped without `TEST_REDIS_URL`. Each test uses a fresh payer, so the in-process - * cache is always cold and every read models a new process. + * real Redis. Each test uses a fresh payer, so the in-process cache is always cold and every read + * models a new process. */ import { type AddressInfo, createServer, type Socket } from 'node:net' @@ -22,6 +22,7 @@ const { transaction, summedPeriods } = vi.hoisted(() => ({ })) const databaseUrl = readTestDatabaseUrl() const redisUrl = readTestRedisUrl() +if (!redisUrl) throw new Error('TEST_REDIS_URL is required') vi.mock('@sim/db', () => { const db = { transaction } @@ -166,7 +167,7 @@ async function exactSum(payer: BillingEntity, period: UsageQueryPeriod): Promise return Number(row?.cost ?? 0) } -describe.runIf(Boolean(redisUrl))('shared reporting usage read', () => { +describe('shared reporting usage read', () => { let redis: Redis let payer: BillingEntity @@ -175,7 +176,7 @@ describe.runIf(Boolean(redisUrl))('shared reporting usage read', () => { } beforeAll(async () => { - redis = new Redis(redisUrl!, { lazyConnect: true, maxRetriesPerRequest: 0 }) + redis = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) await redis.connect() await connection.unsafe(`CREATE SCHEMA "${schemaName}"`) await connection.unsafe(`CREATE TABLE usage_log ( @@ -475,7 +476,7 @@ describe.runIf(Boolean(redisUrl))('shared reporting usage read', () => { }) it('sums the ledger without issuing or queuing a command while Redis is not ready', async () => { - const notReady = new Redis(redisUrl!, { lazyConnect: true, maxRetriesPerRequest: 0 }) + const notReady = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) redisConfigMockFns.mockGetRedisClient.mockReturnValue(notReady) try { await expect(readSoftGateUsageCost(payer, REPORTING)).resolves.toBe(5.75) diff --git a/apps/sim/lib/billing/core/usage-segment-cache.test.ts b/apps/sim/lib/billing/core/usage-segment-cache.test.ts index 71404cd68e5..e36aea55026 100644 --- a/apps/sim/lib/billing/core/usage-segment-cache.test.ts +++ b/apps/sim/lib/billing/core/usage-segment-cache.test.ts @@ -1,3 +1,4 @@ +import { flushMacrotask } from '@sim/testing/helpers/async' import { createMockRedis } from '@sim/testing/mocks/redis.mock' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { generateShortId } from '@sim/utils/id' @@ -56,19 +57,9 @@ describe('readThroughSegments', () => { const stored = new Map() const redis = Object.assign(createMockRedis(), { mget: async (...keys: string[]) => keys.map((key) => stored.get(key) ?? null), - pipeline: () => { - const writes: [string, string][] = [] - const pipeline = { - set: (key: string, value: string) => { - writes.push([key, value]) - return pipeline - }, - exec: async () => { - for (const [key, value] of writes) stored.set(key, value) - return [] - }, - } - return pipeline + set: async (key: string, value: string) => { + stored.set(key, value) + return 'OK' }, }) redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) @@ -76,10 +67,11 @@ describe('readThroughSegments', () => { }) it('reads the ledger when a stored entry does not parse, rather than failing', async () => { - redisConfigMockFns.mockGetRedisClient.mockReturnValue({ - mget: async (...keys: string[]) => keys.map(() => '{not json'), - pipeline: () => ({ set: () => undefined, exec: async () => [] }), - }) + redisConfigMockFns.mockGetRedisClient.mockReturnValue( + Object.assign(createMockRedis(), { + mget: async (...keys: string[]) => keys.map(() => '{not json'), + }) + ) const entries = await read([day('2026-01-05')]) expect(fetchRange).toHaveBeenCalledTimes(1) expect(totalByDay(entries)).toEqual({ '2026-01-05': 24 }) @@ -96,6 +88,7 @@ describe('readThroughSegments', () => { it('serves settled segments from the cache and re-reads only the open hours', async () => { const segments = [day('2026-01-01'), ...hours('2026-01-02', 10)] await read(segments) + await flushMacrotask() fetchRange.mockClear() const entries = await read(segments) @@ -106,6 +99,7 @@ describe('readThroughSegments', () => { it('assembles a newly settled day from its cached hours without a ledger read', async () => { await read(hours('2026-01-02', 23)) + await flushMacrotask() fetchRange.mockClear() expect(totalByDay(await read([day('2026-01-02')]))).toEqual({ '2026-01-02': 24 }) diff --git a/apps/sim/lib/core/config/redis.ts b/apps/sim/lib/core/config/redis.ts index fbb0abe235e..cb535788a70 100644 --- a/apps/sim/lib/core/config/redis.ts +++ b/apps/sim/lib/core/config/redis.ts @@ -17,8 +17,7 @@ const logger = createLogger('Redis') * * For DNS hosts: no override needed, default verification works. */ -function resolveRedisTlsOptions(url: string | undefined): { servername: string } | undefined { - if (!url) return undefined +function resolveRedisTlsOptions(url: string): { servername: string } | undefined { let parsed: URL try { parsed = new URL(url) @@ -55,7 +54,7 @@ export const CONNECT_TIMEOUT_MS = 10_000 /** ioredis's own default, stated so readiness budgets can be derived from it. */ export const DISCONNECT_TIMEOUT_MS = 2_000 -export function getRedisConnectionDefaults(url: string | undefined): RedisConnectionDefaults { +export function getRedisConnectionDefaults(url: string): RedisConnectionDefaults { const tls = resolveRedisTlsOptions(url) return { keepAlive: 1000, @@ -250,15 +249,11 @@ const REDIS_WARMUP_TIMEOUT_MS = coldConnectionBudgetMs({ * Redis channels at module scope without any runtime environment. */ export function getConfiguredRedisUrl(): string { - const redisUrl = env.REDIS_URL - if (!redisUrl) { - throw new Error('REDIS_URL is required') - } const configurationError = getRedisConfigurationError(env) if (configurationError) { throw new Error(configurationError) } - return redisUrl + return env.REDIS_URL } /** diff --git a/apps/sim/lib/core/outbox/retention.integration.ts b/apps/sim/lib/core/outbox/retention.integration.ts index b017e70122d..9363c97a5d5 100644 --- a/apps/sim/lib/core/outbox/retention.integration.ts +++ b/apps/sim/lib/core/outbox/retention.integration.ts @@ -40,6 +40,7 @@ const OCR_CHECKPOINT_EXPIRY = 'knowledge.document.ocr-checkpoint.expire' const EMBEDDING_CHECKPOINT_EXPIRY = 'knowledge.document.embedding-checkpoint.expire' const DEPLOYMENT_EFFECT = 'workflow.deployment.sync-active-side-effects' const redisUrl = readTestRedisUrl() +if (!redisUrl) throw new Error('TEST_REDIS_URL is required') describe('completed outbox retention in PostgreSQL', () => { const schemaName = `outbox_retention_${generateId().replaceAll('-', '')}` @@ -58,22 +59,19 @@ describe('completed outbox retention in PostgreSQL', () => { const cursorKeys = [RECOVER, EMBEDDING_CHECKPOINT_EXPIRY].map( (eventType) => `outbox-prune-cursor:v1:${eventType}` ) - let redis: Redis | undefined + const redis = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) beforeAll(async () => { await connection`CREATE SCHEMA ${connection(schemaName)}` await connection`CREATE TABLE outbox_event (LIKE public.outbox_event INCLUDING ALL)` database.current = testDb - if (redisUrl) { - redis = new Redis(redisUrl, { lazyConnect: true, maxRetriesPerRequest: 0 }) - await redis.connect() - await redis.del(...cursorKeys) - } + await redis.connect() + await redis.del(...cursorKeys) }) afterEach(async () => { redisConfigMockFns.mockGetRedisClient.mockReturnValue(createMockRedis()) - await redis?.del(...cursorKeys) + await redis.del(...cursorKeys) await connection`TRUNCATE outbox_event` }) @@ -82,14 +80,13 @@ describe('completed outbox retention in PostgreSQL', () => { await connection`DROP SCHEMA ${connection(schemaName)} CASCADE` } finally { await connection.end() - await redis?.quit() + await redis.quit() database.current = undefined } }) /** Runs the rest of the test with each type's pass resuming through real Redis. */ function resumeThroughRedis(): Redis { - if (!redis) throw new Error('TEST_REDIS_URL is required for the resume cases') redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) return redis } @@ -180,7 +177,7 @@ describe('completed outbox retention in PostgreSQL', () => { expect(rows.map((row) => [row.id, row.status])).toEqual([[gate.id, 'pending']]) }) - describe.runIf(Boolean(redisUrl))("resuming each type's pass across runs", () => { + describe("resuming each type's pass across runs", () => { /** One full batch plus one newer row, all completed, a millisecond apart, oldest first. */ async function seedFullBatchPlusOne(eventType: string) { const newest = expired().toISOString() diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts b/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts index d1cf401689d..226b5e3c0b8 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts @@ -57,12 +57,12 @@ describe('HostedKeyQueue', () => { }) describe('enqueue', () => { - it('falls back to enabled=false on Redis error', async () => { + it('falls back to enqueued=false on Redis error', async () => { mockRedis.pipeline.exec.mockRejectedValueOnce(new Error('connection lost')) const result = await queue.enqueue(provider, workspaceId, ticketId) - expect(result.enabled).toBe(false) + expect(result.enqueued).toBe(false) }) }) diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts b/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts index fa7a2261119..1b319735711 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/queue.ts @@ -63,7 +63,7 @@ export interface EnqueueResult { /** Position at the moment of enqueue (0 = head, you go next). */ position: number /** Whether the ticket was enqueued — false means the enqueue failed and the queue is skipped. */ - enabled: boolean + enqueued: boolean } /** @@ -99,10 +99,10 @@ export class HostedKeyQueue { // results[0] is the rpush response: [err, length] const length = results?.[0] && typeof results[0][1] === 'number' ? results[0][1] : 1 // Position is length - 1 (just-pushed at the tail). - return { position: length - 1, enabled: true } + return { position: length - 1, enqueued: true } } catch (error) { logger.warn(`Queue enqueue failed for ${listKey}`, { error: toError(error).message }) - return { position: 0, enabled: false } + return { position: 0, enqueued: false } } } @@ -115,7 +115,7 @@ export class HostedKeyQueue { * - "head": you're at the head, proceed to consume from the bucket * - "waiting": someone else is the head and they're alive * - "missing": your ticket isn't in the queue at all (e.g. queue list TTL - * expired); caller should re-enqueue or treat as enabled=false + * expired); caller should re-enqueue or treat as enqueued=false */ async checkHead( provider: string, diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts index 56188cc6ef5..7b9f6e7479e 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts @@ -35,7 +35,7 @@ interface MockQueue { * queue is empty. Tests override per-call to simulate ordering. */ const createMockQueue = (): MockQueue => { const queue: MockQueue = { - enqueue: vi.fn().mockResolvedValue({ position: 0, enabled: true }), + enqueue: vi.fn().mockResolvedValue({ position: 0, enqueued: true }), checkHead: vi.fn().mockResolvedValue('head'), refreshHeartbeat: vi.fn().mockResolvedValue(undefined), dequeue: vi.fn().mockResolvedValue(undefined), diff --git a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts index 75a7c101306..675beb5e383 100644 --- a/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts +++ b/apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.ts @@ -280,7 +280,7 @@ export class HostedKeyRateLimiter { const enqueueResult = await this.queue.enqueue(provider, billingActorId, ticketId) try { - // Wait for our turn at the head of the queue (no-op when Redis unavailable). + // Wait for our turn at the head of the queue (returns at once when the queue fails open). const headStatus = await this.waitForQueueHead( provider, billingActorId, @@ -348,7 +348,7 @@ export class HostedKeyRateLimiter { } const totalWaitedMs = Date.now() - startedAt - if (enqueueResult.enabled && (enqueueResult.position > 0 || totalWaitedMs > 100)) { + if (enqueueResult.enqueued && (enqueueResult.position > 0 || totalWaitedMs > 100)) { // Attribute the wait to its dominant cause: queue depth takes precedence (it's // reported alongside queuePosition), otherwise the bucket phase that actually slept. const reason: 'queue_position' | 'actor_requests' | 'dimension' = @@ -457,7 +457,7 @@ export class HostedKeyRateLimiter { * regular cadence so we don't get reaped as dead. Returns `timedOut: true` once the wait * budget is exhausted before reaching the head (see {@link remainingWaitBudgetMs}). * - * Fails open when a queue Redis call fails (queue.enqueue returns enabled=false and checkHead + * Fails open when a queue Redis call fails (queue.enqueue returns enqueued=false and checkHead * returns 'head'). */ private async waitForQueueHead( diff --git a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts index 17d2ca8e5c9..a7675078567 100644 --- a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts +++ b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts @@ -259,8 +259,8 @@ describe('provider admission', () => { }) expect(consumeTokens).not.toHaveBeenCalled() }) - it('does not interpret a failed rate_limit_bucket query as a reason to retry provider work', async () => { - consumeTokens.mockRejectedValue(new Error('rate_limit_bucket database unavailable')) + it('does not interpret a failed rate-limit storage call as a reason to retry provider work', async () => { + consumeTokens.mockRejectedValue(new Error('rate-limit storage unavailable')) const operation = vi.fn(() => waitForProviderAdmission(INPUT)) await expect(retryWithExponentialBackoff(operation, { maxRetries: 3 })).rejects.toMatchObject({ retryable: false, diff --git a/apps/sim/lib/desktop/application/executor.integration.ts b/apps/sim/lib/desktop/application/executor.integration.ts index 5a3c4899e4a..ee051214449 100644 --- a/apps/sim/lib/desktop/application/executor.integration.ts +++ b/apps/sim/lib/desktop/application/executor.integration.ts @@ -4,16 +4,6 @@ * completion. Calls are persisted the way the run loop persists a bound run's desktop calls, and * Stop runs through the real `requestRunStop`. */ -import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { afterAll, describe, expect, it, vi } from 'vitest' - -const { redisUrl } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - const url = readTestRedisUrl() - /** The real Redis module reads this at import. */ - if (url) process.env.REDIS_URL = url - return { redisUrl: url } -}) import type { SessionPrincipal } from '@sim/auth/principal' import { db } from '@sim/db' @@ -27,11 +17,13 @@ import { user, workspace, } from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { createDeferred } from '@sim/testing/helpers/deferred' import { generateId } from '@sim/utils/id' import { compareStrings } from '@sim/utils/string' import { eq, inArray, sql } from 'drizzle-orm' import type { PgUpdateSetSource } from 'drizzle-orm/pg-core' +import { afterAll, describe, expect, it, vi } from 'vitest' import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' import type { DbTransaction } from '@/lib/db/types' import { @@ -60,7 +52,7 @@ afterAll(async () => { await closeRedisConnection() }) -describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => { +describe('desktop background executor protocol', () => { const userIds: string[] = [] const workspaceIds = new Map() const chatIds: string[] = [] @@ -666,7 +658,7 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => expect(woken?.status).toBe('success') /** The wake-up itself: the confirmation every waiter's pub/sub wakes on, published durably. */ await expect - .poll(() => getRedisClient()?.get(`copilot:tool-confirmation:${toolCallId}`)) + .poll(() => getRedisClient().get(`copilot:tool-confirmation:${toolCallId}`)) .toContain('"status":"success"') await expect( unsealClientToolCompletion(woken?.data, { diff --git a/apps/sim/lib/desktop/application/executor.ts b/apps/sim/lib/desktop/application/executor.ts index 8cff3615344..6f043c96368 100644 --- a/apps/sim/lib/desktop/application/executor.ts +++ b/apps/sim/lib/desktop/application/executor.ts @@ -83,8 +83,7 @@ async function requireBoundDevice(principal: SessionPrincipal, deviceId: string) /** * The device a new turn binds to: the composer's own, but only while the device is registered to - * this very session as an executor. Anything else leaves the turn to the chat view, as before the - * executor existed. + * this very session as an executor. Anything else leaves the turn to the chat view. */ export async function resolveTurnDesktopDevice( principal: SessionPrincipal, diff --git a/apps/sim/lib/desktop/executor/bound-turn.integration.ts b/apps/sim/lib/desktop/executor/bound-turn.integration.ts index f3889fbf328..6e533f6b6cc 100644 --- a/apps/sim/lib/desktop/executor/bound-turn.integration.ts +++ b/apps/sim/lib/desktop/executor/bound-turn.integration.ts @@ -9,8 +9,7 @@ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/ import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') /** Stands in for the agent worker, which Stop also tells to end the stream. */ const server = createServer(async (request, response) => { @@ -21,17 +20,14 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, COPILOT_TOOL_PERMISSIONS_ENABLED: process.env.COPILOT_TOOL_PERMISSIONS_ENABLED, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - /** The real Redis module, the confirmation channel, the permission flag and worker URL read these at import. */ - if (url) process.env.REDIS_URL = url + /** The permission flag and worker URL read these at import. */ process.env.COPILOT_TOOL_PERMISSIONS_ENABLED = 'true' process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: { server } } + return { inheritedEnv, worker: { server } } }) vi.mock('@/lib/auth', () => authMock) @@ -145,7 +141,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))("a turn bound to a desktop's background executor", () => { +describe("a turn bound to a desktop's background executor", () => { const userId = generateId() const workspaceId = generateId() const chatIds: string[] = [] diff --git a/apps/sim/lib/events/pubsub.test.ts b/apps/sim/lib/events/pubsub.test.ts index 9449381d219..1231442da21 100644 --- a/apps/sim/lib/events/pubsub.test.ts +++ b/apps/sim/lib/events/pubsub.test.ts @@ -125,4 +125,25 @@ describe('createPubSubChannel over Redis', () => { subscriber().subscribed[1](null) expect(await settled(channel.ready())).toBe(true) }) + + it('calls a ready listener once the subscription lands, and never one it has released', () => { + const channel = openChannel() + const kept = vi.fn() + const released = vi.fn() + channel.onReady(kept) + const release = channel.onReady(released) + release() + expect(channel.isReady()).toBe(false) + + subscriber().emit('ready') + subscriber().subscribed[0](null) + + expect(channel.isReady()).toBe(true) + expect(kept).toHaveBeenCalledOnce() + expect(released).not.toHaveBeenCalled() + + const late = vi.fn() + channel.onReady(late) + expect(late).toHaveBeenCalledOnce() + }) }) diff --git a/apps/sim/lib/events/pubsub.ts b/apps/sim/lib/events/pubsub.ts index 0be76ba2292..69ce6f7e4c8 100644 --- a/apps/sim/lib/events/pubsub.ts +++ b/apps/sim/lib/events/pubsub.ts @@ -22,6 +22,14 @@ export interface PubSubChannel { * that must not wait indefinitely bounds the wait. */ ready(): Promise + /** Whether this process receives the channel's publications right now. */ + isReady(): boolean + /** + * Calls `listener` once this process next receives the channel's publications (at once if it + * already does). The returned function drops a listener that has not fired, so a caller that + * gives up while the channel cannot subscribe retains nothing. + */ + onReady(listener: () => void): () => void dispose(): void } @@ -41,6 +49,7 @@ class RedisPubSubChannel implements PubSubChannel { private closedConnections = 0 private subscribed: Promise = Promise.resolve() private markSubscribed: () => void = noop + private readyListeners = new Set<() => void>() constructor( redisUrl: string, @@ -83,6 +92,15 @@ class RedisPubSubChannel implements PubSubChannel { this.listening = true logger.info(`Subscribed to ${config.label} channel`) this.markSubscribed() + const listeners = [...this.readyListeners] + this.readyListeners.clear() + for (const listener of listeners) { + try { + listener() + } catch (err) { + logger.error(`Error in ${config.label} ready listener:`, err) + } + } }) }) this.sub.on('close', () => { @@ -127,6 +145,22 @@ class RedisPubSubChannel implements PubSubChannel { return this.subscribed } + isReady(): boolean { + return this.listening + } + + onReady(listener: () => void): () => void { + if (this.disposed) return noop + if (this.listening) { + listener() + return noop + } + this.readyListeners.add(listener) + return () => { + this.readyListeners.delete(listener) + } + } + private awaitSubscription(): void { this.subscribed = new Promise((resolve) => { this.markSubscribed = resolve @@ -136,6 +170,7 @@ class RedisPubSubChannel implements PubSubChannel { dispose(): void { this.disposed = true this.handlers.clear() + this.readyListeners.clear() this.pub.removeAllListeners() this.sub.removeAllListeners() @@ -150,9 +185,10 @@ class RedisPubSubChannel implements PubSubChannel { } /** - * Builds a channel whose Redis connections open on its first `publish`, `subscribe`, or `ready`. + * Builds a channel whose Redis connections open on its first `publish`, `subscribe`, or readiness + * call. * A missing or invalid REDIS_URL surfaces from that first call, never from construction. A - * disposed channel publishes nothing, adds no handlers, and never becomes ready. + * disposed channel publishes nothing, adds no handlers or ready listeners, and never becomes ready. */ export function createPubSubChannel(config: PubSubChannelConfig): PubSubChannel { let connection: RedisPubSubChannel | null = null @@ -178,6 +214,14 @@ export function createPubSubChannel(config: PubSubChannelConfig): PubSubChann if (disposed) return new Promise(noop) return open().ready() }, + isReady() { + if (disposed) return false + return open().isReady() + }, + onReady(listener) { + if (disposed) return noop + return open().onReady(listener) + }, dispose() { disposed = true connection?.dispose() diff --git a/apps/sim/lib/execution/cancel-workflow-execution.test.ts b/apps/sim/lib/execution/cancel-workflow-execution.test.ts index 438a33d71a4..8202db15997 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.test.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.test.ts @@ -1537,7 +1537,6 @@ describe('cancelWorkflowExecution', () => { code: 'conflict', executionId: 'ex-1', executionStatus, - redisAvailable: true, locallyAborted: false, }) ) @@ -1583,7 +1582,6 @@ describe('cancelWorkflowExecution', () => { code: 'conflict', executionId: 'ex-1', executionStatus: 'completed', - redisAvailable: true, locallyAborted: false, }) ) diff --git a/apps/sim/lib/execution/cancel-workflow-execution.ts b/apps/sim/lib/execution/cancel-workflow-execution.ts index ed3a22dbae2..193fd85c982 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.ts @@ -51,8 +51,10 @@ export interface CancelWorkflowExecutionResult { success: boolean executionId: string /** - * Kept on the wire for compatibility. Redis is required, so this is true except on a paused-run - * cancellation whose cancellation event failed to publish. + * Kept on the wire for compatibility; it no longer reports Redis availability. False only when a + * cancellation that must publish its cancellation event (a paused run's, or a reconciled + * workflow-group run's) did not: the stop was not confirmed, the pause could not be reconciled, + * or the publish failed. True otherwise. */ redisAvailable: boolean durablyRecorded: boolean @@ -829,7 +831,6 @@ export async function cancelWorkflowExecution({ throw new WorkflowRunAlreadyTerminalError({ executionId, executionStatus: execution.status, - redisAvailable: true, locallyAborted: false, }) } @@ -1038,7 +1039,6 @@ export async function cancelWorkflowExecution({ throw new WorkflowRunAlreadyTerminalError({ executionId, executionStatus: competingTerminalStatus, - redisAvailable: true, locallyAborted: stopSummary.locallyAborted, }) } diff --git a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts index 4fd00c7d33c..fdeab4f6e0f 100644 --- a/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts +++ b/apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts @@ -1,14 +1,6 @@ import { createHash } from 'node:crypto' import { generateShortId } from '@sim/utils/id' -import { afterAll, describe, expect, it, vi } from 'vitest' - -const { redisUrl } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - const redisUrl = readTestRedisUrl() - if (redisUrl) process.env.REDIS_URL = redisUrl - return { redisUrl } -}) - +import { afterAll, describe, expect, it } from 'vitest' import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' import { encryptSecret } from '@/lib/core/security/encryption' import { @@ -37,7 +29,7 @@ afterAll(async () => { await closeRedisConnection() }) -describe.skipIf(!redisUrl)('physical workbench history with real Redis', () => { +describe('physical workbench history with real Redis', () => { it('atomically retains distinct encrypted inputs under concurrent calls and retries', async () => { const { session, machine } = fixture() await initializeSessionFileProvenance(session, machine) diff --git a/apps/sim/lib/execution/removal-stops.integration.ts b/apps/sim/lib/execution/removal-stops.integration.ts index 9874e8d7824..81e7f441be0 100644 --- a/apps/sim/lib/execution/removal-stops.integration.ts +++ b/apps/sim/lib/execution/removal-stops.integration.ts @@ -8,14 +8,6 @@ import { authMock } from '@sim/testing/mocks/auth.mock' import { afterAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedRedisUrl } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - const url = readTestRedisUrl() - const inheritedRedisUrl = process.env.REDIS_URL - if (url) process.env.REDIS_URL = url - return { redisUrl: url, inheritedRedisUrl } -}) - vi.mock('@/lib/auth', () => authMock) import { db } from '@sim/db' @@ -202,12 +194,7 @@ async function runOutboxWorker(eventIds: string[]) { } } -afterAll(async () => { - if (inheritedRedisUrl === undefined) Reflect.deleteProperty(process.env, 'REDIS_URL') - else process.env.REDIS_URL = inheritedRedisUrl -}) - -describe.runIf(Boolean(redisUrl))('removals stop the workflow runs they orphan', () => { +describe('removals stop the workflow runs they orphan', () => { afterAll(async () => { if (executionIds.length > 0) { await getRedisClient().del(...executionIds.map((id) => `execution:cancel:${id}`)) diff --git a/apps/sim/lib/execution/workflow-run-already-terminal-error.ts b/apps/sim/lib/execution/workflow-run-already-terminal-error.ts index 0ff4966aec4..241e5c671f7 100644 --- a/apps/sim/lib/execution/workflow-run-already-terminal-error.ts +++ b/apps/sim/lib/execution/workflow-run-already-terminal-error.ts @@ -5,7 +5,6 @@ export type WorkflowRunAlreadyTerminalStatus = 'completed' | 'failed' interface WorkflowRunAlreadyTerminalErrorOptions { executionId: string executionStatus: WorkflowRunAlreadyTerminalStatus - redisAvailable: boolean locallyAborted: boolean } @@ -13,7 +12,6 @@ interface WorkflowRunAlreadyTerminalErrorOptions { export class WorkflowRunAlreadyTerminalError extends OrchestrationError { readonly executionId: string readonly executionStatus: WorkflowRunAlreadyTerminalStatus - readonly redisAvailable: boolean readonly locallyAborted: boolean constructor(options: WorkflowRunAlreadyTerminalErrorOptions) { @@ -21,7 +19,6 @@ export class WorkflowRunAlreadyTerminalError extends OrchestrationError { this.name = 'WorkflowRunAlreadyTerminalError' this.executionId = options.executionId this.executionStatus = options.executionStatus - this.redisAvailable = options.redisAvailable this.locallyAborted = options.locallyAborted } } diff --git a/apps/sim/lib/logs/execution/progress-markers.ts b/apps/sim/lib/logs/execution/progress-markers.ts index 9baa19ced36..0a63e81cc86 100644 --- a/apps/sim/lib/logs/execution/progress-markers.ts +++ b/apps/sim/lib/logs/execution/progress-markers.ts @@ -97,7 +97,7 @@ async function setMarker( /** * Persist the last-started-block marker. Returns `false` (caller should fall - * back to the durable SQL path) when Redis is unavailable or the write fails. + * back to the durable SQL path) when the write fails. */ export async function setLastStartedBlock( executionId: string, @@ -109,7 +109,7 @@ export async function setLastStartedBlock( /** * Persist the last-completed-block marker. Returns `false` (caller should fall - * back to the durable SQL path) when Redis is unavailable or the write fails. + * back to the durable SQL path) when the write fails. */ export async function setLastCompletedBlock( executionId: string, diff --git a/apps/sim/lib/mcp/oauth/storage.ts b/apps/sim/lib/mcp/oauth/storage.ts index 2d974a5af45..ab544261dd0 100644 --- a/apps/sim/lib/mcp/oauth/storage.ts +++ b/apps/sim/lib/mcp/oauth/storage.ts @@ -285,8 +285,8 @@ export async function clearState(rowId: string, context = 'unknown'): Promise { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - const redisUrl = readTestRedisUrl() - if (redisUrl) process.env.REDIS_URL = redisUrl - return { redisUrl } -}) vi.mock('@/lib/execution/remote-sandbox/provider', () => remoteSandboxProviderMock) vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) @@ -563,7 +557,7 @@ describe('private native Project writes against PostgreSQL and local storage', ( }) }) -describe.skipIf(!redisUrl)('Project workbench callback across requests', () => { +describe('Project workbench callback across requests', () => { check( 'enforces live callback leases, binds uploads across requests and persists machine source history', async () => { diff --git a/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts b/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts index 5c4c573619f..d0e946788f6 100644 --- a/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts +++ b/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts @@ -7,8 +7,7 @@ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/ */ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') const server = createServer(async (request, response) => { for await (const _chunk of request) { @@ -18,15 +17,12 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - /** The real Redis module and worker URL resolution read these at import. */ - process.env.REDIS_URL = url + /** Worker URL resolution reads this at import. */ process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: { server } } + return { inheritedEnv, worker: { server } } }) import { db } from '@sim/db' @@ -73,11 +69,6 @@ import { /** A recovering controller's first takeover of a run. */ const FIRST_RECOVERY = { attempts: 1, claimedAt: 0, notBefore: 0 } -function redis() { - const client = getRedisClient() - return client -} - /** The sweep's resume point lives in shared Redis; each test and the next suite start fresh. */ async function resetSweepCursor() { await getRedisClient().del('copilot:orphaned-runs:sweep-cursor') @@ -96,7 +87,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { +describe('Chat runs no controller owns', () => { const userId = generateId() const workspaceId = generateId() const chatIds: string[] = [] @@ -463,17 +454,17 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { it('never settles a run whose stream holds its chat lock, or that is still recoverable', async () => { const leased = await admittedRun({ idleMinutes: 90 }) - await redis().set(chatStreamLockKey(leased.chatId), leased.controllerToken!, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(leased.chatId), leased.controllerToken!, 'EX', 60) /** A recovering controller holds the lock under a new token before it claims the run. */ const recovering = await admittedRun({ idleMinutes: 90 }) - await redis().set( + await getRedisClient().set( chatStreamLockKey(recovering.chatId), `${recovering.streamId}\n${generateId()}`, 'EX', 60 ) const replayable = await admittedRun({ idleMinutes: 90 }) - await redis().set(`mothership_stream:${replayable.streamId}:seq`, '4', 'EX', 60) + await getRedisClient().set(`mothership_stream:${replayable.streamId}:seq`, '4', 'EX', 60) const fresh = await admittedRun({ idleMinutes: 5 }) try { @@ -486,7 +477,7 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { expect(current.marker).toBe(run.streamId) } } finally { - await redis().del( + await getRedisClient().del( chatStreamLockKey(leased.chatId), chatStreamLockKey(recovering.chatId), `mothership_stream:${replayable.streamId}:seq` @@ -559,7 +550,7 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { it('leaves a stopped run to the controller of its stream that holds the chat lock', async () => { const owned = await admittedRun() await stop(owned) - await redis().set(chatStreamLockKey(owned.chatId), owned.controllerToken!, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(owned.chatId), owned.controllerToken!, 'EX', 60) try { expect(await settleStoppedRunWithoutController(owned.runId)).toBe(false) @@ -567,7 +558,7 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { expect(run.status).toBe('active') expect(run.marker).toBe(owned.streamId) } finally { - await redis().del(chatStreamLockKey(owned.chatId)) + await getRedisClient().del(chatStreamLockKey(owned.chatId)) } }) @@ -776,7 +767,7 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { updatedAt: sql`now() - interval '2 hours'`, })) ) - const pipeline = redis().pipeline() + const pipeline = getRedisClient().pipeline() for (const blocker of page) { pipeline.set(`mothership_stream:${blocker.streamId}:seq`, '1', 'EX', 600) } @@ -794,7 +785,7 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { .delete(copilotChats) .where(inArray(copilotChats.id, blockerChatIds.slice(start, start + 1000))) } - const pipeline = redis().pipeline() + const pipeline = getRedisClient().pipeline() for (const blocker of blockers) pipeline.del(`mothership_stream:${blocker.streamId}:seq`) await pipeline.exec() } diff --git a/apps/sim/lib/mothership/jobs/lane.integration.ts b/apps/sim/lib/mothership/jobs/lane.integration.ts index 8fd6ee7189a..c099b3d5e25 100644 --- a/apps/sim/lib/mothership/jobs/lane.integration.ts +++ b/apps/sim/lib/mothership/jobs/lane.integration.ts @@ -10,8 +10,7 @@ import { envFlagsMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') type Leg = { text: string; mode: 'end' | 'hang' | 'fail' } const worker = { @@ -84,16 +83,13 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { } await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, COPILOT_API_KEY: process.env.COPILOT_API_KEY, } - if (url) process.env.REDIS_URL = url process.env.COPILOT_API_KEY = 'job-lane-fixture-key' process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: Object.assign(worker, { server, finish }) } + return { inheritedEnv, worker: Object.assign(worker, { server, finish }) } }) vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) @@ -192,11 +188,6 @@ const payer = () => resolveBillingAttribution({ actorUserId: userId, workspaceId const chatIds: string[] = [] const scheduleIds: string[] = [] -function redis() { - const client = getRedisClient() - return client -} - async function waitFor(condition: () => Promise, what: string, timeoutMs = 20_000) { const deadline = Date.now() + timeoutMs while (Date.now() < deadline) { @@ -317,7 +308,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('the scheduled job lane', () => { +describe('the scheduled job lane', () => { beforeAll(async () => { const now = new Date() await db.insert(user).values({ @@ -518,7 +509,10 @@ describe.runIf(Boolean(redisUrl))('the scheduled job lane', () => { controllers.get(occurrenceId)?.abort(new StreamControllerSupersededError()) worker.open.get(occurrenceId)?.response.socket?.destroy() worker.open.delete(occurrenceId) - await waitFor(async () => !(await redis().get(chatStreamLockKey(chatId))), 'the lock to lapse') + await waitFor( + async () => !(await getRedisClient().get(chatStreamLockKey(chatId))), + 'the lock to lapse' + ) /** Before the takeover the turn is unfinished, so the occurrence stays running. */ expect(await finalizeJobRun(occurrenceId)).toBe(false) await db @@ -526,7 +520,7 @@ describe.runIf(Boolean(redisUrl))('the scheduled job lane', () => { .set({ updatedAt: sql`now() - interval '2 minutes'` }) .where(eq(copilotRuns.streamId, occurrenceId)) await waitFor(async () => { - await redis().del(ABANDONED_RUN_CURSOR_KEY) + await getRedisClient().del(ABANDONED_RUN_CURSOR_KEY) await takeOverAbandonedRuns() return legsFor(occurrenceId).length === 2 }, 'the takeover') @@ -736,11 +730,16 @@ describe.runIf(Boolean(redisUrl))('the scheduled job lane', () => { it('skips an occurrence while another turn holds the job chat', async () => { const { chatId, scheduleId, occurrenceId } = await seedJob() - await redis().set(chatStreamLockKey(chatId), `${generateId()}\n${generateId()}`, 'EX', 60) + await getRedisClient().set( + chatStreamLockKey(chatId), + `${generateId()}\n${generateId()}`, + 'EX', + 60 + ) try { await runJobLane(new Date()) } finally { - await redis().del(chatStreamLockKey(chatId)) + await getRedisClient().del(chatStreamLockKey(chatId)) } expect(await runsOf(scheduleId)).toEqual([ expect.objectContaining({ status: 'complete', outcome: 'skipped_busy' }), @@ -825,7 +824,10 @@ describe.runIf(Boolean(redisUrl))('the scheduled job lane', () => { controllers.get(occurrenceId)?.abort(new StreamControllerSupersededError()) worker.open.get(occurrenceId)?.response.socket?.destroy() worker.open.delete(occurrenceId) - await waitFor(async () => !(await redis().get(chatStreamLockKey(chatId))), 'the lock to lapse') + await waitFor( + async () => !(await getRedisClient().get(chatStreamLockKey(chatId))), + 'the lock to lapse' + ) await db.delete(copilotRuns).where(eq(copilotRuns.streamId, occurrenceId)) await reconcileJobRuns(new Date(Date.now() + 6 * 60 * 60_000 + 60_000)) diff --git a/apps/sim/lib/mothership/persistence/tool-confirm/index.ts b/apps/sim/lib/mothership/persistence/tool-confirm/index.ts index a22dc03aec1..9cf55c0db0c 100644 --- a/apps/sim/lib/mothership/persistence/tool-confirm/index.ts +++ b/apps/sim/lib/mothership/persistence/tool-confirm/index.ts @@ -19,11 +19,6 @@ import { MothershipStreamV1ToolOutcome } from '@/lib/mothership/generated/mother const logger = createLogger('CopilotOrchestratorPersistence') const TOOL_CONFIRMATION_TTL_SECONDS = 60 * 10 const DURABLE_CONFIRMATION_POLL_MS = 5_000 -/** - * How long the first durable check waits for the subscription to land. Past it the wait falls - * back to polling, as it must while Redis cannot subscribe. - */ -const SUBSCRIPTION_READY_WAIT_MS = DURABLE_CONFIRMATION_POLL_MS const toolConfirmationKey = (toolCallId: string) => `copilot:tool-confirmation:${toolCallId}` type ToolConfirmGlobal = typeof globalThis & { @@ -132,14 +127,14 @@ export async function waitForToolConfirmation( let settled = false let timeoutId: ReturnType | null = null let pollId: ReturnType | null = null - let readyWaitId: ReturnType | null = null let unsubscribe: (() => void) | null = null + let releaseReadyWait: (() => void) | null = null const cleanup = () => { if (timeoutId) clearTimeout(timeoutId) if (pollId) clearTimeout(pollId) - if (readyWaitId) clearTimeout(readyWaitId) if (unsubscribe) unsubscribe() + if (releaseReadyWait) releaseReadyWait() abortSignal?.removeEventListener('abort', onAbort) } @@ -215,21 +210,16 @@ export async function waitForToolConfirmation( } abortSignal?.addEventListener('abort', onAbort, { once: true }) - // The first durable check runs once the subscription is live. A publisher commits the row - // before it publishes, so a signal sent before then is in the row this check reads, and - // every later one reaches the handler above. - const subscriptionReady = Promise.race([ - toolConfirmationChannel.ready(), - new Promise((resolveWait) => { - readyWaitId = setTimeout(resolveWait, SUBSCRIPTION_READY_WAIT_MS) - }), - ]) - void subscriptionReady.then(async () => { - if (readyWaitId) clearTimeout(readyWaitId) - readyWaitId = null - if (settled) return - await checkDurableConfirmation('subscribe') - scheduleDurablePoll() - }) + // A publisher commits the row before it publishes. When this process is not subscribed yet, a + // signal sent before the subscription lands reaches no handler, so the row is read once more + // when it does. + const subscribedBeforeFirstRead = toolConfirmationChannel.isReady() + void checkDurableConfirmation('subscribe').then(scheduleDurablePoll) + if (!subscribedBeforeFirstRead && !settled) { + releaseReadyWait = toolConfirmationChannel.onReady(() => { + releaseReadyWait = null + if (!settled) void checkDurableConfirmation('subscribe') + }) + } }) } diff --git a/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts b/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts index 8062f59811f..6e51fd88178 100644 --- a/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts +++ b/apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts @@ -5,8 +5,16 @@ import { import { beforeEach, describe, expect, it, vi } from 'vitest' const channelHandlers = new Set<(event: any) => void>() -/** When this process's subscription lands; publishes before then reach no handler. */ -let subscriptionReady: Promise = Promise.resolve() +/** Whether this process's subscription has landed; publishes before then reach no handler. */ +let subscribed = true +const readyListeners = new Set<() => void>() + +function landSubscription() { + subscribed = true + const listeners = [...readyListeners] + readyListeners.clear() + for (const listener of listeners) listener() +} vi.mock('@/lib/mothership/async-runs/repository', () => mothershipAsyncRunsMock) @@ -21,8 +29,18 @@ vi.mock('@/lib/events/pubsub', () => ({ channelHandlers.delete(handler) } }, - ready() { - return subscriptionReady + isReady() { + return subscribed + }, + onReady(listener: () => void) { + if (subscribed) { + listener() + return () => {} + } + readyListeners.add(listener) + return () => { + readyListeners.delete(listener) + } }, dispose() {}, }), @@ -49,7 +67,8 @@ describe('copilot orchestrator persistence', () => { beforeEach(() => { channelHandlers.clear() - subscriptionReady = Promise.resolve() + subscribed = true + readyListeners.clear() revokeExpiredSimToolExecutions.mockResolvedValue([]) row = null getAsyncToolCalls.mockImplementation(async () => (row ? [row] : [])) @@ -190,13 +209,29 @@ describe('copilot orchestrator persistence', () => { }) }) - it('reads the durable row only once its subscription lands, so a detach in between is not missed', async () => { + it('returns a saved result at once while the subscription never lands', async () => { + subscribed = false + row = { + status: 'completed', + error: null, + result: { ok: true }, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + } + + const result = await waitForToolConfirmation('tool-1', 1_000) + + expect(result).toEqual({ + status: 'success', + data: { ok: true }, + timestamp: '2026-01-01T00:00:00.000Z', + }) + expect(readyListeners.size).toBe(0) + }) + + it('reads the durable row again when the subscription lands, so a detach in between is not missed', async () => { vi.useFakeTimers() try { - let landSubscription: () => void = () => {} - subscriptionReady = new Promise((resolve) => { - landSubscription = resolve - }) + subscribed = false row = { status: 'pending', error: null, @@ -210,6 +245,7 @@ describe('copilot orchestrator persistence', () => { result = value }) await vi.advanceTimersByTimeAsync(0) + expect(result).toBeUndefined() // Another pod detaches the call while this one's SUBSCRIBE is still in flight: the row // commits and the publish reaches no handler here. @@ -229,6 +265,23 @@ describe('copilot orchestrator persistence', () => { } }) + it('drops its ready listener when it times out before the subscription lands', async () => { + vi.useFakeTimers() + try { + subscribed = false + const waitPromise = waitForToolConfirmation('tool-1', 1_000) + await vi.advanceTimersByTimeAsync(0) + expect(readyListeners.size).toBe(1) + + await vi.advanceTimersByTimeAsync(1_000) + + await expect(waitPromise).resolves.toBeNull() + expect(readyListeners.size).toBe(0) + } finally { + vi.useRealTimers() + } + }) + it('keeps a no-deadline human wait alive until confirmation arrives', async () => { vi.useFakeTimers() try { diff --git a/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts b/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts index f5055a5ce81..9660ea3a945 100644 --- a/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts +++ b/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts @@ -11,8 +11,7 @@ import { envFlagsMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') const worker = { requests: [] as Array<{ path: string; body: Record }>, @@ -44,14 +43,11 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - if (url) process.env.REDIS_URL = url process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: Object.assign(worker, { server }) } + return { inheritedEnv, worker: Object.assign(worker, { server }) } }) vi.mock('@/lib/auth', () => authMock) @@ -141,11 +137,6 @@ const chatIds: string[] = [] const TOOL_NAME = 'get_credentials' const TOOL_ARGS = {} -function redis() { - const client = getRedisClient() - return client -} - function frameFor(streamId: string, chatId: string) { return (seq: number, type: string, payload: unknown) => ({ v: 1, @@ -334,7 +325,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () => { +describe('taking over a run whose controller died', () => { beforeAll(async () => { const now = new Date() for (const id of [ownerId, userId]) { @@ -371,7 +362,7 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValue(false) workspacePermission.rechecks = 0 workspacePermission.beforeRecheck = undefined - await redis().del(ABANDONED_RUN_CURSOR_KEY) + await getRedisClient().del(ABANDONED_RUN_CURSOR_KEY) await db .insert(permissions) .values({ @@ -417,7 +408,7 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () const { chatId, streamId, runId, toolCallId } = await runAbandonedMidTool() const before = await storedRun(runId) const lock = String(toRecord(before.requestContext).controllerToken) - await redis().set(chatStreamLockKey(chatId), lock, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(chatId), lock, 'EX', 60) try { await takeOverAbandonedRuns() await sleep(500) @@ -426,7 +417,7 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () expect(await storedRun(runId)).toEqual(before) expect((await storedTool(toolCallId)).attempts).toBe(1) } finally { - await redis().del(chatStreamLockKey(chatId)) + await getRedisClient().del(chatStreamLockKey(chatId)) await db.update(copilotRuns).set({ status: 'cancelled' }).where(eq(copilotRuns.id, runId)) } }) @@ -600,7 +591,7 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () ] const abandoned = await runAbandonedMidTool({ runId: ids[2] }) for (const run of live) { - await redis().set(chatStreamLockKey(run.chatId), `${run.streamId}\nlive`, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(run.chatId), `${run.streamId}\nlive`, 'EX', 60) } try { const bound = { pageSize: 2, maxExamined: 2 } @@ -610,7 +601,7 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () ]) } finally { for (const run of [...live, abandoned]) { - await redis().del(chatStreamLockKey(run.chatId)) + await getRedisClient().del(chatStreamLockKey(run.chatId)) await db .update(copilotRuns) .set({ status: 'cancelled' }) @@ -670,7 +661,7 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () expect(await storedRun(runId)).toEqual(before) expect(requestsFor(streamId, '/api/mothership')).toHaveLength(0) expect((await storedTool(toolCallId)).attempts).toBe(1) - expect(await redis().get(chatStreamLockKey(chatId))).toBeNull() + expect(await getRedisClient().get(chatStreamLockKey(chatId))).toBeNull() } finally { await db.update(copilotRuns).set({ status: 'cancelled' }).where(eq(copilotRuns.id, runId)) } diff --git a/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts b/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts index 376a0786b86..ba71a9a7d47 100644 --- a/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts +++ b/apps/sim/lib/mothership/request/application/removal-stops-runs.integration.ts @@ -10,8 +10,7 @@ import { createDeferred } from '@sim/testing/helpers/deferred' import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') const worker = { stops: [] as string[], @@ -34,14 +33,11 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - if (url) process.env.REDIS_URL = url process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: Object.assign(worker, { server }) } + return { inheritedEnv, worker: Object.assign(worker, { server }) } }) vi.mock('@/lib/auth', () => authMock) @@ -485,7 +481,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('removing a run’s chat, owner or access stops the run', () => { +describe('removing a run’s chat, owner or access stops the run', () => { beforeEach(() => { testStreams.clear() worker.unreachable.clear() diff --git a/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts b/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts index 0aefa8fe558..090ad17dcf3 100644 --- a/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts +++ b/apps/sim/lib/mothership/request/go/file-preview-adapter.test.ts @@ -1,4 +1,3 @@ -import { createMockRedis, redisConfigMockFns } from '@sim/testing' import { flushMicrotasks } from '@sim/testing/helpers/async' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { @@ -37,19 +36,6 @@ import type { StreamEvent, } from '@/lib/mothership/request/types' -beforeEach(() => { - // Preview sessions persist through a pipeline; a rejected write would retry on `sleep`, which - // the byte-rate cases' fake timers never advance. - redisConfigMockFns.mockGetRedisClient.mockReturnValue({ - ...createMockRedis(), - pipeline: vi.fn(() => ({ - hset: vi.fn(), - expire: vi.fn(), - exec: vi.fn().mockResolvedValue([]), - })), - }) -}) - const STREAM_ID = 'stream-1' const EDIT_TOOL_CALL_ID = 'edit-content-1' const WORKSPACE_FILE_TOOL_CALL_ID = 'workspace-file-1' diff --git a/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts b/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts index 8fbbe7a48a7..5983487d59a 100644 --- a/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts +++ b/apps/sim/lib/mothership/request/lifecycle/unattended.integration.ts @@ -14,8 +14,7 @@ import { envFlagsMock, resetEnvFlagsMock } from '@sim/testing/mocks/env-flags.mo import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') type Leg = { text: string; end: boolean } const worker = { @@ -69,14 +68,11 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - if (url) process.env.REDIS_URL = url process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: Object.assign(worker, { server }) } + return { inheritedEnv, worker: Object.assign(worker, { server }) } }) vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) @@ -153,11 +149,6 @@ const payer = () => resolveBillingAttribution({ actorUserId: userId, workspaceId const chatIds: string[] = [] const JOB_SECRETS = { secretScope: 'selected' as const, mountedSecrets: ['REPORT_TOKEN'] } -function redis() { - const client = getRedisClient() - return client -} - async function jobChat() { const chatId = generateId() chatIds.push(chatId) @@ -213,7 +204,10 @@ async function killController(chatId: string, streamId: string) { controller.abort(new StreamControllerSupersededError()) worker.open.get(streamId)?.socket?.destroy() worker.open.delete(streamId) - await waitFor(async () => !(await redis().get(chatStreamLockKey(chatId))), 'the lock to lapse') + await waitFor( + async () => !(await getRedisClient().get(chatStreamLockKey(chatId))), + 'the lock to lapse' + ) await db .update(copilotRuns) .set({ updatedAt: sql`now() - interval '2 minutes'` }) @@ -225,7 +219,7 @@ async function takeOverUntilReattached(streamId: string) { const before = legsFor(streamId).length await waitFor( async () => { - await redis().del(ABANDONED_RUN_CURSOR_KEY) + await getRedisClient().del(ABANDONED_RUN_CURSOR_KEY) await takeOverAbandonedRuns() return legsFor(streamId).length > before }, @@ -242,7 +236,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('an unattended turn with no browser attached', () => { +describe('an unattended turn with no browser attached', () => { beforeAll(async () => { const now = new Date() await db.insert(user).values({ @@ -336,7 +330,7 @@ describe.runIf(Boolean(redisUrl))('an unattended turn with no browser attached', 'the run to complete' ) await waitFor( - async () => !(await redis().get(chatStreamLockKey(chatId))), + async () => !(await getRedisClient().get(chatStreamLockKey(chatId))), 'the chat to be released' ) @@ -386,7 +380,7 @@ describe.runIf(Boolean(redisUrl))('an unattended turn with no browser attached', it('answers busy, admitting nothing, while another turn holds the chat', async () => { const chatId = await jobChat() const typedTurn = generateId() - await redis().set(chatStreamLockKey(chatId), `${typedTurn}\n${generateId()}`, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(chatId), `${typedTurn}\n${generateId()}`, 'EX', 60) const occurrenceId = generateId() try { expect( @@ -402,7 +396,7 @@ describe.runIf(Boolean(redisUrl))('an unattended turn with no browser attached', }) ).toBe('busy') } finally { - await redis().del(chatStreamLockKey(chatId)) + await getRedisClient().del(chatStreamLockKey(chatId)) } expect(await storedRun(occurrenceId)).toBeUndefined() expect(await messagesOf(chatId)).toEqual([]) diff --git a/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts b/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts index 40f31f877cd..88b0a927dbc 100644 --- a/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts +++ b/apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts @@ -5,13 +5,9 @@ */ import { afterAll, describe, expect, it, vi } from 'vitest' -const { redisUrl } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - const url = readTestRedisUrl() - process.env.REDIS_URL = url +vi.hoisted(() => { /** Shorter than the lock heartbeat that refreshes a live buffer. */ process.env.COPILOT_STREAM_TTL_SECONDS = '5' - return { redisUrl: url } }) import { sleep } from '@sim/utils/helpers' @@ -50,7 +46,7 @@ async function appendText(streamId: string, text: string): Promise { return seq } -describe.runIf(Boolean(redisUrl))('replay buffer lifetime', () => { +describe('replay buffer lifetime', () => { afterAll(async () => { await closeRedisConnection() }) diff --git a/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts b/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts index f619af25365..8eea001e04f 100644 --- a/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts +++ b/apps/sim/lib/mothership/request/session/recovery-storm.integration.ts @@ -10,8 +10,7 @@ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/ import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { toRecord } = await import('@sim/utils/object') const { createServer } = await import('node:http') interface RunFixture { @@ -78,15 +77,12 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - /** The real Redis module and worker URL resolution read these at import. */ - process.env.REDIS_URL = url + /** Worker URL resolution reads this at import. */ process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: Object.assign(worker, { server }) } + return { inheritedEnv, worker: Object.assign(worker, { server }) } }) vi.mock('@/lib/auth', () => authMock) @@ -180,11 +176,6 @@ const userId = generateId() const workspaceId = generateId() const chatIds: string[] = [] -function redis() { - const client = getRedisClient() - return client -} - /** A run whose controller died after persisting two text frames, its lock long expired. */ async function orphanedRun( options: { @@ -325,14 +316,15 @@ async function reconnect( while (Date.now() - startedAt < windowMs) { const settled = isTerminalStreamStatus((await storedRun(runId)).status) && - !(await redis().get(chatStreamLockKey(chatId))) + !(await getRedisClient().get(chatStreamLockKey(chatId))) if (settled) break await sleep(100) } for (const controller of controllers) controller.abort() await Promise.allSettled(drains) if (waitForLeaseRelease) - for (let i = 0; i < 50 && (await redis().get(chatStreamLockKey(chatId))); i++) await sleep(100) + for (let i = 0; i < 50 && (await getRedisClient().get(chatStreamLockKey(chatId))); i++) + await sleep(100) const requests = fixture.posts.filter((post) => post.at >= startedAt) return { startedAt, @@ -352,7 +344,7 @@ afterAll(async () => { expect(worker.unexpectedRequests).toEqual([]) }) -describe.runIf(Boolean(redisUrl))('reconnecting to an orphaned Chat run', () => { +describe('reconnecting to an orphaned Chat run', () => { beforeAll(async () => { const now = new Date() await db.insert(user).values({ @@ -536,7 +528,7 @@ describe.runIf(Boolean(redisUrl))('reconnecting to an orphaned Chat run', () => it('never takes over a parked run whose controller still holds the chat lock', async () => { const run = await orphanedRun({ status: 'paused_waiting_for_tool' }) const liveController = `${run.streamId}\n${generateId()}` - await redis().set(chatStreamLockKey(run.chatId), liveController, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(run.chatId), liveController, 'EX', 60) try { const { turns } = await reconnect(run, { tails: 3, @@ -548,9 +540,9 @@ describe.runIf(Boolean(redisUrl))('reconnecting to an orphaned Chat run', () => const stored = await storedRun(run.runId) expect(stored.status).toBe('paused_waiting_for_tool') expect(stored.recoveryBackoff).toBeUndefined() - expect(await redis().get(chatStreamLockKey(run.chatId))).toBe(liveController) + expect(await getRedisClient().get(chatStreamLockKey(run.chatId))).toBe(liveController) } finally { - await redis().del(chatStreamLockKey(run.chatId)) + await getRedisClient().del(chatStreamLockKey(run.chatId)) } }, 30_000) diff --git a/apps/sim/lib/mothership/request/session/replay-budget.integration.ts b/apps/sim/lib/mothership/request/session/replay-budget.integration.ts index 3b2714ee464..522fdc7df34 100644 --- a/apps/sim/lib/mothership/request/session/replay-budget.integration.ts +++ b/apps/sim/lib/mothership/request/session/replay-budget.integration.ts @@ -10,8 +10,7 @@ import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' import { toRecord } from '@sim/utils/object' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer: createHttpServer } = await import('node:http') const abortRequests: Array> = [] const hooks = { @@ -46,16 +45,12 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - /** The real Redis module and worker URL resolution read these at import. */ - process.env.REDIS_URL = url + /** Worker URL resolution reads this at import. */ process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` return { - redisUrl: url, inheritedEnv, worker: { server, @@ -177,11 +172,6 @@ import { GET as mothershipChatGET } from '@/app/api/mothership/chats/[chatId]/ro const MB = 1024 * 1024 -function redis() { - const client = getRedisClient() - return client -} - /** A writer that owns a real chat lock, with the frames it delivers to its client. */ async function leasedWriter(options: { lease?: ChatStreamLease; ownLock?: boolean } = {}) { const streamId = generateId() @@ -189,7 +179,7 @@ async function leasedWriter(options: { lease?: ChatStreamLease; ownLock?: boolea key: chatStreamLockKey(generateId()), value: `${streamId}\n${generateId()}`, } - if (options.ownLock !== false) await redis().set(lease.key, lease.value, 'EX', 60) + if (options.ownLock !== false) await getRedisClient().set(lease.key, lease.value, 'EX', 60) const writer = new StreamWriter({ streamId, requestId: generateId(), @@ -221,7 +211,7 @@ async function leasedWriter(options: { lease?: ChatStreamLease; ownLock?: boolea } async function storedMembers(streamId: string): Promise { - return redis().zrange(`mothership_stream:${streamId}:events`, 0, -1) + return getRedisClient().zrange(`mothership_stream:${streamId}:events`, 0, -1) } function toolCall( @@ -256,7 +246,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('leased Chat stream writer with Redis', () => { +describe('leased Chat stream writer with Redis', () => { it('delivers and persists the same bounded copy of a 2 MB tool call, leaving the dispatched event whole', async () => { const { streamId, writer, frames } = await leasedWriter() const stdout = 'o'.repeat(2 * MB) @@ -381,7 +371,7 @@ describe.runIf(Boolean(redisUrl))('leased Chat stream writer with Redis', () => it('still refuses a controller whose lease was taken over', async () => { const lease = { key: chatStreamLockKey(generateId()), value: `stale\n${generateId()}` } - await redis().set(lease.key, `successor\n${generateId()}`, 'EX', 60) + await getRedisClient().set(lease.key, `successor\n${generateId()}`, 'EX', 60) const { writer, frames } = await leasedWriter({ lease, ownLock: false }) const published = writer.publish(text('from the stale controller')) @@ -392,7 +382,7 @@ describe.runIf(Boolean(redisUrl))('leased Chat stream writer with Redis', () => }) }) -describe.runIf(Boolean(redisUrl))('the replay ring trimmed past its byte target', () => { +describe('the replay ring trimmed past its byte target', () => { /** A ring whose counter is already past the byte target, as after a long run. */ async function overTargetRing(seqs: number[]) { const streamId = generateId() @@ -403,7 +393,7 @@ describe.runIf(Boolean(redisUrl))('the replay ring trimmed past its byte target' ) expect(persisted).toEqual({ persisted: true }) const { maxOwnerBytes } = getRedisBudgetLimits('copilot_stream') - await redis().set(ownerBudgetKey, maxOwnerBytes - MB, 'EX', 3600) + await getRedisClient().set(ownerBudgetKey, maxOwnerBytes - MB, 'EX', 3600) return { streamId, ownerBudgetKey } } @@ -425,7 +415,7 @@ describe.runIf(Boolean(redisUrl))('the replay ring trimmed past its byte target' it('trims a replayed member below the ring with it, leaving a contiguous tail', async () => { const { streamId, ownerBudgetKey } = await overTargetRing([2, 3]) - const before = Number(await redis().get(ownerBudgetKey)) + const before = Number(await getRedisClient().get(ownerBudgetKey)) const dropped = await storedMembers(streamId) const newest = textEvent(streamId, 4) @@ -434,7 +424,7 @@ describe.runIf(Boolean(redisUrl))('the replay ring trimmed past its byte target' }) expect(await seqsOf(streamId)).toEqual([4]) - expect(Number(await redis().get(ownerBudgetKey))).toBe( + expect(Number(await getRedisClient().get(ownerBudgetKey))).toBe( before + Buffer.byteLength(JSON.stringify(newest)) - bytesOf(dropped) ) }) @@ -452,13 +442,13 @@ describe.runIf(Boolean(redisUrl))('the replay ring trimmed past its byte target' await appendEvents([textEvent(streamId, 6002)], { streamId }) expect(await seqsOf(streamId)).toEqual([6002]) - expect(Number(await redis().get(ownerBudgetKey))).toBeLessThan( + expect(Number(await getRedisClient().get(ownerBudgetKey))).toBeLessThan( getRedisBudgetLimits('copilot_stream').maxOwnerBytes - MB ) }) }) -describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budget', () => { +describe('a turn whose stream exhausts its replay budget', () => { const userId = generateId() const workspaceId = generateId() const chatId = generateId() @@ -469,7 +459,28 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge attemptedBytes: 0, }).userMessage + /** Resolves when a controller's root span ends, the last step of its teardown. */ + const teardowns = new Map void>() + beforeAll(async () => { + trace.disable() + trace.setGlobalTracerProvider( + new TracerProvider({ + spanProcessors: [ + { + onStart: () => {}, + onEnd: (span) => { + const streamId = span.attributes[TraceAttr.StreamId] + if (span.name === TraceSpan.GenAiAgentExecute && typeof streamId === 'string') { + teardowns.get(streamId)?.() + } + }, + forceFlush: async () => {}, + shutdown: async () => {}, + }, + ], + }) + ) const now = new Date() await db.insert(user).values({ id: userId, @@ -504,39 +515,17 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge await db.delete(permissions).where(eq(permissions.userId, userId)) await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) await db.delete(user).where(eq(user.id, userId)) + trace.disable() }) /** Admits a turn the way the chat POST does, runs its controller, and returns its frames. */ - /** Resolves when a controller's root span ends, the last step of its teardown. */ - const teardowns = new Map void>() - beforeAll(() => { - trace.disable() - trace.setGlobalTracerProvider( - new TracerProvider({ - spanProcessors: [ - { - onStart: () => {}, - onEnd: (span) => { - const streamId = span.attributes[TraceAttr.StreamId] - if (span.name === TraceSpan.GenAiAgentExecute && typeof streamId === 'string') { - teardowns.get(streamId)?.() - } - }, - forceFlush: async () => {}, - shutdown: async () => {}, - }, - ], - }) - ) - }) - afterAll(() => trace.disable()) async function runTurn(script: unknown[], prepare?: (streamId: string) => Promise) { const streamId = generateId() const teardown = new Promise((resolve) => teardowns.set(streamId, resolve)) const runId = generateId() expect(await acquirePendingChatStream(chatId, streamId, 0)).toBe(true) - const controllerToken = (await redis().get(chatStreamLockKey(chatId)))! + const controllerToken = (await getRedisClient().get(chatStreamLockKey(chatId)))! const request = { message: 'Summarize the logs', userId, @@ -600,7 +589,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge const { maxOwnerBytes } = getRedisBudgetLimits('copilot_stream') const [ownerBudgetKey] = getRedisBudgetKeys({ kind: 'copilot_stream', id: streamId }) /** Room for the turn's opening session frame, not for the worker's tool call. */ - await redis().set(ownerBudgetKey, String(maxOwnerBytes - 512), 'EX', 3600) + await getRedisClient().set(ownerBudgetKey, String(maxOwnerBytes - 512), 'EX', 3600) } ) @@ -615,7 +604,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId)) expect(stored).toMatchObject({ status: 'error', error: ownerMessage }) expect(worker.abortRequests).toEqual([expect.objectContaining({ messageId: streamId })]) - expect(await redis().get(chatStreamLockKey(chatId))).toBeNull() + expect(await getRedisClient().get(chatStreamLockKey(chatId))).toBeNull() const reconnect = await streamGET( new NextRequest( @@ -634,7 +623,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge }) expect(recovered?.status).toBe('error') expect(worker.runs).toHaveLength(1) - expect(await redis().get(chatStreamLockKey(chatId))).toBeNull() + expect(await getRedisClient().get(chatStreamLockKey(chatId))).toBeNull() const [controllerRun] = worker.runs expect(controllerRun.dispatched).toEqual([]) @@ -661,18 +650,20 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge const successorToken = `successor\n${generateId()}` const { streamId, runId, frames } = await runTurn([ async () => { - await redis().set(chatStreamLockKey(chatId), successorToken, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(chatId), successorToken, 'EX', 60) }, step, ]) expect(frames.map((frame) => frame.type)).toEqual(['session']) - expect(await redis().ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThan(300) - expect(await redis().get(chatStreamLockKey(chatId))).toBe(successorToken) + expect(await getRedisClient().ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThan( + 300 + ) + expect(await getRedisClient().get(chatStreamLockKey(chatId))).toBe(successorToken) const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId)) expect(stored.status).toBe('active') expect(worker.abortRequests).toEqual([]) - await redis().del(chatStreamLockKey(chatId)) + await getRedisClient().del(chatStreamLockKey(chatId)) } ) @@ -682,7 +673,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge [ async () => { // A corrupt buffer key makes the next append fail while the lease is still held. - await redis().set(eventsKey, 'corrupt', 'EX', 3600) + await getRedisClient().set(eventsKey, 'corrupt', 'EX', 3600) }, toolCall('call-unsaved', 'cli_blocks_get', { command: 'blocks get' }), ], @@ -692,7 +683,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge ) expect(frames.map((frame) => frame.type)).toEqual(['session']) - expect(await redis().ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThan(300) + expect(await getRedisClient().ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThan(300) const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId)) expect(stored.status).toBe('active') expect(worker.abortRequests).toEqual([]) @@ -721,7 +712,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge [ async () => { // A corrupt buffer key makes the terminal append fail while the lease is held. - await redis().set(eventsKey, 'corrupt', 'EX', 3600) + await getRedisClient().set(eventsKey, 'corrupt', 'EX', 3600) }, ], async (id) => { @@ -731,7 +722,9 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId)) expect(stored.status).toBe('complete') - expect(await redis().ttl(`mothership_stream:${streamId}:events`)).toBeLessThanOrEqual(300) + expect(await getRedisClient().ttl(`mothership_stream:${streamId}:events`)).toBeLessThanOrEqual( + 300 + ) }) it('completes a turn that previews a file larger than one replay write', async () => { @@ -820,9 +813,9 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge id: streamId, userId, }) - expect(Number(await redis().get(ownerBudgetKey))).toBe(retainedBytes) + expect(Number(await getRedisClient().get(ownerBudgetKey))).toBe(retainedBytes) expect(retainedBytes).toBeLessThan(maxOwnerBytes) - expect(Number(await redis().get(userBudgetKey))).toBeLessThan(maxUserBytes) + expect(Number(await getRedisClient().get(userBudgetKey))).toBeLessThan(maxUserBytes) const seqs = members.map((member) => JSON.parse(member).seq as number) const oldestSeq = seqs[0] @@ -874,7 +867,12 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge id: generateId(), userId, }) - await redis().set(executionUserKey, getRedisBudgetLimits('execution').maxUserBytes, 'EX', 300) + await getRedisClient().set( + executionUserKey, + getRedisBudgetLimits('execution').maxUserBytes, + 'EX', + 300 + ) try { const { runId, frames } = await runTurn([text('The digest is ready.')]) @@ -883,7 +881,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId)) expect(stored.status).toBe('complete') } finally { - await redis().del(executionUserKey) + await getRedisClient().del(executionUserKey) } }) @@ -936,7 +934,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge { streamId } ) expect(persisted).toEqual({ persisted: true }) - await redis().zremrangebyrank(`mothership_stream:${streamId}:events`, 0, 1) + await getRedisClient().zremrangebyrank(`mothership_stream:${streamId}:events`, 0, 1) worker.runs.length = 0 worker.script = [] return streamId @@ -953,7 +951,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge describe('when a byte trim has removed the head of an unfinished turn', () => { afterAll(async () => { await db.update(copilotChats).set({ conversationId: null }).where(eq(copilotChats.id, chatId)) - await redis().del(chatStreamLockKey(chatId)) + await getRedisClient().del(chatStreamLockKey(chatId)) }) it.each(['after=4', 'after=0'])( @@ -1060,7 +1058,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge it("leaves its successor's stream untouched when the lease is lost while ending a refused turn", async () => { const successorToken = `successor\n${generateId()}` worker.hooks.onAbort = async () => { - await redis().set(chatStreamLockKey(chatId), successorToken, 'EX', 60) + await getRedisClient().set(chatStreamLockKey(chatId), successorToken, 'EX', 60) } try { const { streamId, frames } = await runTurn( @@ -1068,16 +1066,18 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge async (streamId) => { const { maxOwnerBytes } = getRedisBudgetLimits('copilot_stream') const [ownerBudgetKey] = getRedisBudgetKeys({ kind: 'copilot_stream', id: streamId }) - await redis().set(ownerBudgetKey, String(maxOwnerBytes - 512), 'EX', 3600) + await getRedisClient().set(ownerBudgetKey, String(maxOwnerBytes - 512), 'EX', 3600) } ) expect(frames.map((frame) => frame.type)).toEqual(['session', 'error', 'complete']) - expect(await redis().ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThan(300) - expect(await redis().get(chatStreamLockKey(chatId))).toBe(successorToken) + expect(await getRedisClient().ttl(`mothership_stream:${streamId}:events`)).toBeGreaterThan( + 300 + ) + expect(await getRedisClient().get(chatStreamLockKey(chatId))).toBe(successorToken) } finally { worker.hooks.onAbort = undefined - await redis().del(chatStreamLockKey(chatId)) + await getRedisClient().del(chatStreamLockKey(chatId)) } }) @@ -1117,9 +1117,9 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge const controllerToken = `owner\n${generateId()}` const { streamId, runId } = await pausedRun(controllerToken) const lease = { key: chatStreamLockKey(generateId()), value: controllerToken } - await redis().set(lease.key, lease.value, 'EX', 60) + await getRedisClient().set(lease.key, lease.value, 'EX', 60) /** A corrupt buffer key makes every append fail while the lease is still held. */ - await redis().set(`mothership_stream:${streamId}:events`, 'not a sorted set', 'EX', 60) + await getRedisClient().set(`mothership_stream:${streamId}:events`, 'not a sorted set', 'EX', 60) const publisher = new StreamWriter({ streamId, requestId: generateId(), lease }) const failure = await finalizeAsError(publisher, runId).catch((error: unknown) => error) @@ -1128,7 +1128,7 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge expect(failure).not.toBeInstanceOf(StreamControllerSupersededError) const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId)) expect(stored.status).toBe('error') - await redis().del(lease.key) + await getRedisClient().del(lease.key) }) it('leaves its run for a successor when it lost the lease while publishing', async () => { diff --git a/apps/sim/lib/mothership/request/session/replay-gap.integration.ts b/apps/sim/lib/mothership/request/session/replay-gap.integration.ts index d2e72d9cb41..9923d0509e8 100644 --- a/apps/sim/lib/mothership/request/session/replay-gap.integration.ts +++ b/apps/sim/lib/mothership/request/session/replay-gap.integration.ts @@ -7,8 +7,7 @@ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/ import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') const worker = { requests: [] as Array>, @@ -34,17 +33,14 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, COPILOT_STREAM_EVENT_LIMIT: process.env.COPILOT_STREAM_EVENT_LIMIT, } - process.env.REDIS_URL = url process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` /** A ring this small loses the head of every stream below. */ process.env.COPILOT_STREAM_EVENT_LIMIT = '5' - return { redisUrl: url, inheritedEnv, worker: Object.assign(worker, { server }) } + return { inheritedEnv, worker: Object.assign(worker, { server }) } }) vi.mock('@/lib/auth', () => authMock) @@ -144,7 +140,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('reconnects past the replay ring', () => { +describe('reconnects past the replay ring', () => { beforeAll(async () => { const now = new Date() await db.insert(user).values({ diff --git a/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts b/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts index e2860ab6b32..fc5daf8fff9 100644 --- a/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts +++ b/apps/sim/lib/mothership/request/session/stream-recovery.integration.ts @@ -9,8 +9,7 @@ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/ import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') +const { inheritedEnv, worker } = await vi.hoisted(async () => { const { createServer } = await import('node:http') const worker = { requests: [] as Array<{ path: string; body: Record }>, @@ -33,14 +32,11 @@ const { redisUrl, inheritedEnv, worker } = await vi.hoisted(async () => { }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as { port: number } - const url = readTestRedisUrl() const inheritedEnv = { - REDIS_URL: process.env.REDIS_URL, SIM_AGENT_API_URL: process.env.SIM_AGENT_API_URL, } - process.env.REDIS_URL = url process.env.SIM_AGENT_API_URL = `http://127.0.0.1:${port}` - return { redisUrl: url, inheritedEnv, worker: Object.assign(worker, { server }) } + return { inheritedEnv, worker: Object.assign(worker, { server }) } }) vi.mock('@/lib/auth', () => authMock) @@ -187,7 +183,7 @@ afterAll(async () => { } }) -describe.runIf(Boolean(redisUrl))('recovering a run whose ring lost its head', () => { +describe('recovering a run whose ring lost its head', () => { beforeAll(async () => { const now = new Date() await db.insert(user).values({ diff --git a/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts b/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts index c2d0c531d23..c3ced9bd8e6 100644 --- a/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts +++ b/apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts @@ -1,21 +1,3 @@ -import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -/** - * A browser claims a Chat workflow tool, the execute route runs it, and the browser may never - * report back (tab closed, network lost, beacon dropped). Runs against real PostgreSQL and Redis: - * the claim, settlement, execution log lookup, guarded completion, published confirmation and the - * Chat-side waiter are production code. - */ -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' - -const { redisUrl, inheritedEnv } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - const url = readTestRedisUrl() - const inheritedEnv = { REDIS_URL: process.env.REDIS_URL } - /** The real Redis module and the confirmation channel read this at import. */ - process.env.REDIS_URL = url - return { redisUrl: url, inheritedEnv } -}) - import { db } from '@sim/db' import { copilotAsyncToolCalls, @@ -28,8 +10,16 @@ import { workflowExecutionSnapshots, workspace, } from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { generateId } from '@sim/utils/id' import { eq, inArray } from 'drizzle-orm' +/** + * A browser claims a Chat workflow tool, the execute route runs it, and the browser may never + * report back (tab closed, network lost, beacon dropped). Runs against real PostgreSQL and Redis: + * the claim, settlement, execution log lookup, guarded completion, published confirmation and the + * Chat-side waiter are production code. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' import { SIM_TOOL_EXECUTION_VERSION } from '@/lib/mothership/async-runs/lifecycle' import { @@ -84,13 +74,9 @@ afterAll(async () => { channels._toolConfirmationChannel?.dispose() channels._toolConfirmationChannel = undefined await closeRedisConnection() - for (const [key, value] of Object.entries(inheritedEnv)) { - if (value === undefined) delete process.env[key] - else process.env[key] = value - } }) -describe.runIf(Boolean(redisUrl))('settled client-claimed workflow tools', () => { +describe('settled client-claimed workflow tools', () => { const userId = generateId() const workspaceId = generateId() const workflowId = generateId() diff --git a/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts b/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts index cd30a830fb1..0a790a685e0 100644 --- a/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts +++ b/apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts @@ -8,15 +8,6 @@ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/ */ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -const { redisUrl, inheritedRedisUrl } = await vi.hoisted(async () => { - const { readTestRedisUrl } = await import('@sim/db/testing/test-infrastructure') - const url = readTestRedisUrl() - const inheritedRedisUrl = process.env.REDIS_URL - /** The real Redis module reads this at import. */ - if (url) process.env.REDIS_URL = url - return { redisUrl: url, inheritedRedisUrl } -}) - vi.mock('next/server', async (original) => ({ ...(await original()), after: () => {}, @@ -49,11 +40,9 @@ import { POST as wakeRoute } from '@/app/api/mothership/wake/route' afterAll(async () => { chatPubSub.dispose() await closeRedisConnection() - if (inheritedRedisUrl === undefined) Reflect.deleteProperty(process.env, 'REDIS_URL') - else process.env.REDIS_URL = inheritedRedisUrl }) -describe.runIf(Boolean(redisUrl))('task wake retries', () => { +describe('task wake retries', () => { const userId = generateId() const workspaceId = generateId() const chatIds: string[] = [] diff --git a/apps/sim/lib/mothership/tools/handlers/workflow/mutations.test.ts b/apps/sim/lib/mothership/tools/handlers/workflow/mutations.test.ts index d06672353ef..947f1ba43bf 100644 --- a/apps/sim/lib/mothership/tools/handlers/workflow/mutations.test.ts +++ b/apps/sim/lib/mothership/tools/handlers/workflow/mutations.test.ts @@ -267,7 +267,6 @@ describe('workflow mutation Copilot adapters', () => { new WorkflowRunAlreadyTerminalError({ executionId: 'execution-1', executionStatus: 'completed', - redisAvailable: true, locallyAborted: false, }) ) diff --git a/apps/sim/lib/workflows/api/route-policies.ts b/apps/sim/lib/workflows/api/route-policies.ts index bcab1ba5fce..a0e08a4fc60 100644 --- a/apps/sim/lib/workflows/api/route-policies.ts +++ b/apps/sim/lib/workflows/api/route-policies.ts @@ -47,7 +47,7 @@ function v2CancelRunErrorResponse(error: unknown) { v2CancelWorkflowRunDataSchema.parse({ success: false, runId: error.executionId, - redisAvailable: error.redisAvailable, + redisAvailable: true, durablyRecorded: false, locallyAborted: error.locallyAborted, pausedCancelled: false, diff --git a/helm/sim/templates/_helpers.tpl b/helm/sim/templates/_helpers.tpl index db1ecfa5aa4..8ddacdd5cee 100644 --- a/helm/sim/templates/_helpers.tpl +++ b/helm/sim/templates/_helpers.tpl @@ -365,8 +365,10 @@ true {{/* With redis.enabled=false, the enabled components ("app", "realtime", joined by " and ") that no REDIS_URL source reaches. `shared` is whether the app Secret -carries REDIS_URL, which both pods envFrom. Empty when every enabled component -is covered, or when the chart deploys Redis itself. +carries REDIS_URL, which both pods envFrom. An envDefaults REDIS_URL counts +only while no app.env (or, for realtime, realtime.env) REDIS_URL masks it: the +deployments skip inlining a default the operator set there. Empty when every +enabled component is covered, or when the chart deploys Redis itself. */}} {{- define "sim.redisUncoveredComponents" -}} {{- $root := .root -}} @@ -377,11 +379,15 @@ is covered, or when the chart deploys Redis itself. {{- $shared = true -}} {{- end -}} {{- end -}} +{{- $setInAppEnv := include "sim.nonEmptyEnv" (list $root.Values.app.env "REDIS_URL") -}} +{{- $setInRealtimeEnv := include "sim.nonEmptyEnv" (list $root.Values.realtime.env "REDIS_URL") -}} +{{- $appDefault := and (not $setInAppEnv) (include "sim.nonEmptyEnv" (list $root.Values.app.envDefaults "REDIS_URL")) -}} +{{- $realtimeDefault := and (not $setInAppEnv) (not $setInRealtimeEnv) (include "sim.nonEmptyEnv" (list $root.Values.realtime.envDefaults "REDIS_URL")) -}} {{- $uncovered := list -}} -{{- if and $root.Values.app.enabled (not $shared) (not (include "sim.nonEmptyEnv" (list $root.Values.app.envDefaults "REDIS_URL"))) -}} +{{- if and $root.Values.app.enabled (not $shared) (not $appDefault) -}} {{- $uncovered = append $uncovered "app" -}} {{- end -}} -{{- if and $root.Values.realtime.enabled (not $shared) (not (include "sim.nonEmptyEnv" (list $root.Values.realtime.envDefaults "REDIS_URL"))) -}} +{{- if and $root.Values.realtime.enabled (not $shared) (not $realtimeDefault) -}} {{- $uncovered = append $uncovered "realtime" -}} {{- end -}} {{- join " and " $uncovered -}} @@ -414,10 +420,14 @@ than enforced. runtime CrashLoopBackOff with cryptic env errors. Fail at template time if the key is neither set in app.env nor mapped via remoteRefs.app. */}} -{{- $redisInSecret := or (include "sim.nonEmptyEnv" (list $appEnv "REDIS_URL")) (index $remoteRefs "REDIS_URL") -}} +{{- /* + Only remoteRefs reach the synced Secret: app.env / realtime.env must be mapped there too (checked + below for each enabled component), and a disabled component's env is synced nowhere. +*/ -}} +{{- $redisInSecret := index $remoteRefs "REDIS_URL" -}} {{- $redisUncovered := include "sim.redisUncoveredComponents" (dict "root" . "shared" $redisInSecret) -}} {{- if $redisUncovered -}} -{{- fail (printf "Required key 'REDIS_URL' is missing for the %s pod: redis.enabled=false and externalSecrets.enabled=true but the key is neither set in app.env, mapped in externalSecrets.remoteRefs.app, nor supplied by an extraEnvVars entry. Map it via externalSecrets.remoteRefs.app.REDIS_URL='path/in/store' so it is synced into the app Secret." $redisUncovered) }} +{{- fail (printf "Required key 'REDIS_URL' is missing for the %s pod: redis.enabled=false and externalSecrets.enabled=true but the key is neither mapped in externalSecrets.remoteRefs.app nor supplied by an extraEnvVars entry. Map it via externalSecrets.remoteRefs.app.REDIS_URL='path/in/store' so it is synced into the app Secret." $redisUncovered) }} {{- end -}} {{- if .Values.app.enabled -}} {{- $required := list "BETTER_AUTH_SECRET" "ENCRYPTION_KEY" "INTERNAL_API_SECRET" -}} diff --git a/helm/sim/tests/redis_test.yaml b/helm/sim/tests/redis_test.yaml index 009e9d7d3ee..7b37a38c853 100644 --- a/helm/sim/tests/redis_test.yaml +++ b/helm/sim/tests/redis_test.yaml @@ -280,6 +280,51 @@ tests: - failedTemplate: errorPattern: "Required key 'REDIS_URL' is missing for the realtime pod" + # External Secrets syncs only remoteRefs.app, and a disabled app's env is checked and synced nowhere. + - it: does not count an unmapped app.env.REDIS_URL for a realtime-only External Secrets release + template: deployment-realtime.yaml + set: + redis.enabled: false + app.enabled: false + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: my-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: sim/app/better-auth-secret + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: sim/app/encryption-key + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: sim/app/internal-api-secret + externalSecrets.remoteRefs.app.CRON_SECRET: sim/app/cron-secret + externalSecrets.remoteRefs.postgresql.password: sim/postgresql/password + app.env.BETTER_AUTH_SECRET: "" + app.env.ENCRYPTION_KEY: "" + app.env.INTERNAL_API_SECRET: "" + app.env.CRON_SECRET: "" + app.env.REDIS_URL: "redis://redis.example.com:6379" + asserts: + - failedTemplate: + errorPattern: "Required key 'REDIS_URL' is missing for the realtime pod" + + # The realtime deployment skips inlining a default that app.env sets, so app.env masks it. + - it: does not count a realtime.envDefaults REDIS_URL that an unsynced app.env.REDIS_URL masks + template: deployment-realtime.yaml + set: + redis.enabled: false + app.enabled: false + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: my-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: sim/app/better-auth-secret + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: sim/app/encryption-key + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: sim/app/internal-api-secret + externalSecrets.remoteRefs.app.CRON_SECRET: sim/app/cron-secret + externalSecrets.remoteRefs.postgresql.password: sim/postgresql/password + app.env.BETTER_AUTH_SECRET: "" + app.env.ENCRYPTION_KEY: "" + app.env.INTERNAL_API_SECRET: "" + app.env.CRON_SECRET: "" + app.env.REDIS_URL: "redis://redis.example.com:6379" + realtime.envDefaults.REDIS_URL: "redis://redis.example.com:6379" + asserts: + - failedTemplate: + errorPattern: "Required key 'REDIS_URL' is missing for the realtime pod" + - it: syncs the mapped REDIS_URL for a realtime-only External Secrets release template: external-secret-app.yaml set: diff --git a/packages/sim-setup/src/checks.test.ts b/packages/sim-setup/src/checks.test.ts index c2b21dac266..0d96058cd26 100644 --- a/packages/sim-setup/src/checks.test.ts +++ b/packages/sim-setup/src/checks.test.ts @@ -210,6 +210,17 @@ describe('registration mode checks', () => { }) }) +describe('doctor live checks', () => { + beforeEach(() => rmSync(fakeRoot, { recursive: true, force: true })) + + it('leaves the missing env file as the only report instead of failing Redis', async () => { + const findings = await runChecks(loadCheckContext(true), ['live']) + + const redis = findings.filter((finding) => finding.message.startsWith('redis')) + expect(redis).toEqual([{ group: 'live', status: 'skip', message: 'redis: no env file yet' }]) + }) +}) + describe('doctor --fix on env files', () => { const files = { sim: path.join(fakeRoot, 'apps/sim/.env'), diff --git a/packages/sim-setup/src/checks.ts b/packages/sim-setup/src/checks.ts index 39840a89585..64a2cf1265d 100644 --- a/packages/sim-setup/src/checks.ts +++ b/packages/sim-setup/src/checks.ts @@ -682,6 +682,10 @@ async function checkDatabase(sim: EnvFile): Promise { } async function checkRedis(sim: EnvFile): Promise { + // With no env file yet the files check already reports it; a Redis failure would only repeat it. + if (!sim.exists) { + return [{ group: 'live', status: 'skip', message: 'redis: no env file yet' }] + } const redisUrl = sim.vars.get('REDIS_URL') // A Compose install's REDIS_URL comes from the service definition, not the root .env. if (!redisUrl && sim.target === 'root') { diff --git a/packages/sim-setup/src/redis.test.ts b/packages/sim-setup/src/redis.test.ts new file mode 100644 index 00000000000..e782d5d2269 --- /dev/null +++ b/packages/sim-setup/src/redis.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { world, mockSpawnSync, mockDocker, mockConfirm } = vi.hoisted(() => ({ + /** Whether the managed container is up; it publishes 127.0.0.1:6379. */ + world: { running: false }, + mockSpawnSync: vi.fn(), + mockDocker: vi.fn(), + mockConfirm: vi.fn(), +})) + +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, spawnSync: mockSpawnSync } +}) + +vi.mock('./db', () => ({ docker: mockDocker })) + +vi.mock('./docker', () => ({ ensureDocker: vi.fn(async () => true) })) + +vi.mock('./probes', () => ({ + redisPing: vi.fn(async (url: string) => + world.running && new URL(url).host === '127.0.0.1:6379' + ? { ok: true } + : { ok: false, error: 'ECONNREFUSED' } + ), + waitFor: async (probe: () => Promise) => probe(), +})) + +vi.mock('./prompter', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + confirm: mockConfirm, + spinner: () => ({ start: vi.fn(), stop: vi.fn() }), + log: { step: vi.fn(), warn: vi.fn(), info: vi.fn() }, + } +}) + +import type { Detection } from './detect' +import { ensureRedis, resolveRedis } from './redis' + +const CONFIGURED = 'redis://127.0.0.1:6379/5' + +function detection(state: 'running' | 'stopped'): Detection { + return { + redisPortOpen: false, + redisContainer: { state, managed: true }, + } as Detection +} + +describe('reusing the managed Redis container', () => { + beforeEach(() => { + world.running = false + mockConfirm.mockReset() + mockDocker.mockImplementation((args: string[]) => { + if (args[0] === 'start') world.running = true + }) + mockSpawnSync.mockImplementation((command: string, args: string[]) => + command === 'docker' && args[0] === 'port' + ? { status: 0, stdout: '127.0.0.1:6379\n', stderr: '' } + : { status: 0, stdout: '', stderr: '' } + ) + }) + + it('keeps the configured DB index when quick mode restarts a stopped container', async () => { + await expect(ensureRedis(detection('stopped'), CONFIGURED)).resolves.toBe(CONFIGURED) + expect(mockDocker).toHaveBeenCalledWith(['start', 'sim-redis']) + }) + + it('keeps the configured DB index when custom mode restarts a stopped container', async () => { + await expect(resolveRedis(detection('stopped'), CONFIGURED)).resolves.toBe(CONFIGURED) + expect(mockConfirm).not.toHaveBeenCalled() + }) + + it('does not restore a configured URL the user declined to keep', async () => { + world.running = true + mockConfirm.mockResolvedValue(false) + + await expect(resolveRedis(detection('running'), CONFIGURED)).resolves.toBe( + 'redis://127.0.0.1:6379' + ) + }) +}) diff --git a/packages/sim-setup/src/redis.ts b/packages/sim-setup/src/redis.ts index fdd41c7b907..5195bbf4121 100644 --- a/packages/sim-setup/src/redis.ts +++ b/packages/sim-setup/src/redis.ts @@ -21,6 +21,15 @@ function managedRedisUrl(): string | null { return port ? `redis://${LOOPBACK_HOST}:${port}` : null } +/** + * The URL to keep once the managed container answers at `managedUrl`: a configured REDIS_URL that + * answers too (it did not before the restart) wins, so its DB index or credentials survive. + */ +async function keepConfiguredUrl(managedUrl: string, existing?: string): Promise { + if (existing && existing !== managedUrl && (await redisPing(existing)).ok) return existing + return managedUrl +} + async function pingWithSpinner(url: string, label: string): Promise { const spin = p.spinner() spin.start(label) @@ -33,9 +42,11 @@ async function startManagedRedis(detection: Detection): Promise { const hostPort = detection.redisPortOpen ? 6380 : 6379 const url = `redis://${LOOPBACK_HOST}:${hostPort}` // A prior sim-redis (unhealthy, or bound to a stale port) would collide on the - // name. It has no data volume and holds no durable data, so it is replaced - // without a prompt; only short-lived state (live OTPs, OAuth state, rate-limit - // buckets) is dropped. `rm -f` is a no-op when none exists. + // name. It has no data volume, so replacing it drops what it held: live OTPs, + // OAuth state, rate-limit buckets, and idempotency records (kept up to 7 days, + // so a webhook redelivered afterwards can run again). That is acceptable for a + // local dev Redis that is not answering anyway, so it is replaced without a + // prompt. `rm -f` is a no-op when none exists. spawnSync('docker', ['rm', '-f', REDIS_CONTAINER], { stdio: 'ignore' }) docker([ 'run', @@ -96,7 +107,7 @@ export async function ensureRedis(detection: Detection, existing?: string): Prom const managedUrl = managedRedisUrl() if (managedUrl && (await waitFor(async () => (await redisPing(managedUrl)).ok, 15_000, 500))) { p.log.step(`Reusing ${REDIS_CONTAINER}`) - return managedUrl + return keepConfiguredUrl(managedUrl, existing) } } if (!(await ensureDocker(false))) { @@ -115,6 +126,8 @@ export async function ensureRedis(detection: Detection, existing?: string): Prom export async function resolveRedis(detection: Detection, existing?: string): Promise { // A configured REDIS_URL wins over the port scan: it may point at a non-default // port, or at a host that isn't this machine at all. + // A URL the user declined to keep is never restored below. + let configured = existing if ( existing && existing !== LOCAL_URL && @@ -122,6 +135,7 @@ export async function resolveRedis(detection: Detection, existing?: string): Pro ) { const keep = await p.confirm({ message: `Keep using ${existing}?`, initialValue: true }) if (keep) return existing + configured = undefined } if ( @@ -143,10 +157,10 @@ export async function resolveRedis(detection: Detection, existing?: string): Pro managedUrl && (await pingWithSpinner(managedUrl, `Starting existing ${REDIS_CONTAINER} container…`)) ) { - return managedUrl + return keepConfiguredUrl(managedUrl, configured) } p.log.warn( - `${REDIS_CONTAINER} exists but is not answering — starting a fresh one will replace it. It holds no durable data, but live OTPs, OAuth state, and rate-limit buckets are dropped.` + `${REDIS_CONTAINER} exists but is not answering — starting a fresh one will replace it and drop what it held: live OTPs, OAuth state, rate-limit buckets, and idempotency records (so a webhook redelivered afterwards can run again).` ) } diff --git a/packages/sim-setup/src/setup-status.test.ts b/packages/sim-setup/src/setup-status.test.ts index f62a3f3a1ba..e3ba1dad7a2 100644 --- a/packages/sim-setup/src/setup-status.test.ts +++ b/packages/sim-setup/src/setup-status.test.ts @@ -66,11 +66,6 @@ describe('setup status', () => { { REDIS_URL: 'http://localhost:6379' }, { detail: expect.stringContaining('must be a valid redis://') }, ], - [ - 'a rediss:// IP without a TLS server name', - { REDIS_URL: 'rediss://10.0.0.5:6379' }, - { detail: expect.stringContaining('REDIS_TLS_SERVERNAME is required') }, - ], ])('fails when Redis is %s', (_, overrides, issue) => { const report = buildSetupStatusReport(source({ ...CORE_VALUES, ...overrides })) diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index d77f15a4452..e08e743a7cc 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1043,7 +1043,7 @@ "apps/desktop/src/preload/browser/forms.test.ts": 2, "apps/desktop/src/preload/browser/forms.ts": 1, "apps/realtime/src/database/operations.ts": 1, - "apps/realtime/src/handlers/file-doc-store.test.ts": 57, + "apps/realtime/src/handlers/file-doc-store.test.ts": 15, "apps/sim/app/(auth)/login/login-form.tsx": 1, "apps/sim/app/(auth)/oauth/sign-in/route.test.ts": 1, "apps/sim/app/(auth)/verify/use-verification.test.tsx": 1, @@ -1304,7 +1304,6 @@ "apps/sim/lib/billing/application/organization-usage/event-cursor.test.ts": 1, "apps/sim/lib/billing/application/use-cases.test.ts": 1, "apps/sim/lib/billing/calculations/usage-reservation.integration.ts": 1, - "apps/sim/lib/billing/core/reporting-usage-cache.integration.ts": 2, "apps/sim/lib/billing/plan-helpers.ts": 2, "apps/sim/lib/charts/spec.test.ts": 1, "apps/sim/lib/charts/tooltip.test.ts": 3, diff --git a/scripts/check-test-types.baseline.json b/scripts/check-test-types.baseline.json index ec40781e0ec..788bf846153 100644 --- a/scripts/check-test-types.baseline.json +++ b/scripts/check-test-types.baseline.json @@ -1094,16 +1094,13 @@ "apps/sim/lib/core/rate-limiter/hosted-key/rate-limiter.test.ts": { "TS2352": 1 }, - "apps/sim/lib/core/rate-limiter/provider-admission.test.ts": { - "TS2345": 1 - }, "apps/sim/lib/core/rate-limiter/rate-limiter.test.ts": { "TS2307": 4, "TS2352": 1, "TS2664": 2 }, "apps/sim/lib/core/rate-limiter/storage/factory.test.ts": { - "TS2345": 2 + "TS2345": 1 }, "apps/sim/lib/core/security/encryption.test.ts": { "TS2352": 2 @@ -2062,9 +2059,6 @@ "apps/sim/lib/table/dispatcher.test.ts": { "TS2322": 1 }, - "apps/sim/lib/table/events.test.ts": { - "TS2322": 1 - }, "apps/sim/lib/table/jobs/service.test.ts": { "TS2345": 2 }, From 81b621e1e0cfafbe3f81d839a089dab2811a3533 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 19:24:55 -0800 Subject: [PATCH 5/6] fix(redis): keep unit tests off a real Redis and update integration tests for lazy channels --- .../[executionId]/stream/route.test.ts | 23 ++--- .../sim/executor/execution/test-hooks.test.ts | 9 ++ apps/sim/lib/execution/cancellation.test.ts | 17 ++-- apps/sim/lib/execution/signal.test.ts | 2 + .../workbench-file-provenance.integration.ts | 15 ++- .../async-runs/tool-identity.integration.ts | 12 +++ .../backfill-repair.integration.ts | 8 +- .../workflows/executor/execution-core.test.ts | 9 +- .../human-in-the-loop-manager.test.ts | 11 +++ apps/sim/vitest.setup.ts | 2 + .../src/mocks/execution-signal.mock.ts | 93 +++++++++++++++++++ packages/testing/src/mocks/index.ts | 6 ++ 12 files changed, 169 insertions(+), 38 deletions(-) create mode 100644 packages/testing/src/mocks/execution-signal.mock.ts diff --git a/apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts b/apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts index fec090ba11b..cadc90a2c5e 100644 --- a/apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts +++ b/apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts @@ -1,14 +1,13 @@ import { authMockFns, createMockRequest, workflowAuthzMockFns } from '@sim/testing' +import { executionSignalMockFns } from '@sim/testing/mocks/execution-signal.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutionEventEntry } from '@/lib/execution/event-buffer' -const { mockReadExecutionEventsState, mockReadExecutionMetaState, mockSubscribe } = vi.hoisted( - () => ({ - mockReadExecutionEventsState: vi.fn(), - mockReadExecutionMetaState: vi.fn(), - mockSubscribe: vi.fn(), - }) -) +const { mockReadExecutionEventsState, mockReadExecutionMetaState } = vi.hoisted(() => ({ + mockReadExecutionEventsState: vi.fn(), + mockReadExecutionMetaState: vi.fn(), +})) +const mockSubscribe = executionSignalMockFns.mockSubscribe const mockAuthorizeWorkflowByWorkspacePermission = workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission @@ -19,10 +18,6 @@ vi.mock('@/lib/execution/event-buffer', () => ({ readExecutionMetaState: mockReadExecutionMetaState, })) -vi.mock('@/lib/execution/signal', () => ({ - getExecutionSignalHub: () => ({ subscribe: mockSubscribe }), -})) - import { GET } from './route' const mockGetSession = authMockFns.mockGetSession @@ -191,7 +186,7 @@ describe('execution stream reconnect route', () => { let signalHandler: ((reason: 'event') => void) | undefined mockSubscribe.mockImplementationOnce(async (_executionId, handler) => { signalHandler = handler - return vi.fn() + return vi.fn<() => void>() }) mockReadExecutionMetaState .mockResolvedValueOnce({ @@ -253,7 +248,7 @@ describe('execution stream reconnect route', () => { let signalHandler: ((reason: 'event') => void) | undefined mockSubscribe.mockImplementationOnce(async (_executionId, handler) => { signalHandler = handler - return vi.fn() + return vi.fn<() => void>() }) const staleActiveMeta = { status: 'found' as const, @@ -313,7 +308,7 @@ describe('execution stream reconnect route', () => { it('fails the observer closed when its Redis signal subscription becomes unavailable', async () => { mockSubscribe.mockImplementationOnce(async (_executionId, handler) => { queueMicrotask(() => handler('unavailable')) - return vi.fn() + return vi.fn<() => void>() }) const req = createMockRequest( 'GET', diff --git a/apps/sim/executor/execution/test-hooks.test.ts b/apps/sim/executor/execution/test-hooks.test.ts index 85134650631..ec68532c0c5 100644 --- a/apps/sim/executor/execution/test-hooks.test.ts +++ b/apps/sim/executor/execution/test-hooks.test.ts @@ -17,6 +17,15 @@ const { executed, inputsSeen } = vi.hoisted(() => ({ inputsSeen: [] as Array<{ blockId: string; inputs: Record }>, })) +/** A For loop's continuation check is `return Boolean( < )`; nothing here needs a real isolate. */ +vi.mock('@/lib/execution/isolated-vm', () => ({ + executeInIsolatedVM: vi.fn(async ({ code }: { code: string }) => { + const indexCheck = /^return Boolean\((\d+) < (\d+)\)$/.exec(code) + if (!indexCheck) throw new Error(`Unexpected isolated-vm code in test: ${code}`) + return { result: Number(indexCheck[1]) < Number(indexCheck[2]) } + }), +})) + vi.mock('@/executor/handlers/registry', () => ({ createBlockHandlers: () => [ { diff --git a/apps/sim/lib/execution/cancellation.test.ts b/apps/sim/lib/execution/cancellation.test.ts index 291e077853b..e4756d67bdf 100644 --- a/apps/sim/lib/execution/cancellation.test.ts +++ b/apps/sim/lib/execution/cancellation.test.ts @@ -1,29 +1,24 @@ import { redisConfigMockFns } from '@sim/testing' +import { executionSignalMockFns } from '@sim/testing/mocks/execution-signal.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionSignalHandler } from '@/lib/execution/signal' const { mockRedisEval, mockRedisExists, mockRedisPublish, - mockSignalSubscribe, mockSignalUnsubscribe, mockRecordCancellationResult, } = vi.hoisted(() => ({ mockRedisEval: vi.fn(), mockRedisExists: vi.fn(), mockRedisPublish: vi.fn(), - mockSignalSubscribe: vi.fn(), mockSignalUnsubscribe: vi.fn(), mockRecordCancellationResult: vi.fn(), })) const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient - -vi.mock('@/lib/execution/signal', () => ({ - getExecutionSignalChannel: (executionId: string) => `execution:signal:${executionId}`, - getExecutionSignalHub: () => ({ subscribe: mockSignalSubscribe }), - LEGACY_EXECUTION_CANCEL_CHANNEL: 'execution:cancel', -})) +const mockSignalSubscribe = executionSignalMockFns.mockSubscribe vi.mock('@/lib/core/execution-limits/metrics', () => ({ recordExecutionCancellationBackendResult: mockRecordCancellationResult, @@ -195,7 +190,7 @@ describe('subscribeToExecutionCancellation', () => { }) it('re-reads the durable cancellation flag after the signal connection recovers', async () => { - let signalHandler: ((reason: string) => void) | undefined + let signalHandler: ExecutionSignalHandler | undefined mockSignalSubscribe.mockImplementation(async (_executionId, handler) => { signalHandler = handler return mockSignalUnsubscribe @@ -211,7 +206,7 @@ describe('subscribeToExecutionCancellation', () => { }) it('queues a fresh durable read when reconnect occurs during an in-flight read', async () => { - let signalHandler: ((reason: string) => void) | undefined + let signalHandler: ExecutionSignalHandler | undefined mockSignalSubscribe.mockImplementation(async (_executionId, handler) => { signalHandler = handler return mockSignalUnsubscribe @@ -237,7 +232,7 @@ describe('subscribeToExecutionCancellation', () => { }) it('fails closed when the signal subscription cannot be restored', async () => { - let signalHandler: ((reason: string) => void) | undefined + let signalHandler: ExecutionSignalHandler | undefined mockSignalSubscribe.mockImplementation(async (_executionId, handler) => { signalHandler = handler return mockSignalUnsubscribe diff --git a/apps/sim/lib/execution/signal.test.ts b/apps/sim/lib/execution/signal.test.ts index a14265fb133..362e055d511 100644 --- a/apps/sim/lib/execution/signal.test.ts +++ b/apps/sim/lib/execution/signal.test.ts @@ -16,6 +16,8 @@ const { connection, mockRedisUrl, mockSubscribe, mockUnsubscribe } = vi.hoisted( mockUnsubscribe: vi.fn(), })) +vi.unmock('@/lib/execution/signal') + vi.mock('ioredis', () => ({ default: class extends EventEmitter { constructor(_url: string, options: Record) { diff --git a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts index d2b48672a44..b977b5ef369 100644 --- a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts +++ b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts @@ -29,6 +29,13 @@ const redis = redisUrl }) : undefined redis?.on('error', () => {}) +/** A client whose connection has closed: every command rejects the way it does when Redis drops. */ +const closedRedis = redisUrl + ? new Redis(redisUrl, { lazyConnect: true, retryStrategy: () => null }) + : undefined +closedRedis?.on('error', () => {}) +closedRedis?.disconnect() +const STORAGE_FAILURE = 'Connection is closed' const receiptPrefixes = new Set() const historyKeys = new Set() const fixtureWorkspaceId = generateShortId() @@ -354,10 +361,10 @@ describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () await consume(broken.observeUpload(machine, body())) expect(broken.uploadProvenance()).toEqual({ status: 'unknown' }) } - redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(closedRedis!) const unavailable = createWorkbenchFileProvenance(scope) await expect(consume(unavailable.observeUpload(machine, body()))).rejects.toThrow( - 'storage is unavailable' + STORAGE_FAILURE ) expect(() => unavailable.uploadProvenance()).toThrow('has not finished') }) @@ -394,7 +401,7 @@ describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () await expect(reader.read()).rejects.toThrow('stopped') } else { if (outcome === 'storage failure') { - redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null) + redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(closedRedis!) } else if (outcome === 'abort during persistence') { redisConfigMockFns.mockGetRedisClient.mockImplementationOnce(() => { stop.abort(new Error('stopped during persistence')) @@ -406,7 +413,7 @@ describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () expect(await reader.read()).toEqual({ done: true, value: undefined }) } else { await expect(reader.read()).rejects.toThrow( - outcome === 'storage failure' ? 'storage is unavailable' : 'stopped during persistence' + outcome === 'storage failure' ? STORAGE_FAILURE : 'stopped during persistence' ) } } diff --git a/apps/sim/lib/mothership/async-runs/tool-identity.integration.ts b/apps/sim/lib/mothership/async-runs/tool-identity.integration.ts index cab21f5bf1c..8a2cd2126f2 100644 --- a/apps/sim/lib/mothership/async-runs/tool-identity.integration.ts +++ b/apps/sim/lib/mothership/async-runs/tool-identity.integration.ts @@ -139,6 +139,18 @@ describe('Copilot tool identity with PostgreSQL', () => { if (redisUrl) { redisState.current = new Redis(redisUrl, { maxRetriesPerRequest: 1 }) await redisState.current.ping() + const channels = globalThis as typeof globalThis & { + _toolConfirmationChannel?: { ready(): Promise } + _toolPermissionChannel?: { ready(): Promise } + } + if (!channels._toolConfirmationChannel || !channels._toolPermissionChannel) { + throw new Error('Tool confirmation and permission channels were not created on import') + } + /** The channels connect on first use; open both so no publication below precedes them. */ + await Promise.all([ + channels._toolConfirmationChannel.ready(), + channels._toolPermissionChannel.ready(), + ]) await vi.waitFor(async () => { const counts = await redisState.current?.pubsub( 'NUMSUB', diff --git a/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts b/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts index 61bc7120fc9..a685b8ce607 100644 --- a/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts +++ b/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts @@ -339,9 +339,13 @@ describeWithRedis('Operator archive repair against the full compatible schema', const unsubscribe = mcpPubSub.onWorkflowToolsChanged(() => { throw new Error('fixture subscriber failed'); }); const received = []; const unsubscribeHealthy = mcpPubSub.onWorkflowToolsChanged((event) => received.push(event)); + const notified = () => received.some((event) => event.serverId === '${serverId}' && event.workspaceId === 'local-env'); try { + await mcpPubSub.ready(); await repairArchivedProjectEnvironment(client, {workspaceId:'local-env', archivedAt:'2026-05-01 00:00:00', workflowIds:['local-flow']}, 'local-subscriber'); - assert.ok(received.some((event) => event.serverId === '${serverId}' && event.workspaceId === 'local-env'), 'Healthy subscriber missed the archive notification'); + const deadline = Date.now() + 10000; + while (!notified() && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(notified(), 'Healthy subscriber missed the archive notification'); } finally { unsubscribe(); unsubscribeHealthy(); mcpPubSub.dispose(); await client.end(); await Promise.all([...new Set([db.$client,dbReplica.$client])].map(client=>client.end())); @@ -350,7 +354,7 @@ describeWithRedis('Operator archive repair against the full compatible schema', ], { cwd: new URL('../../../', import.meta.url), - env: { ...environment, REDIS_URL: '' }, + env: environment, timeout: 45000, } ) diff --git a/apps/sim/lib/workflows/executor/execution-core.test.ts b/apps/sim/lib/workflows/executor/execution-core.test.ts index 83841247726..cef607dbd34 100644 --- a/apps/sim/lib/workflows/executor/execution-core.test.ts +++ b/apps/sim/lib/workflows/executor/execution-core.test.ts @@ -15,6 +15,7 @@ import { } from '@sim/testing/factories/principal.factory' import { customBlockOperationsMock } from '@sim/testing/mocks/custom-block-operations.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' +import { executionSignalMockFns } from '@sim/testing/mocks/execution-signal.mock' import { largeValueMetadataMock, largeValueMetadataMockFns, @@ -108,13 +109,7 @@ vi.mock('@/lib/execution/cancellation', () => ({ clearExecutionCancellation: clearExecutionCancellationMock, })) -const { connectExecutionSignalHubMock } = vi.hoisted(() => ({ - connectExecutionSignalHubMock: vi.fn(), -})) - -vi.mock('@/lib/execution/signal', () => ({ - connectExecutionSignalHub: connectExecutionSignalHubMock, -})) +const connectExecutionSignalHubMock = executionSignalMockFns.mockConnectExecutionSignalHub vi.mock('@/lib/core/security/encryption', () => encryptionMock) diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts index d6f07c4a21d..41e74a680bd 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts @@ -15,7 +15,10 @@ import { executionPreprocessingMock, executionPreprocessingMockFns, } from '@sim/testing/mocks/execution-preprocessing.mock' +import { deliverExecutionSignal } from '@sim/testing/mocks/execution-signal.mock' import { largeValueMetadataMock } from '@sim/testing/mocks/large-value-metadata.mock' +import { createMockRedis } from '@sim/testing/mocks/redis.mock' +import { redisConfigMockFns, resetRedisConfigMock } from '@sim/testing/mocks/redis-config.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { classifyFailure, logFailureOnce } from '@/lib/core/errors/failure-log' import { createTimeoutAbortController, getExecutionDeadlineAt } from '@/lib/core/execution-limits' @@ -2417,6 +2420,13 @@ describe('stopping an active resume from another replica', () => { }) it('aborts the attempt when Stop records cancellation under the attempt id', async () => { + const redis = createMockRedis() + // Runs the cancellation script's PUBLISH: the durable key write is not what wakes a live attempt. + redis.eval.mockImplementation(async (...args: unknown[]) => { + deliverExecutionSignal(String(args[4]), 'cancelled') + return null + }) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) let attemptSignal: AbortSignal | undefined const runSpy = vi .spyOn(PauseResumeManager as unknown as PauseResumeManagerInternals, 'runResumeExecution') @@ -2450,6 +2460,7 @@ describe('stopping an active resume from another replica', () => { expect(attemptSignal?.aborted).toBe(true) expect(attemptSignal?.reason).toMatchObject({ name: 'AbortError', message: 'user' }) } finally { + resetRedisConfigMock() runSpy.mockRestore() attemptFailedSpy.mockRestore() processSpy.mockRestore() diff --git a/apps/sim/vitest.setup.ts b/apps/sim/vitest.setup.ts index 8cd331b7d11..8861a0c6bed 100644 --- a/apps/sim/vitest.setup.ts +++ b/apps/sim/vitest.setup.ts @@ -3,6 +3,7 @@ import { databaseMock, drizzleOrmMock } from '@sim/testing/mocks/database.mock' import { envMock } from '@sim/testing/mocks/env.mock' import { envFlagsMock } from '@sim/testing/mocks/env-flags.mock' import { environmentUtilsMock } from '@sim/testing/mocks/environment-utils.mock' +import { executionSignalMock } from '@sim/testing/mocks/execution-signal.mock' import { setupGlobalFetchMock } from '@sim/testing/mocks/fetch.mock' import { hybridAuthMock } from '@sim/testing/mocks/hybrid-auth.mock' import { loggerMock } from '@sim/testing/mocks/logger.mock' @@ -53,6 +54,7 @@ vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) vi.mock('@/lib/core/config/env', () => envMock) vi.mock('@/lib/core/utils/urls', () => urlsMock) vi.mock('@/lib/core/config/redis', () => redisConfigMock) +vi.mock('@/lib/execution/signal', () => executionSignalMock) vi.mock('@/lib/environment/utils', () => environmentUtilsMock) vi.mock('@/stores/terminal', () => terminalConsoleMock) diff --git a/packages/testing/src/mocks/execution-signal.mock.ts b/packages/testing/src/mocks/execution-signal.mock.ts new file mode 100644 index 00000000000..babb36cc886 --- /dev/null +++ b/packages/testing/src/mocks/execution-signal.mock.ts @@ -0,0 +1,93 @@ +import { vi } from 'vitest' + +type ExecutionSignalReason = 'event' | 'cancelled' | 'reconnected' | 'unavailable' +type ExecutionSignalHandler = (reason: ExecutionSignalReason) => void + +const EXECUTION_SIGNAL_PREFIX = 'execution:signal:' + +const handlersByChannel = new Map>() + +function getExecutionSignalChannel(executionId: string): string { + return `${EXECUTION_SIGNAL_PREFIX}${executionId}` +} + +/** + * In-process stand-in for the Redis subscriber hub's `subscribe`: registers the handler on the + * execution's channel and resolves at once (the real hub resolves once SUBSCRIBE is acknowledged); + * the returned function unsubscribes it. + */ +async function subscribeInProcess( + executionId: string, + handler: ExecutionSignalHandler +): Promise<() => void> { + const channel = getExecutionSignalChannel(executionId) + let handlers = handlersByChannel.get(channel) + if (!handlers) { + handlers = new Set() + handlersByChannel.set(channel, handlers) + } + handlers.add(handler) + return () => { + const current = handlersByChannel.get(channel) + if (!current) return + current.delete(handler) + if (current.size === 0) handlersByChannel.delete(channel) + } +} + +/** + * Controllable mock functions for `@/lib/execution/signal`. `mockSubscribe` is the hub's + * `subscribe` and defaults to the in-process registry {@link deliverExecutionSignal} publishes to; + * a test that drives subscription outcomes itself overrides it. + * + * @example + * ```ts + * executionSignalMockFns.mockSubscribe.mockRejectedValueOnce(new Error('Redis subscriber connection ended')) + * expect(executionSignalMockFns.mockConnectExecutionSignalHub).toHaveBeenCalledOnce() + * ``` + */ +export const executionSignalMockFns = { + mockSubscribe: vi.fn(subscribeInProcess), + mockConnectExecutionSignalHub: vi.fn(), +} + +const hub = { subscribe: executionSignalMockFns.mockSubscribe } + +/** + * Delivers a Redis `PUBLISH` to the hub's subscribers the way the real subscriber's `message` + * listener does: a `cancelled` message dispatches `cancelled`, anything else `event`. A test whose + * mock Redis client runs the cancellation script calls this with the channel the script publishes to. + * + * @example + * ```ts + * redis.eval.mockImplementation(async (_script, _keys, _key, _expiry, channel) => { + * deliverExecutionSignal(String(channel), 'cancelled') + * }) + * ``` + */ +export function deliverExecutionSignal(channel: string, message: string): void { + const handlers = handlersByChannel.get(channel) + if (!handlers) return + const reason: ExecutionSignalReason = message === 'cancelled' ? 'cancelled' : 'event' + for (const handler of [...handlers]) handler(reason) +} + +/** + * Static mock module for `@/lib/execution/signal`, registered globally in `apps/sim/vitest.setup.ts` + * next to the `@/lib/core/config/redis` mock. The real hub dials its own ioredis subscriber from the + * configured URL rather than going through `getRedisClient`, so without this every unit test that + * runs the executor would wait out the subscriber readiness budget against a Redis that is not + * there. Subscriptions are in-process; {@link deliverExecutionSignal} plays the publish side. + * `lib/execution/signal.test.ts` opts out with `vi.unmock('@/lib/execution/signal')`. + * + * @example + * ```ts + * vi.mock('@/lib/execution/signal', () => executionSignalMock) + * ``` + */ +export const executionSignalMock = { + LEGACY_EXECUTION_CANCEL_CHANNEL: 'execution:cancel', + getExecutionSignalChannel, + getExecutionSignalHub: () => hub, + connectExecutionSignalHub: executionSignalMockFns.mockConnectExecutionSignalHub, +} diff --git a/packages/testing/src/mocks/index.ts b/packages/testing/src/mocks/index.ts index 8bb5f081e68..2d1cfc32285 100644 --- a/packages/testing/src/mocks/index.ts +++ b/packages/testing/src/mocks/index.ts @@ -316,6 +316,12 @@ export { executionPreprocessingMock, executionPreprocessingMockFns, } from './execution-preprocessing.mock' +// Execution signal hub mocks (for @/lib/execution/signal) +export { + deliverExecutionSignal, + executionSignalMock, + executionSignalMockFns, +} from './execution-signal.mock' export { executorPrincipalMock, executorPrincipalMockFns, From a696c09c2c42486e0e4bfe10af7588d89ac5a2b9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 19:50:28 -0800 Subject: [PATCH 6/6] chore(redis): drop stale fallback references after rebasing --- .../e2e/desktop-tools-live-sim.spec.ts | 7 +- .../scripts/fixtures/content-baseline.json | 77 ++++++++++--------- apps/sim/connectors/github/rate-limits.md | 2 +- .../agent/memory-harness.integration.ts | 2 +- .../workbench-file-provenance.integration.ts | 39 +++++----- .../backfill-repair.integration.ts | 3 +- .../lib/workflows/executor/execute-service.ts | 2 +- .../edit-workspace-file-content.ts | 4 +- scripts/check-explicit-any.baseline.json | 1 - scripts/check-unused-exports.baseline.json | 1 - 10 files changed, 70 insertions(+), 68 deletions(-) diff --git a/apps/desktop/e2e/desktop-tools-live-sim.spec.ts b/apps/desktop/e2e/desktop-tools-live-sim.spec.ts index f83e6c61917..c396d8104a4 100644 --- a/apps/desktop/e2e/desktop-tools-live-sim.spec.ts +++ b/apps/desktop/e2e/desktop-tools-live-sim.spec.ts @@ -30,9 +30,10 @@ import { * what the user or the model would observe: the result the model is resumed with, what landed in * the workspace, what Sim persisted, and which requests reached Sim. * - * Sim runs the background executor wherever it has Redis, as it does here. Most tests cover the - * chat view running desktop tools itself, as on an install without Redis: the proxy answers the - * app's registration as such an install does, so the app stays dormant and no turn binds to it. + * Sim runs the background executor for a registered desktop app, as it does here. Most tests cover + * the chat view running desktop tools itself: the proxy answers the app's registration with the + * executor off (as servers predating required Redis did), so the app stays dormant and no turn + * binds to it. * The tests marked as running in the background let Sim's own answer through. */ diff --git a/apps/docs/scripts/fixtures/content-baseline.json b/apps/docs/scripts/fixtures/content-baseline.json index 15d2f0af918..a7cb4d1f3bb 100644 --- a/apps/docs/scripts/fixtures/content-baseline.json +++ b/apps/docs/scripts/fixtures/content-baseline.json @@ -173281,8 +173281,8 @@ { "lang": "text", "meta": null, - "sha256": "a45ddc97179f78324455bd2d0f25917282f721b3962c68e7455d9917e32379f5", - "bytes": 852, + "sha256": "673549d971c13b6c90e7b362eff976c647ad34aa6282346e3b66b738d479478c", + "bytes": 828, "line": 329 } ], @@ -173878,126 +173878,126 @@ "meta": null, "sha256": "297876dd58effad6b4b090076d0ef45be7e8bfed12f88675b87163dd3994bf2d", "bytes": 267, - "line": 35 + "line": 39 }, { "lang": "bash", "meta": null, "sha256": "1207989a26fa955c4393c00573ddef43c081589c29f31ac59e33c21ec5cbcd24", "bytes": 460, - "line": 53 + "line": 57 }, { "lang": "bash", "meta": null, "sha256": "ebd6f30ec1291421eb6a701952e95b5e95d26c11da036967e1f8763ae3e53cb5", "bytes": 62, - "line": 67 + "line": 71 }, { "lang": "bash", "meta": null, "sha256": "ef408cf27c701be6e6a5f6d18e697faaee73fa7c45fc9f10c73d2f27f3ced278", "bytes": 67, - "line": 71 + "line": 75 }, { "lang": "yaml", "meta": null, "sha256": "cf37759439567599919c84244b0ba560f03ef642d1af27396fc1a356b26b90ca", "bytes": 701, - "line": 81 + "line": 85 }, { "lang": "bash", "meta": null, "sha256": "2c02be05f3177ceb5daa6aa1598f3b56d79d929eadb8d3c64463697fc68cbfb4", "bytes": 89, - "line": 125 + "line": 129 }, { "lang": "yaml", "meta": null, "sha256": "df68603f8a69b1564ae8792b1bf46bead93f1635ad2cf6aba1b0e53e250f2276", "bytes": 176, - "line": 134 + "line": 138 }, { "lang": "yaml", "meta": null, "sha256": "ed8c55ae5557bb7dbe7bf3c4e6fea6d0a1afaab691bfaf5452bd19707cf5df59", "bytes": 193, - "line": 156 + "line": 160 }, { "lang": "yaml", "meta": null, "sha256": "49ba0eabaea74095c7bf5b0d0a73d16eb74cc1eba7a112fa2c98311345dd750e", "bytes": 214, - "line": 172 + "line": 176 }, { "lang": "yaml", "meta": null, "sha256": "a1f59ab8940c1e0727969aa6e93265c859abb5faadabf08c4cc9a4a4c0b42dfe", "bytes": 821, - "line": 192 + "line": 196 }, { "lang": "bash", "meta": null, "sha256": "dd8942b9048a34ff56d9db7b7a76c8dad1412c0fc7d5b8aeca28f2d704740e51", "bytes": 130, - "line": 228 + "line": 232 }, { "lang": "bash", "meta": null, "sha256": "78c085f5cd3de3d18a521eef6bbf9039067fad49d81139b1d782d07ebe314cf6", "bytes": 864, - "line": 243 + "line": 247 }, { "lang": "bash", "meta": null, "sha256": "1dfa426d61e4433a888e565f694495f8448d610d39b8ca1155a8618ff6fed976", "bytes": 183, - "line": 272 + "line": 276 }, { "lang": "bash", "meta": null, "sha256": "1ce222972b46042dd9583d7681fd0ddb1895e3b7966efb8e3f692050714bd753", "bytes": 288, - "line": 285 + "line": 289 }, { "lang": "yaml", "meta": null, "sha256": "7829d9f42845f3c88229a162bca612db3a9e1dc7c09c70e530de7daf539c97a7", "bytes": 194, - "line": 299 + "line": 303 }, { "lang": "bash", "meta": null, "sha256": "5c09247e6bc794856c1f448ef7995291c75ad497932c62ba34b939954f5fa008", "bytes": 56, - "line": 318 + "line": 322 }, { "lang": "bash", "meta": null, "sha256": "a31bca6832c9273c0d24c26b45f04bc6dd2e6d0841080971ecc3fa76055eb9bf", "bytes": 117, - "line": 329 + "line": 333 }, { "lang": "bash", "meta": null, "sha256": "486ac558c4262f2f4cf319ccb39b6883911bb16cd173cabf47980a246bd1b32b", "bytes": 40, - "line": 338 + "line": 342 } ], "links": [ @@ -175241,7 +175241,7 @@ "meta": null, "sha256": "84007d710ad43b8238b79e9bf599510bdbe4af4967fdd8ba16e3ea753d186b9c", "bytes": 1381, - "line": 138 + "line": 142 } ], "links": [ @@ -175560,35 +175560,35 @@ "meta": null, "sha256": "6d56d9453ee52a49d2dc12540f7c190354b0624ef1ac23bbded9c7ee40aa639f", "bytes": 131, - "line": 26 + "line": 30 }, { "lang": "yaml", "meta": null, "sha256": "8c382d74d832410c9c2ee0df5bf225951474f715ec638f0fd5ead7648601f1ba", "bytes": 140, - "line": 48 + "line": 52 }, { "lang": "yaml", "meta": null, "sha256": "87dd4fd553ca2c5b50cb2459cc2411cb7a5f2cf5805c7019482f3d6060cca458", "bytes": 43, - "line": 65 + "line": 69 }, { "lang": "bash", "meta": null, "sha256": "973060d6289adb956c0e24441d547751e58f731c47962b3949672a61bee7ea10", "bytes": 71, - "line": 98 + "line": 102 }, { "lang": "yaml", "meta": null, "sha256": "be931c51a1b69c7051582ecc60db3d243f86cf6819c48400b4d4017c6e1f4909", "bytes": 932, - "line": 141 + "line": 145 } ], "links": [ @@ -175937,6 +175937,11 @@ "url": "#collaboration-breaks-with-multiple-replicas", "depth": 2 }, + { + "title": "App refuses to start: REDIS_URL is required", + "url": "#app-refuses-to-start-redis_url-is-required", + "depth": 2 + }, { "title": "App crashes at startup with a REDIS_TLS_SERVERNAME error", "url": "#app-crashes-at-startup-with-a-redis_tls_servername-error", @@ -176071,70 +176076,70 @@ "meta": null, "sha256": "c54df79de3f702f739f9fe5c288c3a9fcc348197d33272df3187cf74b5ad8117", "bytes": 197, - "line": 138 + "line": 142 }, { "lang": "yaml", "meta": null, "sha256": "7b4b8ea041d0669f220e56eeabcd0c5c906738646a647de5bb1a548983ffc2d3", "bytes": 280, - "line": 148 + "line": 152 }, { "lang": "bash", "meta": null, "sha256": "4fd1ce391cf85e5093c6348b4d57dac12c1302939037c308314bd8e12c260b38", "bytes": 117, - "line": 171 + "line": 175 }, { "lang": "bash", "meta": null, "sha256": "7a9aac79c13c638a232f991fc97e9076d7080ec7ac75d80c60eb780b42e1d4bd", "bytes": 76, - "line": 187 + "line": 191 }, { "lang": "bash", "meta": null, "sha256": "ce7ef86fc4fbeab2fdae583b7e60a64259680ac3374b92e8c7641601ca35af93", "bytes": 207, - "line": 200 + "line": 204 }, { "lang": "bash", "meta": null, "sha256": "8560a7167d89cfcd9241ccda8ad1ed21f36e8dc5f108714518fe306aadbd5222", "bytes": 61, - "line": 231 + "line": 235 }, { "lang": "bash", "meta": null, "sha256": "a69235481ad1a0208b7ab7a563c83664dce5cc1ca5ec486ae08b9c10c194ca94", "bytes": 108, - "line": 239 + "line": 243 }, { "lang": "bash", "meta": null, "sha256": "341e83bdf0029a50800e322164b960fd942b1344aa38b563aea95a5640ca103b", "bytes": 190, - "line": 246 + "line": 250 }, { "lang": "bash", "meta": null, "sha256": "62bb3e351950d9f5d7f8ed10bfae6fa61dddb02dcfbaa1a5a27f805d3c861df9", "bytes": 13, - "line": 273 + "line": 277 }, { "lang": "bash", "meta": null, "sha256": "7b2c4142afb1d9e4d4609bd3746ddf70d89307eda5cc804cf471c5527dbc3f3d", "bytes": 65, - "line": 278 + "line": 282 } ], "links": [ diff --git a/apps/sim/connectors/github/rate-limits.md b/apps/sim/connectors/github/rate-limits.md index 3db5e4ab717..a142ac51a17 100644 --- a/apps/sim/connectors/github/rate-limits.md +++ b/apps/sim/connectors/github/rate-limits.md @@ -1,6 +1,6 @@ # GitHub connector request capacity -Repository validation, workspace syncs, member listings, and content hydration use the same request transport. Ordinary admission can wait up to 120 seconds within the 150-second request/body budget, allowing sequential tree and content reads to progress with reduced quota. If quota pacing would exceed that useful request budget, work defers until the reported quota reset instead of repeatedly spending the remaining allowance on bootstrap reads. Actual provider throttles still defer immediately. Every retry acquires a distributed lease; the lease remains held until the response stream is consumed, cancelled, or its deadline expires. Redis is authoritative when configured, with PostgreSQL as the shared backend for installations without Redis. An unavailable configured backend defers work instead of allowing uncontrolled requests. +Repository validation, workspace syncs, member listings, and content hydration use the same request transport. Ordinary admission can wait up to 120 seconds within the 150-second request/body budget, allowing sequential tree and content reads to progress with reduced quota. If quota pacing would exceed that useful request budget, work defers until the reported quota reset instead of repeatedly spending the remaining allowance on bootstrap reads. Actual provider throttles still defer immediately. Every retry acquires a distributed lease; the lease remains held until the response stream is consumed, cancelled, or its deadline expires. Leases live in Redis; if Redis is unavailable, work defers instead of allowing uncontrolled requests. Each token fingerprint has one in-flight request and a 600-request/minute ceiling. Successful responses supply GitHub's remaining request allowance and hourly reset. The shared controller reserves that allowance and spreads requests across the remaining window with 10% headroom, so higher quota tiers retain their throughput and lower or externally consumed allowances slow down. A successful response with zero remaining requests pauses subsequent requests until the reset, including a one-second clock margin. diff --git a/apps/sim/executor/handlers/agent/memory-harness.integration.ts b/apps/sim/executor/handlers/agent/memory-harness.integration.ts index 807d003e79a..896737de95d 100644 --- a/apps/sim/executor/handlers/agent/memory-harness.integration.ts +++ b/apps/sim/executor/handlers/agent/memory-harness.integration.ts @@ -4,7 +4,7 @@ * AGENT_MEMORY_TEST_LIVE=1 uses configured OpenAI/Anthropic credentials and synthetic PDFs. * Otherwise only provider HTTP responses are simulated; storage, SQL, hydration, dispatch, * SDK request construction, response parsing, and memory persistence execute real code. - * Account policy/key lookup use fixtures, Redis uses the in-memory fallback, and no API route runs. + * Account policy/key lookup use fixtures, Redis is the integration test instance, and no API route runs. * AGENT_MEMORY_TEST_REPORT optionally writes SQL snapshots and outgoing attachment hashes. */ diff --git a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts index b977b5ef369..9530dcca33f 100644 --- a/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts +++ b/apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts @@ -20,21 +20,18 @@ import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-tr vi.mock('@/lib/core/config/redis', () => redisConfigMock) const redisUrl = readTestRedisUrl() -const redis = redisUrl - ? new Redis(redisUrl, { - lazyConnect: true, - retryStrategy: () => null, - maxRetriesPerRequest: 1, - connectTimeout: 1000, - }) - : undefined -redis?.on('error', () => {}) +if (!redisUrl) throw new Error('TEST_REDIS_URL is required for this integration suite') +const redis = new Redis(redisUrl, { + lazyConnect: true, + retryStrategy: () => null, + maxRetriesPerRequest: 1, + connectTimeout: 1000, +}) +redis.on('error', () => {}) /** A client whose connection has closed: every command rejects the way it does when Redis drops. */ -const closedRedis = redisUrl - ? new Redis(redisUrl, { lazyConnect: true, retryStrategy: () => null }) - : undefined -closedRedis?.on('error', () => {}) -closedRedis?.disconnect() +const closedRedis = new Redis(redisUrl, { lazyConnect: true, retryStrategy: () => null }) +closedRedis.on('error', () => {}) +closedRedis.disconnect() const STORAGE_FAILURE = 'Connection is closed' const receiptPrefixes = new Set() const historyKeys = new Set() @@ -159,7 +156,7 @@ afterAll(async () => { } }) -describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () => { +describe('trusted workbench byte receipts with real Redis', () => { it.each(['complete', 'pending sibling', 'unknown'] as const)( 'persists settled stdout evidence from a %s registry', async (state) => { @@ -243,7 +240,7 @@ describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () 'isolates receipts by %s when machine history is unavailable', async (field) => { await download() - await redis!.del(...historyKeys) + await redis.del(...historyKeys) const next = createWorkbenchFileProvenance({ ...scope, ...(['workspaceId', 'userId', 'sessionKey'].includes(field) ? { [field]: 'other' } : {}), @@ -356,12 +353,12 @@ describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () provenance: { status: 'unknown' }, }), ]) { - await redis!.set(key, value, 'EX', 60) + await redis.set(key, value, 'EX', 60) const broken = createWorkbenchFileProvenance(scope) await consume(broken.observeUpload(machine, body())) expect(broken.uploadProvenance()).toEqual({ status: 'unknown' }) } - redisConfigMockFns.mockGetRedisClient.mockReturnValue(closedRedis!) + redisConfigMockFns.mockGetRedisClient.mockReturnValue(closedRedis) const unavailable = createWorkbenchFileProvenance(scope) await expect(consume(unavailable.observeUpload(machine, body()))).rejects.toThrow( STORAGE_FAILURE @@ -401,11 +398,11 @@ describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () await expect(reader.read()).rejects.toThrow('stopped') } else { if (outcome === 'storage failure') { - redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(closedRedis!) + redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(closedRedis) } else if (outcome === 'abort during persistence') { redisConfigMockFns.mockGetRedisClient.mockImplementationOnce(() => { stop.abort(new Error('stopped during persistence')) - return redis! + return redis }) } producer.close() @@ -419,7 +416,7 @@ describe.skipIf(!redisUrl)('trusted workbench byte receipts with real Redis', () } if (outcome === 'abort during persistence') { /** Cancellation rejects the reader before the already-issued receipt write settles. */ - await redis!.ping() + await redis.ping() await flushMacrotask() } const entries = await settledWorkspaceAudits() diff --git a/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts b/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts index a685b8ce607..2011aab6f89 100644 --- a/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts +++ b/apps/sim/lib/projects/__integration__/backfill-repair.integration.ts @@ -335,6 +335,7 @@ describeWithRedis('Operator archive repair against the full compatible schema', import { db, dbReplica } from '@sim/db'; import assert from 'node:assert/strict'; import postgres from 'postgres'; + import { sleep } from '@sim/utils/helpers'; const client = postgres(process.env.MIGRATION_DATABASE_URL, {max:1}); const unsubscribe = mcpPubSub.onWorkflowToolsChanged(() => { throw new Error('fixture subscriber failed'); }); const received = []; @@ -344,7 +345,7 @@ describeWithRedis('Operator archive repair against the full compatible schema', await mcpPubSub.ready(); await repairArchivedProjectEnvironment(client, {workspaceId:'local-env', archivedAt:'2026-05-01 00:00:00', workflowIds:['local-flow']}, 'local-subscriber'); const deadline = Date.now() + 10000; - while (!notified() && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 25)); + while (!notified() && Date.now() < deadline) await sleep(25); assert.ok(notified(), 'Healthy subscriber missed the archive notification'); } finally { unsubscribe(); unsubscribeHealthy(); mcpPubSub.dispose(); await client.end(); diff --git a/apps/sim/lib/workflows/executor/execute-service.ts b/apps/sim/lib/workflows/executor/execute-service.ts index 3f6e9ebc697..73ba6b1987c 100644 --- a/apps/sim/lib/workflows/executor/execute-service.ts +++ b/apps/sim/lib/workflows/executor/execute-service.ts @@ -123,7 +123,7 @@ export interface ExecuteWorkflowServiceParams { * The caller may stop waiting before the run settles, leaving it running in this * process. Such a run is admitted like an async one (async rate bucket, reservation, * and timeout), never ends with its caller (so it takes no `abortSignal`), and - * registers an in-process aborter so a run cancel reaches it without Redis. Its + * registers an in-process aborter so a run cancel reaches it directly in this process. Its * execution is otherwise the sync run's own, so a pause resumes as a sync run's does. * Requires `sync-result-stream`, whose pending result the caller stops awaiting. */ diff --git a/apps/sim/lib/workspace-files/application/edit-workspace-file-content.ts b/apps/sim/lib/workspace-files/application/edit-workspace-file-content.ts index 2b99db56fca..e18c77cd3ee 100644 --- a/apps/sim/lib/workspace-files/application/edit-workspace-file-content.ts +++ b/apps/sim/lib/workspace-files/application/edit-workspace-file-content.ts @@ -67,8 +67,8 @@ export interface EditWorkspaceFileContentResult { * * Two guards, because they cover different failures. The advisory lock * serializes concurrent edits to one file so two agents do not both read the - * same text and each write a version missing the other's change; it is - * best-effort and no-ops without Redis. `expectedUpdatedAt` is the guard that + * same text and each write a version missing the other's change; it is a TTL + * lease, so it can lapse under a slow edit. `expectedUpdatedAt` is the guard that * actually holds: the manager compares it under `select ... for update` and * refuses a stale write outright. It is deliberately `contentUpdatedAt` and not * `updatedAt`, because a rename or a move bumps the latter and would fail an diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index e08e743a7cc..f8d833f30f5 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1486,7 +1486,6 @@ "apps/sim/lib/mothership/agent-cli/resource-owner.test.ts": 1, "apps/sim/lib/mothership/agent-cli/run-cli-files.test.ts": 1, "apps/sim/lib/mothership/agent-cli/saved-run-read.live.test.ts": 18, - "apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts": 4, "apps/sim/lib/mothership/agent-cli/workbench-file-provenance.ts": 2, "apps/sim/lib/mothership/assistant/tool-policy.test.ts": 2, "apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts": 9, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index ce22c5a35ba..b16b16fc140 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -4300,7 +4300,6 @@ "apps/sim/lib/execution/remote-sandbox/workspace-sandboxes.ts#isSandboxNameTaken", "apps/sim/lib/execution/remote-sandbox/workspace-sandboxes.ts#isWorkspaceSandboxNameConflictError", "apps/sim/lib/execution/remote-sandbox/workspace-sandboxes.ts#scheduleSandboxBuild", - "apps/sim/lib/execution/signal.ts#ExecutionSignalHandler", "apps/sim/lib/file-parsers/errors.ts#FILE_PARSER_ERROR_CODES", "apps/sim/lib/file-parsers/index.ts#parseFile", "apps/sim/lib/file-parsers/json-parser.ts#stripJsonComments",