diff --git a/apps/sim/lib/core/redis/byte-budget.server.test.ts b/apps/sim/lib/core/redis/byte-budget.server.test.ts index 1787f8fdcd5..8d53972157e 100644 --- a/apps/sim/lib/core/redis/byte-budget.server.test.ts +++ b/apps/sim/lib/core/redis/byte-budget.server.test.ts @@ -17,30 +17,16 @@ describe('getRedisBudgetKeys', () => { /** * These keys are shared with counters written before this module existed, so the - * layout is a wire contract: a change here strands every counter in flight. + * execution layout is a wire contract: a change here strands every counter in flight. */ - it('separates owner kinds without disturbing the execution key layout', () => { + it('gives a stream its own owner and user counters', () => { expect( getRedisBudgetKeys({ kind: 'copilot_stream', id: 'stream-1', userId: 'user-1' }) ).toEqual([ 'execution:redis-budget:copilot_stream:stream-1', - 'execution:redis-budget:user:user-1', + 'execution:redis-budget:copilot_stream_user:user-1', ]) }) - - it('shares one user ceiling across owner kinds', () => { - const [, executionUserKey] = getRedisBudgetKeys({ - kind: 'execution', - id: 'exec-1', - userId: 'user-1', - }) - const [, streamUserKey] = getRedisBudgetKeys({ - kind: 'copilot_stream', - id: 'stream-1', - userId: 'user-1', - }) - expect(streamUserKey).toBe(executionUserKey) - }) }) describe('getRedisBudgetLimits', () => { diff --git a/apps/sim/lib/core/redis/byte-budget.server.ts b/apps/sim/lib/core/redis/byte-budget.server.ts index c8554ffb400..80b5e46689c 100644 --- a/apps/sim/lib/core/redis/byte-budget.server.ts +++ b/apps/sim/lib/core/redis/byte-budget.server.ts @@ -36,15 +36,27 @@ import type { Logger } from '@sim/logger' */ const REDIS_BUDGET_PREFIX = 'execution:redis-budget:' -/** What a budget is charged to. One counter per owner, plus one per user across owners. */ +/** What a budget is charged to. One counter per owner, plus one per user per kind. */ export type RedisBudgetOwnerKind = 'execution' | 'copilot_stream' +/** + * The user counter's key segment for each kind. + * + * Each kind has its own user ceiling, so each needs its own counter: a shared one lets + * workflow runs, within their larger ceiling, lock the user out of Chat for the window. + * `user` is the execution counter's original segment, kept so in-flight counters still apply. + */ +const USER_KEY_SEGMENTS: Record = { + execution: 'user', + copilot_stream: 'copilot_stream_user', +} + export interface RedisBudgetScope { kind: RedisBudgetOwnerKind /** The owner's id — an execution id, a stream id, a table id. */ id: string /** - * Charges the write to a second, user-wide counter as well. Omitted where the + * Charges the write to the user's counter for this kind as well. Omitted where the * writer has no user in scope; the owner counter still applies. */ userId?: string @@ -64,12 +76,12 @@ export interface RedisBudgetLimits { * itself, so sliding its TTL on every write keeps the counter and the bytes it * represents in step. * - * A user counter aggregates across every owner that user writes to. Sliding it on - * each write would keep it alive indefinitely for anyone who stays active while the - * data underneath it keeps expiring — so the counter would accrue bytes Redis has - * already dropped and eventually pin the user at their ceiling until they went a full - * window without writing. User counters therefore get a fixed window: set on - * creation, never extended. + * A user counter aggregates across every owner of its kind that user writes to. + * Sliding it on each write would keep it alive indefinitely for anyone who stays + * active while the data underneath it keeps expiring — so the counter would accrue + * bytes Redis has already dropped and eventually pin the user at their ceiling until + * they went a full window without writing. User counters therefore get a fixed + * window: set on creation, never extended. * * Because a trim refunds both counters, the user counter bounds bytes HELD across a * user's owners, not bytes written per hour: a single long copilot stream holds at most @@ -113,7 +125,7 @@ export function getRedisBudgetLimits(kind: RedisBudgetOwnerKind): RedisBudgetLim export function getRedisBudgetKeys(scope: RedisBudgetScope): string[] { const keys = [`${REDIS_BUDGET_PREFIX}${scope.kind}:${scope.id}`] if (scope.userId) { - keys.push(`${REDIS_BUDGET_PREFIX}user:${scope.userId}`) + keys.push(`${REDIS_BUDGET_PREFIX}${USER_KEY_SEGMENTS[scope.kind]}:${scope.userId}`) } return keys } diff --git a/apps/sim/lib/execution/resource-errors.ts b/apps/sim/lib/execution/resource-errors.ts index 3cd2f61bad9..e734a6596be 100644 --- a/apps/sim/lib/execution/resource-errors.ts +++ b/apps/sim/lib/execution/resource-errors.ts @@ -1,3 +1,5 @@ +import { formatLargeValueSize } from '@/lib/execution/payloads/large-value-ref' + export const EXECUTION_RESOURCE_LIMIT_CODE = 'execution_resource_limit_exceeded' as const export type ExecutionResourceLimitResource = @@ -14,6 +16,26 @@ export interface ExecutionResourceLimitDetails { statusCode?: number } +const GENERIC_LIMIT_MESSAGE = 'Execution memory limit exceeded. Reduce payload size and try again.' + +/** + * A payload refusal names the value's size and the limit, so whoever fixes the workflow + * (a person or Chat) can see it is a structural problem rather than retrying it. A Redis + * budget's byte counts describe a write chunk, which means nothing to a user. A size + * that reads the same as the limit (only known to exceed it, or just over) is left unstated. + */ +function getResourceLimitMessage(details: ExecutionResourceLimitDetails): string { + if (details.resource !== 'execution_payload_bytes') return GENERIC_LIMIT_MESSAGE + const limit = formatSize(details.limitBytes) + const attempted = formatSize(details.attemptedBytes) + const size = attempted === limit ? 'is larger than' : `is ${attempted}, more than` + return `Execution memory limit exceeded: the data ${size} the ${limit} limit. Pass a smaller field or file, or have the blocks that produce it output less.` +} + +function formatSize(bytes: number): string { + return bytes < 1024 * 1024 ? `${Math.ceil(bytes / 1024)} KB` : formatLargeValueSize(bytes) +} + export class ExecutionResourceLimitError extends Error { readonly code = EXECUTION_RESOURCE_LIMIT_CODE readonly statusCode: number @@ -23,7 +45,7 @@ export class ExecutionResourceLimitError extends Error { readonly currentBytes?: number constructor(details: ExecutionResourceLimitDetails) { - super('Execution memory limit exceeded. Reduce payload size and try again.') + super(getResourceLimitMessage(details)) this.name = 'ExecutionResourceLimitError' this.resource = details.resource this.attemptedBytes = details.attemptedBytes diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index da0e15b0c4d..62433c6e3e6 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -2775,6 +2775,39 @@ describe('Function execution request', () => { expect(data.error).toContain('File secret provenance is unavailable') expect(JSON.stringify(data)).not.toContain('private file content') }) + + it('tells user code how large a referenced value is when it is too large to load', async () => { + const ref = { + __simLargeValueRef: true, + version: 1, + id: 'lv_loopresult01', + kind: 'array', + size: 28 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_loopresult01.json', + executionId: 'execution-1', + } + mockExecuteInIsolatedVM.mockImplementationOnce(async (_input, options) => ({ + result: await options.brokers['sim.values.read']({ ref }).catch( + (error: Error) => error.message + ), + stdout: '', + })) + + const response = await POST( + createMockRequest('POST', { + code: 'return 1', + language: 'javascript', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + ) + + const data = await response.json() + expect(data.output.result).toBe( + 'Execution memory limit exceeded: the data is 28.0 MB, more than the 16.0 MB limit. Pass a smaller field or file, or have the blocks that produce it output less.' + ) + }) }) describe('Template Variable Resolution', () => { diff --git a/apps/sim/lib/mothership/request/session/buffer.test.ts b/apps/sim/lib/mothership/request/session/buffer.test.ts index 1c614b27879..aafb74c15ba 100644 --- a/apps/sim/lib/mothership/request/session/buffer.test.ts +++ b/apps/sim/lib/mothership/request/session/buffer.test.ts @@ -310,7 +310,9 @@ describe('mothership-stream-outbox', () => { mockRedis.eval.mockClear() await appendEvents([envelope], { streamId: 'stream-1', userId: 'user-1' }) expect(mockRedis.eval.mock.calls[0][1]).toBe(4) - expect(mockRedis.eval.mock.calls[0][5]).toBe('execution:redis-budget:user:user-1') + expect(mockRedis.eval.mock.calls[0][5]).toBe( + 'execution:redis-budget:copilot_stream_user:user-1' + ) }) it('clears persisted stream state during teardown cleanup', async () => { @@ -443,13 +445,15 @@ describe('mothership-stream-outbox', () => { expect(mockRedis.del.mock.calls[0]).toContain('execution:redis-budget:copilot_stream:stream-1') }) - it('never touches the shared user counter when clearing a buffer', async () => { + it('never touches the user counter when clearing a buffer', async () => { // An owner id is not proof of who wrote the bytes, so crediting the user counter here would // let anyone who can name a stream decrement a ceiling they never charged. await clearBuffer('stream-1') const keys = mockRedis.del.mock.calls[0] as string[] - expect(keys.some((key) => key.includes('redis-budget:user:'))).toBe(false) + expect(keys.filter((key) => key.startsWith('execution:redis-budget:'))).toEqual([ + 'execution:redis-budget:copilot_stream:stream-1', + ]) }) }) diff --git a/apps/sim/lib/mothership/request/session/buffer.ts b/apps/sim/lib/mothership/request/session/buffer.ts index 883f7ba3d5c..bd89f4c3680 100644 --- a/apps/sim/lib/mothership/request/session/buffer.ts +++ b/apps/sim/lib/mothership/request/session/buffer.ts @@ -142,7 +142,7 @@ export async function clearBuffer(streamId: string, operation = 'clear_outbox'): its reservation already erased. One variadic DEL is a single atomic command, so no script is needed to get that. - The shared user counter is deliberately untouched: an owner id is not proof of who wrote + The user's stream counter is deliberately untouched: an owner id is not proof of who wrote the bytes, so crediting it here would let anyone able to name a stream decrement a ceiling they never charged — and a counter driven down grants writes rather than denying them. Its fixed window settles it instead, over-counting in the safe direction meanwhile. diff --git a/apps/sim/lib/mothership/request/session/replay-budget.integration.ts b/apps/sim/lib/mothership/request/session/replay-budget.integration.ts index 51a94f73518..270bf37d5a4 100644 --- a/apps/sim/lib/mothership/request/session/replay-budget.integration.ts +++ b/apps/sim/lib/mothership/request/session/replay-budget.integration.ts @@ -867,6 +867,25 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge ]) }, 180_000) + it("is not refused for bytes the user's workflow runs hold", async () => { + const [, executionUserKey] = getRedisBudgetKeys({ + kind: 'execution', + id: generateId(), + userId, + }) + await redis().set(executionUserKey, getRedisBudgetLimits('execution').maxUserBytes, 'EX', 300) + try { + const { runId, frames } = await runTurn([text('The digest is ready.')]) + + expect(frames.map((frame) => frame.type)).not.toContain('error') + expect(frames.at(-1)).toMatchObject({ type: 'complete', payload: { status: 'complete' } }) + const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId)) + expect(stored.status).toBe('complete') + } finally { + await redis().del(executionUserKey) + } + }) + /** * An unfinished run with no live controller whose ring a byte trim has advanced past * its head: seqs 1–2 are gone and 3–4 remain.