Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 3 additions & 17 deletions apps/sim/lib/core/redis/byte-budget.server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,16 @@ describe('getRedisBudgetKeys', () => {

/**
* These keys are shared with counters written before this module existed, so the
* layout is a wire contract: a change here strands every counter in flight.
* execution layout is a wire contract: a change here strands every counter in flight.
*/
it('separates owner kinds without disturbing the execution key layout', () => {
it('gives a stream its own owner and user counters', () => {
expect(
getRedisBudgetKeys({ kind: 'copilot_stream', id: 'stream-1', userId: 'user-1' })
).toEqual([
'execution:redis-budget:copilot_stream:stream-1',
'execution:redis-budget:user:user-1',
'execution:redis-budget:copilot_stream_user:user-1',
])
})

it('shares one user ceiling across owner kinds', () => {
const [, executionUserKey] = getRedisBudgetKeys({
kind: 'execution',
id: 'exec-1',
userId: 'user-1',
})
const [, streamUserKey] = getRedisBudgetKeys({
kind: 'copilot_stream',
id: 'stream-1',
userId: 'user-1',
})
expect(streamUserKey).toBe(executionUserKey)
})
})

describe('getRedisBudgetLimits', () => {
Expand Down
30 changes: 21 additions & 9 deletions apps/sim/lib/core/redis/byte-budget.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,27 @@ import type { Logger } from '@sim/logger'
*/
const REDIS_BUDGET_PREFIX = 'execution:redis-budget:'

/** What a budget is charged to. One counter per owner, plus one per user across owners. */
/** What a budget is charged to. One counter per owner, plus one per user per kind. */
export type RedisBudgetOwnerKind = 'execution' | 'copilot_stream'

/**
* The user counter's key segment for each kind.
*
* Each kind has its own user ceiling, so each needs its own counter: a shared one lets
* workflow runs, within their larger ceiling, lock the user out of Chat for the window.
* `user` is the execution counter's original segment, kept so in-flight counters still apply.
*/
const USER_KEY_SEGMENTS: Record<RedisBudgetOwnerKind, string> = {
execution: 'user',
copilot_stream: 'copilot_stream_user',
}

export interface RedisBudgetScope {
kind: RedisBudgetOwnerKind
/** The owner's id — an execution id, a stream id, a table id. */
id: string
/**
* Charges the write to a second, user-wide counter as well. Omitted where the
* Charges the write to the user's counter for this kind as well. Omitted where the
* writer has no user in scope; the owner counter still applies.
*/
userId?: string
Expand All @@ -64,12 +76,12 @@ export interface RedisBudgetLimits {
* itself, so sliding its TTL on every write keeps the counter and the bytes it
* represents in step.
*
* A user counter aggregates across every owner that user writes to. Sliding it on
* each write would keep it alive indefinitely for anyone who stays active while the
* data underneath it keeps expiring — so the counter would accrue bytes Redis has
* already dropped and eventually pin the user at their ceiling until they went a full
* window without writing. User counters therefore get a fixed window: set on
* creation, never extended.
* A user counter aggregates across every owner of its kind that user writes to.
* Sliding it on each write would keep it alive indefinitely for anyone who stays
* active while the data underneath it keeps expiring — so the counter would accrue
* bytes Redis has already dropped and eventually pin the user at their ceiling until
* they went a full window without writing. User counters therefore get a fixed
* window: set on creation, never extended.
*
* Because a trim refunds both counters, the user counter bounds bytes HELD across a
* user's owners, not bytes written per hour: a single long copilot stream holds at most
Expand Down Expand Up @@ -113,7 +125,7 @@ export function getRedisBudgetLimits(kind: RedisBudgetOwnerKind): RedisBudgetLim
export function getRedisBudgetKeys(scope: RedisBudgetScope): string[] {
const keys = [`${REDIS_BUDGET_PREFIX}${scope.kind}:${scope.id}`]
if (scope.userId) {
keys.push(`${REDIS_BUDGET_PREFIX}user:${scope.userId}`)
keys.push(`${REDIS_BUDGET_PREFIX}${USER_KEY_SEGMENTS[scope.kind]}:${scope.userId}`)
}
return keys
}
Expand Down
24 changes: 23 additions & 1 deletion apps/sim/lib/execution/resource-errors.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { formatLargeValueSize } from '@/lib/execution/payloads/large-value-ref'

export const EXECUTION_RESOURCE_LIMIT_CODE = 'execution_resource_limit_exceeded' as const

export type ExecutionResourceLimitResource =
Expand All @@ -14,6 +16,26 @@ export interface ExecutionResourceLimitDetails {
statusCode?: number
}

const GENERIC_LIMIT_MESSAGE = 'Execution memory limit exceeded. Reduce payload size and try again.'

/**
* A payload refusal names the value's size and the limit, so whoever fixes the workflow
* (a person or Chat) can see it is a structural problem rather than retrying it. A Redis
* budget's byte counts describe a write chunk, which means nothing to a user. A size
* that reads the same as the limit (only known to exceed it, or just over) is left unstated.
*/
function getResourceLimitMessage(details: ExecutionResourceLimitDetails): string {
if (details.resource !== 'execution_payload_bytes') return GENERIC_LIMIT_MESSAGE
const limit = formatSize(details.limitBytes)
const attempted = formatSize(details.attemptedBytes)
const size = attempted === limit ? 'is larger than' : `is ${attempted}, more than`
return `Execution memory limit exceeded: the data ${size} the ${limit} limit. Pass a smaller field or file, or have the blocks that produce it output less.`
}

function formatSize(bytes: number): string {
return bytes < 1024 * 1024 ? `${Math.ceil(bytes / 1024)} KB` : formatLargeValueSize(bytes)
}

export class ExecutionResourceLimitError extends Error {
readonly code = EXECUTION_RESOURCE_LIMIT_CODE
readonly statusCode: number
Expand All @@ -23,7 +45,7 @@ export class ExecutionResourceLimitError extends Error {
readonly currentBytes?: number

constructor(details: ExecutionResourceLimitDetails) {
super('Execution memory limit exceeded. Reduce payload size and try again.')
super(getResourceLimitMessage(details))
this.name = 'ExecutionResourceLimitError'
this.resource = details.resource
this.attemptedBytes = details.attemptedBytes
Expand Down
33 changes: 33 additions & 0 deletions apps/sim/lib/function-execution/execute-request.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2775,6 +2775,39 @@ describe('Function execution request', () => {
expect(data.error).toContain('File secret provenance is unavailable')
expect(JSON.stringify(data)).not.toContain('private file content')
})

it('tells user code how large a referenced value is when it is too large to load', async () => {
const ref = {
__simLargeValueRef: true,
version: 1,
id: 'lv_loopresult01',
kind: 'array',
size: 28 * 1024 * 1024,
key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_loopresult01.json',
executionId: 'execution-1',
}
mockExecuteInIsolatedVM.mockImplementationOnce(async (_input, options) => ({
result: await options.brokers['sim.values.read']({ ref }).catch(
(error: Error) => error.message
),
stdout: '',
}))

const response = await POST(
createMockRequest('POST', {
code: 'return 1',
language: 'javascript',
workspaceId: 'workspace-1',
workflowId: 'workflow-1',
executionId: 'execution-1',
})
)

const data = await response.json()
expect(data.output.result).toBe(
'Execution memory limit exceeded: the data is 28.0 MB, more than the 16.0 MB limit. Pass a smaller field or file, or have the blocks that produce it output less.'
)
})
})

describe('Template Variable Resolution', () => {
Expand Down
10 changes: 7 additions & 3 deletions apps/sim/lib/mothership/request/session/buffer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,9 @@ describe('mothership-stream-outbox', () => {
mockRedis.eval.mockClear()
await appendEvents([envelope], { streamId: 'stream-1', userId: 'user-1' })
expect(mockRedis.eval.mock.calls[0][1]).toBe(4)
expect(mockRedis.eval.mock.calls[0][5]).toBe('execution:redis-budget:user:user-1')
expect(mockRedis.eval.mock.calls[0][5]).toBe(
'execution:redis-budget:copilot_stream_user:user-1'
)
})

it('clears persisted stream state during teardown cleanup', async () => {
Expand Down Expand Up @@ -443,13 +445,15 @@ describe('mothership-stream-outbox', () => {
expect(mockRedis.del.mock.calls[0]).toContain('execution:redis-budget:copilot_stream:stream-1')
})

it('never touches the shared user counter when clearing a buffer', async () => {
it('never touches the user counter when clearing a buffer', async () => {
// An owner id is not proof of who wrote the bytes, so crediting the user counter here would
// let anyone who can name a stream decrement a ceiling they never charged.
await clearBuffer('stream-1')

const keys = mockRedis.del.mock.calls[0] as string[]
expect(keys.some((key) => key.includes('redis-budget:user:'))).toBe(false)
expect(keys.filter((key) => key.startsWith('execution:redis-budget:'))).toEqual([
'execution:redis-budget:copilot_stream:stream-1',
])
})
})

Expand Down
2 changes: 1 addition & 1 deletion apps/sim/lib/mothership/request/session/buffer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ export async function clearBuffer(streamId: string, operation = 'clear_outbox'):
its reservation already erased. One variadic DEL is a single atomic command, so no script
is needed to get that.

The shared user counter is deliberately untouched: an owner id is not proof of who wrote
The user's stream counter is deliberately untouched: an owner id is not proof of who wrote
the bytes, so crediting it here would let anyone able to name a stream decrement a ceiling
they never charged — and a counter driven down grants writes rather than denying them. Its
fixed window settles it instead, over-counting in the safe direction meanwhile.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,25 @@ describe.runIf(Boolean(redisUrl))('a turn whose stream exhausts its replay budge
])
}, 180_000)

it("is not refused for bytes the user's workflow runs hold", async () => {
const [, executionUserKey] = getRedisBudgetKeys({
kind: 'execution',
id: generateId(),
userId,
})
await redis().set(executionUserKey, getRedisBudgetLimits('execution').maxUserBytes, 'EX', 300)
try {
const { runId, frames } = await runTurn([text('The digest is ready.')])

expect(frames.map((frame) => frame.type)).not.toContain('error')
expect(frames.at(-1)).toMatchObject({ type: 'complete', payload: { status: 'complete' } })
const [stored] = await db.select().from(copilotRuns).where(eq(copilotRuns.id, runId))
expect(stored.status).toBe('complete')
} finally {
await redis().del(executionUserKey)
}
})

/**
* An unfinished run with no live controller whose ring a byte trim has advanced past
* its head: seqs 1–2 are gone and 3–4 remain.
Expand Down
Loading