Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 0 additions & 19 deletions apps/sim/executor/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,6 @@ export const SPECIAL_REFERENCE_PREFIXES = [
*/
export const RESERVED_BLOCK_NAMES = RESERVED_WORKFLOW_BLOCK_NAMES

export const LOOP_REFERENCE = {
ITERATION: 'iteration',
INDEX: 'index',
ITEM: 'item',
INDEX_PATH: 'loop.index',
} as const

export const DEFAULTS = {
BLOCK_TYPE: 'unknown',
BLOCK_TITLE: 'Untitled Block',
Expand Down Expand Up @@ -389,18 +382,6 @@ export function isAnnotationOnlyBlock(blockType: string | undefined): boolean {
return blockType === BlockType.NOTE
}

export function buildReference(path: string): string {
return `${REFERENCE.START}${path}${REFERENCE.END}`
}

export function buildLoopReference(property: string): string {
return buildReference(`${REFERENCE.PREFIX.LOOP}${REFERENCE.PATH_DELIMITER}${property}`)
}

export function buildLoopIndexCondition(maxIterations: number): string {
return `${buildLoopReference(LOOP_REFERENCE.INDEX)} < ${maxIterations}`
}

export function isReference(value: string): boolean {
return value.startsWith(REFERENCE.START) && value.endsWith(REFERENCE.END)
}
Expand Down
61 changes: 10 additions & 51 deletions apps/sim/executor/execution/condition-loop-ceiling.test.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,8 @@
import type { SessionPrincipal } from '@sim/auth/principal'
import {
createSerializedBlock,
createSerializedWorkflow,
} from '@sim/testing/factories/serialized-block.factory'
import {
evaluateLoopIndexCheck,
isolatedVmMock,
isolatedVmMockFns,
} from '@sim/testing/mocks/isolated-vm.mock'
import { isolatedVmMock, isolatedVmMockFns } from '@sim/testing/mocks/isolated-vm.mock'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { BlockType, DEFAULTS } from '@/executor/constants'
import { DAGExecutor } from '@/executor/execution/executor'
import type { ExecutionResult } from '@/executor/types'
import { hasExecutionResult } from '@/executor/utils/errors'
import type { SerializedLoop, SerializedWorkflow } from '@/serializer/types'
import { DEFAULTS } from '@/executor/constants'
import { loopWorkflow, runLoopWorkflow } from '@/executor/execution/loop-workflow.test-helpers'
import type { SerializedWorkflow } from '@/serializer/types'

const { bodyRuns, conditionResult, runaway } = vi.hoisted(() => ({
bodyRuns: { count: 0 },
Expand All @@ -37,50 +26,20 @@ vi.mock('@/executor/handlers/registry', () => ({
],
}))

const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' }
const CEILING = DEFAULTS.MAX_CONDITION_LOOP_ITERATIONS

function loopWorkflow(loop: Partial<Omit<SerializedLoop, 'id' | 'nodes'>>): SerializedWorkflow {
const blocks = [
createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }),
createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }),
createSerializedBlock({ id: 'tick', name: 'tick', type: BlockType.FUNCTION }),
]
return {
...createSerializedWorkflow(blocks, [
{ source: 'start', target: 'loop' },
{ source: 'loop', target: 'tick', sourceHandle: 'loop-start-source' },
]),
loops: { loop: { id: 'loop', nodes: ['tick'], iterations: 0, ...loop } },
}
}

/** Runs the workflow and returns its result, including the one a failed run attaches to its throw. */
async function run(workflow: SerializedWorkflow): Promise<ExecutionResult> {
try {
return await new DAGExecutor({
workflow,
contextExtensions: {
workspaceId: 'ws',
executionId: 'exec-1',
principal: PRINCIPAL,
abortSignal: runaway.controller.signal,
},
}).execute('wf')
} catch (error) {
if (hasExecutionResult(error)) return error.executionResult
throw error
}
}
/** Runs under the runaway guard, so a loop the ceiling fails to stop is cancelled instead of hanging. */
const run = (workflow: SerializedWorkflow) =>
runLoopWorkflow(workflow, { abortSignal: runaway.controller.signal })

describe('condition-driven loop iteration ceiling', () => {
beforeEach(() => {
bodyRuns.count = 0
conditionResult.stopAfter = Number.POSITIVE_INFINITY
runaway.controller = new AbortController()
/** Count conditions evaluate; user conditions stay opaque and hold until `stopAfter`. */
isolatedVmMockFns.mockExecuteInIsolatedVM.mockImplementation(async ({ code }) => ({
result: evaluateLoopIndexCheck(code) ?? bodyRuns.count < conditionResult.stopAfter,
/** Only user conditions reach the sandbox; they stay opaque and hold until `stopAfter`. */
isolatedVmMockFns.mockExecuteInIsolatedVM.mockImplementation(async () => ({
result: bodyRuns.count < conditionResult.stopAfter,
}))
})

Expand Down
51 changes: 51 additions & 0 deletions apps/sim/executor/execution/loop-workflow.test-helpers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import type { SessionPrincipal } from '@sim/auth/principal'
import {
createSerializedBlock,
createSerializedWorkflow,
} from '@sim/testing/factories/serialized-block.factory'
import { BlockType } from '@/executor/constants'
import { DAGExecutor } from '@/executor/execution/executor'
import type { ExecutionResult } from '@/executor/types'
import { hasExecutionResult } from '@/executor/utils/errors'
import type { SerializedLoop, SerializedWorkflow } from '@/serializer/types'

const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' }

/** A Start block feeding a loop named `loop` whose body is one block, `tick`. */
export function loopWorkflow(
loop: Partial<Omit<SerializedLoop, 'id' | 'nodes'>>
): SerializedWorkflow {
const blocks = [
createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }),
createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }),
createSerializedBlock({ id: 'tick', name: 'tick', type: BlockType.FUNCTION }),
]
return {
...createSerializedWorkflow(blocks, [
{ source: 'start', target: 'loop' },
{ source: 'loop', target: 'tick', sourceHandle: 'loop-start-source' },
]),
loops: { loop: { id: 'loop', nodes: ['tick'], iterations: 0, ...loop } },
}
}

/** Runs the workflow and returns its result, including the one a failed run attaches to its throw. */
export async function runLoopWorkflow(
workflow: SerializedWorkflow,
options: { abortSignal?: AbortSignal } = {}
): Promise<ExecutionResult> {
try {
return await new DAGExecutor({
workflow,
contextExtensions: {
workspaceId: 'ws',
executionId: 'exec-1',
principal: PRINCIPAL,
abortSignal: options.abortSignal,
},
}).execute('wf')
} catch (error) {
if (hasExecutionResult(error)) return error.executionResult
throw error
}
}
184 changes: 182 additions & 2 deletions apps/sim/executor/orchestrators/loop.test.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
import { createLogger } from '@sim/logger'
import { isolatedVmMock, isolatedVmMockFns } from '@sim/testing/mocks/isolated-vm.mock'
import {
isolatedVmMock,
isolatedVmMockFns,
resetIsolatedVmMock,
} from '@sim/testing/mocks/isolated-vm.mock'
import { storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock'
import { uploadsMock } from '@sim/testing/mocks/uploads.mock'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { classifyFailure, logFailureOnce } from '@/lib/core/errors/failure-log'
import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache'
import { EDGE } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import { loopWorkflow, runLoopWorkflow } from '@/executor/execution/loop-workflow.test-helpers'
import type { BlockStateController } from '@/executor/execution/types'
import { LoopOrchestrator } from '@/executor/orchestrators/loop'
import type { ExecutionContext } from '@/executor/types'
import type { ExecutionContext, ExecutionResult } from '@/executor/types'
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'

const mockUploadFile = storageServiceMockFns.mockUploadFile
Expand All @@ -24,6 +30,20 @@ vi.mock('@/lib/execution/isolated-vm', () => isolatedVmMock)

vi.mock('@/lib/uploads', () => uploadsMock)

const { bodyRuns } = vi.hoisted(() => ({ bodyRuns: { count: 0 } }))

vi.mock('@/executor/handlers/registry', () => ({
createBlockHandlers: () => [
{
canHandle: () => true,
execute: async (_ctx: unknown, block: { id: string }) => {
if (block.id === 'tick') bodyRuns.count++
return { ok: true }
},
},
],
}))

function createNode(id: string): DAGNode {
return {
id,
Expand Down Expand Up @@ -333,3 +353,163 @@ describe('LoopOrchestrator', () => {
expect(scope.condition).toBe('true')
})
})

const SANDBOX_AT_CAPACITY = {
result: null,
stdout: '',
error: {
message: 'Code execution is at capacity. Please try again in a moment.',
name: 'Error',
isSystemError: true,
},
}

const CONDITION_SYNTAX_ERROR = {
result: null,
stdout: '',
error: { message: "Unexpected token ')'", name: 'SyntaxError' },
}

function loopLog(result: ExecutionResult) {
return result.logs?.find((log) => log.blockId === 'loop')
}

const sandboxAtCapacity = () => mockExecuteInIsolatedVM.mockResolvedValue(SANDBOX_AT_CAPACITY)
const sandboxRejects = () =>
mockExecuteInIsolatedVM.mockRejectedValue(new Error('REDIS_URL is required'))

describe('loop continuation when the sandbox cannot evaluate', () => {
beforeEach(() => {
bodyRuns.count = 0
resetIsolatedVmMock()
clearLargeValueCacheForTests()
mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey }))
})

it.each([
['For', 'is at capacity', { loopType: 'for', iterations: 5 }, sandboxAtCapacity],
['For', 'rejects', { loopType: 'for', iterations: 5 }, sandboxRejects],
[
'ForEach',
'is at capacity',
{ loopType: 'forEach', forEachItems: ['a', 'b', 'c', 'd', 'e'] },
sandboxAtCapacity,
],
[
'count-driven Do While',
'is at capacity',
{ loopType: 'doWhile', iterations: 5 },
sandboxAtCapacity,
],
] as const)(
'runs every iteration of a %s loop when the sandbox %s',
async (_label, _mode, loop, failSandbox) => {
failSandbox()

const result = await runLoopWorkflow(loopWorkflow(loop))

expect(result.success).toBe(true)
expect(bodyRuns.count).toBe(5)
expect(loopLog(result)?.output?.results).toHaveLength(5)
}
)

it('fails a While loop whose condition has a syntax error instead of skipping its body', async () => {
mockExecuteInIsolatedVM.mockResolvedValue(CONDITION_SYNTAX_ERROR)

const result = await runLoopWorkflow(
loopWorkflow({ loopType: 'while', whileCondition: '<loop.index> < 3)' })
)

expect(result.success).toBe(false)
expect(bodyRuns.count).toBe(0)
expect(result.error).toContain(
"While loop condition could not be evaluated: Unexpected token ')'"
)
expect(loopLog(result)).toMatchObject({
success: false,
error: expect.stringContaining('condition could not be evaluated'),
})
})

it.each([
['answers with a system error', sandboxAtCapacity, /at capacity/],
['rejects', sandboxRejects, /REDIS_URL is required/],
] as const)(
'fails a Do While loop mid-run when the sandbox %s instead of ending it early',
async (_label, failSandbox, message) => {
mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: true })
mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: true })
failSandbox()

const result = await runLoopWorkflow(
loopWorkflow({ loopType: 'doWhile', doWhileCondition: '<loop.index> < 10' })
)

expect(result.success).toBe(false)
expect(bodyRuns.count).toBe(3)
expect(result.error).toMatch(/^Do While loop condition could not be evaluated: /)
expect(result.error).toMatch(message)
expect(loopLog(result)).toMatchObject({ success: false })
}
)

it('attributes a broken condition to the author and a sandbox failure to Sim', async () => {
const { orchestrator } = createOrchestrator()
const whileScope = () => ({
iteration: 0,
currentIterationOutputs: new Map(),
allIterationOutputs: [],
loopType: 'while',
condition: 'x <',
})

mockExecuteInIsolatedVM.mockResolvedValueOnce(CONDITION_SYNTAX_ERROR)
const userError = await orchestrator
.evaluateInitialCondition(createContext(whileScope()), 'loop-1')
.catch((error: unknown) => error)

mockExecuteInIsolatedVM.mockResolvedValueOnce(SANDBOX_AT_CAPACITY)
const systemError = await orchestrator
.evaluateInitialCondition(createContext(whileScope()), 'loop-1')
.catch((error: unknown) => error)

expect(classifyFailure(userError)).toBe('user')
expect(classifyFailure(systemError)).toBe('internal')
expect(logFailureOnce(createLogger('probe'), 'probe', userError)).toBeUndefined()
expect(logFailureOnce(createLogger('probe'), 'probe', systemError)).toBeUndefined()
})

it('keeps a resolved secret out of the persisted condition error', async () => {
const secret = 'resolved-condition-secret-value'
const { orchestrator } = createOrchestrator()
const scope = {
iteration: 0,
currentIterationOutputs: new Map(),
allIterationOutputs: [],
loopType: 'while',
condition: '<condition.output>',
} as Record<string, unknown>
const ctx = createContext(scope)
const registry = new ResolvedSecretTraceRegistry([
{ name: 'CONDITION_SECRET', plaintext: secret, encryptedValue: 'encrypted-condition-secret' },
])
registry.recordResolved('CONDITION_SECRET', secret)
ctx.resolvedSecretTraceRegistry = registry
mockExecuteInIsolatedVM.mockResolvedValueOnce({
result: null,
stdout: '',
error: { message: `ReferenceError: ${secret} is not defined`, name: 'ReferenceError' },
})

const error = await orchestrator
.evaluateInitialCondition(ctx, 'loop-1')
.catch((thrown: unknown) => thrown)

expect(error).toBeInstanceOf(Error)
expect((error as Error).message).toMatch(/^While loop condition could not be evaluated/)
expect(
JSON.stringify([(error as Error).message, ctx.blockLogs, scope.validationError])
).not.toContain(secret)
})
})
Loading
Loading