From 2317d9aa389c685198b17dc1b47305b6f2df8780 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 20:35:57 -0800 Subject: [PATCH 1/3] fix(executor): a loop whose condition cannot be evaluated fails instead of ending early as success For, ForEach, and count-driven Do While loops decide continuation from their count instead of evaluating ' < N' in the isolated VM, so a sandbox failure can no longer truncate them. A While or Do While condition that fails to evaluate (syntax or runtime error, unresolvable reference, sandbox failure) now fails the loop with a loop error log and a secret-projected message instead of being read as false. Author errors are attributed to the user; sandbox failures stay internal. --- apps/sim/executor/constants.ts | 19 -- .../execution/condition-loop-ceiling.test.ts | 12 +- apps/sim/executor/orchestrators/loop.test.ts | 216 +++++++++++++++++- apps/sim/executor/orchestrators/loop.ts | 145 +++++++----- packages/testing/src/mocks/index.ts | 1 - .../testing/src/mocks/isolated-vm.mock.ts | 32 +-- scripts/check-unused-exports.baseline.json | 3 - 7 files changed, 311 insertions(+), 117 deletions(-) diff --git a/apps/sim/executor/constants.ts b/apps/sim/executor/constants.ts index 7c0a87bcf11..8d7ec391cf5 100644 --- a/apps/sim/executor/constants.ts +++ b/apps/sim/executor/constants.ts @@ -184,13 +184,6 @@ export const SPECIAL_REFERENCE_PREFIXES = [ */ export const RESERVED_BLOCK_NAMES = RESERVED_WORKFLOW_BLOCK_NAMES -export const LOOP_REFERENCE = { - ITERATION: 'iteration', - INDEX: 'index', - ITEM: 'item', - INDEX_PATH: 'loop.index', -} as const - export const DEFAULTS = { BLOCK_TYPE: 'unknown', BLOCK_TITLE: 'Untitled Block', @@ -389,18 +382,6 @@ export function isAnnotationOnlyBlock(blockType: string | undefined): boolean { return blockType === BlockType.NOTE } -export function buildReference(path: string): string { - return `${REFERENCE.START}${path}${REFERENCE.END}` -} - -export function buildLoopReference(property: string): string { - return buildReference(`${REFERENCE.PREFIX.LOOP}${REFERENCE.PATH_DELIMITER}${property}`) -} - -export function buildLoopIndexCondition(maxIterations: number): string { - return `${buildLoopReference(LOOP_REFERENCE.INDEX)} < ${maxIterations}` -} - export function isReference(value: string): boolean { return value.startsWith(REFERENCE.START) && value.endsWith(REFERENCE.END) } diff --git a/apps/sim/executor/execution/condition-loop-ceiling.test.ts b/apps/sim/executor/execution/condition-loop-ceiling.test.ts index 59fabad0c57..88f6d7efd5c 100644 --- a/apps/sim/executor/execution/condition-loop-ceiling.test.ts +++ b/apps/sim/executor/execution/condition-loop-ceiling.test.ts @@ -3,11 +3,7 @@ import { createSerializedBlock, createSerializedWorkflow, } from '@sim/testing/factories/serialized-block.factory' -import { - evaluateLoopIndexCheck, - isolatedVmMock, - isolatedVmMockFns, -} from '@sim/testing/mocks/isolated-vm.mock' +import { isolatedVmMock, isolatedVmMockFns } from '@sim/testing/mocks/isolated-vm.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import { BlockType, DEFAULTS } from '@/executor/constants' import { DAGExecutor } from '@/executor/execution/executor' @@ -78,9 +74,9 @@ describe('condition-driven loop iteration ceiling', () => { bodyRuns.count = 0 conditionResult.stopAfter = Number.POSITIVE_INFINITY runaway.controller = new AbortController() - /** Count conditions evaluate; user conditions stay opaque and hold until `stopAfter`. */ - isolatedVmMockFns.mockExecuteInIsolatedVM.mockImplementation(async ({ code }) => ({ - result: evaluateLoopIndexCheck(code) ?? bodyRuns.count < conditionResult.stopAfter, + /** Only user conditions reach the sandbox; they stay opaque and hold until `stopAfter`. */ + isolatedVmMockFns.mockExecuteInIsolatedVM.mockImplementation(async () => ({ + result: bodyRuns.count < conditionResult.stopAfter, })) }) diff --git a/apps/sim/executor/orchestrators/loop.test.ts b/apps/sim/executor/orchestrators/loop.test.ts index a30dd7d0341..6a8e7198f66 100644 --- a/apps/sim/executor/orchestrators/loop.test.ts +++ b/apps/sim/executor/orchestrators/loop.test.ts @@ -1,16 +1,25 @@ +import type { SessionPrincipal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' +import { + createSerializedBlock, + createSerializedWorkflow, +} from '@sim/testing/factories/serialized-block.factory' import { isolatedVmMock, isolatedVmMockFns } from '@sim/testing/mocks/isolated-vm.mock' import { storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { uploadsMock } from '@sim/testing/mocks/uploads.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { classifyFailure } from '@/lib/core/errors/failure-log' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' -import { EDGE } from '@/executor/constants' +import { BlockType, EDGE } from '@/executor/constants' import type { DAG, DAGNode } from '@/executor/dag/builder' import type { EdgeManager } from '@/executor/execution/edge-manager' +import { DAGExecutor } from '@/executor/execution/executor' import type { BlockStateController } from '@/executor/execution/types' import { LoopOrchestrator } from '@/executor/orchestrators/loop' -import type { ExecutionContext } from '@/executor/types' +import type { ExecutionContext, ExecutionResult } from '@/executor/types' +import { hasExecutionResult } from '@/executor/utils/errors' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import type { SerializedLoop, SerializedWorkflow } from '@/serializer/types' const mockUploadFile = storageServiceMockFns.mockUploadFile @@ -24,6 +33,20 @@ vi.mock('@/lib/execution/isolated-vm', () => isolatedVmMock) vi.mock('@/lib/uploads', () => uploadsMock) +const { bodyRuns } = vi.hoisted(() => ({ bodyRuns: { count: 0 } })) + +vi.mock('@/executor/handlers/registry', () => ({ + createBlockHandlers: () => [ + { + canHandle: () => true, + execute: async (_ctx: unknown, block: { id: string }) => { + if (block.id === 'tick') bodyRuns.count++ + return { ok: true } + }, + }, + ], +})) + function createNode(id: string): DAGNode { return { id, @@ -333,3 +356,192 @@ describe('LoopOrchestrator', () => { expect(scope.condition).toBe('true') }) }) + +const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } + +const SANDBOX_AT_CAPACITY = { + result: null, + stdout: '', + error: { + message: 'Code execution is at capacity. Please try again in a moment.', + name: 'Error', + isSystemError: true, + }, +} + +const CONDITION_SYNTAX_ERROR = { + result: null, + stdout: '', + error: { message: "Unexpected token ')'", name: 'SyntaxError' }, +} + +function loopWorkflow(loop: Partial>): SerializedWorkflow { + const blocks = [ + createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }), + createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }), + createSerializedBlock({ id: 'tick', name: 'tick', type: BlockType.FUNCTION }), + ] + return { + ...createSerializedWorkflow(blocks, [ + { source: 'start', target: 'loop' }, + { source: 'loop', target: 'tick', sourceHandle: 'loop-start-source' }, + ]), + loops: { loop: { id: 'loop', nodes: ['tick'], iterations: 0, ...loop } }, + } +} + +/** Runs the workflow and returns its result, including the one a failed run attaches to its throw. */ +async function runWorkflow(workflow: SerializedWorkflow): Promise { + try { + return await new DAGExecutor({ + workflow, + contextExtensions: { workspaceId: 'ws', executionId: 'exec-1', principal: PRINCIPAL }, + }).execute('wf') + } catch (error) { + if (hasExecutionResult(error)) return error.executionResult + throw error + } +} + +function loopLog(result: ExecutionResult) { + return result.logs?.find((log) => log.blockId === 'loop') +} + +describe('loop continuation when the sandbox cannot evaluate', () => { + beforeEach(() => { + bodyRuns.count = 0 + clearLargeValueCacheForTests() + mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) + }) + + it.each([ + ['For', { loopType: 'for', iterations: 5 }], + ['ForEach', { loopType: 'forEach', forEachItems: ['a', 'b', 'c', 'd', 'e'] }], + ['count-driven Do While', { loopType: 'doWhile', iterations: 5 }], + ] as const)( + 'runs every iteration of a %s loop while the sandbox is at capacity', + async (_label, loop) => { + mockExecuteInIsolatedVM.mockResolvedValue(SANDBOX_AT_CAPACITY) + + const result = await runWorkflow(loopWorkflow(loop)) + + expect(result.success).toBe(true) + expect(bodyRuns.count).toBe(5) + expect(loopLog(result)?.output?.results).toHaveLength(5) + } + ) + + it('runs every iteration of a For loop when the sandbox call rejects', async () => { + mockExecuteInIsolatedVM.mockRejectedValue(new Error('REDIS_URL is required')) + + const result = await runWorkflow(loopWorkflow({ loopType: 'for', iterations: 5 })) + + expect(result.success).toBe(true) + expect(bodyRuns.count).toBe(5) + }) + + it('fails a While loop whose condition has a syntax error instead of skipping its body', async () => { + mockExecuteInIsolatedVM.mockResolvedValue(CONDITION_SYNTAX_ERROR) + + const result = await runWorkflow( + loopWorkflow({ loopType: 'while', whileCondition: ' < 3)' }) + ) + + expect(result.success).toBe(false) + expect(bodyRuns.count).toBe(0) + expect(result.error).toContain( + "While loop condition could not be evaluated: Unexpected token ')'" + ) + expect(loopLog(result)).toMatchObject({ + success: false, + error: expect.stringContaining('condition could not be evaluated'), + }) + }) + + it.each([ + [ + 'answers with a system error', + () => mockExecuteInIsolatedVM.mockResolvedValue(SANDBOX_AT_CAPACITY), + /at capacity/, + ], + [ + 'rejects', + () => mockExecuteInIsolatedVM.mockRejectedValue(new Error('REDIS_URL is required')), + /REDIS_URL is required/, + ], + ] as const)( + 'fails a Do While loop mid-run when the sandbox %s instead of ending it early', + async (_label, failSandbox, message) => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: true }) + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: true }) + failSandbox() + + const result = await runWorkflow( + loopWorkflow({ loopType: 'doWhile', doWhileCondition: ' < 10' }) + ) + + expect(result.success).toBe(false) + expect(bodyRuns.count).toBe(3) + expect(result.error).toMatch(/^Do While loop condition could not be evaluated: /) + expect(result.error).toMatch(message) + expect(loopLog(result)).toMatchObject({ success: false }) + } + ) + + it('attributes a broken condition to the author and a sandbox failure to Sim', async () => { + const { orchestrator } = createOrchestrator() + const whileScope = () => ({ + iteration: 0, + currentIterationOutputs: new Map(), + allIterationOutputs: [], + loopType: 'while', + condition: 'x <', + }) + + mockExecuteInIsolatedVM.mockResolvedValueOnce(CONDITION_SYNTAX_ERROR) + const userError = await orchestrator + .evaluateInitialCondition(createContext(whileScope()), 'loop-1') + .catch((error: unknown) => error) + + mockExecuteInIsolatedVM.mockResolvedValueOnce(SANDBOX_AT_CAPACITY) + const systemError = await orchestrator + .evaluateInitialCondition(createContext(whileScope()), 'loop-1') + .catch((error: unknown) => error) + + expect(classifyFailure(userError)).toBe('user') + expect(classifyFailure(systemError)).toBe('internal') + }) + + it('keeps a resolved secret out of the persisted condition error', async () => { + const secret = 'resolved-condition-secret-value' + const { orchestrator } = createOrchestrator() + const scope = { + iteration: 0, + currentIterationOutputs: new Map(), + allIterationOutputs: [], + loopType: 'while', + condition: '', + } as Record + const ctx = createContext(scope) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'CONDITION_SECRET', plaintext: secret, encryptedValue: 'encrypted-condition-secret' }, + ]) + registry.recordResolved('CONDITION_SECRET', secret) + ctx.resolvedSecretTraceRegistry = registry + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: '', + error: { message: `ReferenceError: ${secret} is not defined`, name: 'ReferenceError' }, + }) + + const error = await orchestrator + .evaluateInitialCondition(ctx, 'loop-1') + .catch((thrown: unknown) => thrown) + + expect(error).toBeInstanceOf(Error) + expect((error as Error).message).toMatch(/^While loop condition could not be evaluated/) + expect( + JSON.stringify([(error as Error).message, ctx.blockLogs, scope.validationError]) + ).not.toContain(secret) + }) +}) diff --git a/apps/sim/executor/orchestrators/loop.ts b/apps/sim/executor/orchestrators/loop.ts index 945f76ec36e..ca92caf03fb 100644 --- a/apps/sim/executor/orchestrators/loop.ts +++ b/apps/sim/executor/orchestrators/loop.ts @@ -1,16 +1,15 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { + classifyFailure, + inheritFailureMarks, + markFailureKind, +} from '@/lib/core/errors/failure-log' import { generateRequestId } from '@/lib/core/utils/request' import { executeInIsolatedVM } from '@/lib/execution/isolated-vm' import { compactSubflowResults } from '@/lib/execution/payloads/serializer' import { isLikelyReferenceSegment } from '@/lib/workflows/sanitization/references' -import { - buildLoopIndexCondition, - CONTROL_BACK_EDGE_HANDLES, - DEFAULTS, - EDGE, - PARALLEL, -} from '@/executor/constants' +import { CONTROL_BACK_EDGE_HANDLES, DEFAULTS, EDGE, PARALLEL } from '@/executor/constants' import type { DAG } from '@/executor/dag/builder' import type { EdgeManager } from '@/executor/execution/edge-manager' import type { LoopScope } from '@/executor/execution/state' @@ -98,7 +97,6 @@ export class LoopOrchestrator { loopType, currentIterationOutputs: new Map(), allIterationOutputs: [], - condition: 'false', validationError: errorMessage, } if (!ctx.loopExecutions) { @@ -121,7 +119,6 @@ export class LoopOrchestrator { scope.loopType = 'for' const requestedIterations = loopConfig.iterations || DEFAULTS.DEFAULT_LOOP_ITERATIONS scope.maxIterations = requestedIterations - scope.condition = buildLoopIndexCondition(scope.maxIterations) break } @@ -141,7 +138,6 @@ export class LoopOrchestrator { scope.items = [] scope.maxIterations = 0 scope.validationError = errorMessage - scope.condition = buildLoopIndexCondition(0) ctx.loopExecutions?.set(loopId, scope) throw new Error(errorMessage) } @@ -181,7 +177,6 @@ export class LoopOrchestrator { scope.items = [] scope.maxIterations = 0 scope.validationError = persistedErrorMessage - scope.condition = buildLoopIndexCondition(0) ctx.loopExecutions?.set(loopId, scope) throw new Error(persistedErrorMessage) } @@ -194,7 +189,6 @@ export class LoopOrchestrator { } scope.maxIterations = items.length scope.item = items[0] - scope.condition = buildLoopIndexCondition(scope.maxIterations) break } @@ -210,7 +204,6 @@ export class LoopOrchestrator { } else { const requestedIterations = loopConfig.iterations || DEFAULTS.DEFAULT_LOOP_ITERATIONS scope.maxIterations = requestedIterations - scope.condition = buildLoopIndexCondition(scope.maxIterations) } break } @@ -316,7 +309,7 @@ export class LoopOrchestrator { return await this.createExitResult(ctx, loopId, scope) } - if (!(await this.evaluateCondition(ctx, scope, scope.iteration + 1))) { + if (!(await this.evaluateCondition(ctx, loopId, scope, scope.iteration + 1))) { return await this.createExitResult(ctx, loopId, scope) } @@ -407,28 +400,32 @@ export class LoopOrchestrator { } } + /** + * Decides whether the loop runs `nextIteration`. For, ForEach, and count-driven Do While loops + * compare against their count; only a user-written While or Do While condition needs the sandbox. + */ private async evaluateCondition( ctx: ExecutionContext, + loopId: string, scope: LoopScope, - iteration?: number + nextIteration: number ): Promise { + if (!isConditionDriven(scope)) { + return nextIteration < (scope.maxIterations ?? 0) + } + if (!scope.condition) { - logger.warn('No condition defined for loop') + logger.warn('No condition defined for loop', { loopId }) return false } const currentIteration = scope.iteration - if (iteration !== undefined) { - scope.iteration = iteration - } - - const result = await this.evaluateWhileCondition(ctx, scope.condition, scope) - - if (iteration !== undefined) { + scope.iteration = nextIteration + try { + return await this.evaluateWhileCondition(ctx, loopId, scope.condition, scope) + } finally { scope.iteration = currentIteration } - - return result } clearLoopExecutionState(loopId: string, ctx: ExecutionContext): void { @@ -749,7 +746,7 @@ export class LoopOrchestrator { return false } - const result = await this.evaluateWhileCondition(ctx, scope.condition, scope) + const result = await this.evaluateWhileCondition(ctx, loopId, scope.condition, scope) logger.info('While loop initial condition evaluation', { loopId, conditionLength: scope.condition.length, @@ -769,6 +766,7 @@ export class LoopOrchestrator { private async evaluateWhileCondition( ctx: ExecutionContext, + loopId: string, condition: string, scope: LoopScope ): Promise { @@ -776,13 +774,14 @@ export class LoopOrchestrator { return false } - try { - logger.info('Evaluating loop condition', { - conditionLength: condition.length, - iteration: scope.iteration, - workflowVariableCount: Object.keys(ctx.workflowVariables ?? {}).length, - }) + logger.info('Evaluating loop condition', { + conditionLength: condition.length, + iteration: scope.iteration, + workflowVariableCount: Object.keys(ctx.workflowVariables ?? {}).length, + }) + let vmResult: Awaited> + try { const evaluatedCondition = await replaceLoopConditionReferences(condition, async (match) => { const resolved = await this.resolver.resolveSingleReference(ctx, '', match, scope) logger.debug('Resolved variable reference in loop condition', { @@ -805,44 +804,70 @@ export class LoopOrchestrator { return match }) - const requestId = generateRequestId() - const code = `return Boolean(${evaluatedCondition})` - - const vmResult = await executeInIsolatedVM({ - code, + vmResult = await executeInIsolatedVM({ + code: `return Boolean(${evaluatedCondition})`, params: {}, envVars: {}, contextVariables: {}, timeoutMs: LOOP_CONDITION_TIMEOUT_MS, - requestId, + requestId: generateRequestId(), ownerKey: `user:${ctx.userId}`, ownerWeight: 1, }) + } catch (error) { + return await this.failConditionEvaluation(ctx, loopId, scope, error) + } - if (vmResult.error) { - const isSystemError = vmResult.error.isSystemError === true - const logFn = isSystemError ? logger.error.bind(logger) : logger.warn.bind(logger) - logFn('Failed to evaluate loop condition', { - isSystemError, - ...projectResolvedSecretDiagnosticError(vmResult.error, ctx.resolvedSecretTraceRegistry), - }) - return false - } + if (vmResult.error) { + const cause = new Error(vmResult.error.message) + cause.name = vmResult.error.name + if (vmResult.error.isSystemError !== true) markFailureKind(cause, 'user') + return await this.failConditionEvaluation(ctx, loopId, scope, cause) + } - const result = Boolean(vmResult.result) + const result = Boolean(vmResult.result) - logger.info('Loop condition evaluation result', { - conditionLength: condition.length, - result, - }) + logger.info('Loop condition evaluation result', { + conditionLength: condition.length, + result, + }) - return result - } catch (error) { - logger.error('Failed to evaluate loop condition', { - conditionLength: condition.length, - ...projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry), - }) - return false - } + return result + } + + /** + * Fails the run when a While or Do While condition cannot be evaluated: a syntax or runtime + * error in the condition, a reference that does not resolve, or a sandbox that could not run it. + * Reading the failure as `false` would end the loop as if its condition had, handing downstream + * blocks a truncated result on a run that reports success. + */ + private async failConditionEvaluation( + ctx: ExecutionContext, + loopId: string, + scope: LoopScope, + cause: unknown + ): Promise { + const loopType = scope.loopType === 'doWhile' ? 'doWhile' : 'while' + const label = loopType === 'doWhile' ? 'Do While' : 'While' + const errorMessage = `${label} loop condition could not be evaluated: ${toError(cause).message}` + const registry = ctx.resolvedSecretTraceRegistry + const errorDiagnostic = projectResolvedSecretDiagnosticError( + new Error(errorMessage), + registry, + { + loopId, + } + ) + const persistedErrorMessage = registry + ? typeof errorDiagnostic.error === 'string' + ? errorDiagnostic.error + : `${label} loop condition could not be evaluated` + : errorMessage + const isUserFailure = classifyFailure(cause) === 'user' + const logFn = isUserFailure ? logger.warn.bind(logger) : logger.error.bind(logger) + logFn('Failed to evaluate loop condition', { ...errorDiagnostic, isUserFailure }) + await this.addLoopErrorLog(ctx, loopId, loopType, persistedErrorMessage, {}) + scope.validationError = persistedErrorMessage + throw inheritFailureMarks(new Error(persistedErrorMessage), cause) } } diff --git a/packages/testing/src/mocks/index.ts b/packages/testing/src/mocks/index.ts index 44eff70eb5c..df08a46c9c0 100644 --- a/packages/testing/src/mocks/index.ts +++ b/packages/testing/src/mocks/index.ts @@ -332,7 +332,6 @@ export { } from './feature-flags.mock' // Isolated VM mocks (for @/lib/execution/isolated-vm) export { - evaluateLoopIndexCheck, isolatedVmMock, isolatedVmMockFns, resetIsolatedVmMock, diff --git a/packages/testing/src/mocks/isolated-vm.mock.ts b/packages/testing/src/mocks/isolated-vm.mock.ts index b561a7e889e..1065ed392cf 100644 --- a/packages/testing/src/mocks/isolated-vm.mock.ts +++ b/packages/testing/src/mocks/isolated-vm.mock.ts @@ -1,39 +1,23 @@ import { vi } from 'vitest' -/** The continuation check the loop orchestrator compiles for count-driven loops. */ -const LOOP_INDEX_CHECK = /^return Boolean\((\d+) < (\d+)\)$/ - -/** - * Evaluates a loop's `return Boolean( < )` continuation check, or returns `null` - * for any other code so a test can decide what that code resolves to. - */ -export function evaluateLoopIndexCheck(code: string): boolean | null { - const match = LOOP_INDEX_CHECK.exec(code) - return match ? Number(match[1]) < Number(match[2]) : null -} - -async function executeLoopIndexCheck({ code }: { code: string }) { - const result = evaluateLoopIndexCheck(code) - if (result === null) throw new Error(`Unexpected isolated-vm code in test: ${code}`) - return { result } +async function rejectUnexpectedCode({ code }: { code: string }): Promise { + throw new Error(`Unexpected isolated-vm code in test: ${code}`) } /** * Controllable mock functions for `@/lib/execution/isolated-vm`. Executor tests mock it because * the real sandbox acquires a Redis-backed lease, which the shared Redis mock answers as - * over-limit. By default only loop continuation checks evaluate; any other code throws, so a - * test that needs more must say what it resolves to. + * over-limit. Count-driven loops never reach the sandbox, so by default any call throws: a test + * whose code does reach it (a While condition, a Function block) must say what it resolves to. * * @example * ```ts - * isolatedVmMockFns.mockExecuteInIsolatedVM.mockImplementation(async ({ code }) => ({ - * result: evaluateLoopIndexCheck(code) ?? userConditionResult, - * })) + * isolatedVmMockFns.mockExecuteInIsolatedVM.mockResolvedValue({ result: userConditionResult }) * ``` */ export const isolatedVmMockFns = { /** Untyped like a bare `vi.fn()`, so each test resolves the fields its code path reads. */ - mockExecuteInIsolatedVM: vi.fn().mockImplementation(executeLoopIndexCheck), + mockExecuteInIsolatedVM: vi.fn().mockImplementation(rejectUnexpectedCode), } /** @@ -48,7 +32,7 @@ export const isolatedVmMock = { executeInIsolatedVM: isolatedVmMockFns.mockExecuteInIsolatedVM, } -/** Restores the default loop-check-only implementation. */ +/** Restores the default implementation, which rejects every call. */ export function resetIsolatedVmMock(): void { - isolatedVmMockFns.mockExecuteInIsolatedVM.mockReset().mockImplementation(executeLoopIndexCheck) + isolatedVmMockFns.mockExecuteInIsolatedVM.mockReset().mockImplementation(rejectUnexpectedCode) } diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index b16b16fc140..9aaebcaea94 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -901,12 +901,9 @@ "apps/sim/enrichments/index.ts#EnrichmentProviderFailureProjection", "apps/sim/enrichments/index.ts#EnrichmentRegistry", "apps/sim/enrichments/index.ts#EnrichmentRunContext", - "apps/sim/executor/constants.ts#LOOP_REFERENCE", "apps/sim/executor/constants.ts#METADATA_ONLY_BLOCK_TYPES", "apps/sim/executor/constants.ts#TRIGGER_BLOCK_TYPES", "apps/sim/executor/constants.ts#TRIGGER_INTERNAL_KEYS", - "apps/sim/executor/constants.ts#buildLoopReference", - "apps/sim/executor/constants.ts#buildReference", "apps/sim/executor/constants.ts#extractReferenceContent", "apps/sim/executor/handlers/pi/cloud/babysit/backend.ts#resolveBabysitExecutionBudgetMs", "apps/sim/executor/handlers/pi/cloud/babysit/github.ts#BabysitCheckDisposition", From d424f3440e06412ac72c364d98a092ce5062e17f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:03:44 -0800 Subject: [PATCH 2/3] refactor(executor): share loop error projection and loop-workflow test helpers Extract projectLoopErrorMessage for the ForEach-resolution and condition-evaluation failure paths, drop the count check that the native continuation now covers, and move loopWorkflow/runLoopWorkflow into a shared test helper. --- .../execution/condition-loop-ceiling.test.ts | 49 ++------- .../execution/loop-workflow.test-helpers.ts | 51 +++++++++ apps/sim/executor/orchestrators/loop.test.ts | 100 ++++++------------ apps/sim/executor/orchestrators/loop.ts | 73 ++++++------- 4 files changed, 123 insertions(+), 150 deletions(-) create mode 100644 apps/sim/executor/execution/loop-workflow.test-helpers.ts diff --git a/apps/sim/executor/execution/condition-loop-ceiling.test.ts b/apps/sim/executor/execution/condition-loop-ceiling.test.ts index 88f6d7efd5c..6c155fa8cbc 100644 --- a/apps/sim/executor/execution/condition-loop-ceiling.test.ts +++ b/apps/sim/executor/execution/condition-loop-ceiling.test.ts @@ -1,15 +1,8 @@ -import type { SessionPrincipal } from '@sim/auth/principal' -import { - createSerializedBlock, - createSerializedWorkflow, -} from '@sim/testing/factories/serialized-block.factory' import { isolatedVmMock, isolatedVmMockFns } from '@sim/testing/mocks/isolated-vm.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { BlockType, DEFAULTS } from '@/executor/constants' -import { DAGExecutor } from '@/executor/execution/executor' -import type { ExecutionResult } from '@/executor/types' -import { hasExecutionResult } from '@/executor/utils/errors' -import type { SerializedLoop, SerializedWorkflow } from '@/serializer/types' +import { DEFAULTS } from '@/executor/constants' +import { loopWorkflow, runLoopWorkflow } from '@/executor/execution/loop-workflow.test-helpers' +import type { SerializedWorkflow } from '@/serializer/types' const { bodyRuns, conditionResult, runaway } = vi.hoisted(() => ({ bodyRuns: { count: 0 }, @@ -33,41 +26,11 @@ vi.mock('@/executor/handlers/registry', () => ({ ], })) -const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } const CEILING = DEFAULTS.MAX_CONDITION_LOOP_ITERATIONS -function loopWorkflow(loop: Partial>): SerializedWorkflow { - const blocks = [ - createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }), - createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }), - createSerializedBlock({ id: 'tick', name: 'tick', type: BlockType.FUNCTION }), - ] - return { - ...createSerializedWorkflow(blocks, [ - { source: 'start', target: 'loop' }, - { source: 'loop', target: 'tick', sourceHandle: 'loop-start-source' }, - ]), - loops: { loop: { id: 'loop', nodes: ['tick'], iterations: 0, ...loop } }, - } -} - -/** Runs the workflow and returns its result, including the one a failed run attaches to its throw. */ -async function run(workflow: SerializedWorkflow): Promise { - try { - return await new DAGExecutor({ - workflow, - contextExtensions: { - workspaceId: 'ws', - executionId: 'exec-1', - principal: PRINCIPAL, - abortSignal: runaway.controller.signal, - }, - }).execute('wf') - } catch (error) { - if (hasExecutionResult(error)) return error.executionResult - throw error - } -} +/** Runs under the runaway guard, so a loop the ceiling fails to stop is cancelled instead of hanging. */ +const run = (workflow: SerializedWorkflow) => + runLoopWorkflow(workflow, { abortSignal: runaway.controller.signal }) describe('condition-driven loop iteration ceiling', () => { beforeEach(() => { diff --git a/apps/sim/executor/execution/loop-workflow.test-helpers.ts b/apps/sim/executor/execution/loop-workflow.test-helpers.ts new file mode 100644 index 00000000000..7fdf4186821 --- /dev/null +++ b/apps/sim/executor/execution/loop-workflow.test-helpers.ts @@ -0,0 +1,51 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { + createSerializedBlock, + createSerializedWorkflow, +} from '@sim/testing/factories/serialized-block.factory' +import { BlockType } from '@/executor/constants' +import { DAGExecutor } from '@/executor/execution/executor' +import type { ExecutionResult } from '@/executor/types' +import { hasExecutionResult } from '@/executor/utils/errors' +import type { SerializedLoop, SerializedWorkflow } from '@/serializer/types' + +const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } + +/** A Start block feeding a loop named `loop` whose body is one block, `tick`. */ +export function loopWorkflow( + loop: Partial> +): SerializedWorkflow { + const blocks = [ + createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }), + createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }), + createSerializedBlock({ id: 'tick', name: 'tick', type: BlockType.FUNCTION }), + ] + return { + ...createSerializedWorkflow(blocks, [ + { source: 'start', target: 'loop' }, + { source: 'loop', target: 'tick', sourceHandle: 'loop-start-source' }, + ]), + loops: { loop: { id: 'loop', nodes: ['tick'], iterations: 0, ...loop } }, + } +} + +/** Runs the workflow and returns its result, including the one a failed run attaches to its throw. */ +export async function runLoopWorkflow( + workflow: SerializedWorkflow, + options: { abortSignal?: AbortSignal } = {} +): Promise { + try { + return await new DAGExecutor({ + workflow, + contextExtensions: { + workspaceId: 'ws', + executionId: 'exec-1', + principal: PRINCIPAL, + abortSignal: options.abortSignal, + }, + }).execute('wf') + } catch (error) { + if (hasExecutionResult(error)) return error.executionResult + throw error + } +} diff --git a/apps/sim/executor/orchestrators/loop.test.ts b/apps/sim/executor/orchestrators/loop.test.ts index 6a8e7198f66..1d393a776ab 100644 --- a/apps/sim/executor/orchestrators/loop.test.ts +++ b/apps/sim/executor/orchestrators/loop.test.ts @@ -1,25 +1,22 @@ -import type { SessionPrincipal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { - createSerializedBlock, - createSerializedWorkflow, -} from '@sim/testing/factories/serialized-block.factory' -import { isolatedVmMock, isolatedVmMockFns } from '@sim/testing/mocks/isolated-vm.mock' + isolatedVmMock, + isolatedVmMockFns, + resetIsolatedVmMock, +} from '@sim/testing/mocks/isolated-vm.mock' import { storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { uploadsMock } from '@sim/testing/mocks/uploads.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' import { classifyFailure } from '@/lib/core/errors/failure-log' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' -import { BlockType, EDGE } from '@/executor/constants' +import { EDGE } from '@/executor/constants' import type { DAG, DAGNode } from '@/executor/dag/builder' import type { EdgeManager } from '@/executor/execution/edge-manager' -import { DAGExecutor } from '@/executor/execution/executor' +import { loopWorkflow, runLoopWorkflow } from '@/executor/execution/loop-workflow.test-helpers' import type { BlockStateController } from '@/executor/execution/types' import { LoopOrchestrator } from '@/executor/orchestrators/loop' import type { ExecutionContext, ExecutionResult } from '@/executor/types' -import { hasExecutionResult } from '@/executor/utils/errors' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' -import type { SerializedLoop, SerializedWorkflow } from '@/serializer/types' const mockUploadFile = storageServiceMockFns.mockUploadFile @@ -357,8 +354,6 @@ describe('LoopOrchestrator', () => { }) }) -const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } - const SANDBOX_AT_CAPACITY = { result: null, stdout: '', @@ -375,55 +370,43 @@ const CONDITION_SYNTAX_ERROR = { error: { message: "Unexpected token ')'", name: 'SyntaxError' }, } -function loopWorkflow(loop: Partial>): SerializedWorkflow { - const blocks = [ - createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }), - createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }), - createSerializedBlock({ id: 'tick', name: 'tick', type: BlockType.FUNCTION }), - ] - return { - ...createSerializedWorkflow(blocks, [ - { source: 'start', target: 'loop' }, - { source: 'loop', target: 'tick', sourceHandle: 'loop-start-source' }, - ]), - loops: { loop: { id: 'loop', nodes: ['tick'], iterations: 0, ...loop } }, - } -} - -/** Runs the workflow and returns its result, including the one a failed run attaches to its throw. */ -async function runWorkflow(workflow: SerializedWorkflow): Promise { - try { - return await new DAGExecutor({ - workflow, - contextExtensions: { workspaceId: 'ws', executionId: 'exec-1', principal: PRINCIPAL }, - }).execute('wf') - } catch (error) { - if (hasExecutionResult(error)) return error.executionResult - throw error - } -} - function loopLog(result: ExecutionResult) { return result.logs?.find((log) => log.blockId === 'loop') } +const sandboxAtCapacity = () => mockExecuteInIsolatedVM.mockResolvedValue(SANDBOX_AT_CAPACITY) +const sandboxRejects = () => + mockExecuteInIsolatedVM.mockRejectedValue(new Error('REDIS_URL is required')) + describe('loop continuation when the sandbox cannot evaluate', () => { beforeEach(() => { bodyRuns.count = 0 + resetIsolatedVmMock() clearLargeValueCacheForTests() mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) }) it.each([ - ['For', { loopType: 'for', iterations: 5 }], - ['ForEach', { loopType: 'forEach', forEachItems: ['a', 'b', 'c', 'd', 'e'] }], - ['count-driven Do While', { loopType: 'doWhile', iterations: 5 }], + ['For', 'is at capacity', { loopType: 'for', iterations: 5 }, sandboxAtCapacity], + ['For', 'rejects', { loopType: 'for', iterations: 5 }, sandboxRejects], + [ + 'ForEach', + 'is at capacity', + { loopType: 'forEach', forEachItems: ['a', 'b', 'c', 'd', 'e'] }, + sandboxAtCapacity, + ], + [ + 'count-driven Do While', + 'is at capacity', + { loopType: 'doWhile', iterations: 5 }, + sandboxAtCapacity, + ], ] as const)( - 'runs every iteration of a %s loop while the sandbox is at capacity', - async (_label, loop) => { - mockExecuteInIsolatedVM.mockResolvedValue(SANDBOX_AT_CAPACITY) + 'runs every iteration of a %s loop when the sandbox %s', + async (_label, _mode, loop, failSandbox) => { + failSandbox() - const result = await runWorkflow(loopWorkflow(loop)) + const result = await runLoopWorkflow(loopWorkflow(loop)) expect(result.success).toBe(true) expect(bodyRuns.count).toBe(5) @@ -431,19 +414,10 @@ describe('loop continuation when the sandbox cannot evaluate', () => { } ) - it('runs every iteration of a For loop when the sandbox call rejects', async () => { - mockExecuteInIsolatedVM.mockRejectedValue(new Error('REDIS_URL is required')) - - const result = await runWorkflow(loopWorkflow({ loopType: 'for', iterations: 5 })) - - expect(result.success).toBe(true) - expect(bodyRuns.count).toBe(5) - }) - it('fails a While loop whose condition has a syntax error instead of skipping its body', async () => { mockExecuteInIsolatedVM.mockResolvedValue(CONDITION_SYNTAX_ERROR) - const result = await runWorkflow( + const result = await runLoopWorkflow( loopWorkflow({ loopType: 'while', whileCondition: ' < 3)' }) ) @@ -459,16 +433,8 @@ describe('loop continuation when the sandbox cannot evaluate', () => { }) it.each([ - [ - 'answers with a system error', - () => mockExecuteInIsolatedVM.mockResolvedValue(SANDBOX_AT_CAPACITY), - /at capacity/, - ], - [ - 'rejects', - () => mockExecuteInIsolatedVM.mockRejectedValue(new Error('REDIS_URL is required')), - /REDIS_URL is required/, - ], + ['answers with a system error', sandboxAtCapacity, /at capacity/], + ['rejects', sandboxRejects, /REDIS_URL is required/], ] as const)( 'fails a Do While loop mid-run when the sandbox %s instead of ending it early', async (_label, failSandbox, message) => { @@ -476,7 +442,7 @@ describe('loop continuation when the sandbox cannot evaluate', () => { mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: true }) failSandbox() - const result = await runWorkflow( + const result = await runLoopWorkflow( loopWorkflow({ loopType: 'doWhile', doWhileCondition: ' < 10' }) ) diff --git a/apps/sim/executor/orchestrators/loop.ts b/apps/sim/executor/orchestrators/loop.ts index ca92caf03fb..8c1e9dc45d0 100644 --- a/apps/sim/executor/orchestrators/loop.ts +++ b/apps/sim/executor/orchestrators/loop.ts @@ -52,6 +52,25 @@ async function replaceLoopConditionReferences( return result + condition.slice(cursor) } +/** + * Projects a loop failure message through the run's resolved-secret registry. `persistedMessage` + * is safe for block logs and the thrown error: it falls back to `fallback` whenever projection + * cannot prove the message free of resolved secrets. + */ +function projectLoopErrorMessage( + message: string, + fallback: string, + registry: ExecutionContext['resolvedSecretTraceRegistry'], + loopId: string +): { diagnostic: Record; persistedMessage: string } { + const diagnostic = projectResolvedSecretDiagnosticError(new Error(message), registry, { loopId }) + if (!registry) return { diagnostic, persistedMessage: message } + return { + diagnostic, + persistedMessage: typeof diagnostic.error === 'string' ? diagnostic.error : fallback, + } +} + /** While loops, and Do While loops given a condition, stop only when their condition does. */ function isConditionDriven(scope: LoopScope): boolean { return ( @@ -155,17 +174,13 @@ export class LoopOrchestrator { buildLoopSentinelStartId(loopId) ) } catch (error) { - const errorMessage = `ForEach loop resolution failed: ${toError(error).message}` - const errorDiagnostic = projectResolvedSecretDiagnosticError( - new Error(errorMessage), - resolutionCtx.resolvedSecretTraceRegistry, - { loopId } - ) - const persistedErrorMessage = resolutionCtx.resolvedSecretTraceRegistry - ? typeof errorDiagnostic.error === 'string' - ? errorDiagnostic.error - : 'ForEach loop resolution failed' - : errorMessage + const { diagnostic: errorDiagnostic, persistedMessage: persistedErrorMessage } = + projectLoopErrorMessage( + `ForEach loop resolution failed: ${toError(error).message}`, + 'ForEach loop resolution failed', + resolutionCtx.resolvedSecretTraceRegistry, + loopId + ) logger.error('ForEach loop resolution failed', errorDiagnostic) await this.addLoopErrorLog(resolutionCtx, loopId, loopType, persistedErrorMessage, { inputType: Array.isArray(loopConfig.forEachItems) @@ -305,10 +320,6 @@ export class LoopOrchestrator { scope.currentIterationOutputs.clear() - if (this.hasReachedConfiguredIterationLimit(scope, scope.iteration + 1)) { - return await this.createExitResult(ctx, loopId, scope) - } - if (!(await this.evaluateCondition(ctx, loopId, scope, scope.iteration + 1))) { return await this.createExitResult(ctx, loopId, scope) } @@ -330,13 +341,6 @@ export class LoopOrchestrator { } } - private hasReachedConfiguredIterationLimit(scope: LoopScope, nextIteration: number): boolean { - if (scope.loopType !== 'doWhile' || scope.maxIterations === undefined) { - return false - } - return nextIteration >= scope.maxIterations - } - /** * Fails the run once a condition-driven loop has used its iteration ceiling with the condition * still true. Exiting quietly instead would hand downstream blocks a truncated result as if @@ -770,10 +774,6 @@ export class LoopOrchestrator { condition: string, scope: LoopScope ): Promise { - if (!condition) { - return false - } - logger.info('Evaluating loop condition', { conditionLength: condition.length, iteration: scope.iteration, @@ -849,20 +849,13 @@ export class LoopOrchestrator { ): Promise { const loopType = scope.loopType === 'doWhile' ? 'doWhile' : 'while' const label = loopType === 'doWhile' ? 'Do While' : 'While' - const errorMessage = `${label} loop condition could not be evaluated: ${toError(cause).message}` - const registry = ctx.resolvedSecretTraceRegistry - const errorDiagnostic = projectResolvedSecretDiagnosticError( - new Error(errorMessage), - registry, - { - loopId, - } - ) - const persistedErrorMessage = registry - ? typeof errorDiagnostic.error === 'string' - ? errorDiagnostic.error - : `${label} loop condition could not be evaluated` - : errorMessage + const { diagnostic: errorDiagnostic, persistedMessage: persistedErrorMessage } = + projectLoopErrorMessage( + `${label} loop condition could not be evaluated: ${toError(cause).message}`, + `${label} loop condition could not be evaluated`, + ctx.resolvedSecretTraceRegistry, + loopId + ) const isUserFailure = classifyFailure(cause) === 'user' const logFn = isUserFailure ? logger.warn.bind(logger) : logger.error.bind(logger) logFn('Failed to evaluate loop condition', { ...errorDiagnostic, isUserFailure }) From 889981adb681a3fe9aa55a55e60bfa8e93d48b45 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:20:29 -0800 Subject: [PATCH 3/3] fix(executor): mark a loop condition failure logged so the engine does not log it again --- apps/sim/executor/orchestrators/loop.test.ts | 4 +++- apps/sim/executor/orchestrators/loop.ts | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/sim/executor/orchestrators/loop.test.ts b/apps/sim/executor/orchestrators/loop.test.ts index 1d393a776ab..8aa43144156 100644 --- a/apps/sim/executor/orchestrators/loop.test.ts +++ b/apps/sim/executor/orchestrators/loop.test.ts @@ -7,7 +7,7 @@ import { import { storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { uploadsMock } from '@sim/testing/mocks/uploads.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { classifyFailure } from '@/lib/core/errors/failure-log' +import { classifyFailure, logFailureOnce } from '@/lib/core/errors/failure-log' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { EDGE } from '@/executor/constants' import type { DAG, DAGNode } from '@/executor/dag/builder' @@ -476,6 +476,8 @@ describe('loop continuation when the sandbox cannot evaluate', () => { expect(classifyFailure(userError)).toBe('user') expect(classifyFailure(systemError)).toBe('internal') + expect(logFailureOnce(createLogger('probe'), 'probe', userError)).toBeUndefined() + expect(logFailureOnce(createLogger('probe'), 'probe', systemError)).toBeUndefined() }) it('keeps a resolved secret out of the persisted condition error', async () => { diff --git a/apps/sim/executor/orchestrators/loop.ts b/apps/sim/executor/orchestrators/loop.ts index 8c1e9dc45d0..d2507fadec0 100644 --- a/apps/sim/executor/orchestrators/loop.ts +++ b/apps/sim/executor/orchestrators/loop.ts @@ -4,6 +4,7 @@ import { classifyFailure, inheritFailureMarks, markFailureKind, + markFailureLogged, } from '@/lib/core/errors/failure-log' import { generateRequestId } from '@/lib/core/utils/request' import { executeInIsolatedVM } from '@/lib/execution/isolated-vm' @@ -861,6 +862,9 @@ export class LoopOrchestrator { logFn('Failed to evaluate loop condition', { ...errorDiagnostic, isUserFailure }) await this.addLoopErrorLog(ctx, loopId, loopType, persistedErrorMessage, {}) scope.validationError = persistedErrorMessage - throw inheritFailureMarks(new Error(persistedErrorMessage), cause) + /** Logged above, so the engine does not log the same failure again as it propagates. */ + const failure = inheritFailureMarks(new Error(persistedErrorMessage), cause) + markFailureLogged(failure) + throw failure } }