From 21765b06ef0652ea48e50caa721ff47102b2bb04 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 12:54:39 -0800 Subject: [PATCH 01/51] fix(workflows): match child selected outputs by normalized block name in sync runs findChildSpanOutput compared the normalized span name with the caller's raw block ref, so a capitalised or spaced child block name was silently omitted while the parent and streaming paths matched it. --- .../execute-workflow-selected-outputs.integration.ts | 11 +++++++++++ apps/sim/lib/workflows/executor/execute-service.ts | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts b/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts index 602bb458739..81c802b1ef2 100644 --- a/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts +++ b/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts @@ -221,6 +221,17 @@ describe('selectedOutputs on a run that calls a child workflow', () => { }) }) + it('matches a child block name regardless of case and spacing, as the parent does', async () => { + const childSelector = `${childWorkflowId}.Child Done.data.answer` + const result = await run([childSelector, 'Done.data.answer']) + + expect(result).toMatchObject({ + ok: true, + status: 'completed', + blockOutputs: { [childSelector]: 'from child', 'Done.data.answer': 'from child' }, + }) + }) + it('returns a child block output selected by child block ID', async () => { const childSelector = `${childWorkflowId}.${blockIds.childDone}.data.answer` const result = await run([childSelector]) diff --git a/apps/sim/lib/workflows/executor/execute-service.ts b/apps/sim/lib/workflows/executor/execute-service.ts index 3f6e9ebc697..6c9652676f5 100644 --- a/apps/sim/lib/workflows/executor/execute-service.ts +++ b/apps/sim/lib/workflows/executor/execute-service.ts @@ -1042,12 +1042,13 @@ function findChildSpanOutput( selector: ChildOutputSelector, inSelectedWorkflow: boolean ): Record | undefined { + const selectedName = normalizeName(selector.blockId) let found: Record | undefined for (const span of spans) { if ( inSelectedWorkflow && span.output !== undefined && - (span.blockId === selector.blockId || normalizeName(span.name) === selector.blockId) + (span.blockId === selector.blockId || normalizeName(span.name) === selectedName) ) { found = span.output } From 8fc90c321b039a087f1c0f9d4b2e7024ea22114b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 12:57:08 -0800 Subject: [PATCH 02/51] fix(executor): a join condition inside a loop reads the branch that ran this iteration resolveSourceBlockId took the first incoming block whose state said executed, and a block state keeps that flag across loop iterations, so the edge drawn first won. It now reads executedBlocks, which the loop clears between iterations. --- .../execution/condition-join-loop.test.ts | 100 ++++++++++++++++++ .../handlers/condition/handler.test.ts | 1 + .../executor/handlers/condition/handler.ts | 5 +- 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 apps/sim/executor/execution/condition-join-loop.test.ts diff --git a/apps/sim/executor/execution/condition-join-loop.test.ts b/apps/sim/executor/execution/condition-join-loop.test.ts new file mode 100644 index 00000000000..0de9dca244c --- /dev/null +++ b/apps/sim/executor/execution/condition-join-loop.test.ts @@ -0,0 +1,100 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { + createSerializedBlock, + createSerializedWorkflow, +} from '@sim/testing/factories/serialized-block.factory' +import { describe, expect, it, vi } from 'vitest' +import { BlockType } from '@/executor/constants' +import { DAGExecutor } from '@/executor/execution/executor' +import { ConditionBlockHandler } from '@/executor/handlers/condition/handler' +import type { SerializedWorkflow } from '@/serializer/types' + +/** Condition code runs as plain JavaScript instead of through the sandboxed function tool. */ +vi.mock('@/tools', () => ({ + executeTool: vi.fn(async (_toolId: string, params: { code: string }) => ({ + success: true, + output: { result: new Function(params.code)() }, + })), +})) + +/** Real condition handler; every other block answers with its own name. */ +vi.mock('@/executor/handlers/registry', () => ({ + createBlockHandlers: () => [ + new ConditionBlockHandler(), + { + canHandle: () => true, + execute: async (_ctx: unknown, block: { metadata?: { name?: string } }) => ({ + result: block.metadata?.name, + }), + }, + ], +})) + +const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } + +function condition(id: string, conditions: Array<{ id: string; title: string; value: string }>) { + return createSerializedBlock({ + id, + name: id, + type: BlockType.CONDITION, + params: { conditions: JSON.stringify(conditions) }, + }) +} + +/** + * A two-iteration loop where `lc1` sends iteration 0 to `la` and iteration 1 to `lb`, both + * joining at `lc2`, whose passthrough output carries the branch it read. + */ +function joinInLoop(branchEdgeOrder: ['la' | 'lb', 'la' | 'lb']): SerializedWorkflow { + const blocks = [ + createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }), + createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }), + condition('lc1', [ + { id: 'first', title: 'if', value: ' == 0' }, + { id: 'rest', title: 'else', value: '' }, + ]), + createSerializedBlock({ id: 'la', name: 'LA', type: BlockType.FUNCTION }), + createSerializedBlock({ id: 'lb', name: 'LB', type: BlockType.FUNCTION }), + condition('lc2', [{ id: 'always', title: 'if', value: 'true' }]), + createSerializedBlock({ id: 'lout', name: 'LOut', type: BlockType.FUNCTION }), + ] + return { + ...createSerializedWorkflow(blocks, [ + { source: 'start', target: 'loop' }, + { source: 'loop', target: 'lc1', sourceHandle: 'loop-start-source' }, + { source: 'lc1', target: 'la', sourceHandle: 'condition-first' }, + { source: 'lc1', target: 'lb', sourceHandle: 'condition-rest' }, + ...branchEdgeOrder.map((source) => ({ source, target: 'lc2' })), + { source: 'lc2', target: 'lout', sourceHandle: 'condition-always' }, + ]), + loops: { + loop: { + id: 'loop', + nodes: ['lc1', 'la', 'lb', 'lc2', 'lout'], + iterations: 2, + loopType: 'for', + }, + }, + } +} + +describe('a condition joining two branches inside a loop', () => { + it.each([ + ['la', 'lb'], + ['lb', 'la'], + ] as const)( + 'reads the branch that ran in each iteration (edges into the join: %s first)', + async (first, second) => { + const result = await new DAGExecutor({ + workflow: joinInLoop([first, second]), + contextExtensions: { workspaceId: 'ws', executionId: 'exec-1', principal: PRINCIPAL }, + }).execute('wf') + + expect(result.success).toBe(true) + const joinReads = (result.logs ?? []) + .filter((log) => log.blockId.startsWith('lc2')) + .map((log) => log.output?.result) + expect(joinReads).toEqual(['LA', 'LB']) + } + ) +}) diff --git a/apps/sim/executor/handlers/condition/handler.test.ts b/apps/sim/executor/handlers/condition/handler.test.ts index 6535bb2ec7f..1f49b471731 100644 --- a/apps/sim/executor/handlers/condition/handler.test.ts +++ b/apps/sim/executor/handlers/condition/handler.test.ts @@ -811,6 +811,7 @@ describe('ConditionBlockHandler', () => { workspaceId: 'test-workspace-id', workflow: parallelWorkflow, blockStates: parallelBlockStates, + executedBlocks: new Set(parallelBlockStates.keys()), blockLogs: [], completedBlocks: new Set(), decisions: { diff --git a/apps/sim/executor/handlers/condition/handler.ts b/apps/sim/executor/handlers/condition/handler.ts index eb9076af9f7..52f4694676e 100644 --- a/apps/sim/executor/handlers/condition/handler.ts +++ b/apps/sim/executor/handlers/condition/handler.ts @@ -569,7 +569,8 @@ export class ConditionBlockHandler implements BlockHandler { * The block whose output the condition evaluates against and passes on: of the blocks with an * edge into it (each as its own branch's copy inside a parallel, when that copy ran), the first * that ran in this execution, else the first. A condition joining an if/else so reads the - * branch that ran, whichever edge was drawn first. + * branch that ran, whichever edge was drawn first. "Ran" reads `executedBlocks`, which a loop + * clears between iterations; a block state keeps `executed` from earlier iterations. */ private resolveSourceBlockId( ctx: ExecutionContext, @@ -583,7 +584,7 @@ export class ConditionBlockHandler implements BlockHandler { const virtualSourceId = buildBranchNodeId(conn.source, branchIndex) return ctx.blockStates.has(virtualSourceId) ? virtualSourceId : conn.source }) - return sources.find((id) => ctx.blockStates.get(id)?.executed) ?? sources[0] + return sources.find((id) => ctx.executedBlocks.has(id)) ?? sources[0] } private findConnectionForCondition( From f0ce8626ba6a310fa3268657b1db42426462ef3a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:02:19 -0800 Subject: [PATCH 03/51] fix(workflows): an operations-batch block rename rewrites references to the block The ops path (Copilot, API, sim workflows operations apply) only set block.name, leaving references dangling, so runs emitted the raw template. The reference rewrite moves to lib/workflows/references/block-rename.ts and the editor rename, workflow import, and ops rename all use it; the editor store's inline copy (unescaped regex, stateful global test) is gone. --- ...y-workflow-operations-edits.integration.ts | 135 ++++++++++++++++++ apps/sim/lib/workflows/editing/operations.ts | 19 ++- .../lib/workflows/references/block-rename.ts | 46 ++++++ apps/sim/stores/workflows/utils.ts | 43 +----- apps/sim/stores/workflows/workflow/store.ts | 33 +---- 5 files changed, 205 insertions(+), 71 deletions(-) create mode 100644 apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts create mode 100644 apps/sim/lib/workflows/references/block-rename.ts diff --git a/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts new file mode 100644 index 00000000000..c5602a6aab9 --- /dev/null +++ b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts @@ -0,0 +1,135 @@ +/** + * Real PostgreSQL coverage for what an operations batch persists when it renames a block, read + * back from the normalized tables. + */ +import { db } from '@sim/db' +import { auditLog, permissions, user, workflow, workspace } from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { featureFlagsMock } from '@sim/testing/mocks/feature-flags.mock' +import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' +import { generateId } from '@sim/utils/id' +import { saveWorkflowToNormalizedTables } from '@sim/workflow-persistence/save' +import type { BlockState, WorkflowState } from '@sim/workflow-types/workflow' +import { eq } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' +import { applyWorkflowOperations } from '@/lib/workflows/application/apply-workflow-operations' +import type { EditWorkflowOperation } from '@/lib/workflows/editing/types' +import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) +vi.mock('@/lib/realtime/notify', () => realtimeNotifyMock) + +const ownerId = generateId() +const workspaceId = generateId() +const principal = createSessionPrincipal({ userId: ownerId, sessionId: generateId() }) + +function block(id: string, type: string, name: string, code?: string): BlockState { + return { + id, + type, + name, + position: { x: 0, y: 0 }, + subBlocks: code === undefined ? {} : { code: { id: 'code', type: 'code', value: code } }, + outputs: {}, + enabled: true, + data: {}, + } +} + +async function seedWorkflow(blocks: BlockState[]): Promise { + const id = generateId() + const now = new Date() + await db.insert(workflow).values({ + id, + userId: ownerId, + workspaceId, + name: `Operations edits ${id}`, + lastSynced: now, + createdAt: now, + updatedAt: now, + }) + const state: WorkflowState = { + blocks: Object.fromEntries(blocks.map((entry) => [entry.id, entry])), + edges: [], + loops: {}, + parallels: {}, + } + const saved = await saveWorkflowToNormalizedTables(id, state) + if (!saved.success) throw new Error(`Seeding the workflow failed: ${saved.error}`) + return id +} + +function apply(workflowId: string, operations: EditWorkflowOperation[], atomic = false) { + return withPermissionGroupScope(() => + applyWorkflowOperations.execute({ principal, input: { workflowId, operations, atomic } }) + ) +} + +async function stored(workflowId: string) { + const loaded = await loadWorkflowFromNormalizedTables(workflowId) + if (!loaded) throw new Error('The workflow did not load') + return loaded +} + +beforeAll(async () => { + const now = new Date() + await db.insert(user).values({ + id: ownerId, + name: 'Operations edits fixture', + email: `${ownerId}@operations-edits.invalid`, + emailVerified: true, + createdAt: now, + updatedAt: now, + }) + await insertWorkspaceFixture(db, { + id: workspaceId, + name: 'Operations edits', + ownerId, + billedAccountUserId: ownerId, + }) + await db.insert(permissions).values({ + id: generateId(), + userId: ownerId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) +}) + +afterAll(async () => { + await db.delete(auditLog).where(eq(auditLog.workspaceId, workspaceId)) + await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) + await db.delete(user).where(eq(user.id, ownerId)) +}) + +describe('renaming a block through an operations batch', () => { + it('rewrites the references other blocks hold to it, as the editor does', async () => { + const save = generateId() + const respond = generateId() + const unrelated = generateId() + const workflowId = await seedWorkflow([ + block(generateId(), 'starter', 'Start'), + block(save, 'function', 'Save Order', 'return { row: 1 }'), + block( + respond, + 'function', + 'Respond', + 'return { saved: , all: }' + ), + block(unrelated, 'function', 'Unrelated', 'return '), + ]) + + await apply(workflowId, [ + { operation_type: 'edit', block_id: save, params: { name: 'Save Order Edited' } }, + ]) + + const { blocks } = await stored(workflowId) + expect(blocks[save].name).toBe('Save Order Edited') + expect(blocks[respond].subBlocks.code.value).toBe( + 'return { saved: , all: }' + ) + expect(blocks[unrelated].subBlocks.code.value).toBe('return ') + }) +}) diff --git a/apps/sim/lib/workflows/editing/operations.ts b/apps/sim/lib/workflows/editing/operations.ts index c0879c35537..ab80b20ea4b 100644 --- a/apps/sim/lib/workflows/editing/operations.ts +++ b/apps/sim/lib/workflows/editing/operations.ts @@ -4,6 +4,7 @@ import { isNonEmptyValue } from '@sim/workflow-code/catalog/canonical-index' import type { BlockState } from '@sim/workflow-types/workflow' import { isEqual } from 'es-toolkit' import { checkBlockPlacement } from '@/lib/workflows/editing/gates' +import { rewriteBlockNameReferences } from '@/lib/workflows/references/block-rename' import { isValidKey } from '@/lib/workflows/sanitization/key-validation' import { getTransitiveSubBlockDependents } from '@/lib/workflows/subblocks/dependencies' import { buildSubBlockValues } from '@/lib/workflows/subblocks/visibility' @@ -65,6 +66,22 @@ function refuseNestedPlacement( return true } +/** + * Renames a block and rewrites the `, blockId: string, name: string): void { + const nameMap = new Map([[normalizeName(blocks[blockId].name), normalizeName(name)]]) + blocks[blockId].name = name + for (const [id, block] of Object.entries(blocks)) { + if (id === blockId) continue + for (const subBlock of Object.values(block.subBlocks ?? {})) { + if (subBlock.value === undefined || subBlock.value === null) continue + subBlock.value = rewriteBlockNameReferences(subBlock.value, nameMap) as typeof subBlock.value + } + } +} + /** * Applies loop/parallel container config from `inputs` onto a block state (data.loopType, etc.). */ @@ -677,7 +694,7 @@ export function handleEditOperation(op: EditWorkflowOperation, ctx: OperationCon }, }) } else { - block.name = params.name + renameBlock(modifiedState.blocks, block_id, params.name) } } } diff --git a/apps/sim/lib/workflows/references/block-rename.ts b/apps/sim/lib/workflows/references/block-rename.ts new file mode 100644 index 00000000000..09e91fe877a --- /dev/null +++ b/apps/sim/lib/workflows/references/block-rename.ts @@ -0,0 +1,46 @@ +import { escapeRegExp } from '@sim/utils/string' + +/** + * `value` with every ` +): unknown { + if (typeof value === 'string') { + let updatedValue = value + nameMap.forEach((newName, oldName) => { + /** + * A rename to itself is a no-op, so skip the scan entirely. This is the + * whole map on the import path (`regenerateWorkflowIds` seeds it with + * `name -> name`), which turns an O(names x values) rescan of every + * sub-block string into nothing. + */ + if (oldName === newName) return + + /** + * `oldName` is a block name, which reaches this function straight from + * imported workflow JSON — `normalizeWorkflowBlockName` only lowercases + * and strips whitespace/dots, so regex metacharacters survive. Without + * escaping, a name like `a*a*a*a*b` compiles to a catastrophically + * backtracking pattern that pins the event loop on a sub-kilobyte input. + */ + const regex = new RegExp(`<${escapeRegExp(oldName)}\\.`, 'g') + updatedValue = updatedValue.replace(regex, `<${newName}.`) + }) + return updatedValue + } + if (Array.isArray(value)) { + return value.map((item) => rewriteBlockNameReferences(item, nameMap)) + } + if (value && typeof value === 'object') { + const result: Record = {} + for (const [key, val] of Object.entries(value)) { + result[key] = rewriteBlockNameReferences(val, nameMap) + } + return result + } + return value +} diff --git a/apps/sim/stores/workflows/utils.ts b/apps/sim/stores/workflows/utils.ts index ede20204e4a..ff64e4885e0 100644 --- a/apps/sim/stores/workflows/utils.ts +++ b/apps/sim/stores/workflows/utils.ts @@ -1,11 +1,11 @@ import { generateId } from '@sim/utils/id' -import { escapeRegExp } from '@sim/utils/string' import { mergeSubblockStateWithValues } from '@sim/workflow-persistence/subblocks' import { filterUniqueWorkflowEdges } from '@sim/workflow-types/workflow' import type { Edge } from '@xyflow/react' import { DEFAULT_DUPLICATE_OFFSET } from '@/lib/workflows/autolayout/constants' import { remapConditionBlockIds, remapConditionEdgeHandle } from '@/lib/workflows/condition-ids' import { isDynamicHandleSubblock } from '@/lib/workflows/dynamic-handle-topology' +import { rewriteBlockNameReferences } from '@/lib/workflows/references/block-rename' import { normalizeName } from '@/executor/constants' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' @@ -112,43 +112,6 @@ export function mergeSubblockState( return mergeSubblockStateWithValues(blocks, workflowSubblockValues, blockId) } -function updateValueReferences(value: unknown, nameMap: Map): unknown { - if (typeof value === 'string') { - let updatedValue = value - nameMap.forEach((newName, oldName) => { - /** - * A rename to itself is a no-op, so skip the scan entirely. This is the - * whole map on the import path (`regenerateWorkflowIds` seeds it with - * `name -> name`), which turns an O(names x values) rescan of every - * sub-block string into nothing. - */ - if (oldName === newName) return - - /** - * `oldName` is a block name, which reaches this function straight from - * imported workflow JSON — `normalizeWorkflowBlockName` only lowercases - * and strips whitespace/dots, so regex metacharacters survive. Without - * escaping, a name like `a*a*a*a*b` compiles to a catastrophically - * backtracking pattern that pins the event loop on a sub-kilobyte input. - */ - const regex = new RegExp(`<${escapeRegExp(oldName)}\\.`, 'g') - updatedValue = updatedValue.replace(regex, `<${newName}.`) - }) - return updatedValue - } - if (Array.isArray(value)) { - return value.map((item) => updateValueReferences(item, nameMap)) - } - if (value && typeof value === 'object') { - const result: Record = {} - for (const [key, val] of Object.entries(value)) { - result[key] = updateValueReferences(val, nameMap) - } - return result - } - return value -} - /** * Clears a cloned block's `triggerPath` so it derives a fresh webhook URL from its own block id. * @@ -198,7 +161,7 @@ function updateBlockReferences( } if (subBlock.value !== undefined && subBlock.value !== null) { - const updatedValue = updateValueReferences( + const updatedValue = rewriteBlockNameReferences( subBlock.value, nameMap ) as SubBlockState['value'] @@ -494,7 +457,7 @@ export function regenerateBlockIds( Object.entries(newSubBlockValues).forEach(([_, blockValues]) => { Object.keys(blockValues).forEach((subBlockId) => { - blockValues[subBlockId] = updateValueReferences(blockValues[subBlockId], nameMap) + blockValues[subBlockId] = rewriteBlockNameReferences(blockValues[subBlockId], nameMap) }) }) diff --git a/apps/sim/stores/workflows/workflow/store.ts b/apps/sim/stores/workflows/workflow/store.ts index 716ade55cd5..ea211a357bd 100644 --- a/apps/sim/stores/workflows/workflow/store.ts +++ b/apps/sim/stores/workflows/workflow/store.ts @@ -19,6 +19,7 @@ import { getDynamicHandleSubblockType, isDynamicHandleSubblock, } from '@/lib/workflows/dynamic-handle-topology' +import { rewriteBlockNameReferences } from '@/lib/workflows/references/block-rename' import { normalizeName } from '@/executor/constants' import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { @@ -711,34 +712,7 @@ export const useWorkflowStore = create()( const workflowValues = subBlockStore.workflowValues[activeWorkflowId] || {} const updatedWorkflowValues = { ...workflowValues } - // Helper function to recursively update references in any data structure - function updateReferences(value: any, regex: RegExp, replacement: string): any { - // Handle string values - if (typeof value === 'string') { - return regex.test(value) ? value.replace(regex, replacement) : value - } - - // Handle arrays - if (Array.isArray(value)) { - return value.map((item) => updateReferences(item, regex, replacement)) - } - - // Handle objects - if (value !== null && typeof value === 'object') { - const result = { ...value } - for (const key in result) { - result[key] = updateReferences(result[key], regex, replacement) - } - return result - } - - // Return unchanged for other types - return value - } - - const oldBlockName = normalizeName(oldBlock.name) - const newBlockName = normalizeName(name) - const regex = new RegExp(`<${oldBlockName}\\.`, 'g') + const nameMap = new Map([[normalizeName(oldBlock.name), normalizeName(name)]]) // Loop through blocks Object.entries(workflowValues).forEach(([blockId, blockValues]) => { @@ -749,8 +723,7 @@ export const useWorkflowStore = create()( // Loop through subblocks and update references Object.entries(blockValues).forEach(([subBlockId, value]) => { - // Use a recursive function to handle all object types - const updatedValue = updateReferences(value, regex, `<${newBlockName}.`) + const updatedValue = rewriteBlockNameReferences(value, nameMap) // Check if the value actually changed if (JSON.stringify(updatedValue) !== JSON.stringify(value)) { From f449dfd8942b5653c2c07d36a1b464dbcfa65dcb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:02:50 -0800 Subject: [PATCH 04/51] fix(workflows): coerce a numeric-string loop iterations or parallel count on the ops path An operations batch stored iterations: "3" or count: "2" as a string, which the loader then reset to its default (1 iterations, 5 branches) with no error. A numeric string is now stored as its number, as slider inputs coerce; any other value is an input validation error that keeps the current count, so --atomic refuses it. --- ...y-workflow-operations-edits.integration.ts | 68 +++++++++++- apps/sim/lib/workflows/editing/operations.ts | 105 +++++++++++++++--- 2 files changed, 156 insertions(+), 17 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts index c5602a6aab9..43560d445ba 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts @@ -1,6 +1,6 @@ /** - * Real PostgreSQL coverage for what an operations batch persists when it renames a block, read - * back from the normalized tables. + * Real PostgreSQL coverage for what an operations batch persists when it renames a block or + * sets a loop or parallel count from a string, read back from the normalized tables. */ import { db } from '@sim/db' import { auditLog, permissions, user, workflow, workspace } from '@sim/db/schema' @@ -133,3 +133,67 @@ describe('renaming a block through an operations batch', () => { expect(blocks[unrelated].subBlocks.code.value).toBe('return ') }) }) + +describe('a loop or parallel count given as a string', () => { + it('stores a numeric string as its number on add and on edit', async () => { + const workflowId = await seedWorkflow([block(generateId(), 'starter', 'Start')]) + const loop = generateId() + const parallel = generateId() + + await apply( + workflowId, + [ + { + operation_type: 'add', + block_id: loop, + params: { type: 'loop', name: 'Repeat', inputs: { loopType: 'for', iterations: '3' } }, + }, + { + operation_type: 'add', + block_id: parallel, + params: { + type: 'parallel', + name: 'Fan Out', + inputs: { parallelType: 'count', count: '2' }, + }, + }, + ], + true + ) + + const added = await stored(workflowId) + expect(added.loops[loop]).toMatchObject({ iterations: 3 }) + expect(added.parallels[parallel]).toMatchObject({ count: 2 }) + + await apply(workflowId, [ + { operation_type: 'edit', block_id: loop, params: { inputs: { iterations: ' 4 ' } } }, + ]) + + expect((await stored(workflowId)).loops[loop]).toMatchObject({ iterations: 4 }) + }) + + it('refuses a non-numeric count instead of silently resetting it', async () => { + const workflowId = await seedWorkflow([block(generateId(), 'starter', 'Start')]) + + await expect( + apply( + workflowId, + [ + { + operation_type: 'add', + block_id: generateId(), + params: { + type: 'loop', + name: 'Repeat', + inputs: { loopType: 'for', iterations: 'three' }, + }, + }, + ], + true + ) + ).rejects.toMatchObject({ + droppedInputs: [expect.objectContaining({ field: 'iterations', value: 'three' })], + }) + expect(Object.keys((await stored(workflowId)).loops)).toEqual([]) + }) +}) diff --git a/apps/sim/lib/workflows/editing/operations.ts b/apps/sim/lib/workflows/editing/operations.ts index ab80b20ea4b..907ebadad14 100644 --- a/apps/sim/lib/workflows/editing/operations.ts +++ b/apps/sim/lib/workflows/editing/operations.ts @@ -23,7 +23,7 @@ import { updateCanonicalModesForInputs, } from './builders' import { VALID_LOOP_TYPES, VALID_PARALLEL_TYPES } from './container-types' -import type { EditWorkflowOperation, OperationContext } from './types' +import type { EditWorkflowOperation, OperationContext, ValidationError } from './types' import { logSkippedItem } from './types' import { findBlockWithDuplicateNormalizedName, validateInputsForBlock } from './validation' @@ -82,10 +82,39 @@ function renameBlock(blocks: Record, blockId: string, name: } } +/** + * A loop's `iterations` or a parallel's `count` as stored: a number as given, a numeric string as + * its number (the way a slider input coerces), and anything else refused as an input validation + * error, keeping `current`, since the loader would otherwise reset a non-number to its default. + */ +function containerCount( + block: BlockState, + field: 'iterations' | 'count', + value: unknown, + validationErrors: ValidationError[], + current?: unknown +): unknown { + if (typeof value === 'number') return value + const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : Number.NaN + if (Number.isFinite(parsed)) return parsed + validationErrors.push({ + blockId: block.id, + blockType: block.type, + field, + value, + error: `Invalid ${field} "${String(value)}" for ${block.type} "${block.name}" - must be a number`, + }) + return current +} + /** * Applies loop/parallel container config from `inputs` onto a block state (data.loopType, etc.). */ -function applyLoopOrParallelContainerData(block: any, params: Record): void { +function applyLoopOrParallelContainerData( + block: any, + params: Record, + validationErrors: ValidationError[] +): void { if (params.type === 'loop') { const loopType = params.inputs?.loopType && VALID_LOOP_TYPES.includes(params.inputs.loopType) @@ -96,7 +125,16 @@ function applyLoopOrParallelContainerData(block: any, params: Record): void { +function updateLoopOrParallelContainerData( + block: any, + params: Record, + validationErrors: ValidationError[] +): void { if (block.type === 'loop') { block.data = block.data || {} if (params.inputs?.loopType) { @@ -195,7 +246,13 @@ function updateLoopOrParallelContainerData(block: any, params: Record Date: Fri, 20 Nov 2026 13:21:17 -0800 Subject: [PATCH 05/51] test(executor): use the central tools mock in the loop join test; shrink the explicit-any baseline --- .../execution/condition-join-loop.test.ts | 21 ++++++++++++------- scripts/check-explicit-any.baseline.json | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/apps/sim/executor/execution/condition-join-loop.test.ts b/apps/sim/executor/execution/condition-join-loop.test.ts index 0de9dca244c..2552774177d 100644 --- a/apps/sim/executor/execution/condition-join-loop.test.ts +++ b/apps/sim/executor/execution/condition-join-loop.test.ts @@ -3,19 +3,14 @@ import { createSerializedBlock, createSerializedWorkflow, } from '@sim/testing/factories/serialized-block.factory' -import { describe, expect, it, vi } from 'vitest' +import { toolsMock, toolsMockFns } from '@sim/testing/mocks/tools.mock' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { BlockType } from '@/executor/constants' import { DAGExecutor } from '@/executor/execution/executor' import { ConditionBlockHandler } from '@/executor/handlers/condition/handler' import type { SerializedWorkflow } from '@/serializer/types' -/** Condition code runs as plain JavaScript instead of through the sandboxed function tool. */ -vi.mock('@/tools', () => ({ - executeTool: vi.fn(async (_toolId: string, params: { code: string }) => ({ - success: true, - output: { result: new Function(params.code)() }, - })), -})) +vi.mock('@/tools', () => toolsMock) /** Real condition handler; every other block answers with its own name. */ vi.mock('@/executor/handlers/registry', () => ({ @@ -79,6 +74,16 @@ function joinInLoop(branchEdgeOrder: ['la' | 'lb', 'la' | 'lb']): SerializedWork } describe('a condition joining two branches inside a loop', () => { + /** Condition code runs as plain JavaScript instead of through the sandboxed function tool. */ + beforeEach(() => { + toolsMockFns.mockExecuteTool.mockImplementation( + async (_toolId: string, params: { code: string }) => ({ + success: true, + output: { result: new Function(params.code)() }, + }) + ) + }) + it.each([ ['la', 'lb'], ['lb', 'la'], diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 7b91a1b044a..82f2ac9d67a 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -326,7 +326,7 @@ "apps/sim/stores/workflow-diff/utils.ts": 2, "apps/sim/stores/workflows/subblock/store.ts": 5, "apps/sim/stores/workflows/utils.ts": 1, - "apps/sim/stores/workflows/workflow/store.ts": 6, + "apps/sim/stores/workflows/workflow/store.ts": 4, "apps/sim/stores/workflows/workflow/types.ts": 2, "apps/sim/tools/ahrefs/anchors.ts": 1, "apps/sim/tools/ahrefs/backlinks.ts": 1, From c6c72766b69544dcbd50bad90fe6ac5c05e89a48 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 12:58:21 -0800 Subject: [PATCH 06/51] fix(workspace-source): a promotion deletes what the directory deleted since the last one A resource the promotion record holds that the directory no longer declares stays in the base, so the apply plans its delete and refuses it without --allow-delete, as an apply to the directory's own workspace does. Only resources no promotion left in the target are named W_PROMOTE_DELETES_TARGET_ONLY. --- .../content/docs/workflows/workspace-code.mdx | 2 +- .../application/promote.integration.ts | 57 ++++++++++++++++++- .../protocol/workspace-source-apply.test.ts | 15 ++++- .../protocol/workspace-source-apply.ts | 9 ++- packages/workspace-code/DESIGN.md | 7 ++- packages/workspace-code/src/promote.ts | 32 ++++++++--- 6 files changed, 106 insertions(+), 16 deletions(-) diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index bfe38d33471..f6dfb519361 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -93,7 +93,7 @@ sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --dry sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --yes ``` -Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`, which deletes every one of them: the plan names each with a `W_PROMOTE_DELETES_TARGET_ONLY` warning. The directory stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. +Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`, which deletes every one of them: the plan names each with a `W_PROMOTE_DELETES_TARGET_ONLY` warning. A resource you deleted from the directory since the last promotion is deleted in the target too, and like any delete it needs `--allow-delete`. The directory stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. Each promotion records what it left in the target in `.sim/targets/.json`, and the next promotion compares with that record: diff --git a/apps/sim/lib/workspace-source/application/promote.integration.ts b/apps/sim/lib/workspace-source/application/promote.integration.ts index 6f878503ba4..c2042f82ce9 100644 --- a/apps/sim/lib/workspace-source/application/promote.integration.ts +++ b/apps/sim/lib/workspace-source/application/promote.integration.ts @@ -110,7 +110,12 @@ async function promote(dev: Read, target: string, options: { allowDelete?: boole * Promotes `dev` to `target` as `apply --to` does with the target's promotion record, returning * the apply's result and the record it leaves. */ -async function promoteRecorded(dev: Read, target: string, record: PromotionRecord | undefined) { +async function promoteRecorded( + dev: Read, + target: string, + record: PromotionRecord | undefined, + options: { allowDelete?: boolean } = {} +) { const prod = await runner.get(target) const composed = promotionSubmission({ files: dev.files, @@ -118,11 +123,16 @@ async function promoteRecorded(dev: Read, target: string, record: PromotionRecor kinds: dev.kinds, target: prod, record, - allowDelete: false, + allowDelete: options.allowDelete === true, + }) + const result = await runner.apply(target, { + files: composed.files, + base: composed.base, + ...options, }) - const result = await runner.apply(target, { files: composed.files, base: composed.base }) return { result, + composed, record: promotionRecord({ workspaceId: target, previous: record, @@ -418,4 +428,45 @@ export default defineTable({ ).toEqual(['delete table:audit']) } ) + + check( + 'a resource deleted in the directory since the last promotion is a delete: refused without --allow-delete, and not named as only in the target', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: DEV }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + expect(first.result.failed).toBeUndefined() + const prodRead = await runner.get(prod) + await runner.apply(prod, { + files: { + 'tables/leads.ts': prodRead.files['tables/leads.ts'], + 'tables/audit.ts': LEADS.replace("name: 'leads'", "name: 'audit'"), + }, + base: prodRead.base, + kinds: ['table'], + }) + const devRead = await runner.get(dev) + await runner.apply(dev, { + files: Object.fromEntries( + Object.entries(devRead.files).filter(([path]) => path !== 'skills/tone.md') + ), + base: devRead.base, + allowDelete: true, + }) + const deletedInDev = await runner.get(dev) + + const refused = await refusal(promoteRecorded(deletedInDev, prod, first.record)) + expect(codes(refused.diagnostics)).toEqual(['E_APPLY_DELETE_NOT_ALLOWED']) + expect((await runner.get(prod)).files['skills/tone.md']).toContain('Be brief.') + + const allowed = await promoteRecorded(deletedInDev, prod, first.record, { allowDelete: true }) + expect( + allowed.result.applied + .map((change) => `${change.action} ${change.kind}:${change.key}`) + .sort() + ).toEqual(['delete skill:tone', 'delete table:audit']) + expect(allowed.composed.targetOnly).toEqual(['table:audit']) + } + ) }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 9a4705c5b00..4679588b3cd 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -474,6 +474,7 @@ describe('workspaces source apply', () => { it('a --to dry run warns of what the target deleted since the last promotion, gives stale findings a promotion hint, and names each target-only delete', async () => { const leads = { id: 'tbl_prod', fingerprint: `rf1:${'a'.repeat(64)}` } + const dropped = { id: 'tbl_dropped', fingerprint: `rf1:${'d'.repeat(64)}` } write( '.sim/targets/ws_prod.json', JSON.stringify({ @@ -482,6 +483,7 @@ describe('workspaces source apply', () => { resources: { 'table:leads': leads, 'table:gone': { id: 'tbl_gone', fingerprint: `rf1:${'c'.repeat(64)}` }, + 'table:dropped': dropped, }, workflows: {}, }) @@ -493,12 +495,20 @@ describe('workspaces source apply', () => { data: { workspaceId: 'ws_prod', kinds: ['table'], - files: { 'tables/leads.ts': LEADS, 'tables/prod-only.ts': LEADS }, + files: { + 'tables/leads.ts': LEADS, + 'tables/prod-only.ts': LEADS, + 'tables/dropped.ts': LEADS, + }, blobs: [], base: { version: 1, workspaceId: 'ws_prod', - resources: { 'table:leads': leads, 'table:prod-only': { id: 'tbl_only' } }, + resources: { + 'table:leads': leads, + 'table:prod-only': { id: 'tbl_only' }, + 'table:dropped': dropped, + }, }, skipped: [], diagnostics: [], @@ -512,6 +522,7 @@ describe('workspaces source apply', () => { { kind: 'table', key: 'gone', action: 'noop' }, { kind: 'table', key: 'leads', action: 'noop' }, { kind: 'table', key: 'prod-only', action: 'delete' }, + { kind: 'table', key: 'dropped', action: 'delete' }, ], diagnostics: [ { diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 66ca8109d33..797dbe58650 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -409,6 +409,8 @@ interface Promotion { target: string /** What the target deleted since the last promotion that the directory declares (addresses). */ deletedInTarget: string[] + /** What only the target holds: undeclared, and no promotion from the directory left it there. */ + targetOnly: string[] } /** @@ -456,6 +458,7 @@ async function promoted( record, target, deletedInTarget: composed.deletedInTarget, + targetOnly: composed.targetOnly, } } @@ -556,9 +559,13 @@ function promotionWarnings( }, ] }) + const onlyInTarget = new Set(promotion.targetOnly) const targetOnly = outcome.plan .filter( - (change) => change.action === 'delete' && (!outcome.failed || wasWritten(change, outcome)) + (change) => + change.action === 'delete' && + onlyInTarget.has(`${change.kind}:${change.key}`) && + (!outcome.failed || wasWritten(change, outcome)) ) .map( (change): Finding => ({ diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index 7e40ef9d54d..d4077c63954 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -332,8 +332,11 @@ to another (prod) without rebinding the directory. The CLI reads the target as a (`GET /source` for the managed kinds, no blob downloads) and composes the submission with `promotionSubmission` (`src/promote.ts`): the directory's text with every workflow folder whole, each folder's `.sim/lock.json` and `.sim/base.json` replaced by the target's folder of the same key -(or dropped, so the key is created), and the target's root base, narrowed to the keys the -directory declares unless `--allow-delete`. The server sees an ordinary apply against its own base, +(or dropped, so the key is created), and the target's root base. A base entry the directory does +not declare stays in it when the promotion record holds it for the same id (the directory deleted +it since, so the apply refuses it without `--allow-delete` as it would at home); one only the target +ever held (`targetOnly`) leaves it unless `--allow-delete`, and the CLI names each such delete +`W_PROMOTE_DELETES_TARGET_ONLY`. The server sees an ordinary apply against its own base, so every rule holds unchanged: a resource bound under the key is updated in place (a workflow keeps its id and block ids through the target's lock), an unbound one of the same name is adopted, a missing one created, and secrets and credentials bind by name and key to the target's own. The diff --git a/packages/workspace-code/src/promote.ts b/packages/workspace-code/src/promote.ts index a122ee238df..5e0141c4194 100644 --- a/packages/workspace-code/src/promote.ts +++ b/packages/workspace-code/src/promote.ts @@ -36,13 +36,22 @@ import { kindOfPath } from './layout' export interface PromotionSubmission { /** The directory's text files, each workflow folder whole, with the target's record files. */ files: Record - /** The target's base, narrowed to what the directory declares unless deletes are allowed. */ + /** + * The target's base, without what only the target holds unless deletes are allowed (see + * {@link PromotionInput.allowDelete}). + */ base: WorkspaceBaseV1 /** * The resources (by address) the last promotion left in the target that the target deleted * since and the directory still declares: the apply keeps them deleted, so the promotion warns. */ deletedInTarget: string[] + /** + * The resources (by address) the target holds that the directory does not declare and no + * promotion from it left there: only the target ever held them, so `--allow-delete` deleting one + * deletes something the directory never had. + */ + targetOnly: string[] } export interface PromotionInput { @@ -57,9 +66,11 @@ export interface PromotionInput { /** What the last promotion from this directory left in the target, when one is recorded. */ record?: PromotionRecord /** - * Whether the promotion may delete what the target holds and the directory does not declare. - * Without it, those resources leave the base, so the apply keeps them with a warning instead of - * refusing for want of `--allow-delete`. + * Whether the promotion may delete what only the target holds. Without it, those resources leave + * the base, so the apply keeps them with a warning instead of refusing for want of + * `--allow-delete`. A resource the last promotion left that the directory no longer declares + * stays in the base either way: the directory deleted it, which needs `--allow-delete` as an + * apply to its own workspace does. */ allowDelete: boolean } @@ -201,7 +212,7 @@ export function parsePromotionRecord( * The submission that promotes a directory to the target its get answer describes: the * directory's source with every workflow folder's lock and base replaced by the target's folder * of the same key (or dropped, so a key the target lacks creates a workflow), and the target's - * base, narrowed to what the directory declares unless deletes are allowed. With a + * base, without what only the target holds unless deletes are allowed. With a * {@link PromotionRecord}, each entry the record holds for the same id stands in for the target's * own, and a workflow folder records the fingerprint the last promotion left: sent as its base * alone (a no-op) when the directory has not changed it since, else whole, so a target edit since @@ -256,15 +267,21 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission const lock = input.target.files[lockPath] if (lock !== undefined) files[lockPath] = lock } + const targetOnly: string[] = [] const resources = Object.fromEntries( Object.entries(input.target.base.resources).flatMap(([address, live]) => { const parsed = parseResourceAddress(address) if (!parsed) return [] const keys = declared.get(parsed.kind) - if (keys === undefined || !(input.allowDelete || keys.has(parsed.key))) return [] + if (keys === undefined) return [] const recorded: BaseResource | undefined = input.record?.resources[resourceAddress(parsed.kind, parsed.key)] - return [[address, recorded?.id === live.id ? recorded : live] as const] + const promoted = recorded?.id === live.id ? recorded : undefined + if (!keys.has(parsed.key) && !promoted) { + targetOnly.push(resourceAddress(parsed.kind, parsed.key)) + if (!input.allowDelete) return [] + } + return [[address, promoted ?? live] as const] }) ) // A resource the last promotion left in the target that the target no longer holds was deleted @@ -282,6 +299,7 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission files, base: { ...input.target.base, resources }, deletedInTarget: [...new Set(deletedInTarget)].sort(compareStrings), + targetOnly: targetOnly.sort(compareStrings), } } From b0830381c44d56cb057d953eff70d99e135eb277 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 12:58:32 -0800 Subject: [PATCH 07/51] fix(sim-cli): the promotion stale hint no longer suggests deleting the record Deleting .sim/targets/.json made the next promotion overwrite every change made in the target since, rows its workflows wrote included, while the hint read as keeping the target's version. The hint now offers only the flows that keep the target's data and says what deleting the record does. --- apps/docs/content/docs/workflows/workspace-code.mdx | 2 +- .../src/commands/protocol/workspace-source-apply.test.ts | 2 ++ .../sim-cli/src/commands/protocol/workspace-source-apply.ts | 2 +- packages/workspace-code/DESIGN.md | 5 +++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index f6dfb519361..299c7f4201a 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -102,7 +102,7 @@ Each promotion records what it left in the target in `.sim/targets/ - A resource deleted in the target since the last promotion stays deleted, with a `W_PROMOTE_TARGET_DELETED` warning. Remove its entry from the record to create it again. - The first promotion, and any resource the record does not hold (one created in the target since), updates the target to match the directory. Review the dry run first. -Commit the record with the rest of the directory, so a promotion from CI or another clone compares with it. Deleting it makes the next promotion take the target as it stands. +Commit the record with the rest of the directory, so a promotion from CI or another clone compares with it. Don't delete it to get past a stale refusal: without it, the next promotion overwrites every change made in the target since, including rows its workflows wrote. ## Git and CI diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 4679588b3cd..fffffa36f0c 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -559,6 +559,8 @@ describe('workspaces source apply', () => { const [stale] = byCode('E_APPLY_STALE') expect(stale.hint).toContain('.sim/targets/ws_prod.json') expect(stale.hint).not.toContain('fresh directory') + expect(stale.hint).toContain('sim workspaces source get --workspace ws_prod') + expect(stale.hint).not.toContain('.sim/targets/ws_prod.json first') const stderr = vi.mocked(console.error).mock.calls.flat().join('\n') expect(stderr).toContain(' ! gone (deleted on the server; kept as the server has it)') expect(stderr).not.toContain('resource(s) that only workspace ws_prod holds') diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 797dbe58650..78a7e1549a0 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -503,7 +503,7 @@ function resourceWhere(kind: WorkspaceKind, key: string): string { function promotionHint(code: string, target: string): string | undefined { const record = promotionRecordPath(target) ?? '.sim/targets/.json' if (code === 'E_APPLY_STALE') { - return `Workspace ${target} changed this resource since the last promotion from this directory. Carry that change into the directory, or undo it in ${target}, then promote again; to take ${target} as it stands now, delete ${record} first.` + return `Workspace ${target} changed this resource since the last promotion from this directory. Carry that change into the directory (\`sim workspaces source get --workspace ${target} --dir \` prints it), or undo it in ${target}, then promote again. Don't delete ${record} to get past this: the next promotion would then overwrite every change made in ${target} since, rows its workflows wrote included.` } if (code === 'E_APPLY_NO_BASE') { return `Workspace ${target} holds this resource, but no promotion from this directory recorded it. Rename the declaration, or rename or delete the resource in ${target}, then promote again.` diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index d4077c63954..bce3f1c8c98 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -349,8 +349,9 @@ with what the last promotion left (a target edit since is kept when the director alone, `E_APPLY_STALE` when it changed it too); a workflow folder whose digest is unchanged is sent as its base alone (a no-op), and an edited one whole with the recorded fingerprint, so a target edit since is the folder's 409. Without a record entry (the first promotion, or a resource created in -the target since) the live entry is the base and the target is updated to match the directory; -deleting the record re-takes the target as it stands. Read alone, the target's live base could +the target since) the live entry is the base and the target is updated to match the directory, so +deleting the record makes the next promotion overwrite every change made in the target since, +rows its runs wrote included: the stale hint never suggests it. Read alone, the target's live base could never look changed. Without `--to`, the CLI refuses a directory read from another workspace (`SOURCE_OTHER_WORKSPACE`); `--workspace ` instead applies it as a copy whose base the server ignores (`W_BASE_OTHER_WORKSPACE`), which only a first From 4a6604d117bedb168fc1050d21abf5c48e61d7ef Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:02:28 -0800 Subject: [PATCH 08/51] fix(table): a source reshape strips a dropped column's cells in its own transaction The strip ran after commit in the background and moved the managed rows' version after the apply recorded it, so the next status reported server row drift and a delete of the table was refused as stale. For a managed table the strip now commits with the reshape, so the recorded version holds it. --- apps/sim/lib/table/application/reshape.ts | 11 ++++- apps/sim/lib/table/columns/service.ts | 44 +++++++++++++------ .../application/rows.integration.ts | 43 ++++++++++++++++++ 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/apps/sim/lib/table/application/reshape.ts b/apps/sim/lib/table/application/reshape.ts index f61f12172e4..a1bb94527d3 100644 --- a/apps/sim/lib/table/application/reshape.ts +++ b/apps/sim/lib/table/application/reshape.ts @@ -357,8 +357,17 @@ export const reshapeTableUseCase = defineAuthorizedTableUseCase({ const progress: TableFieldsProgress = { current: locked, applied: [], changed: [] } if (input.table) await writeTableFields(progress, input.table, resolution?.folderId, tx) let table = progress.current + // Managed rows' version is the base source compares with, so the drop's cell strip + // commits with it rather than moving the version after the apply recorded it. + const managed = + input.sourceRows === undefined + ? locked.metadata?.sourceRows?.mode === 'managed' + : input.sourceRows?.mode === 'managed' for (const columnName of input.drop ?? []) { - table = await deleteColumn({ tableId: table.id, columnName }, requestId, options) + table = await deleteColumn({ tableId: table.id, columnName }, requestId, { + ...options, + stripRowsInTransaction: managed, + }) } // With rows to write, a column made required is made so after them (an add, below, too). const deferredUpdates: string[] = [] diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index 961d8ca0736..8280cdff913 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -387,6 +387,19 @@ function stripColumnIdsFromMetadata( return next } +/** Removes `columnIds`' cells from every row of the table, in `trx`. */ +async function stripColumnData( + trx: DbTransaction, + tableId: string, + workspaceId: string, + columnIds: string[] +): Promise { + await updateTableRowsWithDerivedSecretProvenance(trx, { + rowWhere: and(eq(userTableRows.tableId, tableId), eq(userTableRows.workspaceId, workspaceId))!, + transformation: { mode: 'remove-columns', columnIds }, + }) +} + /** * Fire-and-forget reclamation of a deleted column's row storage. The column is * already gone from the schema, so reads never surface the orphaned id — @@ -409,13 +422,7 @@ function stripColumnDataInBackground( perRowMs: 2 * columnIds.length, }) await setTableTxTimeouts(trx, { statementMs }) - await updateTableRowsWithDerivedSecretProvenance(trx, { - rowWhere: and( - eq(userTableRows.tableId, tableId), - eq(userTableRows.workspaceId, workspaceId) - )!, - transformation: { mode: 'remove-columns', columnIds }, - }) + await stripColumnData(trx, tableId, workspaceId, columnIds) }) logger.info( `Background-stripped deleted column data [${columnIds.join(', ')}] from table ${tableId}`, @@ -432,9 +439,9 @@ function stripColumnDataInBackground( } /** - * Deletes a column from a table's schema. When id-keyed, returns once the schema - * is updated and reclaims the column's row-data storage in the background - * (fire-and-forget); the legacy path strips the row key synchronously. + * Deletes a column from a table's schema. Returns once the schema is updated and + * reclaims the column's row-data storage in the background (fire-and-forget), + * unless `stripRowsInTransaction` strips it in the caller's transaction. * * @param data - Delete column data * @param requestId - Request ID for logging @@ -444,7 +451,13 @@ function stripColumnDataInBackground( export async function deleteColumn( data: DeleteColumnData, requestId: string, - options?: ColumnMutationOptions + options?: ColumnMutationOptions & { + /** + * Strip the column's cells in the caller's `tx`, so the row version the commit leaves already + * holds that write (workspace source records it for managed rows, which stay small). + */ + stripRowsInTransaction?: boolean + } ): Promise { const { def, stripKey } = await withLockedTable( data.tableId, @@ -534,9 +547,12 @@ export async function deleteColumn( def.rowCount ?? 0, requestId ) - // Inside a caller's transaction the drop may still roll back, so the cells stay until it commits. - if (options?.tx) options.tx.afterCommit(strip) - else strip() + if (options?.tx && options.stripRowsInTransaction) { + await stripColumnData(options.tx.trx, data.tableId, def.workspaceId, [stripKey]) + } else if (options?.tx) { + // Inside a caller's transaction the drop may still roll back, so the cells stay until it commits. + options.tx.afterCommit(strip) + } else strip() return def } diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index f528ff4d4b3..f9da38924e7 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -1115,4 +1115,47 @@ export default defineTable({ const settled = await apply(workspaceId, result.files, { base: result.base }) expect(settled.applied).toEqual([]) }) + + check( + 'a column the apply drops leaves no row drift: the next plan sees no server change, and a delete of the table is not stale', + async () => { + const workspaceId = await newWorkspace() + const read = await managedPlans(workspaceId) + const tierId = + (await storedTable(workspaceId)).schema.columns.find((column) => column.name === 'tier') + ?.id ?? '' + const dropped = await apply( + workspaceId, + { + 'tables/plans.ts': read.files['tables/plans.ts'].replace( + " tier: column.select({ options: ['basic', 'pro'] }),\n", + '' + ), + 'tables/plans.rows.jsonl': '{"code":"solo","seats":1}\n{"code":"team","seats":10}\n', + }, + { base: read.base, allowDelete: true } + ) + expect(dropped.failed).toBeUndefined() + // Once the dropped column's cells are gone from every row, nothing is left to write them. + const table = await storedTable(workspaceId) + for (let waited = 0; waited < 50; waited++) { + const [{ holding }] = await db + .select({ holding: sql`count(*)::int` }) + .from(userTableRows) + .where(and(eq(userTableRows.tableId, table.id), sql`${userTableRows.data} ? ${tierId}`)) + if (holding === 0) break + await new Promise((resolve) => setTimeout(resolve, 100)) + } + + const planned = await apply(workspaceId, dropped.files, { + base: dropped.base, + dryRun: true, + }) + expect(codes(planned.diagnostics, 'warning')).not.toContain('W_SERVER_CHANGED') + const deleted = await apply(workspaceId, {}, { base: dropped.base, allowDelete: true }) + expect(deleted.applied.map((change) => `${change.action} ${change.key}`)).toEqual([ + 'delete plans', + ]) + } + ) }) From d633eafd2849438de9189b39ca5a8f1e020f2d8e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:07:42 -0800 Subject: [PATCH 09/51] fix(workspace-source): name-taken, table-folder and credential findings say what to do A workflow create refused for a taken name gets its file and a hint, without the use case's 'or in another folder', which a directory cannot express; the workspace guide's exit table names E_NAME_TAKEN. E_TABLE_FOLDER and E_KB_FOLDER name the folders create command, and the credential finding no longer reads 'has no a slack credential'. --- .../application/apply.integration.ts | 9 ++++++++ .../workspace-source/application/errors.ts | 23 ++++++++++++++++--- .../workspace-source/application/findings.ts | 2 +- .../application/kinds/credentials/adapter.ts | 2 +- .../commands/protocol/source-exit-codes.ts | 4 +++- 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/apply.integration.ts b/apps/sim/lib/workspace-source/application/apply.integration.ts index ce7f98db0e2..15c8e169523 100644 --- a/apps/sim/lib/workspace-source/application/apply.integration.ts +++ b/apps/sim/lib/workspace-source/application/apply.integration.ts @@ -2255,6 +2255,15 @@ export default workflow('Probe', {}, () => { key: 'ingest', file: entry, }) + const created = 'workflows/copy/workflow.ts' + const createTaken = await apply( + workspaceId, + { ...read.files, [created]: INGEST.replace("workflow('Ingest'", "workflow('Intake'") }, + { base: read.base, dryRun: true } + ) + expect( + createTaken.diagnostics.find((finding) => finding.code === 'E_NAME_TAKEN') + ).toMatchObject({ kind: 'workflow', key: 'copy', file: created, hint: expect.any(String) }) const result = await apply(workspaceId, renamedTo('Ingest v2'), { base: read.base }) expect(result.failed).toBeUndefined() diff --git a/apps/sim/lib/workspace-source/application/errors.ts b/apps/sim/lib/workspace-source/application/errors.ts index 08589fe64f9..981688a2f9b 100644 --- a/apps/sim/lib/workspace-source/application/errors.ts +++ b/apps/sim/lib/workspace-source/application/errors.ts @@ -1,5 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' +import { SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' import { refusalClass, type WorkspaceKind } from '@sim/workspace-code' +import { workflowFolder } from '@sim/workspace-code/kinds/workflow' import { OrchestrationError } from '@/lib/core/orchestration/types' import { KnowledgeBaseConflictError } from '@/lib/knowledge/service' import { TableRowsChangedError } from '@/lib/table/application/errors' @@ -113,11 +115,26 @@ export function refusalDiagnostic( ) { return finding('E_APPLY_STALE', getErrorMessage(error), { hint: GET_HINT, ...resource }) } + if (error instanceof WorkflowSourceNameTakenError && resource.key !== undefined) { + // Workflow folders are not placed in a folder by source, so the use case's "or in another + // folder" is advice a directory cannot follow. + const file = `${workflowFolder(resource.key)}/${SOURCE_ENTRY_FILE}` + return finding( + 'E_NAME_TAKEN', + `A workflow named ${JSON.stringify(error.workflowName)} already exists in this workspace.`, + { + hint: `Rename the workflow in ${file}, or rename or delete the one that holds the name in Sim.`, + file, + ...resource, + } + ) + } if (error instanceof ResourceKeyTakenError || isNameTaken(error)) { return finding('E_NAME_TAKEN', error.message, { - ...(error instanceof ResourceKeyTakenError - ? { hint: 'Rename the key, or delete the resource that holds it.' } - : {}), + hint: + error instanceof ResourceKeyTakenError + ? 'Rename the key, or delete the resource that holds it.' + : 'Rename it in its declaration, or rename or delete the resource that holds the name in Sim.', ...resource, }) } diff --git a/apps/sim/lib/workspace-source/application/findings.ts b/apps/sim/lib/workspace-source/application/findings.ts index 8f93d232cb8..e1f2f47af61 100644 --- a/apps/sim/lib/workspace-source/application/findings.ts +++ b/apps/sim/lib/workspace-source/application/findings.ts @@ -145,7 +145,7 @@ export function missingFolderFinding( kind === 'table' ? 'E_TABLE_FOLDER' : 'E_KB_FOLDER', `The folder ${folder} does not exist in this workspace.`, { - hint: `Create the folder first, or drop \`folder\` to keep the ${WORKSPACE_KIND_SPECS[kind].noun} at the root.`, + hint: `Create it first (\`sim ${kind === 'table' ? 'tables' : 'knowledge'} folders create "${folder.replace(/^\//, '')}"\`), or drop \`folder\` to keep the ${WORKSPACE_KIND_SPECS[kind].noun} at the root.`, } ) } diff --git a/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts index 9926380a5e4..fc5c2d5b869 100644 --- a/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts @@ -106,7 +106,7 @@ function unavailable( hiddenBinding: boolean ): CredentialChange { const { key } = change - const wanted = `a ${entry.provider} credential${entry.name !== undefined ? ` named "${entry.name}"` : ''}` + const wanted = `${entry.provider} credential${entry.name !== undefined ? ` named "${entry.name}"` : ''}` const ambiguous = candidates.length > 1 const holders = heldOutside .slice(0, NAMED_CREDENTIALS) diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts index 0f7643a9edb..c991a51918f 100644 --- a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts @@ -70,7 +70,9 @@ function sourceExitCodeRows(prefix: SourceCommandGroup): SourceExitCodeRow[] { { code: 1, meaning: - 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed', + prefix === 'sim workspaces source' + ? 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another resource holds (`E_NAME_TAKEN`), a cancelled prompt, or the command failed' + : 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed', next: 'Fix what it printed and run it again', help: { 'workflows source apply': From b978304cfa0cd649e27762411c7eb0e2facbb14c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:08:32 -0800 Subject: [PATCH 10/51] fix(sim-cli): a stale resource is not printed as kept, and promotion drift hints the target The plan line of a resource refused as stale said 'kept as the server has it'; it now says the edit is refused as stale. A promotion's W_SERVER_CHANGED pointed at a get of the directory's own workspace; it now names a get of the target. --- .../protocol/workspace-source-apply.test.ts | 12 ++++++++++++ .../src/commands/protocol/workspace-source-apply.ts | 13 ++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index fffffa36f0c..59905fb7ecb 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -523,8 +523,17 @@ describe('workspaces source apply', () => { { kind: 'table', key: 'leads', action: 'noop' }, { kind: 'table', key: 'prod-only', action: 'delete' }, { kind: 'table', key: 'dropped', action: 'delete' }, + { kind: 'table', key: 'hotfixed', action: 'noop' }, ], diagnostics: [ + { + code: 'W_SERVER_CHANGED', + severity: 'warning', + message: 'tables/hotfixed changed on the server since this directory was read.', + hint: 'Run `sim workspaces source get` into a fresh directory to see what changed.', + kind: 'table', + key: 'hotfixed', + }, { code: 'W_SERVER_DELETED', severity: 'warning', @@ -561,8 +570,11 @@ describe('workspaces source apply', () => { expect(stale.hint).not.toContain('fresh directory') expect(stale.hint).toContain('sim workspaces source get --workspace ws_prod') expect(stale.hint).not.toContain('.sim/targets/ws_prod.json first') + const [drifted] = byCode('W_SERVER_CHANGED') + expect(drifted.hint).toContain('sim workspaces source get --workspace ws_prod') const stderr = vi.mocked(console.error).mock.calls.flat().join('\n') expect(stderr).toContain(' ! gone (deleted on the server; kept as the server has it)') + expect(stderr).toContain(' ! leads (changed on the server; your edit is refused as stale)') expect(stderr).not.toContain('resource(s) that only workspace ws_prod holds') }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 78a7e1549a0..e58762ef35c 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -178,6 +178,11 @@ function printPlan( ) const driftOf = (change: Change) => change.action === 'noop' ? drifted.get(`${change.kind}:${change.key}`) : undefined + const stale = new Set( + diagnostics + .filter((finding) => finding.code === 'E_APPLY_STALE') + .map((finding) => `${finding.kind}:${finding.key}`) + ) for (const kind of APPLY_ORDER) { const changes = plan.filter((change) => change.kind === kind) if (changes.length === 0) continue @@ -197,7 +202,10 @@ function printPlan( for (const change of changes) { const drift = driftOf(change) if (drift) { - printError(` ! ${change.key} (${drift} on the server; kept as the server has it)`) + const fate = stale.has(`${change.kind}:${change.key}`) + ? 'your edit is refused as stale' + : 'kept as the server has it' + printError(` ! ${change.key} (${drift} on the server; ${fate})`) continue } if (change.action === 'noop') continue @@ -505,6 +513,9 @@ function promotionHint(code: string, target: string): string | undefined { if (code === 'E_APPLY_STALE') { return `Workspace ${target} changed this resource since the last promotion from this directory. Carry that change into the directory (\`sim workspaces source get --workspace ${target} --dir \` prints it), or undo it in ${target}, then promote again. Don't delete ${record} to get past this: the next promotion would then overwrite every change made in ${target} since, rows its workflows wrote included.` } + if (code === 'W_SERVER_CHANGED') { + return `Workspace ${target} changed this resource since the last promotion from this directory, and this promotion keeps that change. To carry it into the directory, \`sim workspaces source get --workspace ${target} --dir \` prints it.` + } if (code === 'E_APPLY_NO_BASE') { return `Workspace ${target} holds this resource, but no promotion from this directory recorded it. Rename the declaration, or rename or delete the resource in ${target}, then promote again.` } From 0e73aaf7370e27cc7d380b475eee9ae0302d70be Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:11:12 -0800 Subject: [PATCH 11/51] fix(workspace-source): a real apply lists each finding of a written workflow once A workflow's commit runs its pipeline again and re-reported every finding its plan had found, so a real apply listed each warning twice. A write's finding now replaces the plan's (matched without the block id the commit gave it). --- .../application/apply-workspace-source.ts | 16 +++++++++++++- .../application/apply.integration.ts | 22 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/workspace-source/application/apply-workspace-source.ts b/apps/sim/lib/workspace-source/application/apply-workspace-source.ts index 7b92a8a36b0..3603b2028cc 100644 --- a/apps/sim/lib/workspace-source/application/apply-workspace-source.ts +++ b/apps/sim/lib/workspace-source/application/apply-workspace-source.ts @@ -2,6 +2,7 @@ import { AuditAction, AuditResourceType, withAuditCorrelation } from '@sim/audit import { createLogger } from '@sim/logger' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { stableStringify } from '@sim/workflow-code' import { APPLY_ORDER, type FlatDiagnostic, @@ -396,12 +397,25 @@ async function writeAll( base: WorkspaceBaseV1, result: ApplyWorkspaceSourceResult ): Promise { + // A write re-reports what its plan already found (a workflow's commit runs its pipeline + // again): each finding is listed once, with the ids the write gave it. + const findingKey = (diagnostic: WorkspaceSourceDiagnostic) => + stableStringify({ ...diagnostic, blockId: undefined }) + const reported = new Map(result.diagnostics.map((diagnostic, at) => [findingKey(diagnostic), at])) for (const change of writeOrder(changes)) { try { await lease.confirm() const written = await write(change, writeCtx) Object.assign(result.files, written.text) - result.diagnostics.push(...(written.diagnostics ?? [])) + for (const diagnostic of written.diagnostics ?? []) { + const at = reported.get(findingKey(diagnostic)) + if (at !== undefined) { + result.diagnostics[at] = diagnostic + continue + } + reported.set(findingKey(diagnostic), result.diagnostics.length) + result.diagnostics.push(diagnostic) + } result.applied.push(summarize(change, written.id)) if (written.keyTaken) { // Written but not bound to its key, so the base keeps what it held, and a later write diff --git a/apps/sim/lib/workspace-source/application/apply.integration.ts b/apps/sim/lib/workspace-source/application/apply.integration.ts index 15c8e169523..1d2b441e71e 100644 --- a/apps/sim/lib/workspace-source/application/apply.integration.ts +++ b/apps/sim/lib/workspace-source/application/apply.integration.ts @@ -252,6 +252,28 @@ describe('workspace source apply', () => { } } + check( + 'a real apply reports each finding of a written workflow once, as its dry run does', + async () => { + const workspaceId = await newWorkspace() + const files = { + ...STARTER, + 'workflows/ingest/workflow.ts': INGEST.replace( + 'data: { email: start.email },', + 'data: { email: start.email, note: env.QA_NOT_SET },' + ).replace('import { table,', 'import { env, table,'), + } + const once = (diagnostics: ReadonlyArray<{ code: string; severity: string }>) => + codes(diagnostics, 'warning').filter((code) => code === 'W_ENV_UNKNOWN') + + const planned = await apply(workspaceId, files, { dryRun: true }) + expect(once(planned.diagnostics)).toEqual(['W_ENV_UNKNOWN']) + const written = await apply(workspaceId, files) + expect(written.failed).toBeUndefined() + expect(once(written.diagnostics)).toEqual(['W_ENV_UNKNOWN']) + } + ) + check( 'every audit entry an apply writes carries its apply id, and the next apply has another', async () => { From a53cd3ba4a85d77c1ee42bf0bb065ab83f53250a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:12:07 -0800 Subject: [PATCH 12/51] fix(sim-cli): a workspace get says why it skipped a resource A table over 10,000 rows (or any resource the get cannot read as source) was only counted in the result, with the reason in .sim/skipped.json. The get now prints each skipped resource with its reason. --- .../src/commands/protocol/workspace-source-get.test.ts | 3 +++ .../sim-cli/src/commands/protocol/workspace-source-get.ts | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts index 431278db1aa..fae2e3b2b13 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts @@ -220,6 +220,9 @@ describe('workspaces source get', () => { }, })) await get() + expect(vi.mocked(console.error).mock.calls.flat().join('\n')).toContain( + 'Skipped file big.bin: over the size limit' + ) serve({ 'tables/leads.ts': LEADS }) await get('--only', 'table') diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index 554edda745a..0a25b1fb408 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -374,6 +374,11 @@ export function attachWorkspaceSourceGet(source: Command): void { const result = await writeWorkspaceSource(client, options.dir, data, options.force === true) const findings = findingsOutput(profile.output) findings.print(data.diagnostics) + for (const { kind, key, reason } of data.skipped) { + printError( + `Skipped ${kind} ${key}: ${reason} It is not written here, and an apply keeps it as the server has it (${WORKSPACE_SKIPPED_FILE} lists it).` + ) + } printTreeResult(profile.output, { workspaceId: data.workspaceId, dir: options.dir, From c7ce6bfcd140031db8db72ae9a10a436f945b939 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:13:56 -0800 Subject: [PATCH 13/51] fix(workspace-source): a stale workflow folder gets the hint every other kind's stale finding has The workflow pipeline's E_APPLY_STALE carried its single-directory advice in the message and no hint; a workspace plan now reports it as staleFinding does for every other kind, naming the folder and the get to run. --- .../application/apply.integration.ts | 26 +++++++++++++++++++ .../application/kinds/workflow/adapter.ts | 9 ++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/workspace-source/application/apply.integration.ts b/apps/sim/lib/workspace-source/application/apply.integration.ts index 1d2b441e71e..9b55df147d5 100644 --- a/apps/sim/lib/workspace-source/application/apply.integration.ts +++ b/apps/sim/lib/workspace-source/application/apply.integration.ts @@ -274,6 +274,32 @@ describe('workspace source apply', () => { } ) + check( + 'a workflow edited on both sides is refused as stale with the hint every other kind gives', + async () => { + const workspaceId = await newWorkspace() + await apply(workspaceId, STARTER) + const read = await get(workspaceId) + const entry = 'workflows/ingest/workflow.ts' + const renamedTo = (name: string) => ({ + ...read.files, + [entry]: read.files[entry].replace("'Save Lead'", `'${name}'`), + }) + await apply(workspaceId, renamedTo('Store Lead'), { base: read.base }) + + const stale = await refusal(apply(workspaceId, renamedTo('Keep Lead'), { base: read.base })) + + expect(stale.diagnostics).toEqual([ + expect.objectContaining({ + code: 'E_APPLY_STALE', + kind: 'workflow', + key: 'ingest', + hint: expect.stringContaining('sim workspaces source get'), + }), + ]) + } + ) + check( 'every audit entry an apply writes carries its apply id, and the next apply has another', async () => { diff --git a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts index 1d1f072379f..8b12136e809 100644 --- a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts @@ -83,6 +83,7 @@ import { planUndeclared, refuseDeclaredTwice, serverGoneFinding, + staleFinding, } from '@/lib/workspace-source/application/findings' import { bindStored, claimKey, listUpTo } from '@/lib/workspace-source/application/keys' import { liveTriggerCounts } from '@/lib/workspace-source/application/kinds/deployment/triggers' @@ -434,7 +435,13 @@ async function planUpdate( }, }) const result = summarizePlan(outcome) - change.diagnostics.push(...inFolder(key, result.diagnostics)) + // The pipeline words a stale draft for a single-workflow directory; a workspace names the + // folder and its own next step, as every other kind's stale finding does. + change.diagnostics.push( + ...inFolder(key, result.diagnostics).map((finding) => + finding.code === 'E_APPLY_STALE' ? staleFinding('workflow', key) : finding + ) + ) if (result.workflowName !== undefined && result.diff.meta?.includes('name')) { renamed.set(key, result.workflowName) } From 20fc629245898c95d066efce6bc290e44737853a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:14:38 -0800 Subject: [PATCH 14/51] fix(sim-cli): a delete refused for want of --allow-delete is not printed as unchanged --- .../protocol/workspace-source-apply.test.ts | 27 +++++++++++++++++++ .../protocol/workspace-source-apply.ts | 27 +++++++++++++------ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 59905fb7ecb..7902fbd3cc0 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -430,6 +430,33 @@ describe('workspaces source apply', () => { expect(stderr).toContain('2 change(s) delete data that applying an older directory') }) + it('prints a delete refused for want of --allow-delete as refused, not unchanged', async () => { + request.mockResolvedValue( + answer({ + dryRun: true, + plan: [ + { kind: 'skill', key: 'tone', action: 'noop' }, + { kind: 'skill', key: 'brief', action: 'noop' }, + ], + diagnostics: [ + { + code: 'E_APPLY_DELETE_NOT_ALLOWED', + severity: 'error', + message: 'skills/tone is no longer declared; deleting "tone" needs --allow-delete.', + kind: 'skill', + key: 'tone', + }, + ], + }) + ) + + await apply('--dry-run') + + const stderr = vi.mocked(console.error).mock.calls.flat().join('\n') + expect(stderr).toContain('skills: 1 delete refused, 1 unchanged') + expect(stderr).toContain(' - tone — refused, needs --allow-delete') + }) + it('prints a --to dry run with its findings in the JSON result', async () => { request.mockImplementation(async (path: string, options: { body?: unknown }) => { if (path === '/api/v2/workspaces/ws_prod/source' && options.body === undefined) { diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index e58762ef35c..c455354bac4 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -178,11 +178,16 @@ function printPlan( ) const driftOf = (change: Change) => change.action === 'noop' ? drifted.get(`${change.kind}:${change.key}`) : undefined - const stale = new Set( - diagnostics - .filter((finding) => finding.code === 'E_APPLY_STALE') - .map((finding) => `${finding.kind}:${finding.key}`) - ) + const addressesOf = (code: string) => + new Set( + diagnostics + .filter((finding) => finding.code === code) + .map((finding) => `${finding.kind}:${finding.key}`) + ) + const stale = addressesOf('E_APPLY_STALE') + const deleteRefused = addressesOf('E_APPLY_DELETE_NOT_ALLOWED') + const refusedDelete = (change: Change) => + change.action === 'noop' && deleteRefused.has(`${change.kind}:${change.key}`) for (const kind of APPLY_ORDER) { const changes = plan.filter((change) => change.kind === kind) if (changes.length === 0) continue @@ -191,9 +196,11 @@ function printPlan( const drift = driftOf(change) const label = drift ? `${drift} on the server` - : change.action === 'noop' - ? 'unchanged' - : change.action + : refusedDelete(change) + ? 'delete refused' + : change.action === 'noop' + ? 'unchanged' + : change.action counts.set(label, (counts.get(label) ?? 0) + 1) } printError( @@ -208,6 +215,10 @@ function printPlan( printError(` ! ${change.key} (${drift} on the server; ${fate})`) continue } + if (refusedDelete(change)) { + printError(` - ${change.key} — refused, needs --allow-delete`) + continue + } if (change.action === 'noop') continue const fields = change.fields && change.fields.length > 0 ? ` (${change.fields.join(', ')})` : '' From db74b55975b4652a7a33909d484c6810db601ffc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:31:54 -0800 Subject: [PATCH 15/51] test(workspace-source): wait for the column strip with sleep --- apps/sim/lib/workspace-source/application/rows.integration.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index f9da38924e7..c245411a8b9 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -15,6 +15,7 @@ import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/featur import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' import { tableTriggerMock, tableTriggerMockFns } from '@sim/testing/mocks/table-trigger.mock' +import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { compareStrings } from '@sim/utils/string' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' @@ -1144,7 +1145,7 @@ export default defineTable({ .from(userTableRows) .where(and(eq(userTableRows.tableId, table.id), sql`${userTableRows.data} ? ${tierId}`)) if (holding === 0) break - await new Promise((resolve) => setTimeout(resolve, 100)) + await sleep(100) } const planned = await apply(workspaceId, dropped.files, { From edb89a5c8c6ca8d5ddfd61e62f3fdc211a68efbf Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 12:55:08 -0800 Subject: [PATCH 16/51] fix(workflow-code): tie a falsy-valued variable rename by its moved references --- .../source-pipeline/evaluate/refusals.ts | 2 +- .../workflow-code/src/materialize/identity.ts | 34 ++++++++++++++++++- .../src/materialize/index.test.ts | 33 ++++++++++++++++++ .../workflow-code/src/materialize/index.ts | 32 +++++++++++++++-- 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts index 94d2a521c2b..323edd4ffd7 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts @@ -94,7 +94,7 @@ function variableDeleteRefusal( const [next] = added const rename = removed.length === 1 && added.length === 1 && former.type === next.type - ? `If ${next.name} renames ${former.name}, apply the new key with the value ${former.name} holds first and change the value in a second apply, so the variable keeps its id; or pass --allow-delete (allowDelete=true) to replace it.` + ? `If ${next.name} renames ${former.name}, keep the value ${former.name} holds and move every vars reference to the new key in this apply (change the value in a second one), so the variable keeps its id. A variable no block reads, holding 0, false or an empty value, cannot be tied: pass --allow-delete (allowDelete=true) to replace it.` : undefined return diagnostic( 'E_APPLY_VARIABLE_DELETE', diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index b8a9528b49f..7d325c35052 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -8,7 +8,13 @@ import type { MaterializeDiagnosticCode } from '../diagnostics/codes' import { diagnostic } from '../diagnostics/factory' import type { Diagnostic } from '../diagnostics/frame' import { pointer, type WorkflowLockBlock, type WorkflowLockV1 } from '../lock' -import type { BlockKey, CanonicalBlock, CanonicalWorkflow } from '../model' +import { + type BlockKey, + type CanonicalBlock, + type CanonicalWorkflow, + isSimNode, + type VarKey, +} from '../model' import { deepEqual, getOwn } from '../records' import type { MaterializeHooks } from './context' @@ -427,6 +433,32 @@ function referencesKey(block: CanonicalBlock, key: BlockKey): boolean { return false } +/** + * Whether a block reads variable `key`: a `` reference in any field a reference can + * sit in, or a variable node (a Variables block's assignment target). + */ +export function referencesVariable(block: CanonicalBlock, key: VarKey): boolean { + const reads = (value: unknown): boolean => { + if (typeof value === 'string') { + return findWorkflowReferenceTokens(value).some((token) => { + if (token.kind !== 'workflow') return false + const [head, name] = parseStoredPath(token.value.slice(1, -1)) ?? [] + return ( + head?.kind === 'name' && + head.name === 'variable' && + name?.kind === 'name' && + normalizeWorkflowBlockName(name.name) === key + ) + }) + } + if (Array.isArray(value)) return value.some(reads) + if (value === null || typeof value !== 'object') return false + if (isSimNode(value)) return value.$sim === 'var' && value.key === key + return Object.values(value).some(reads) + } + return reads([block.params, block.branches, block.subflow]) +} + /** * The strong identity signal that ties a new block to a vacated stored one, if any: `settings`, * equal settings (references to the old key read as the new); `connections`, most of their diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 7828aa36e89..14f14365258 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -975,6 +975,39 @@ describe('materialize: identity', () => { } }) + it.each([ + ['number', 0], + ['boolean', false], + ['plain', ''], + ] as const)( + 'keeps the id of a variable holding %s %j renamed with every reference to it', + (type, value) => { + const pulled = workflowState( + [ + blockState('a1', 'function', 'Read', { code: 'return + 1' }), + blockState('a2', 'function', 'Other', { code: 'return 2' }), + ], + { variables: { 'var-1': { id: 'var-1', name: 'Retries', type, value } } } + ) + const rename = (code: string, renamedValue: number | boolean | string = value) => + edit(canonical(pulled).workflow, (draft) => { + draft.variables = { + attempts: { key: 'attempts', name: 'Attempts', type, value: renamedValue }, + } + draft.blocks.read.params.code = { kind: 'value', value: code } + }) + const ids = (workflow: CanonicalWorkflow) => + Object.keys(run(workflow, null, { prev: pulled }).state.variables ?? {}) + + expect(ids(rename('return + 1'))).toEqual(['var-1']) + // A reference left behind, or a changed value, ties nothing. + expect(ids(rename('return 1'))).toEqual(['new-1']) + expect(ids(rename('return + 1', type === 'number' ? 5 : 'x'))).toEqual([ + 'new-1', + ]) + } + ) + it('leaves an inputFormat id two fields share out of the identity items', () => { const prev = workflowState([ blockState('start', 'start_trigger', 'Start', { diff --git a/packages/workflow-code/src/materialize/index.ts b/packages/workflow-code/src/materialize/index.ts index 8f1d8884a6a..0043bb4aca6 100644 --- a/packages/workflow-code/src/materialize/index.ts +++ b/packages/workflow-code/src/materialize/index.ts @@ -25,7 +25,12 @@ import { deepEqual, getOwn, invertRecord, setOwn } from '../records' import { type BuiltBlock, buildBlock } from './block' import type { MaterializeContext, MaterializeHooks } from './context' import { type BranchHandles, writeEdges } from './edges' -import { type IdentityEntry, type LockFreshness, resolveIdentity } from './identity' +import { + type IdentityEntry, + type LockFreshness, + referencesVariable, + resolveIdentity, +} from './identity' import { restorePreserved } from './preserve' import { sharesEvidence, soleRename } from './rename' import { loopEntry, parallelEntry } from './subflows' @@ -120,13 +125,36 @@ function renamedVariable( former !== undefined && variable !== undefined && former.type === variable.type && - sharesEvidence(former, variable, ['value']) + (sharesEvidence(former, variable, ['value']) || + (deepEqual(former.value, variable.value) && + referencesFollow(prevCanonical.workflow, workflow, formerKey, key))) ) } ) return renamed && { key: renamed.item, id: renamed.id } } +/** + * Whether every block that read variable `formerKey` now reads `key` instead: the evidence a + * rename carries when the value itself says nothing (`0`, `false`, `''`), as it does for counters + * and flags. + */ +function referencesFollow( + prev: CanonicalWorkflow, + workflow: CanonicalWorkflow, + formerKey: VarKey, + key: VarKey +): boolean { + const readers = Object.values(prev.blocks).filter((block) => referencesVariable(block, formerKey)) + return ( + readers.length > 0 && + readers.every((block) => { + const now = getOwn(workflow.blocks, block.key) + return now !== undefined && referencesVariable(now, key) + }) + ) +} + /** * Each matched block with the preserved UI state of the stored block it continues * (`restorePreserved`): source holds what executes, and the rest comes from prev. From fc06cc0cbb201e69e87a9d2ef776c53c60f65015 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 12:58:36 -0800 Subject: [PATCH 17/51] fix(sim-cli): prune only function files the CLI recorded writing --- .../src/commands/protocol/source-dir/layout.ts | 18 ++++++++++++------ .../commands/protocol/source-dir/write-back.ts | 8 +++++++- .../commands/protocol/source-dir/written.ts | 5 +++-- .../protocol/workflow-source-roundtrip.test.ts | 18 ++++++++++++++++++ .../protocol/workspace-source-apply.test.ts | 17 +++++++++++++++-- .../protocol/workspace-source-apply.ts | 17 ++++++++++++++--- 6 files changed, 69 insertions(+), 14 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-dir/layout.ts b/packages/sim-cli/src/commands/protocol/source-dir/layout.ts index 82966b52d6e..d86e42fac9b 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/layout.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/layout.ts @@ -122,20 +122,26 @@ export function lockCodeFiles(lock: unknown): Set { } /** - * The function files a write leaves behind: named by the lock the directory held (so the CLI - * wrote them), no longer named by the new lock, and not being written now. A file the person - * added and no lock ever named is never pruned, nor is an entry that is no code file (such as - * `functions/`), whose removal would take the whole folder. + * The function files a write leaves behind: named by the lock the directory held, recorded as + * written by the CLI (`recorded`, the written record's paths relative to the same directory), no + * longer named by the new lock, and not being written now. A file the CLI never wrote is never + * pruned, even when a hand-edited lock names it (say as a trigger's `codeFile`), nor is an entry + * that is no code file (such as `functions/`), whose removal would take the whole folder. */ export function orphanedCodeFiles( previousLock: unknown, nextLock: unknown, - written: Iterable + written: Iterable, + recorded: Readonly> ): string[] { const kept = lockCodeFiles(nextLock) for (const path of written) kept.add(path) return [...lockCodeFiles(previousLock)].filter( - (path) => path.startsWith(`${SOURCE_FUNCTIONS_DIR}/`) && isCodeFilePath(path) && !kept.has(path) + (path) => + path.startsWith(`${SOURCE_FUNCTIONS_DIR}/`) && + isCodeFilePath(path) && + Object.hasOwn(recorded, path) && + !kept.has(path) ) } diff --git a/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts b/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts index 53c21b97b4b..bba48bcac50 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts @@ -27,6 +27,7 @@ import { digest, inWorkspace, type PrintedSourceAnswer, + readFolderWritten, readWritten, writtenFiles, writtenRecord, @@ -94,7 +95,12 @@ export async function writeBack( [SOURCE_BASE_FILE, `${JSON.stringify(answer.base, null, 2)}\n`], ] const write = [...printed, ...(folder ? [] : writtenFiles(writtenRecord(files))), ...extra.write] - const orphaned = orphanedCodeFiles(previousLock, answer.lock, Object.keys(answer.files)) + const orphaned = orphanedCodeFiles( + previousLock, + answer.lock, + Object.keys(answer.files), + (await readFolderWritten(dir)).digests + ) const unnamed = await unnamedCodeFiles(dir, previousLock, answer, orphaned) await updateSourceDir(dir, { write, diff --git a/packages/sim-cli/src/commands/protocol/source-dir/written.ts b/packages/sim-cli/src/commands/protocol/source-dir/written.ts index 76b8de80b0a..4823a24b9cc 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/written.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/written.ts @@ -214,10 +214,11 @@ export async function locallyEditedFiles( previousLock: unknown ): Promise { const touched = new Map(Object.entries(answer.files)) - for (const path of orphanedCodeFiles(previousLock, answer.lock, touched.keys())) { + const { digests } = await readFolderWritten(dir) + for (const path of orphanedCodeFiles(previousLock, answer.lock, touched.keys(), digests)) { touched.set(path, null) } - return editedFiles(dir, touched, (await readFolderWritten(dir)).digests) + return editedFiles(dir, touched, digests) } /** `path` of the workflow folder `key`, relative to its workspace directory. */ diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 1fabd5501e3..3c2e9ac19ee 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -617,6 +617,24 @@ describe('pruning function files', () => { expect(stderr.join('\n')).not.toContain('functions/score.js is no longer named') }) + it('never prunes a file of the author a hand-edited lock gives to a block', async () => { + await get() + writeFileSync(join(dir, 'functions/mine.js'), 'mine') + const handEdited = JSON.parse(read('.sim/lock.json')) + handEdited.blocks.start = { + id: 'b-start', + parent: null, + ident: 'start', + order: 9, + codeFile: 'functions/mine.js', + } + writeFileSync(join(dir, '.sim/lock.json'), JSON.stringify(handEdited)) + + await apply() + + expect(read('functions/mine.js')).toBe('mine') + }) + it('prunes, on a fresh get, a function file the previous get wrote', async () => { server.source = () => json(sourceResponse(twoFiles, twoLock)) await get() diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 7902fbd3cc0..996670d70e6 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -738,13 +738,26 @@ describe('workspaces source apply', () => { expect(Object.keys(bodies()[1].files)).not.toContain('tools/lookup.js') }) - it('removes the function files an applied workflow no longer uses, keeping files no lock named', async () => { + it('removes the function files an applied workflow no longer uses, keeping files the CLI never wrote', async () => { const lock = (codeFiles: string[]) => `${JSON.stringify({ version: 1, blocks: Object.fromEntries(codeFiles.map((file, index) => [`b${index}`, { id: `blk_${index}`, codeFile: file }])), variables: {} })}\n` - write('workflows/ingest/.sim/lock.json', lock(['functions/score.js', 'functions/old.js'])) + // A hand-edited lock names mine.js too; the CLI never wrote it, so it is never pruned. + write( + 'workflows/ingest/.sim/lock.json', + lock(['functions/score.js', 'functions/old.js', 'functions/mine.js']) + ) write('workflows/ingest/functions/score.js', 'return 1\n') write('workflows/ingest/functions/old.js', 'return 2\n') write('workflows/ingest/functions/mine.js', 'return 3\n') + write( + '.sim/written.json', + JSON.stringify({ + files: { + 'workflows/ingest/functions/score.js': digest('return 1\n'), + 'workflows/ingest/functions/old.js': digest('return 2\n'), + }, + }) + ) request.mockResolvedValue( answer({ applied: [{ kind: 'workflow', key: 'ingest', action: 'update', id: 'wf_1' }], diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index c455354bac4..7bb267a5842 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -270,7 +270,11 @@ const WORKFLOW_LOCK_PATH = new RegExp( * a folder whose new lock does not read as one: without it nothing tells a dropped file from a * kept one. */ -async function orphanedWorkflowFiles(dir: string, files: Record) { +async function orphanedWorkflowFiles( + dir: string, + files: Record, + recorded: Written['digests'] +) { const filesByFolder = new Map() for (const path of Object.keys(files)) { if (!path.startsWith(`${KIND_DIRS.workflow}/`)) continue @@ -292,7 +296,14 @@ async function orphanedWorkflowFiles(dir: string, files: Record) } catch {} if (!isRecordLike(next) || !isRecordLike(next.blocks)) continue const previous = await readSourceJson(dir, path) - orphaned.push(...orphanedCodeFiles(previous, next, written).map((file) => `${folder}/${file}`)) + const inFolder = Object.fromEntries( + Object.entries(recorded).flatMap(([recordedPath, sha]) => + recordedPath.startsWith(`${folder}/`) ? [[recordedPath.slice(folder.length + 1), sha]] : [] + ) + ) + orphaned.push( + ...orphanedCodeFiles(previous, next, written, inFolder).map((file) => `${folder}/${file}`) + ) } return orphaned } @@ -385,7 +396,7 @@ async function saveApplied( ): Promise { assertWorkspacePaths(Object.keys(data.files)) const orphaned = [ - ...(await orphanedWorkflowFiles(dir, data.files)), + ...(await orphanedWorkflowFiles(dir, data.files, written.digests)), ...droppedResourceFiles(submission.files, data.files), ] const kept = { ...written.digests, ...appliedBlobDigests(submission, data.base) } From bfb0295ef823353e06567b47168a0a662e9414e9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 12:59:47 -0800 Subject: [PATCH 18/51] fix(sim-cli): status leaves out function files no codeFile() names and drops the stale plan line --- .../src/commands/protocol/source-status.ts | 9 ++++++ .../src/commands/protocol/workflow-dir.ts | 4 +-- .../protocol/workflow-source-status.test.ts | 30 +++++++++++++++++-- .../protocol/workflow-source-status.ts | 27 ++++++++++------- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-status.ts b/packages/sim-cli/src/commands/protocol/source-status.ts index efc6083e713..6a876e1efec 100644 --- a/packages/sim-cli/src/commands/protocol/source-status.ts +++ b/packages/sim-cli/src/commands/protocol/source-status.ts @@ -61,6 +61,8 @@ export interface StatusReport extends SourceStatus { server: ServerCheck /** Why the server was not checked, or what it left unchecked. */ note?: string + /** Files in the directory no source names, which applying never sends. */ + unnamed?: string[] plan: Plan | null next: NextStep | null } @@ -192,6 +194,13 @@ export function printStatusReport( } else { lines.push(s.yellow(`Server not checked${report.note ? `: ${sanitize(report.note)}` : '.'}`)) } + if (report.unnamed?.length) { + lines.push( + s.dim( + `Not named by any codeFile(), so applying never sends ${report.unnamed.length === 1 ? 'it' : 'them'}: ${sanitize(report.unnamed.join(', '))}` + ) + ) + } if (report.server === 'checked' && report.note) lines.push(s.dim(sanitize(report.note))) if (report.plan) lines.push(planLine(report.plan)) if (report.next) { diff --git a/packages/sim-cli/src/commands/protocol/workflow-dir.ts b/packages/sim-cli/src/commands/protocol/workflow-dir.ts index edfe121dbe4..34ff9c012c4 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-dir.ts @@ -93,9 +93,9 @@ async function readJson(directory: string, path: string, hint: string): Promise< * reference to any other file never uploads it; the server reports a referenced file that is * missing or of another kind. */ -function codeFilePaths(entry: string, lock: WorkflowLockV1 | null): string[] { +export function codeFilePaths(entry: string | null, lock: unknown): string[] { const candidates = lockCodeFiles(lock) - for (const literal of codeFileReferences(entry)) { + for (const literal of entry === null ? [] : codeFileReferences(entry)) { const path = sourceRelativePath(literal) if (path) candidates.add(path) } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts index e491378fbfa..22cc3d3861b 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts @@ -179,7 +179,7 @@ describe('workflows source status', () => { await run(root, 'status') const { frame: _frame, ...listed } = warning - expect(printed()).toMatchObject({ plan: { warnings: 1 }, diagnostics: [listed] }) + expect(printed()).toMatchObject({ plan: null, diagnostics: [listed] }) contextMockState.output = 'text' mockProfileFrom.mockReturnValue({ @@ -217,7 +217,11 @@ describe('workflows source status', () => { }) it('reports an edit, a deleted function file and a new one, exit 11', async () => { - write(root, 'workflow.ts', `${ENTRY}// edited\n`) + write( + root, + 'workflow.ts', + ENTRY.replace('})\n})', "})\n fn('Rank', { code: codeFile('./functions/rank.js') })\n})") + ) unlinkSync(join(root, 'functions/score.js')) write(root, 'functions/rank.js', 'return 2\n') mockRequest.mockResolvedValue(dryRun([], 1)) @@ -241,6 +245,25 @@ describe('workflows source status', () => { expect(process.exitCode).toBe(11) }) + it('leaves out a function file no codeFile() names, which applying never sends, and says so', async () => { + write(root, 'functions/notes.js', 'return 3\n') + mockRequest.mockResolvedValue(dryRun()) + + await run(root, 'status') + + expect(printed()).toMatchObject({ + state: 'clean', + local: [], + unnamed: ['functions/notes.js'], + next: null, + }) + expect(process.exitCode ?? 0).toBe(0) + + vi.mocked(console.log).mockClear() + await run(root, 'diff', '--server') + expect(printed()).toMatchObject({ files: [] }) + }) + it('reports drift when the workflow changed on the server and nothing here, exit 12', async () => { mockRequest.mockResolvedValue(dryRun([STALE])) @@ -249,7 +272,8 @@ describe('workflows source status', () => { expect(printed()).toMatchObject({ state: 'drift', drift: [{ kind: 'workflow', key: WORKFLOW_ID, server: 'changed' }], - plan: { errors: 0 }, + // Applying is refused as stale, so no plan of what it would change is shown. + plan: null, next: { command: `sim workflows source get ${WORKFLOW_ID} --dir ${root}` }, }) expect(process.exitCode).toBe(12) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts index 1656c0f2dba..826ce9b7740 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts @@ -27,6 +27,7 @@ import { digest, listSourceFiles, readFolderWritten, + readPreviousLock, readSourceText, type Written, } from './source-dir' @@ -51,7 +52,7 @@ import { } from './source-status' import { refreshWorkflowWorkspaceTypes, withSourceExitCode } from './source-tree' import type { Diagnostic } from './source-ui' -import { readBase, readLock, readSourceFiles } from './workflow-dir' +import { codeFilePaths, readBase, readLock, readSourceFiles } from './workflow-dir' type PlanData = PlanWorkflowSourceResponse['data'] @@ -62,23 +63,24 @@ interface LocalWorkflow { base: Awaited> workflowId: string written: Written - /** `workflow.ts` and every file under `functions/`, by path. */ + /** `workflow.ts` and every file under `functions/` an apply sends, by path. */ texts: Record changed: ChangedFile[] + /** Files under `functions/` that neither `workflow.ts` nor the lock names: an apply never sends them. */ + unnamed: string[] } async function readLocalWorkflow(dir: string): Promise { const base = await readBase(dir, undefined) const { workflowId } = base const written = await readFolderWritten(dir) - const paths = [ - SOURCE_ENTRY_FILE, - ...(await listSourceFiles(dir, SOURCE_CODE_DIR)).filter( - (path) => !path.split('/').some((segment) => segment.startsWith('.')) - ), - ] - const texts: Record = {} - for (const path of paths) { + const entry = await readSourceText(dir, SOURCE_ENTRY_FILE) + const sent = new Set(codeFilePaths(entry, await readPreviousLock(dir))) + const listed = (await listSourceFiles(dir, SOURCE_CODE_DIR)).filter( + (path) => !path.split('/').some((segment) => segment.startsWith('.')) + ) + const texts: Record = entry === null ? {} : { [SOURCE_ENTRY_FILE]: entry } + for (const path of listed.filter((each) => sent.has(each))) { const text = await readSourceText(dir, path) if (text !== null) texts[path] = text } @@ -91,6 +93,7 @@ async function readLocalWorkflow(dir: string): Promise { written, texts, changed: changedFiles(current, written.digests, () => true), + unnamed: listed.filter((path) => !sent.has(path)), } } @@ -216,13 +219,15 @@ export function attachWorkflowSourceStatus(source: Command): void { } } const status = sourceStatus(locals, serverChanges) - const plan = data ? planSummary(data) : null + // A plan against a base the server moved past says nothing: applying it is refused as stale. + const plan = data && serverChanges.length === 0 ? planSummary(data) : null const report: StatusReport & { workflowId: string } = { workflowId, dir: options.dir, server, ...(note ? { note } : {}), ...status, + ...(local.unnamed.length > 0 ? { unnamed: local.unnamed } : {}), plan, next: nextStep(status, { prefix: PREFIX, From 5a106b96dfecb9c0d0274fe5a783b6e1d6ead9ae Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:01:06 -0800 Subject: [PATCH 19/51] fix(workflow-code): lay out a block moved into or out of a loop or parallel --- packages/workflow-code/src/compile/SYNTAX.md | 2 +- .../workflow-code/src/materialize/block.ts | 12 +++++--- .../src/materialize/index.test.ts | 29 +++++++++++++++++++ .../workflow-code/src/materialize/index.ts | 2 +- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index 56ed953e179..17e87e3e5e1 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -496,7 +496,7 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `W_MAT_INFERRED_RENAME` | A rename inferred from type, parent and edges (the key and the ident both changed). | | `W_MAT_RETYPE_RECREATED` | A key kept with a new type: delete plus create, so webhook URLs and schedules change. | | `W_MAT_LOCK_ID_CLAIMED` | A lock id another workflow holds; a new id was minted. | -| `I_MAT_NEEDS_LAYOUT` | A new block with no position; placed by autolayout. | +| `I_MAT_NEEDS_LAYOUT` | A new block with no position, or one moved into or out of a loop or parallel; placed by autolayout. | **Apply** diff --git a/packages/workflow-code/src/materialize/block.ts b/packages/workflow-code/src/materialize/block.ts index c07f386f662..951de5e8288 100644 --- a/packages/workflow-code/src/materialize/block.ts +++ b/packages/workflow-code/src/materialize/block.ts @@ -274,7 +274,8 @@ export function buildBlock(inputs: BlockInputs, context: MaterializeContext): Bu } } - if (isNew || inputs.parentId !== prev?.parentId) { + const reparented = isNew || inputs.parentId !== prev?.parentId + if (reparented) { if (inputs.parentId !== null) { data.parentId = inputs.parentId data.extent = 'parent' @@ -304,9 +305,12 @@ export function buildBlock(inputs: BlockInputs, context: MaterializeContext): Bu let built = block let needsLayout = false - if (isNew) { - if (inputs.layout) built = applyLayout(block, inputs.layout) - else needsLayout = true + if (isNew && inputs.layout) built = applyLayout(block, inputs.layout) + else if (reparented) { + // A child's position is relative to its container, so a move into or out of one keeps a + // position from the wrong frame: send it to autolayout, as the server does an unplaced block. + if (!isNew) built = { ...block, position: { x: 0, y: 0 } } + needsLayout = true } const items: Record = {} diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 14f14365258..5c1df6fdebc 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -729,6 +729,35 @@ describe('materialize: identity', () => { expect(codes(result)).toContain('I_MAT_NEEDS_LAYOUT') }) + it('sends a block moved into or out of a container to autolayout: its position was relative to the old one', () => { + const prev = workflowState( + [ + blockState('l1', 'loop', 'Each', {}, { data: { loopType: 'for', count: 2 } }), + blockState( + 'f1', + 'function', + 'Inner', + { code: 'x' }, + { data: { parentId: 'l1', extent: 'parent' } } + ), + blockState('f2', 'function', 'Outer', { code: 'y' }, { position: { x: 900, y: 400 } }), + ], + { loops: { l1: { id: 'l1', nodes: ['f1'], iterations: 2, loopType: 'for', enabled: true } } } + ) + const lock = lockFor(prev) + const target = edit(canonical(prev, lock).workflow, (draft) => { + draft.blocks.inner.parent = null + draft.blocks.outer.parent = 'each' + }) + + const result = run(target, lock, { prev }) + + expect(result.state.blocks.f1.position).toEqual({ x: 0, y: 0 }) + expect(result.state.blocks.f2.position).toEqual({ x: 0, y: 0 }) + expect([...result.needsLayout].sort()).toEqual(['inner', 'outer']) + expect(result.identity.blocks.inner.id).toBe('f1') + }) + it('recreates a block whose type changed under the same key, with a warning', () => { const prev = base() const target = edit(canonical(prev).workflow, (draft) => { diff --git a/packages/workflow-code/src/materialize/index.ts b/packages/workflow-code/src/materialize/index.ts index 0043bb4aca6..86349e5be13 100644 --- a/packages/workflow-code/src/materialize/index.ts +++ b/packages/workflow-code/src/materialize/index.ts @@ -340,7 +340,7 @@ export function materialize( context.diagnostics.push( diagnostic( 'I_MAT_NEEDS_LAYOUT', - `Block "${key}" has no stored or lock position; it is placed by autolayout`, + `Block "${key}" has no position in its container yet; it is placed by autolayout`, { blockKey: key, } From 9f5a412829a929dcd8697b519a3f8d9d9becf3a5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:09:07 -0800 Subject: [PATCH 20/51] fix(workflows): word the variable rename hint so its first step can work --- .../workflows/application/source-pipeline/evaluate/refusals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts index 323edd4ffd7..d0e97ebff3c 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts @@ -94,7 +94,7 @@ function variableDeleteRefusal( const [next] = added const rename = removed.length === 1 && added.length === 1 && former.type === next.type - ? `If ${next.name} renames ${former.name}, keep the value ${former.name} holds and move every vars reference to the new key in this apply (change the value in a second one), so the variable keeps its id. A variable no block reads, holding 0, false or an empty value, cannot be tied: pass --allow-delete (allowDelete=true) to replace it.` + ? `If ${next.name} renames ${former.name}, keep the value ${former.name} holds and move every vars reference to the new key in this apply, then change the value in a second apply, so the variable keeps its id; a variable no block reads that holds 0, false or an empty value cannot be tied, so pass --allow-delete (allowDelete=true) to replace it.` : undefined return diagnostic( 'E_APPLY_VARIABLE_DELETE', From 2c839c63e65a585885f28d30002323807ae2f41c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:09:09 -0800 Subject: [PATCH 21/51] fix(workflow-code): refuse an unknown dropdown option on an edited block in check and apply alike --- .../apply-workflow-source.integration.ts | 25 +++++++++++++-- .../source-pipeline/evaluate/finalize.ts | 31 ++++++++++++++----- .../protocol/workflow-source-check.test.ts | 20 ++++++++++-- .../workflow-code/src/diagnostics/refusals.ts | 23 ++++++++++++++ 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index 8f72c8c20cb..051fda6973b 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -1542,12 +1542,31 @@ describe('source apply edge cases', () => { 'E_APPLY_INVALID_SUBBLOCK_FIELD' ) - expect( - error.diagnostics.find((d) => d.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD') - ).toMatchObject({ blockId: score.id, param: 'language' }) + const refusals = error.diagnostics.filter((d) => d.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD') + expect(refusals).toHaveLength(1) + expect(refusals[0]).toMatchObject({ blockId: score.id, param: 'language' }) + expect(error.diagnostics.map((d) => d.code)).not.toContain('W_UNKNOWN_OPTION') expect(await snapshot(workflowId)).toEqual(before) }) + check( + 'an edited block keeps a value outside its options the stored block already holds', + async () => { + const start = seedBlock('start_trigger', 'Start') + const score = seedBlock('function', 'Score', { + code: ['code', 'return 1'], + language: ['dropdown', 'cobol'], + }) + const workflowId = await seedGraph([start, score]) + const source = await getInlineSource(workflowId) + + const result = await apply(source, edit(source.files, 'return 1', 'return 2')) + + expect(source.files['workflow.ts']).toContain("language: raw('cobol')") + expect(result.applied).toBe(true) + } + ) + check( 'a value the field normalizes is stored normalized, and untouched blocks keep their bytes', async () => { diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts index 6ede61f3432..62b726fd821 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts @@ -202,16 +202,31 @@ export async function finalizePlan( pending, keyOf: keyOfId, }) + // A field the compile already refused (with its span and options) needs no second refusal from + // the value gate. + const refusedFields = new Set( + compileDiagnostics.flatMap((finding) => + finding.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD' + ? [`${finding.blockKey}\u0000${finding.param}`] + : [] + ) + ) + const refusedAgain = (finding: { code: string; blockId?: string; param?: string }) => + finding.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD' && + finding.blockId !== undefined && + refusedFields.has(`${keyOfId(finding.blockId)}\u0000${finding.param}`) const diagnostics = [ ...compileDiagnostics, - ...resolved.keptDiagnostics, - ...resolved.materializeDiagnostics, - ...refusals.sourceRefusals, - ...refusals.renameWarnings, - ...gated.gateFindings, - ...normalized.diagnostics, - ...foreignIds, - ...lintDiagnostics(compiledLint, changedIds, keyOfId), + ...[ + ...resolved.keptDiagnostics, + ...resolved.materializeDiagnostics, + ...refusals.sourceRefusals, + ...refusals.renameWarnings, + ...gated.gateFindings, + ...normalized.diagnostics, + ...foreignIds, + ...lintDiagnostics(compiledLint, changedIds, keyOfId), + ].filter((finding) => !refusedAgain(finding)), ].map(locate) if (hasErrors(diagnostics)) { return { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index b26d38e116f..a4c24b2c76e 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -481,7 +481,10 @@ describe('sim workflows source check', () => { writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) await check() - expect(findingLines()).toMatch(/error W_UNKNOWN_OPTION/) + expect(findingLines()).toMatch( + /error E_APPLY_INVALID_SUBBLOCK_FIELD: method has no option FETCH\./ + ) + expect(findingLines()).not.toContain('kept as written') expect(process.exitCode).toBe(1) process.exitCode = undefined @@ -974,12 +977,25 @@ export default workflow('W', {}, () => { expect(process.exitCode).toBe(1) process.exitCode = undefined - writeLock([]) await checkIn(dir, '--no-strict') expect(result()).toMatchObject({ errors: 0, warnings: 1 }) expect(process.exitCode).toBeUndefined() }) + it('fails with --no-strict on a dropdown value a new block sets, as the server refuses it', async () => { + cacheCatalog() + writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) + writeLock([]) + + await checkIn(dir, '--no-strict') + + expect(result()).toMatchObject({ errors: 1, warnings: 0 }) + expect(findingLines()).toMatch( + /error E_APPLY_INVALID_SUBBLOCK_FIELD: method has no option FETCH\./ + ) + expect(process.exitCode).toBe(1) + }) + it('passes what a get prints from stored state the server keeps: a loop error edge, a cycle, an unwired branch', async () => { cacheCatalog() writeFileSync(join(dir, 'workflow.ts'), STORED_SHAPES) diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index 90c6256d488..c082e7cee2d 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -65,6 +65,7 @@ export type EditRefusalCode = | 'E_APPLY_UNRESOLVED_REF' | 'E_APPLY_REF_OUT_OF_SCOPE' | 'E_APPLY_CODE_FILE_REF_SYNTAX' + | 'E_APPLY_INVALID_SUBBLOCK_FIELD' /** What an edit changed, as the caller can tell it. */ export interface EditedBlocks { @@ -86,6 +87,17 @@ const CHANGED_BLOCK_REFUSALS: ReadonlyMap = new Map([ ['W_REF_OUT_OF_SCOPE', 'E_APPLY_REF_OUT_OF_SCOPE'], ]) +/** + * In `workflow.ts` on a block the edit changed: a dropdown value none of its options names, which + * the server's field gate refuses. A value the stored block already holds outside its options + * prints as `raw()`, which the compile does not check, so only a value the author wrote is + * refused. `operation` is left out: an unknown one is already an error, and its warning means the + * stored block holds a retired operation, which the gate keeps. + */ +const PARAM_REFUSALS: ReadonlyMap = new Map([ + ['W_UNKNOWN_OPTION', 'E_APPLY_INVALID_SUBBLOCK_FIELD'], +]) + /** In `workflow.ts` on a block the edit changed: a path no output of the block has. */ const ENTRY_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OUTPUT_PATH', 'E_APPLY_UNRESOLVED_REF'], @@ -108,6 +120,7 @@ const CODE_FILE_REFUSALS: ReadonlyMap = new Map([ * refusal. */ export const REFUSABLE_WARNING_CODES: ReadonlySet = new Set([ + 'W_UNKNOWN_OPTION', 'W_REF_OUT_OF_SCOPE', 'W_UNKNOWN_OUTPUT_PATH', 'W_UNRESOLVED_REF', @@ -125,6 +138,11 @@ export const EDIT_REFUSALS: ReadonlyArray<{ refusal, where: 'changed block' as const, })), + ...[...PARAM_REFUSALS].map(([warning, refusal]) => ({ + warning, + refusal, + where: 'workflow.ts' as const, + })), ...[...ENTRY_REFUSALS].map(([warning, refusal]) => ({ warning, refusal, @@ -158,6 +176,11 @@ function refusalOf(finding: LocatedDiagnostic, edit: EditedBlocks): EditRefusalC const edited = edit.changed(blockKey) || (!inEntryFile && edit.codeEdited(blockKey, span.file)) return edited ? onChanged : undefined } + const onParam = PARAM_REFUSALS.get(finding.code) + if (onParam) { + if (!inEntryFile || finding.param === 'operation') return undefined + return edit.changed(blockKey) ? onParam : undefined + } if (inEntryFile) { const inEntry = ENTRY_REFUSALS.get(finding.code) return inEntry && edit.changed(blockKey) ? inEntry : undefined From e4394ecea6f2f968950e807e1599cf046e6bfbfc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:10:07 -0800 Subject: [PATCH 22/51] fix(sim-cli): release the directory lock when Ctrl-C ends a deferred write --- .../src/commands/protocol/source-apply-interrupt.test.ts | 4 +++- .../sim-cli/src/commands/protocol/source-dir/command-lock.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts index ee5a49fda00..825607bb8e6 100644 --- a/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts +++ b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts @@ -1,5 +1,5 @@ import { spawn } from 'node:child_process' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import type { AddressInfo } from 'node:net' import { tmpdir } from 'node:os' @@ -143,6 +143,7 @@ describe('Ctrl-C while a workflow apply is committing', () => { expect(run.stderr).toContain('the apply was already sent') expect(savedFingerprint()).toBe('sf1:b') expect(readFileSync(join(dir, 'src/workflow.ts'), 'utf8')).toContain('// stored') + expect(existsSync(join(dir, 'src/.sim/command.lock'))).toBe(false) }, 30_000) it('stops at a second Ctrl-C and says the apply may have been saved and how to recover', async () => { @@ -154,5 +155,6 @@ describe('Ctrl-C while a workflow apply is committing', () => { expect(run.stderr).toContain('Stopped before the apply answered. It may have been saved.') expect(run.stderr).toContain(`sim workflows source get ${WORKFLOW_ID}`) expect(savedFingerprint()).toBe('sf1:a') + expect(existsSync(join(dir, 'src/.sim/command.lock'))).toBe(false) }, 30_000) }) diff --git a/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts b/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts index 3716ccc37b0..1550fe498dc 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts @@ -189,10 +189,12 @@ export async function withSourceDirLock( await writeFile(join(lock, LOCK_OWNER_FILE), JSON.stringify(owner)) return await run() } finally { - disposeSignalRelease() + // The signal release stays installed until the lock is gone: an interrupt a deferred write + // held is raised again as `run` ends, and with no handler it would end the process mid-removal. await rm(await checkedPath(root, SOURCE_DIR_LOCK, new Set()), { recursive: true, force: true }) // Folders the lock alone created go with it; one the command filled stays. await removeCreatedFolders() + disposeSignalRelease() } } From 107c2f3bca9bd4c0a7caec3465cd1cd6bbb6f7a9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:11:51 -0800 Subject: [PATCH 23/51] fix(sim-cli): leave a codeFile() path with a NUL byte to the compile's E_CODE_FILE_PATH --- .../src/commands/protocol/source-dir/layout.ts | 4 +++- .../protocol/workflow-source-check.test.ts | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/sim-cli/src/commands/protocol/source-dir/layout.ts b/packages/sim-cli/src/commands/protocol/source-dir/layout.ts index d86e42fac9b..e9d1e637466 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/layout.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/layout.ts @@ -91,11 +91,13 @@ export function assertSendableCodeFiles(files: Record): void { /** * A path inside the source directory, normalized (`./functions/x.js` → `functions/x.js`), or - * `null` for an absolute path, a drive letter or a `..` escape. + * `null` for an absolute path, a drive letter, a `..` escape or a NUL byte (which no file name + * holds, so the compile refuses the path as `E_CODE_FILE_PATH`). */ export function sourceRelativePath(path: string): string | null { const normalized = posix.normalize(path.replace(/\\/g, '/')) if ( + normalized.includes('\0') || normalized.startsWith('/') || /^[A-Za-z]:/.test(normalized) || normalized === '.' || diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index a4c24b2c76e..c4f03ad129e 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -284,6 +284,22 @@ describe('sim workflows source check', () => { expect(process.exitCode).toBe(1) }) + it('refuses a codeFile() path holding a NUL byte by its code, reading no such file', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace('import { response,', 'import { codeFile, fn, response,').replace( + ' start.to(done)', + " const step = fn('Step', { code: codeFile('./functions/\\u0000x.js') })\n start.to(done)" + ) + ) + + await check() + + expect(findingLines()).toMatch(/error E_CODE_FILE_PATH/) + expect(process.exitCode).toBe(1) + }) + it('puts each finding in the JSON on stdout', async () => { cacheCatalog() writeFileSync( From 4f0ba50d3bcb197eba62c2490f1018f9ee87d1af Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:16:01 -0800 Subject: [PATCH 24/51] fix(sim-cli): say the truncation marker leaves out the info findings a result hides --- .../src/commands/protocol/source-ui.ts | 18 ++++++++++----- .../protocol/workflow-source-apply.test.ts | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-ui.ts b/packages/sim-cli/src/commands/protocol/source-ui.ts index 11f0b60d55c..d0b0005bc1e 100644 --- a/packages/sim-cli/src/commands/protocol/source-ui.ts +++ b/packages/sim-cli/src/commands/protocol/source-ui.ts @@ -154,13 +154,21 @@ function resultDiagnostics( verbose = false ): { diagnostics?: ResultDiagnostic[] } { if (!isMachine(format)) return {} + const hidden = (finding: Diagnostic) => + !verbose && finding.severity === 'info' && finding.code !== 'I_FINDINGS_TRUNCATED' + const hiddenCount = diagnostics.filter(hidden).length + // The marker counts what the answer left out; the infos hidden here are left out on top of it. + const marked = (finding: Diagnostic): Diagnostic => + finding.code === 'I_FINDINGS_TRUNCATED' && hiddenCount > 0 + ? { + ...finding, + message: `${finding.message} ${hiddenCount} info finding${hiddenCount === 1 ? '' : 's'} listed ${hiddenCount === 1 ? 'is' : 'are'} hidden too; pass --verbose to see ${hiddenCount === 1 ? 'it' : 'them'}.`, + } + : finding return { diagnostics: diagnostics - .filter( - (finding) => - verbose || finding.severity !== 'info' || finding.code === 'I_FINDINGS_TRUNCATED' - ) - .map(({ frame: _frame, ...finding }) => finding), + .filter((finding) => !hidden(finding)) + .map(({ frame: _frame, ...finding }) => marked(finding)), } } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index c34537eecdd..9488778a598 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -780,6 +780,28 @@ describe('sim workflows source apply', () => { ]) }) + it('without --verbose, says the truncation marker leaves out the infos it hides too', async () => { + setTerminal(false) + const truncated = { + code: 'I_FINDINGS_TRUNCATED', + severity: 'info', + message: 'There are 140 findings; 40 more are left out.', + } + request.mockResolvedValue( + outcome({ dryRun: true, diagnostics: [layout, { ...layout, line: 4 }, truncated] }) + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stdoutJson().diagnostics).toEqual([ + { + ...truncated, + message: + 'There are 140 findings; 40 more are left out. 2 info findings listed are hidden too; pass --verbose to see them.', + }, + ]) + }) + it('in a display format, prints the first warning of a code whole and the rest on one line', async () => { setTerminal(false) contextMockState.output = 'text' From fcc865b61f54b5b7b5aa8105e80dcc2d8228dd43 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:16:03 -0800 Subject: [PATCH 25/51] fix(sim-cli): count every compile finding a check fails on, past the per-code cap --- .../src/commands/protocol/source-check.ts | 31 ++++++++++++++++--- .../protocol/workflow-source-check.test.ts | 24 ++++++++++++++ .../protocol/workflow-source-check.ts | 3 +- .../protocol/workspace-source-check.ts | 5 ++- packages/workflow-code/src/compile/index.ts | 6 +++- packages/workflow-code/src/compile/report.ts | 28 +++++++++++++---- .../workflow-code/src/diagnostics/index.ts | 1 + .../workflow-code/src/diagnostics/refusals.ts | 5 +++ packages/workflow-code/src/index.ts | 1 + 9 files changed, 90 insertions(+), 14 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-check.ts b/packages/sim-cli/src/commands/protocol/source-check.ts index 298816947fd..ca192bffa07 100644 --- a/packages/sim-cli/src/commands/protocol/source-check.ts +++ b/packages/sim-cli/src/commands/protocol/source-check.ts @@ -20,6 +20,7 @@ import { parseWorkflowLock, raisedMessage, refuseEditedFindings, + refusesEdited, scriptName, startInputs, type Value, @@ -435,7 +436,7 @@ export async function checkWorkflowDir( dir: string, catalog: BlockCatalog, options: { declarations?: WorkspaceDeclarations; strict: StrictMode; prefix?: string } -): Promise<{ findings: CheckFinding[]; entry: string }> { +): Promise<{ findings: CheckFinding[]; entry: string; unlisted: UnlistedCounts }> { const { lock, files, compiled, storedBlocks, printedWith } = await compileWorkflowDir( dir, catalog, @@ -576,7 +577,25 @@ export async function checkWorkflowDir( ...(span ? { file: `${prefix}${span.file}`, line: span.line, column: span.column } : {}), } }) - return { findings, entry: entryText } + // The warnings the compile's caps left out are counted as the listed ones would be: one a + // refusal raised is listed already. + const unlisted: UnlistedCounts = { errors: 0, warnings: 0 } + const edit = { changed, codeEdited } + for (const { code, blockKey } of [ + ...compiled.omitted, + ...compiled.withheld.filter((finding) => !refusable(finding) || !refusesEdited(finding, edit)), + ]) { + if (!reported(code, blockKey)) continue + if (fails(code, blockKey)) unlisted.errors++ + else unlisted.warnings++ + } + return { findings, entry: entryText, unlisted } +} + +/** Findings a check counts but does not list: those the compile's caps left out. */ +interface UnlistedCounts { + errors: number + warnings: number } /** @@ -588,17 +607,19 @@ export function reportCheck( output: Parameters[0], findings: CheckFinding[], result: Record, - options: { verbose?: boolean; bound?: AnswerBound } = {} + options: { verbose?: boolean; bound?: AnswerBound; unlisted?: UnlistedCounts } = {} ): void { const shown = findingsOutput(output, options.verbose) const listed = boundAnswerFindings(findings, options.bound) shown.print(listed) - const errors = findings.filter((finding) => finding.severity === 'error').length + const unlisted = options.unlisted ?? { errors: 0, warnings: 0 } + const errors = findings.filter((finding) => finding.severity === 'error').length + unlisted.errors if (errors > 0) setSoftExitCode(1) printProtocolResult(output, { checked: 'compile', errors, - warnings: findings.filter((finding) => finding.severity === 'warning').length, + warnings: + findings.filter((finding) => finding.severity === 'warning').length + unlisted.warnings, ...result, ...shown.result(listed), }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index c4f03ad129e..9dda4da57e6 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -469,6 +469,30 @@ describe('sim workflows source check', () => { expect(codes.filter((code) => code === 'I_FINDINGS_TRUNCATED')).toHaveLength(1) }) + it('counts every literal secret it fails on, past the 20 of one code it lists', async () => { + cacheCatalog() + const bearer = `Bearer ${'sk-proj-'}${'Abc123Def456Ghi789Jkl012'}` + const calls = Array.from( + { length: 25 }, + (_, index) => + ` const call${index} = api('Call ${index}', { url: 'https://api.example.com/v1', method: 'GET', headers: [{ cells: { Key: 'Authorization', Value: '${bearer}' } }] })\n start.to(call${index})` + ).join('\n') + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace('import { response,', 'import { api, response,').replace( + ' start.to(done)', + `${calls}\n start.to(done)` + ) + ) + + await check() + + const answer = result() + expect(answer).toMatchObject({ errors: 25 }) + const codes = (answer.diagnostics as Array<{ code: string }>).map((finding) => finding.code) + expect(codes.filter((code) => code === 'W_LITERAL_SECRET')).toHaveLength(20) + }) + it('fails a row write naming a column or select option the table does not declare', async () => { cacheCatalog() writeWorkspace() diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index df197ef165d..ba4ae9b7188 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -102,7 +102,7 @@ export function attachWorkflowSourceCheck(source: Command): void { workflowInputs: await workspaceWorkflowInputs(workspace.root, catalog), } : undefined - const { findings, entry } = await checkWorkflowDir(options.dir, catalog, { + const { findings, entry, unlisted } = await checkWorkflowDir(options.dir, catalog, { strict: strictMode(options), ...(declarations ? { declarations } : {}), }) @@ -127,6 +127,7 @@ export function attachWorkflowSourceCheck(source: Command): void { { catalogHash: catalog.hash }, { verbose: options.verbose === true, + unlisted, } ) } diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts index a07d8deae35..33c0f9458d3 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts @@ -132,6 +132,7 @@ export function attachWorkspaceSourceCheck(source: Command): void { } const keys = await workspaceWorkflowKeys(options.dir) const findings: CheckFinding[] = [] + const unlisted = { errors: 0, warnings: 0 } for (const key of keys) { const checked = await checkWorkflowDir( join(options.dir, KIND_DIRS.workflow, key), @@ -143,6 +144,8 @@ export function attachWorkspaceSourceCheck(source: Command): void { } ) findings.push(...checked.findings) + unlisted.errors += checked.unlisted.errors + unlisted.warnings += checked.unlisted.warnings } for (const { code, @@ -172,7 +175,7 @@ export function attachWorkspaceSourceCheck(source: Command): void { profile.output, findings, { workflows: keys.length, catalogHash: catalog.hash }, - { verbose: options.verbose === true, bound: WORKSPACE_ANSWER_BOUND } + { verbose: options.verbose === true, bound: WORKSPACE_ANSWER_BOUND, unlisted } ) }) } diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 8744c7f4a17..96b9028b7f6 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -36,7 +36,7 @@ import { EdgeCollector } from './edges' import { referencedResultKind } from './function-returns' import { didYouMean, HINTS, hintForNode, nodeLabel } from './hints' import { type CompileLimits, DEFAULT_LIMITS, parseWorkflowSource, sdkHeader } from './parse' -import { ErrorLimitReached, type Range, Reporter } from './report' +import { ErrorLimitReached, type OmittedFinding, type Range, Reporter } from './report' import { checkSuspiciousName, collectBody, collisionSuffix } from './scope' import { checkCodeFileRefs, @@ -115,6 +115,8 @@ export interface CompileResult { * what `refuseEditedFindings` reads beside `diagnostics`, so a cap never hides a refusal. */ withheld: LocatedDiagnostic[] + /** Every other warning the caps left out, by code and block, so a caller counts every finding. */ + omitted: OmittedFinding[] /** Where each anchor sits, to map later server findings back to lines. */ spans: Partial> /** @@ -428,6 +430,7 @@ function emptyResult(reporter: Reporter, sdkMajor: number | null): CompileResult workflow: null, diagnostics: reporter.diagnostics, withheld: reporter.withheld, + omitted: reporter.omitted, spans: {}, metaSpans: {}, hints: { blocks: {} }, @@ -793,6 +796,7 @@ function compileEntry( workflow, diagnostics: reporter.diagnostics, withheld: reporter.withheld, + omitted: reporter.omitted, spans, metaSpans: ctx.metaSpans, hints, diff --git a/packages/workflow-code/src/compile/report.ts b/packages/workflow-code/src/compile/report.ts index 30e33e930ad..3c4a6a9d393 100644 --- a/packages/workflow-code/src/compile/report.ts +++ b/packages/workflow-code/src/compile/report.ts @@ -108,6 +108,11 @@ export class Reporter { * `refuseEditedFindings`) while the listing stays capped. */ readonly withheld: LocatedDiagnostic[] = [] + /** + * Every other warning a cap left out, by code and block only (no text, span or frame is built + * for one), so a caller can still count every finding. + */ + readonly omitted: OmittedFinding[] = [] /** The code, block and file of each refusable warning listed or withheld. */ private readonly refusableHeld = new Set() private readonly indexes = new Map() @@ -170,7 +175,7 @@ export class Reporter { } else if (severity === 'info') { if (this.infoCount >= MAX_LISTED) return } else if (this.warningCount >= MAX_LISTED) { - this.withhold(code, message, at, extras) + this.omit(code, message, at, extras) if (this.warningCount === MAX_LISTED) { this.truncated( `Listed ${MAX_LISTED} warnings; the rest are omitted. The next: ${message}`, @@ -183,7 +188,7 @@ export class Reporter { const ofCode = this.warningsByCode.get(code) ?? 0 this.warningsByCode.set(code, ofCode + 1) if (ofCode >= MAX_WARNINGS_PER_CODE) { - this.withhold(code, message, at, extras) + this.omit(code, message, at, extras) if (ofCode === MAX_WARNINGS_PER_CODE) { this.truncated( `Listed ${MAX_WARNINGS_PER_CODE} ${code} warnings; the rest of them are omitted. The next: ${message}`, @@ -200,17 +205,22 @@ export class Reporter { } /** - * Keeps a refusable warning a cap left out in {@link withheld}, unless one of its code, block - * and file is already listed or withheld. + * Keeps a warning a cap left out: in {@link withheld} when it is refusable and none of its code, + * block and file is listed or withheld yet, else in {@link omitted}. */ - private withhold( + private omit( code: CompileDiagnosticCode, message: string, at: Range, extras: ReportExtras ): void { const held = refusableKey(code, at, extras) - if (held === undefined || this.refusableHeld.has(held)) return + if (held === undefined || this.refusableHeld.has(held)) { + this.omitted.push( + extras.blockKey === undefined ? { code } : { code, blockKey: extras.blockKey } + ) + return + } this.refusableHeld.add(held) const { hint, ...listed } = extras const diagnostic = this.build(code, message, at, listed, framesSource(code)) @@ -286,6 +296,12 @@ export class Reporter { } } +/** A warning a compile's caps left out of every list, as {@link Reporter.omitted} keeps it. */ +export interface OmittedFinding { + code: CompileDiagnosticCode + blockKey?: BlockKey +} + /** The code, block and file a refusable warning is withheld by; none for any other finding. */ function refusableKey( code: CompileDiagnosticCode, diff --git a/packages/workflow-code/src/diagnostics/index.ts b/packages/workflow-code/src/diagnostics/index.ts index 148943b79d8..f93b31ccd74 100644 --- a/packages/workflow-code/src/diagnostics/index.ts +++ b/packages/workflow-code/src/diagnostics/index.ts @@ -20,5 +20,6 @@ export { raisedMessage, refusalClassifier, refuseEditedFindings, + refusesEdited, } from './refusals' export type { DiagnosticSeverity, SeverityOf } from './severity' diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index c082e7cee2d..f55bc0d203a 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -189,6 +189,11 @@ function refusalOf(finding: LocatedDiagnostic, edit: EditedBlocks): EditRefusalC return inCode && edit.codeEdited(blockKey, span.file) ? inCode : undefined } +/** Whether the apply refuses `finding` (a compile warning), given what the edit changed. */ +export function refusesEdited(finding: LocatedDiagnostic, edit: EditedBlocks): boolean { + return refusalOf(finding, edit) !== undefined +} + /** * A compile's findings with every warning the apply refuses on an edited block raised to its * error code (see the module doc). `withheld`, the refusable warnings the compile's caps left diff --git a/packages/workflow-code/src/index.ts b/packages/workflow-code/src/index.ts index 275d1a751bc..8ac917625a4 100644 --- a/packages/workflow-code/src/index.ts +++ b/packages/workflow-code/src/index.ts @@ -57,6 +57,7 @@ export { type LocatedDiagnostic, raisedMessage, refuseEditedFindings, + refusesEdited, type SourceSpan, splitLines, } from './diagnostics' From 99a139626ccc29af031fd896ce800eace8bfb7e6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:16:37 -0800 Subject: [PATCH 26/51] chore(rules): load the workflow-code rule for the folders wave 2 moved its code into --- .claude/rules/sim-workflow-code.md | 12 +++++++++++- .cursor/rules/sim-workflow-code.mdc | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.claude/rules/sim-workflow-code.md b/.claude/rules/sim-workflow-code.md index 752c3d67731..8bf5712ad06 100644 --- a/.claude/rules/sim-workflow-code.md +++ b/.claude/rules/sim-workflow-code.md @@ -4,13 +4,23 @@ paths: - "packages/workflow-code/**" - "packages/workspace-code/**" - "packages/workflow-sdk/**" + - "packages/table-cells/**" + - "packages/workflow-persistence/src/save.ts" - "packages/sim-cli/src/commands/protocol/*source*" + - "packages/sim-cli/src/commands/protocol/source-dir/**" + - "packages/sim-cli/src/commands/protocol/workflow-dir.ts" + - "packages/sim-cli/src/commands/protocol/stored-edges.ts" + - "packages/sim-cli/src/commands/protocol/local-errors.ts" - "apps/sim/lib/workflows/code/**" - "apps/sim/lib/workflows/application/*source*" + - "apps/sim/lib/workflows/application/source-pipeline/**" + - "apps/sim/lib/workflows/api/source-*" - "apps/sim/lib/workspace-source/**" - "apps/sim/blocks/sdk-names/**" - "apps/sim/app/workspace/**/panel/components/code/**" + - "apps/sim/app/api/workflows/*/source/**" - "apps/sim/app/api/v2/**/source/**" + - "apps/sim/lib/api/contracts/workflow-source.ts" - "apps/sim/lib/api/contracts/v2/*source*" - "apps/sim/app/api/cron/cleanup-source-staging/**" --- @@ -33,7 +43,7 @@ workspace directory). | `packages/workflow-sdk` | Authoring declarations (`src/core.d.ts`, `src/workspace/*.d.ts`), type codegen from the catalog (`src/codegen/`), the cookbook | | `packages/sim-cli/src/commands/protocol/` | `workflow-source-*` and `workspace-source-*` commands, which never import each other's command modules; `source-dir/`, `workflow-dir.ts`, `source-check.ts`, `source-write.ts`, `source-tree.ts`, `source-ui.ts` and `workspace-source-dir.ts` (the workspace directory reader a workflow check also reads declarations through) are shared | | `apps/sim/lib/workflows/code/` | Server glue: catalog projection from the registry (`catalog/`), source-apply gates (`apply/`), stored canonical form, the feature flag, the proof corpus (`corpus/`) | -| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources | +| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources; `source-pipeline/` holds the one plan → preview/commit flow an apply and a create share | | `apps/sim/lib/workspace-source/` | Workspace source use cases (`application/`), one adapter per kind (`application/kinds//`), route policies (`api/`) | | `apps/sim/blocks/sdk-names/.json` | Frozen SDK builder names, one file per block type | | `apps/sim/app/workspace/**/panel/components/code/` | The read-only Code tab | diff --git a/.cursor/rules/sim-workflow-code.mdc b/.cursor/rules/sim-workflow-code.mdc index aaf39757082..c631f32be1b 100644 --- a/.cursor/rules/sim-workflow-code.mdc +++ b/.cursor/rules/sim-workflow-code.mdc @@ -1,6 +1,6 @@ --- description: "Workflows and workspaces as code (workflow source, workspace source, the workflow SDK, the Code tab)" -globs: ["packages/workflow-code/**","packages/workspace-code/**","packages/workflow-sdk/**","packages/sim-cli/src/commands/protocol/*source*","apps/sim/lib/workflows/code/**","apps/sim/lib/workflows/application/*source*","apps/sim/lib/workspace-source/**","apps/sim/blocks/sdk-names/**","apps/sim/app/workspace/**/panel/components/code/**","apps/sim/app/api/v2/**/source/**","apps/sim/lib/api/contracts/v2/*source*","apps/sim/app/api/cron/cleanup-source-staging/**"] +globs: ["packages/workflow-code/**","packages/workspace-code/**","packages/workflow-sdk/**","packages/table-cells/**","packages/workflow-persistence/src/save.ts","packages/sim-cli/src/commands/protocol/*source*","packages/sim-cli/src/commands/protocol/source-dir/**","packages/sim-cli/src/commands/protocol/workflow-dir.ts","packages/sim-cli/src/commands/protocol/stored-edges.ts","packages/sim-cli/src/commands/protocol/local-errors.ts","apps/sim/lib/workflows/code/**","apps/sim/lib/workflows/application/*source*","apps/sim/lib/workflows/application/source-pipeline/**","apps/sim/lib/workflows/api/source-*","apps/sim/lib/workspace-source/**","apps/sim/blocks/sdk-names/**","apps/sim/app/workspace/**/panel/components/code/**","apps/sim/app/api/workflows/*/source/**","apps/sim/app/api/v2/**/source/**","apps/sim/lib/api/contracts/workflow-source.ts","apps/sim/lib/api/contracts/v2/*source*","apps/sim/app/api/cron/cleanup-source-staging/**"] --- @@ -23,7 +23,7 @@ workspace directory). | `packages/workflow-sdk` | Authoring declarations (`src/core.d.ts`, `src/workspace/*.d.ts`), type codegen from the catalog (`src/codegen/`), the cookbook | | `packages/sim-cli/src/commands/protocol/` | `workflow-source-*` and `workspace-source-*` commands, which never import each other's command modules; `source-dir/`, `workflow-dir.ts`, `source-check.ts`, `source-write.ts`, `source-tree.ts`, `source-ui.ts` and `workspace-source-dir.ts` (the workspace directory reader a workflow check also reads declarations through) are shared | | `apps/sim/lib/workflows/code/` | Server glue: catalog projection from the registry (`catalog/`), source-apply gates (`apply/`), stored canonical form, the feature flag, the proof corpus (`corpus/`) | -| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources | +| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources; `source-pipeline/` holds the one plan → preview/commit flow an apply and a create share | | `apps/sim/lib/workspace-source/` | Workspace source use cases (`application/`), one adapter per kind (`application/kinds//`), route policies (`api/`) | | `apps/sim/blocks/sdk-names/.json` | Frozen SDK builder names, one file per block type | | `apps/sim/app/workspace/**/panel/components/code/` | The read-only Code tab | From 490eba5fb53fec8ae9de781548cb82de9e0232cf Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:16:54 -0800 Subject: [PATCH 27/51] fix(workflow-code): reword the renamed-block hint, which an apply that keeps the id already follows --- packages/workflow-code/src/compile/grammar.test.ts | 2 +- packages/workflow-code/src/compile/values.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/workflow-code/src/compile/grammar.test.ts b/packages/workflow-code/src/compile/grammar.test.ts index 10fa4003e98..488b90128db 100644 --- a/packages/workflow-code/src/compile/grammar.test.ts +++ b/packages/workflow-code/src/compile/grammar.test.ts @@ -1159,7 +1159,7 @@ describe('errors, each with its code, location and hint', () => { const finding = result.diagnostics.find((d) => d.code === 'W_UNRESOLVED_REF') expect(finding?.hint).toBe( - '$ref.lookup is a block this workflow had that workflow.ts no longer declares under that key: if Fetch Page is it renamed, write $ref.fetchpage.data.' + '$ref.lookup is a block this workflow had that workflow.ts no longer declares under that key: if it was renamed to Fetch Page, an apply that keeps its id rewrites this reference to $ref.fetchpage.data, so no edit is needed.' ) }) diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index bda93e592c5..86c06ecc451 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -1062,7 +1062,7 @@ function formerBlockHint( const lead = `${spell([{ kind: 'name', name: formerKey }])} is a block this workflow had that ${SOURCE_ENTRY_FILE} no longer declares under that key` const [only] = unbound return unbound.length === 1 && only - ? `${lead}: if ${only.name} is it renamed, write ${rewritten(only.key)}.` + ? `${lead}: if it was renamed to ${only.name}, an apply that keeps its id rewrites this reference to ${rewritten(only.key)}, so no edit is needed.` : `${lead}: if you renamed it, write ${spell([{ kind: 'name', name: 'newkey' }])}, where newkey is its display name lowercased with whitespace and dots removed.` } From 2d78307d715f689d2a74a71f5b1af534877002dc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:17:51 -0800 Subject: [PATCH 28/51] fix(sim-cli): pin the deploy an apply names as its next step to the draft it saved --- packages/sim-cli/src/commands/protocol/workflow-dir.ts | 2 +- .../src/commands/protocol/workflow-source-apply.test.ts | 8 ++++++-- .../src/commands/protocol/workflow-source-apply.ts | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/workflow-dir.ts b/packages/sim-cli/src/commands/protocol/workflow-dir.ts index 34ff9c012c4..c983e1ef687 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-dir.ts @@ -212,7 +212,7 @@ The primer above is enough to edit; read a section here when a step needs it. 2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. 3. \`${typecheckCommand()}\`: about a second; catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It runs the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply: a builder from another block fails with \`has no exported member\` until you run step 2 (or use \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine. 4. \`sim workflows source status --dir .\`: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\` until it exits 0. -5. \`sim workflows source apply --dir . --yes\`: saves the draft, refusing errors, block deletions without \`--allow-delete\` and a stale directory, so it needs no separate dry run (without \`--yes\` and a terminal it prints the plan and exits ${SOURCE_PLANNED_EXIT_CODE}). \`--deploy\` also deploys it; without it, a deployed workflow's next \`sim workflows deploy \` is printed. It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out). +5. \`sim workflows source apply --dir . --yes\`: saves the draft, refusing errors, block deletions without \`--allow-delete\` and a stale directory, so it needs no separate dry run (without \`--yes\` and a terminal it prints the plan and exits ${SOURCE_PLANNED_EXIT_CODE}). \`--deploy\` also deploys it; without it, a deployed workflow's next \`sim workflows deploy --expected-draft-fingerprint \` is printed, which deploys only the draft this apply saved. It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out). 6. Exit ${SOURCE_STALE_EXIT_CODE} (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get --dir \`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit ${SOURCE_STALE_EXIT_CODE} right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare. Only when it holds your edit, run \`sim workflows source get --dir . --force\`; otherwise reapply your edit in the fresh directory. Editing the \`workflow('…')\` name or description renames or redescribes the workflow; a name another workflow in its folder holds is refused with \`E_NAME_TAKEN\`. Lines starting \`note:\` explain something the write did differently than the source says; read them. diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index 9488778a598..aed6ea4f181 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -953,7 +953,9 @@ describe('sim workflows source apply', () => { expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH]) expect(stdoutJson()).toMatchObject({ needsRedeployment: true, - next: { command: `sim workflows deploy ${WORKFLOW_ID}` }, + next: { + command: `sim workflows deploy ${WORKFLOW_ID} --expected-draft-fingerprint sf1:b`, + }, }) }) @@ -984,7 +986,9 @@ describe('sim workflows source apply', () => { expect(error).toMatchObject({ status: 403, code: 'FORBIDDEN' }) expect((error as Error).message).toContain(`The apply was saved into ${dir}`) - expect((error as Error).message).toContain(`sim workflows deploy ${WORKFLOW_ID}`) + expect((error as Error).message).toContain( + `sim workflows deploy ${WORKFLOW_ID} --expected-draft-fingerprint sf1:b` + ) expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(`${ENTRY}// stored\n`) // The saved write's result still prints, before the deploy's failure. expect(stdoutJson()).toMatchObject({ workflowId: WORKFLOW_ID, applied: true }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 0e35fbabf7f..1b790c3484e 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -238,7 +238,8 @@ async function deployStep( data: SourceApplyData, options: { deploy?: boolean; dir: string; verb: string; machine: boolean } ): Promise { - const deployCommand = `sim workflows deploy ${data.workflowId}` + // Pinned to the draft this write stored, as `--deploy` is: an edit made since is never deployed. + const deployCommand = `sim workflows deploy ${data.workflowId} --expected-draft-fingerprint ${data.base.fingerprint}` if (!options.deploy) { if (!data.needsRedeployment) return {} return { From 34ef52b1e7c2d14bb2237cd5a1509379c8776936 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:19:07 -0800 Subject: [PATCH 29/51] fix(sim-cli): deploy nothing after a no-op apply whose draft the live version already matches --- .../protocol/workflow-source-apply.test.ts | 43 +++++++++++++++++++ .../protocol/workflow-source-apply.ts | 32 +++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index aed6ea4f181..8ddfe50f8f8 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -944,6 +944,49 @@ describe('sim workflows source apply', () => { }) }) + it('deploys no new version when a no-op apply leaves the live version matching the draft', async () => { + setTerminal(false) + const STATUS_PATH = `/api/v2/workflows/${WORKFLOW_ID}/deployment` + const noop = outcome() + noop.data.applied = false + noop.data.diff.noop = true + noop.data.needsRedeployment = false + const status = (isDeployed: boolean) => ({ + data: { + ...deployed.data, + isDeployed, + needsRedeployment: false, + isPublicApi: false, + webhooks: [], + activeDeployment: isDeployed + ? { deploymentVersionId: 'dv-7', version: 7, deployedAt: '2026-11-20T00:00:00Z' } + : null, + }, + }) + let live = true + request.mockImplementation(async (path: string) => + path === STATUS_PATH ? status(live) : path === DEPLOY_PATH ? deployed : noop + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH, STATUS_PATH]) + expect(stdoutJson()).toMatchObject({ + deployed: { version: 7, isDeployed: true }, + needsRedeployment: false, + }) + + // A workflow never deployed is deployed, no-op or not. + live = false + request.mockClear() + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + expect(request.mock.calls.map((call) => call[0])).toEqual([ + APPLY_PATH, + STATUS_PATH, + DEPLOY_PATH, + ]) + }) + it('without --deploy, names the deploy command when the deployed workflow now differs', async () => { setTerminal(false) request.mockResolvedValue(outcome()) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 1b790c3484e..66c62c5a69c 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -8,6 +8,7 @@ import type { ApplyWorkflowSourceResponse, CreateWorkflowFromSourceBody, DeployWorkflowResponse, + GetWorkflowDeploymentResponse, } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' @@ -122,7 +123,7 @@ async function writeWorkflowSource(args: mode: WriteMode creating: boolean question: string - client: OperationClient + client: OperationClient operation: Operation params: Record body: ApplyWorkflowSourceBody | CreateWorkflowFromSourceBody @@ -234,7 +235,7 @@ interface DeployOutcome { * deploy says the write itself was saved. */ async function deployStep( - client: OperationClient<'deployWorkflow'>, + client: OperationClient<'deployWorkflow' | 'getWorkflowDeployment'>, data: SourceApplyData, options: { deploy?: boolean; dir: string; verb: string; machine: boolean } ): Promise { @@ -249,6 +250,31 @@ async function deployStep( }, } } + if (!data.applied && !data.needsRedeployment) { + // A no-op write changed no draft: when the live version already matches it, a deploy would + // only publish the same version again. + const status = await client + .request('getWorkflowDeployment', { + params: { workflowId: data.workflowId }, + }) + .then( + (response) => response.data, + (error: unknown) => { + throw deployFailure(error, options, deployCommand) + } + ) + if (status.isDeployed && !status.needsRedeployment) { + if (!options.machine) + printError('The live version already matches the draft; nothing was deployed.') + return { + deployed: { + ...(status.activeDeployment ? { version: status.activeDeployment.version } : {}), + isDeployed: true, + }, + needsRedeployment: false, + } + } + } const deployed = await client .request('deployWorkflow', { params: { workflowId: data.workflowId }, @@ -317,6 +343,7 @@ export function attachWorkflowSourceApply(source: Command): void { const [command, connectApply] = apiCommand(source, 'apply', [ 'applyWorkflowSource', 'deployWorkflow', + 'getWorkflowDeployment', 'getSourceResources', ]) absoluteDirOption(command) @@ -402,6 +429,7 @@ export function attachWorkflowSourceCreate(source: Command): void { 'getSourceCatalog', 'getSourceResources', 'deployWorkflow', + 'getWorkflowDeployment', ]) absoluteDirOption(command) verboseFlag(command) From c1e4188897965d9f5322ed78c09069fb86c0cb32 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:31:40 -0800 Subject: [PATCH 30/51] fix(workflows): say what a block bound to when its claimed lock id is ignored --- .../apply-workflow-source.integration.ts | 33 ++++++++++++++++++- .../source-pipeline/evaluate/resolve.ts | 27 ++++++++------- packages/workflow-code/src/compile/SYNTAX.md | 2 +- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index 051fda6973b..bfe4f1cfc8e 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -2054,6 +2054,33 @@ export default workflow('Fresh', {}, () => { } ) + check( + 'a lock copied from another workflow is ignored where it claims ids, keeping the stored ones', + async () => { + const target = await seedWorkflow() + const other = await seedWorkflow() + const source = await getSource(target.workflowId) + const foreign = await getSource(other.workflowId) + + // Block ids only: the foreign lock's variable ids are another matter. + const result = await apply(source, edit(source.files, '41 + 1', '41 + 2'), { + lock: { ...foreign.lock, variables: source.lock.variables }, + }) + + expect(result.applied).toBe(true) + const claims = result.diagnostics.filter((d) => d.code === 'W_MAT_LOCK_ID_CLAIMED') + expect(claims.length).toBeGreaterThan(0) + for (const claim of claims) { + expect(claim.message).toContain('the block keeps the id of the stored block it matches') + } + expect(result.mintedIds).toEqual({}) + const after = await snapshot(target.workflowId) + expect(Object.keys(after.state?.blocks ?? {}).sort()).toEqual( + [target.startId, target.scoreId].sort() + ) + } + ) + check('a lock id another workflow already stores is dropped and a new id is minted', async () => { const target = await seedWorkflow() const other = await seedWorkflow() @@ -2064,7 +2091,11 @@ export default workflow('Fresh', {}, () => { expect(result.applied).toBe(true) expect(result.diagnostics).toContainEqual( - expect.objectContaining({ code: 'W_MAT_LOCK_ID_CLAIMED', blockKey: 'extra' }) + expect.objectContaining({ + code: 'W_MAT_LOCK_ID_CLAIMED', + blockKey: 'extra', + message: expect.stringContaining('a new id was minted'), + }) ) const minted = result.mintedIds.extra expect(minted).toBeDefined() diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts index 6cacb3d0f07..7e744f3a846 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts @@ -276,20 +276,25 @@ export async function resolveSubmission( const claimed = new Set(claimedIds) if (claimed.size > 0) { const blocks: Record = {} + const claimedKeys: string[] = [] for (const [key, entry] of Object.entries(lock.blocks)) { - if (claimed.has(entry.id)) { - claimedDiagnostics.push( - diagnostic( - 'W_MAT_LOCK_ID_CLAIMED', - `The lock id for block "${key}" belongs to another workflow; a new id was minted`, - { blockKey: key } - ) - ) - } else { - blocks[key] = entry - } + if (claimed.has(entry.id)) claimedKeys.push(key) + else blocks[key] = entry } materialized = materializeWith({ ...lock, blocks }) + // Without its lock id the block binds as a lockless one would: to the stored block of its + // name, or to a new id. + const outcomes = new Map(materialized.entries.map((entry) => [entry.key, entry])) + for (const key of claimedKeys) { + const kept = outcomes.get(key)?.status === 'matched' + claimedDiagnostics.push( + diagnostic( + 'W_MAT_LOCK_ID_CLAIMED', + `The lock id for block "${key}" belongs to another workflow, so it was ignored; ${kept ? 'the block keeps the id of the stored block it matches' : 'a new id was minted'}`, + { blockKey: key } + ) + ) + } } const idByKey = new Map( diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index 17e87e3e5e1..e4e68d66e3e 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -495,7 +495,7 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `W_MAT_DELETED_SINCE_PULL` | A lock entry whose block was deleted since the read; re-created. | | `W_MAT_INFERRED_RENAME` | A rename inferred from type, parent and edges (the key and the ident both changed). | | `W_MAT_RETYPE_RECREATED` | A key kept with a new type: delete plus create, so webhook URLs and schedules change. | -| `W_MAT_LOCK_ID_CLAIMED` | A lock id another workflow holds; a new id was minted. | +| `W_MAT_LOCK_ID_CLAIMED` | A lock id another workflow holds, ignored: the block binds to the stored block it matches, or gets a new id. | | `I_MAT_NEEDS_LAYOUT` | A new block with no position, or one moved into or out of a loop or parallel; placed by autolayout. | **Apply** From ee120dbfcb392037167b3a10383b5123afc37583 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:32:29 -0800 Subject: [PATCH 31/51] fix(workflow-code): report the code-file total-size cap once, on the file that crosses it --- packages/workflow-code/src/compile/blocks.ts | 3 +++ packages/workflow-code/src/compile/hostile.test.ts | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 1d43d1f8a17..0e486dbb5c2 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -415,8 +415,11 @@ function parseCodeFile( decl: BlockDecl ): string | null { const { limits } = ctx + const before = ctx.codeFiles.bytes ctx.codeFiles.bytes += utf8Bytes(text, limits.maxTotalBytes) if (ctx.codeFiles.bytes > limits.maxTotalBytes) { + // Reported once, on the file that crosses the cap; each file after it is skipped silently. + if (before > limits.maxTotalBytes) return null ctx.reporter.report( 'E_LIMIT_EXCEEDED', `The workflow and its code files total more than ${limits.maxTotalBytes} bytes.`, diff --git a/packages/workflow-code/src/compile/hostile.test.ts b/packages/workflow-code/src/compile/hostile.test.ts index 80ed340b098..a8e7c91e488 100644 --- a/packages/workflow-code/src/compile/hostile.test.ts +++ b/packages/workflow-code/src/compile/hostile.test.ts @@ -379,7 +379,7 @@ describe('hostile sources produce diagnostics only', () => { expect(errorCodes(result)).toEqual(['E_INVALID_CHARACTER']) }) - it('caps the total bytes of distinct code files', () => { + it('caps the total bytes of distinct code files with one diagnostic, on the file that crosses it', () => { const files = Object.fromEntries( Array.from({ length: 4 }, (_, i) => [ `workflows/functions/f${i}.js`, @@ -391,7 +391,8 @@ describe('hostile sources produce diagnostics only', () => { { maxTotalBytes: 2_000 }, files ) - expect(codes(result)).toContain('E_LIMIT_EXCEEDED') + const limits = result.diagnostics.filter((d) => d.code === 'E_LIMIT_EXCEEDED') + expect(limits.map((d) => d.blockKey)).toEqual(['f2']) expect(result.workflow).toBeNull() }) From 4b6962ada1b0411c8c530d31fa0c08246ae786c7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 13:53:08 -0800 Subject: [PATCH 32/51] docs(cli): regenerate the CLI reference --- apps/docs/content/docs/cli/reference.mdx | 8 ++++---- apps/docs/content/docs/cli/workflows.mdx | 6 +++--- apps/docs/content/docs/cli/workspaces.mdx | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index f3c7c515d3d..7c8e7c4a43d 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8586,7 +8586,7 @@ sim workflows source apply [workflowId] [options] Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment), [getSourceResources](/api-reference/workflows/getSourceResources). ### sim workflows source create @@ -8614,7 +8614,7 @@ sim workflows source create [options] Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment). ### sim workflows source check @@ -8641,7 +8641,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -10017,7 +10017,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index a376ef22740..7e76fc8249d 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1367,7 +1367,7 @@ sim workflows source create [options] Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment). ### Apply a workflow source directory to a workflow draft @@ -1402,7 +1402,7 @@ sim workflows source apply [workflowId] [options] Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment), [getSourceResources](/api-reference/workflows/getSourceResources). ### Show the block catalog workflow source is checked against @@ -1445,7 +1445,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index e0002bfa79b..2676c9565a6 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -786,7 +786,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. From bbaeea3eaa4b72f6f56422323b85afbd1bee69f6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 14:48:31 -0800 Subject: [PATCH 33/51] fix(workspace-code): keep target-only resources on promote --allow-delete; delete them only with --delete-target-only Plain --allow-delete on a promotion now deletes only what the promotion record proves the directory deleted. Resources only the target holds stay out of the base, so the server keeps them and names each (W_UNDECLARED, with a promotion hint). --delete-target-only, which needs --to and --allow-delete, opts into deleting them. A target-only delete the apply held back is no longer written into the promotion record, where it read as a directory delete on the next promotion. --- apps/docs/content/docs/cli/reference.mdx | 3 +- apps/docs/content/docs/cli/workspaces.mdx | 3 +- .../content/docs/workflows/workspace-code.mdx | 2 +- .../application/promote.integration.ts | 50 ++++++-- .../protocol/workspace-source-apply.test.ts | 107 +++++++++++++++++- .../protocol/workspace-source-apply.ts | 76 ++++++++++--- .../commands/protocol/workspace-source-dir.ts | 2 +- packages/workspace-code/DESIGN.md | 9 +- packages/workspace-code/src/promote.test.ts | 47 +++++++- packages/workspace-code/src/promote.ts | 38 ++++--- 10 files changed, 283 insertions(+), 54 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 7c8e7c4a43d..b91e32700b9 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -9982,7 +9982,8 @@ sim workspaces source apply [options] | `--dir ` | No | Workspace source directory written by `workspaces source get`. Defaults to `.`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | -| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | +| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares (with --to, only what it deleted since the last promotion: the target's own resources are kept). | +| `--delete-target-only` | No | With --to and --allow-delete, also delete the target's resources this directory never declared and no promotion from it left there (prod-only tables and their rows, skills made in prod). Without it a promotion keeps them, naming each with a warning. | | `--only ` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | | `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 2676c9565a6..64e39c047d7 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -753,7 +753,8 @@ sim workspaces source apply [options] | `--dir ` | No | Workspace source directory written by `workspaces source get`. Defaults to `.`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | -| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | +| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares (with --to, only what it deleted since the last promotion: the target's own resources are kept). | +| `--delete-target-only` | No | With --to and --allow-delete, also delete the target's resources this directory never declared and no promotion from it left there (prod-only tables and their rows, skills made in prod). Without it a promotion keeps them, naming each with a warning. | | `--only ` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | | `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index 299c7f4201a..c19b2711db9 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -93,7 +93,7 @@ sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --dry sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --yes ``` -Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`, which deletes every one of them: the plan names each with a `W_PROMOTE_DELETES_TARGET_ONLY` warning. A resource you deleted from the directory since the last promotion is deleted in the target too, and like any delete it needs `--allow-delete`. The directory stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. +Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. A resource you deleted from the directory since the last promotion is deleted in the target too, and like any delete it needs `--allow-delete`. Resources only the target holds, such as a table created in prod and the rows its runs wrote, are kept, with or without `--allow-delete`: the plan names each with a `W_UNDECLARED` warning. To delete them as well, pass `--delete-target-only` beside `--allow-delete`; the plan then names each with a `W_PROMOTE_DELETES_TARGET_ONLY` warning, so run it with `--dry-run` first. The directory stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. Each promotion records what it left in the target in `.sim/targets/.json`, and the next promotion compares with that record: diff --git a/apps/sim/lib/workspace-source/application/promote.integration.ts b/apps/sim/lib/workspace-source/application/promote.integration.ts index c2042f82ce9..9e151e20077 100644 --- a/apps/sim/lib/workspace-source/application/promote.integration.ts +++ b/apps/sim/lib/workspace-source/application/promote.integration.ts @@ -93,17 +93,27 @@ const DEV: Record = { /** The text files a get wrote (no blobs here), as a directory holds them. */ type Read = Awaited> +/** `apply --to`'s `--allow-delete` and `--delete-target-only`. */ +interface PromoteOptions { + allowDelete?: boolean + deleteTargetOnly?: boolean +} + /** What `apply --to` sends for `dev` (a get of the dev workspace) to `target`. */ -async function promote(dev: Read, target: string, options: { allowDelete?: boolean } = {}) { +async function promote(dev: Read, target: string, options: PromoteOptions = {}) { const prod = await runner.get(target) const composed = promotionSubmission({ files: dev.files, blobPaths: [], kinds: dev.kinds, target: prod, + deleteTargetOnly: options.deleteTargetOnly === true, + }) + return runner.apply(target, { + files: composed.files, + base: composed.base, allowDelete: options.allowDelete === true, }) - return runner.apply(target, { files: composed.files, base: composed.base, ...options }) } /** @@ -114,7 +124,7 @@ async function promoteRecorded( dev: Read, target: string, record: PromotionRecord | undefined, - options: { allowDelete?: boolean } = {} + options: PromoteOptions = {} ) { const prod = await runner.get(target) const composed = promotionSubmission({ @@ -123,12 +133,12 @@ async function promoteRecorded( kinds: dev.kinds, target: prod, record, - allowDelete: options.allowDelete === true, + deleteTargetOnly: options.deleteTargetOnly === true, }) const result = await runner.apply(target, { files: composed.files, base: composed.base, - ...options, + allowDelete: options.allowDelete === true, }) return { result, @@ -394,7 +404,7 @@ export default defineTable({ }) check( - "the target's own resources are kept, with a warning, unless the promotion allows deletes", + "the target's own resources are kept, with a warning, even with --allow-delete, and deleted only with --delete-target-only", async () => { const dev = await workspaces.create({ name: 'Dev', ownerId }) const prod = await workspaces.create({ name: 'Prod', ownerId }) @@ -422,7 +432,18 @@ export default defineTable({ ) expect(prodTables.map((table) => table.name).sort()).toEqual(['audit', 'leads']) - const deleted = await promote(await runner.get(dev), prod, { allowDelete: true }) + const allowed = await promote(await runner.get(dev), prod, { allowDelete: true }) + expect(allowed.applied).toEqual([]) + expect( + allowed.diagnostics + .filter((finding) => finding.code === 'W_UNDECLARED') + .map((finding) => finding.key) + ).toEqual(['audit']) + + const deleted = await promote(await runner.get(dev), prod, { + allowDelete: true, + deleteTargetOnly: true, + }) expect( deleted.applied.map((change) => `${change.action} ${change.kind}:${change.key}`) ).toEqual(['delete table:audit']) @@ -430,7 +451,7 @@ export default defineTable({ ) check( - 'a resource deleted in the directory since the last promotion is a delete: refused without --allow-delete, and not named as only in the target', + 'a resource deleted in the directory since the last promotion is a delete: refused without --allow-delete, and --allow-delete deletes it alone, keeping what only the target holds', async () => { const dev = await workspaces.create({ name: 'Dev', ownerId }) const prod = await workspaces.create({ name: 'Prod', ownerId }) @@ -462,11 +483,16 @@ export default defineTable({ const allowed = await promoteRecorded(deletedInDev, prod, first.record, { allowDelete: true }) expect( - allowed.result.applied - .map((change) => `${change.action} ${change.kind}:${change.key}`) - .sort() - ).toEqual(['delete skill:tone', 'delete table:audit']) + allowed.result.applied.map((change) => `${change.action} ${change.kind}:${change.key}`) + ).toEqual(['delete skill:tone']) expect(allowed.composed.targetOnly).toEqual(['table:audit']) + expect( + allowed.result.diagnostics + .filter((finding) => finding.code === 'W_UNDECLARED') + .map((finding) => finding.key) + ).toEqual(['audit']) + expect(allowed.record.resources['table:audit']).toBeUndefined() + expect((await runner.get(prod)).files['tables/audit.ts']).toBeDefined() } ) }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 996670d70e6..11300b73418 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -580,7 +580,15 @@ describe('workspaces source apply', () => { }) }) - await apply('--to', 'ws_prod', '--only', 'tables', '--allow-delete', '--dry-run') + await apply( + '--to', + 'ws_prod', + '--only', + 'tables', + '--allow-delete', + '--delete-target-only', + '--dry-run' + ) const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) const byCode = (code: string) => @@ -605,6 +613,93 @@ describe('workspaces source apply', () => { expect(stderr).not.toContain('resource(s) that only workspace ws_prod holds') }) + it('a --to with --allow-delete keeps what only the target holds, naming it, and points at --delete-target-only', async () => { + const leads = { id: 'tbl_prod', fingerprint: `rf1:${'a'.repeat(64)}` } + write( + '.sim/targets/ws_prod.json', + JSON.stringify({ + version: 1, + workspaceId: 'ws_prod', + resources: { 'table:leads': leads, 'table:dropped': { id: 'tbl_dropped' } }, + workflows: {}, + }) + ) + const sent: unknown[] = [] + request.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path === '/api/v2/workspaces/ws_prod/source' && options.body === undefined) { + return { + data: { + workspaceId: 'ws_prod', + kinds: ['table'], + files: { + 'tables/leads.ts': LEADS, + 'tables/dropped.ts': LEADS, + 'tables/audit.ts': LEADS, + }, + blobs: [], + base: { + version: 1, + workspaceId: 'ws_prod', + resources: { + 'table:leads': leads, + 'table:dropped': { id: 'tbl_dropped' }, + 'table:audit': { id: 'tbl_audit' }, + }, + }, + skipped: [], + diagnostics: [], + }, + } + } + sent.push(options.body) + return answer({ + workspaceId: 'ws_prod', + dryRun: true, + plan: [ + { kind: 'table', key: 'leads', action: 'noop' }, + { kind: 'table', key: 'dropped', action: 'delete' }, + { kind: 'table', key: 'audit', action: 'noop' }, + ], + diagnostics: [ + { + code: 'W_UNDECLARED', + severity: 'warning', + message: + '"audit" (table audit) is on the server but not in this directory; it is kept.', + kind: 'table', + key: 'audit', + }, + ], + }) + }) + + await apply('--to', 'ws_prod', '--only', 'tables', '--allow-delete', '--dry-run') + + const [body] = sent as Array<{ base: { resources: Record } }> + expect(Object.keys(body.base.resources).sort()).toEqual(['table:dropped', 'table:leads']) + const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) + const [kept] = summary.diagnostics.filter( + (finding: { code: string }) => finding.code === 'W_UNDECLARED' + ) + expect(kept).toMatchObject({ key: 'audit' }) + expect(kept.hint).toContain('--delete-target-only') + expect( + summary.diagnostics.filter( + (finding: { code: string }) => finding.code === 'W_PROMOTE_DELETES_TARGET_ONLY' + ) + ).toEqual([]) + }) + + it('refuses --delete-target-only without --to or without --allow-delete', async () => { + await expect(apply('--to', 'ws_prod', '--delete-target-only', '--yes')).rejects.toThrow( + /needs --allow-delete/ + ) + await expect(apply('--allow-delete', '--delete-target-only', '--yes')).rejects.toThrow( + /needs --to/ + ) + expect(request).not.toHaveBeenCalled() + }) + it('names only the target-only deletes a promotion that stopped part-way wrote', async () => { const target = { version: 1, @@ -637,7 +732,15 @@ describe('workspaces source apply', () => { }) }) - await apply('--to', 'ws_prod', '--only', 'tables', '--allow-delete', '--yes') + await apply( + '--to', + 'ws_prod', + '--only', + 'tables', + '--allow-delete', + '--delete-target-only', + '--yes' + ) const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) expect( diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 7bb267a5842..9c901d7fc70 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -33,7 +33,7 @@ import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' import { searchTasks } from '../search' import { gitProvenance } from './git-provenance' -import { SourceDirError } from './local-errors' +import { CliUsageError, SourceDirError } from './local-errors' import { printProtocolResult } from './result' import { absoluteDirOption, @@ -92,6 +92,7 @@ interface ApplyOptions { dryRun?: boolean yes?: boolean allowDelete?: boolean + deleteTargetOnly?: boolean only?: string to?: string } @@ -455,7 +456,7 @@ async function promoted( dir: string, submission: WorkspaceSubmission, target: string, - allowDelete: boolean + deleteTargetOnly: boolean ): Promise { if (baseWorkspaceOf(submission.base) === target) { throw new SourceDirError(`${dir} was read from workspace ${target}; apply it without --to.`, { @@ -480,7 +481,7 @@ async function promoted( kinds: submission.kinds, target: { files: data.files, base: targetBase }, record, - allowDelete, + deleteTargetOnly, }) return { submission: { ...submission, files: composed.files, base: composed.base }, @@ -513,6 +514,7 @@ async function savePromotion( submission: { files: promotion.submission.files, base: parseWorkspaceBase(promotion.submission.base).base ?? base, + targetOnly: promotion.targetOnly, }, applied: { base, files: data.files, plan: data.plan }, }) @@ -530,7 +532,7 @@ function resourceWhere(kind: WorkspaceKind, key: string): string { * The hint a promotion to `target` gives a refusal the server words for an apply to the * directory's own workspace ("get it into a fresh directory"), which a promotion cannot follow. */ -function promotionHint(code: string, target: string): string | undefined { +function promotionHint(code: string, target: string, targetOnly: boolean): string | undefined { const record = promotionRecordPath(target) ?? '.sim/targets/.json' if (code === 'E_APPLY_STALE') { return `Workspace ${target} changed this resource since the last promotion from this directory. Carry that change into the directory (\`sim workspaces source get --workspace ${target} --dir \` prints it), or undo it in ${target}, then promote again. Don't delete ${record} to get past this: the next promotion would then overwrite every change made in ${target} since, rows its workflows wrote included.` @@ -538,19 +540,26 @@ function promotionHint(code: string, target: string): string | undefined { if (code === 'W_SERVER_CHANGED') { return `Workspace ${target} changed this resource since the last promotion from this directory, and this promotion keeps that change. To carry it into the directory, \`sim workspaces source get --workspace ${target} --dir \` prints it.` } + if (code === 'E_APPLY_DELETE_NOT_ALLOWED') { + return `This directory no longer declares it. Pass --allow-delete to delete it in ${target} too: that deletes only what this directory dropped, and resources only ${target} holds are kept. Or restore it to keep it.` + } + if (code === 'W_UNDECLARED' && targetOnly) { + return `Only workspace ${target} holds it: this directory does not declare it and no promotion from it left it there, so the promotion keeps it, with or without --allow-delete. To delete it there, add --delete-target-only beside --allow-delete; to manage it from this directory, declare it here.` + } if (code === 'E_APPLY_NO_BASE') { return `Workspace ${target} holds this resource, but no promotion from this directory recorded it. Rename the declaration, or rename or delete the resource in ${target}, then promote again.` } return undefined } -/** `finding` with the hint a promotion to `target` gives its code, when it gives one. */ -function withPromotionHint( +/** `finding` with the hint `promotion` gives its code, when it gives one. */ +function withPromotionHint( finding: F, code: string, - target: string + promotion: Promotion ): F { - const hint = promotionHint(code, target) + const targetOnly = promotion.targetOnly.includes(`${finding.kind}:${finding.key}`) + const hint = promotionHint(code, promotion.target, targetOnly) return hint === undefined ? finding : { ...finding, hint } } @@ -564,14 +573,15 @@ function promotionDiagnostics(diagnostics: readonly Finding[], promotion: Promot return diagnostics.flatMap((finding) => finding.code === 'W_SERVER_DELETED' && warned.has(`${finding.kind}:${finding.key}`) ? [] - : [withPromotionHint(finding, finding.code, promotion.target)] + : [withPromotionHint(finding, finding.code, promotion)] ) } /** * The warnings only a promotion knows to give: each declared resource the target deleted since the - * last promotion (kept deleted), and each resource only the target holds that `--allow-delete` - * deletes there (of an apply that stopped part-way, each it deleted). + * last promotion (kept deleted), and each resource only the target holds that + * `--delete-target-only` deletes there (of an apply that stopped part-way, each it deleted). One + * it keeps is the server's `W_UNDECLARED`, with the promotion's hint. */ function promotionWarnings( promotion: Promotion, @@ -604,8 +614,8 @@ function promotionWarnings( (change): Finding => ({ code: 'W_PROMOTE_DELETES_TARGET_ONLY', severity: 'warning', - message: `${resourceWhere(change.kind, change.key)} is only in workspace ${target}, not in this directory, so --allow-delete deletes it there.`, - hint: 'Promote without --allow-delete to keep it, or declare it in this directory.', + message: `${resourceWhere(change.kind, change.key)} is only in workspace ${target}, not in this directory, so --delete-target-only deletes it there.`, + hint: 'Promote without --delete-target-only to keep it, or declare it in this directory.', kind: change.kind, key: change.key, }) @@ -625,7 +635,7 @@ function rehinted(refusal: unknown, promotion: Promotion): unknown { ...refusal.details, diagnostics: diagnostics.map((finding) => isRecordLike(finding) && typeof finding.code === 'string' - ? withPromotionHint(finding, finding.code, promotion.target) + ? withPromotionHint(finding, finding.code, promotion) : finding ), } @@ -667,7 +677,14 @@ export function attachWorkspaceSourceApply(source: Command): void { .option('--dir ', 'Workspace source directory written by `workspaces source get`', '.') .option('--dry-run', 'Plan and check without writing anything') .option('-y, --yes', 'Apply without asking for confirmation') - .option('--allow-delete', 'Delete resources, columns and data the directory no longer declares') + .option( + '--allow-delete', + "Delete resources, columns and data the directory no longer declares (with --to, only what it deleted since the last promotion: the target's own resources are kept)" + ) + .option( + '--delete-target-only', + "With --to and --allow-delete, also delete the target's resources this directory never declared and no promotion from it left there (prod-only tables and their rows, skills made in prod). Without it a promotion keeps them, naming each with a warning" + ) .option('--only ', `Apply only these kinds (${ONLY_KINDS_HELP})`) .option( '--to ', @@ -680,6 +697,7 @@ export function attachWorkspaceSourceApply(source: Command): void { `${sourceExitCodesHelp('workspaces source apply')}${SOURCE_WRITE_EXIT_CODES_HELP}` ) .action(async (options: ApplyOptions) => { + assertTargetOnlyDeletable(options) const mode = writeMode(options) const only = parseKinds(options.only) const { client, profile } = connect() @@ -770,7 +788,13 @@ export function attachWorkspaceSourceApply(source: Command): void { } assertBlobsFetched(options.dir, read, written.digests) promotion = options.to - ? await promoted(client, options.dir, read, options.to, options.allowDelete === true) + ? await promoted( + client, + options.dir, + read, + options.to, + options.deleteTargetOnly === true + ) : null const submission = promotion?.submission ?? read /** `data` as this apply reports it: a promotion's own wording and warnings added. */ @@ -869,9 +893,29 @@ function applyCommand(options: ApplyOptions): string { ...(options.only ? [`--only ${shellWord(options.only)}`] : []), ...(options.to ? [`--to ${shellWord(options.to)}`] : []), ...(options.allowDelete ? ['--allow-delete'] : []), + ...(options.deleteTargetOnly ? ['--delete-target-only'] : []), ].join(' ') } +/** + * Refuses `--delete-target-only` without `--to` (only a promotion has a target of its own) or + * without `--allow-delete` (the server deletes nothing without it): the flag never stands in for + * consent to the directory's own deletes. + */ +function assertTargetOnlyDeletable(options: ApplyOptions): void { + if (!options.deleteTargetOnly) return + if (!options.to) { + throw new CliUsageError( + "--delete-target-only deletes a promotion target's own resources: it needs --to ." + ) + } + if (!options.allowDelete) { + throw new CliUsageError( + '--delete-target-only needs --allow-delete beside it: add it, and the promotion also deletes what this directory deleted since the last promotion.' + ) + } +} + /** * A `W_PATH_NOT_SENT` warning for each file under a kind folder the CLI does not send: a hidden * name, or a character no workspace path holds. diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index 5c44370f2d8..e44363ca12a 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -186,7 +186,7 @@ const ADVANCED_GUIDE = `# Sim workspace source: advanced ## Promote to another workspace -\`sim workspaces source apply --dir . --to --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name). Each promote records what it left in the target in \`.sim/targets/.json\` (commit it), and the next promote compares with that: a resource changed in the target since is kept when this directory left it alone, and refused as stale (exit ${SOURCE_STALE_EXIT_CODE}) when this directory changed it too. A resource with no record (the first promote, or one created in the target since) is updated to match this directory; delete the record to take the target as it stands again. Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w \`) and connect its credentials, or the workflows using them are held back. This directory stays bound to the workspace it was read from. Without \`--to\`, \`check\`, \`status\`, \`diff\` and \`apply\` act on the workspace this directory was read from, whatever the profile's default; a \`--workspace\` naming another is refused (\`SOURCE_OTHER_WORKSPACE\`). +\`sim workspaces source apply --dir . --to --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name). Each promote records what it left in the target in \`.sim/targets/.json\` (commit it), and the next promote compares with that: a resource changed in the target since is kept when this directory left it alone, and refused as stale (exit ${SOURCE_STALE_EXIT_CODE}) when this directory changed it too. A resource with no record (the first promote, or one created in the target since) is updated to match this directory; delete the record to take the target as it stands again. Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory never declared (and no promote left there) are kept with a warning, even with \`--allow-delete\`; deleting them too takes \`--delete-target-only\` beside \`--allow-delete\`. A resource this directory deleted since the last promote is deleted in the target only with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w \`) and connect its credentials, or the workflows using them are held back. This directory stays bound to the workspace it was read from. Without \`--to\`, \`check\`, \`status\`, \`diff\` and \`apply\` act on the workspace this directory was read from, whatever the profile's default; a \`--workspace\` naming another is refused (\`SOURCE_OTHER_WORKSPACE\`). ` /** The workspace directory's tree beside the printed source. */ diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index bce3f1c8c98..d462e7d66b1 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -335,8 +335,13 @@ each folder's `.sim/lock.json` and `.sim/base.json` replaced by the target's fol (or dropped, so the key is created), and the target's root base. A base entry the directory does not declare stays in it when the promotion record holds it for the same id (the directory deleted it since, so the apply refuses it without `--allow-delete` as it would at home); one only the target -ever held (`targetOnly`) leaves it unless `--allow-delete`, and the CLI names each such delete -`W_PROMOTE_DELETES_TARGET_ONLY`. The server sees an ordinary apply against its own base, +ever held (`targetOnly`) leaves it, so the apply keeps it (`W_UNDECLARED`) whether or not it allows +deletes. Deleting those takes `--delete-target-only` beside `--allow-delete` (the CLI refuses it +alone): `--allow-delete` is consent to the directory's own deletes, and must not also reach the +target's prod-only tables, their rows, or skills made there. The CLI names each such delete +`W_PROMOTE_DELETES_TARGET_ONLY`, and `promotionRecord` never records a target-only entry (a held +delete keeps its submitted base entry, which would otherwise read as the directory's delete on +the next promotion). The server sees an ordinary apply against its own base, so every rule holds unchanged: a resource bound under the key is updated in place (a workflow keeps its id and block ids through the target's lock), an unbound one of the same name is adopted, a missing one created, and secrets and credentials bind by name and key to the target's own. The diff --git a/packages/workspace-code/src/promote.test.ts b/packages/workspace-code/src/promote.test.ts index ac47d5c83ac..5a1976a1b7f 100644 --- a/packages/workspace-code/src/promote.test.ts +++ b/packages/workspace-code/src/promote.test.ts @@ -34,7 +34,7 @@ function firstRecord(plan: Array<{ kind: string; key: string; action: string; he blobPaths: [], kinds: ['workflow'], target: TARGET, - allowDelete: false, + deleteTargetOnly: false, }) return promotionRecord({ workspaceId: 'ws_prod', @@ -57,7 +57,7 @@ describe('promotion records', () => { kinds: ['workflow'], target: TARGET, record, - allowDelete: false, + deleteTargetOnly: false, }) expect(next.files['workflows/ingest/workflow.ts']).toBe(DIR['workflows/ingest/workflow.ts']) }) @@ -77,7 +77,7 @@ describe('promotion records', () => { }, }, record, - allowDelete: false, + deleteTargetOnly: false, }) expect(Object.keys(promote(DIR).files)).toEqual(['workflows/ingest/.sim/base.json']) @@ -87,6 +87,47 @@ describe('promotion records', () => { ) }) + it('keeps what only the target holds out of the base, even with deletes allowed, unless its deletion is asked for, and never records it', () => { + const target = { + files: TARGET.files, + base: { + ...TARGET.base, + resources: { ...TARGET.base.resources, 'skill:audit': { id: 'sk_audit' } }, + }, + } + const promote = (deleteTargetOnly: boolean, record?: PromotionRecord) => + promotionSubmission({ + files: DIR, + blobPaths: [], + kinds: ['workflow', 'skill'], + target, + record, + deleteTargetOnly, + }) + + expect(promote(false).targetOnly).toEqual(['skill:audit']) + expect(promote(false).base.resources['skill:audit']).toBeUndefined() + const deleting = promote(true) + expect(deleting.base.resources['skill:audit']).toEqual({ id: 'sk_audit' }) + + // The delete was held back (a target workflow uses it): the answer's base keeps the entry. + const record = promotionRecord({ + workspaceId: 'ws_prod', + previous: undefined, + kinds: ['workflow', 'skill'], + files: DIR, + submission: deleting, + applied: { + base: deleting.base, + files: {}, + plan: [{ kind: 'skill', key: 'audit', action: 'delete', held: true }], + }, + }) + expect(record.resources['skill:audit']).toBeUndefined() + expect(promote(false, record).targetOnly).toEqual(['skill:audit']) + expect(promote(false, record).base.resources['skill:audit']).toBeUndefined() + }) + it('ignores a record for another target, or one this build cannot read', () => { const record: PromotionRecord = { version: 1, diff --git a/packages/workspace-code/src/promote.ts b/packages/workspace-code/src/promote.ts index 5e0141c4194..987f1def997 100644 --- a/packages/workspace-code/src/promote.ts +++ b/packages/workspace-code/src/promote.ts @@ -37,8 +37,8 @@ export interface PromotionSubmission { /** The directory's text files, each workflow folder whole, with the target's record files. */ files: Record /** - * The target's base, without what only the target holds unless deletes are allowed (see - * {@link PromotionInput.allowDelete}). + * The target's base, without what only the target holds unless its deletion is asked for (see + * {@link PromotionInput.deleteTargetOnly}). */ base: WorkspaceBaseV1 /** @@ -48,8 +48,8 @@ export interface PromotionSubmission { deletedInTarget: string[] /** * The resources (by address) the target holds that the directory does not declare and no - * promotion from it left there: only the target ever held them, so `--allow-delete` deleting one - * deletes something the directory never had. + * promotion from it left there: only the target ever held them. They are kept, and the server + * names each (`W_UNDECLARED`), unless {@link PromotionInput.deleteTargetOnly}. */ targetOnly: string[] } @@ -66,13 +66,13 @@ export interface PromotionInput { /** What the last promotion from this directory left in the target, when one is recorded. */ record?: PromotionRecord /** - * Whether the promotion may delete what only the target holds. Without it, those resources leave - * the base, so the apply keeps them with a warning instead of refusing for want of - * `--allow-delete`. A resource the last promotion left that the directory no longer declares - * stays in the base either way: the directory deleted it, which needs `--allow-delete` as an - * apply to its own workspace does. + * Whether the promotion deletes what only the target holds (`--delete-target-only`, which the + * apply's `--allow-delete` must accompany). Without it, those resources leave the base, so the + * apply keeps them with a warning whether or not it allows deletes. A resource the last + * promotion left that the directory no longer declares stays in the base either way: the + * directory deleted it, which needs `--allow-delete` as an apply to its own workspace does. */ - allowDelete: boolean + deleteTargetOnly: boolean } /** A workflow folder's record files: what binds it to one workspace's workflow and block ids. */ @@ -212,7 +212,7 @@ export function parsePromotionRecord( * The submission that promotes a directory to the target its get answer describes: the * directory's source with every workflow folder's lock and base replaced by the target's folder * of the same key (or dropped, so a key the target lacks creates a workflow), and the target's - * base, without what only the target holds unless deletes are allowed. With a + * base, without what only the target holds unless its deletion is asked for. With a * {@link PromotionRecord}, each entry the record holds for the same id stands in for the target's * own, and a workflow folder records the fingerprint the last promotion left: sent as its base * alone (a no-op) when the directory has not changed it since, else whole, so a target edit since @@ -279,7 +279,7 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission const promoted = recorded?.id === live.id ? recorded : undefined if (!keys.has(parsed.key) && !promoted) { targetOnly.push(resourceAddress(parsed.kind, parsed.key)) - if (!input.allowDelete) return [] + if (!input.deleteTargetOnly) return [] } return [[address, promoted ?? live] as const] }) @@ -309,7 +309,9 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission * plan). A resource's entry is the answer's base entry; a workflow folder's is what the apply * wrote, or what was sent whole and planned as a no-op, else the `previous` entry (a folder sent * as its base alone, or held back, or never reached). The record of a kind the promotion did not - * manage is kept. + * manage is kept. A resource only the target held (`targetOnly`) is never recorded: a delete of it + * the apply held back keeps its submitted base entry, and recording that would make the next + * promotion read it as one the directory deleted. */ export function promotionRecord(input: { workspaceId: string @@ -317,7 +319,7 @@ export function promotionRecord(input: { kinds: readonly WorkspaceKind[] /** The directory's text files, as {@link PromotionInput} took them. */ files: Readonly> - submission: Pick + submission: Pick applied: { base: WorkspaceBaseV1 files: Readonly> @@ -332,7 +334,13 @@ export function promotionRecord(input: { resources[resourceAddress(parsed.kind, parsed.key)] = entry } } - Object.assign(resources, input.applied.base.resources) + const targetOnly = new Set(input.submission.targetOnly) + for (const [address, entry] of Object.entries(input.applied.base.resources)) { + const parsed = parseResourceAddress(address) + if (parsed && !targetOnly.has(address)) { + resources[resourceAddress(parsed.kind, parsed.key)] = entry + } + } const workflows: Record = {} if (!managed.has('workflow')) { Object.assign(workflows, input.previous?.workflows) From 18433644086b6054140a65cd93bf96e8687d1abe Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 14:52:17 -0800 Subject: [PATCH 34/51] fix(table): strip a source apply's dropped managed columns once, under a row-scaled timeout The in-transaction strip ran once per dropped column under withLockedTable's 10 s default statement timeout. reshape now collects the dropped column ids and strips them in one pass with stripDroppedColumnData, which scales the statement and idle timeouts per row as the sibling column rewrites do. The carried-over non-null assertion on the row filter is gone. --- apps/sim/lib/table/application/reshape.ts | 10 +++- apps/sim/lib/table/columns/service.ts | 53 ++++++++++++------ .../application/rows.integration.ts | 55 +++++++++++++++++++ 3 files changed, 99 insertions(+), 19 deletions(-) diff --git a/apps/sim/lib/table/application/reshape.ts b/apps/sim/lib/table/application/reshape.ts index a1bb94527d3..2d19b0ac2e9 100644 --- a/apps/sim/lib/table/application/reshape.ts +++ b/apps/sim/lib/table/application/reshape.ts @@ -33,7 +33,7 @@ import { rowDataNameToId, } from '@/lib/table/column-keys' import { columnTypeById } from '@/lib/table/column-types' -import { addTableColumn, deleteColumn } from '@/lib/table/columns/service' +import { addTableColumn, deleteColumn, stripDroppedColumnData } from '@/lib/table/columns/service' import { signalTableRowsChanged, signalTableSchemaChanged } from '@/lib/table/events' import { performUpdateTableColumn } from '@/lib/table/orchestration' import { readCurrentRowsVersion } from '@/lib/table/row-changes' @@ -363,12 +363,18 @@ export const reshapeTableUseCase = defineAuthorizedTableUseCase({ input.sourceRows === undefined ? locked.metadata?.sourceRows?.mode === 'managed' : input.sourceRows?.mode === 'managed' + const droppedIds: string[] = [] for (const columnName of input.drop ?? []) { + const dropped = table.schema.columns.find((column) => + columnMatchesRef(column, columnName) + ) table = await deleteColumn({ tableId: table.id, columnName }, requestId, { ...options, - stripRowsInTransaction: managed, + callerStripsRows: managed, }) + if (managed && dropped) droppedIds.push(getColumnId(dropped)) } + await stripDroppedColumnData(tx, table, droppedIds) // With rows to write, a column made required is made so after them (an add, below, too). const deferredUpdates: string[] = [] const updates = (input.update ?? []).flatMap((entry) => { diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index 8280cdff913..14fb8a0a5fb 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -20,7 +20,7 @@ import { userTableDefinitions, userTableRows } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { resolveCurrencyCode } from '@sim/table-cells/currency' import { omit } from '@sim/utils/object' -import { and, asc, count, eq, gt, sql } from 'drizzle-orm' +import { and, asc, count, eq, gt, type SQL, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { columnMatchesRef, generateColumnId, getColumnId } from '@/lib/table/column-keys' import { @@ -395,11 +395,35 @@ async function stripColumnData( columnIds: string[] ): Promise { await updateTableRowsWithDerivedSecretProvenance(trx, { - rowWhere: and(eq(userTableRows.tableId, tableId), eq(userTableRows.workspaceId, workspaceId))!, + rowWhere: and( + eq(userTableRows.tableId, tableId), + eq(userTableRows.workspaceId, workspaceId) + ) as SQL, transformation: { mode: 'remove-columns', columnIds }, }) } +/** The statement timeout a strip of `columnCount` columns from `rowCount` rows gets. */ +function stripTimeoutMs(rowCount: number, columnCount: number): number { + return scaledStatementTimeoutMs(rowCount, { baseMs: 60_000, perRowMs: 2 * columnCount }) +} + +/** + * Removes the cells of `columnIds`, columns {@link deleteColumn} dropped in `tx` with + * `callerStripsRows`, from every row of `table`, in one pass under a row-count-scaled timeout, so + * the row version the commit leaves already holds that write. + */ +export async function stripDroppedColumnData( + tx: TableWriteTx, + table: Pick, + columnIds: string[] +): Promise { + if (columnIds.length === 0) return + const timeoutMs = stripTimeoutMs(table.rowCount ?? 0, columnIds.length) + await setTableTxTimeouts(tx.trx, { statementMs: timeoutMs, idleMs: timeoutMs }) + await stripColumnData(tx.trx, table.id, table.workspaceId, columnIds) +} + /** * Fire-and-forget reclamation of a deleted column's row storage. The column is * already gone from the schema, so reads never surface the orphaned id — @@ -417,11 +441,7 @@ function stripColumnDataInBackground( void (async () => { try { await db.transaction(async (trx) => { - const statementMs = scaledStatementTimeoutMs(rowCount, { - baseMs: 60_000, - perRowMs: 2 * columnIds.length, - }) - await setTableTxTimeouts(trx, { statementMs }) + await setTableTxTimeouts(trx, { statementMs: stripTimeoutMs(rowCount, columnIds.length) }) await stripColumnData(trx, tableId, workspaceId, columnIds) }) logger.info( @@ -441,7 +461,8 @@ function stripColumnDataInBackground( /** * Deletes a column from a table's schema. Returns once the schema is updated and * reclaims the column's row-data storage in the background (fire-and-forget), - * unless `stripRowsInTransaction` strips it in the caller's transaction. + * unless `callerStripsRows` leaves that to the caller's transaction + * ({@link stripDroppedColumnData}). * * @param data - Delete column data * @param requestId - Request ID for logging @@ -453,10 +474,11 @@ export async function deleteColumn( requestId: string, options?: ColumnMutationOptions & { /** - * Strip the column's cells in the caller's `tx`, so the row version the commit leaves already - * holds that write (workspace source records it for managed rows, which stay small). + * Leave the column's cells for the caller to strip in its `tx` with + * {@link stripDroppedColumnData}, once for every column it drops, so the row version the + * commit leaves already holds that write (workspace source records it for managed rows). */ - stripRowsInTransaction?: boolean + callerStripsRows?: boolean } ): Promise { const { def, stripKey } = await withLockedTable( @@ -547,12 +569,9 @@ export async function deleteColumn( def.rowCount ?? 0, requestId ) - if (options?.tx && options.stripRowsInTransaction) { - await stripColumnData(options.tx.trx, data.tableId, def.workspaceId, [stripKey]) - } else if (options?.tx) { - // Inside a caller's transaction the drop may still roll back, so the cells stay until it commits. - options.tx.afterCommit(strip) - } else strip() + if (!options?.tx) strip() + // Inside a caller's transaction the drop may still roll back, so the cells stay until it commits. + else if (!options.callerStripsRows) options.tx.afterCommit(strip) return def } diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index c245411a8b9..ae0e734f316 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -1117,6 +1117,61 @@ export default defineTable({ expect(settled.applied).toEqual([]) }) + check( + "columns one apply drops from managed rows are stripped in one pass, under a timeout scaled past the table write's 10 s default", + async () => { + const workspaceId = await newWorkspace() + const read = await managedPlans(workspaceId) + const table = await storedTable(workspaceId) + // A probe on this table's row updates records each one that rewrites cells, and the timeout it + // ran under. + const probe = `strip_probe_${table.id.replace(/[^a-z0-9]/gi, '').toLowerCase()}` + await db.execute( + sql.raw(`create table ${probe} (row_id text not null, timeout_ms numeric not null)`) + ) + await db.execute( + sql.raw( + `create function ${probe}_fn() returns trigger language plpgsql as $$ begin insert into ${probe} values (new.id, extract(epoch from current_setting('statement_timeout')::interval) * 1000); return new; end $$` + ) + ) + await db.execute( + sql.raw( + `create trigger ${probe}_trg before update on user_table_rows for each row when (new.table_id = '${table.id}' and old.data is distinct from new.data) execute function ${probe}_fn()` + ) + ) + try { + const dropped = await apply( + workspaceId, + { + 'tables/plans.ts': read.files['tables/plans.ts'] + .replace(' seats: column.number(),\n', '') + .replace(" tier: column.select({ options: ['basic', 'pro'] }),\n", ''), + 'tables/plans.rows.jsonl': '{"code":"solo"}\n{"code":"team"}\n', + }, + { base: read.base, allowDelete: true } + ) + expect(dropped.failed).toBeUndefined() + const updates = [ + ...(await db.execute<{ passes: number; timeout_ms: number }>( + sql.raw( + `select count(*)::int as passes, min(timeout_ms)::int as timeout_ms from ${probe} group by row_id` + ) + )), + ] + expect(updates).toHaveLength(2) + for (const update of updates) { + expect(update.passes).toBe(1) + expect(update.timeout_ms).toBeGreaterThanOrEqual(60_000) + } + expect(Object.keys((await storedRows(workspaceId))[0])).toEqual(['code']) + } finally { + await db.execute(sql.raw(`drop trigger if exists ${probe}_trg on user_table_rows`)) + await db.execute(sql.raw(`drop function if exists ${probe}_fn()`)) + await db.execute(sql.raw(`drop table if exists ${probe}`)) + } + } + ) + check( 'a column the apply drops leaves no row drift: the next plan sees no server change, and a delete of the table is not stale', async () => { From 2de66fe7a59eb582bbcb4ff1f2e78a9bd2bf1d81 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 14:54:07 -0800 Subject: [PATCH 35/51] fix(sim-cli): workspace status and apply leave out function files no codeFile() names readWorkspaceDir drops each file under a workflow folder's functions/ that neither its workflow.ts nor its lock names, through the same codeFilePaths rule the workflow directory status uses. A workspace status no longer reports such a file as a local change (exit 11 forever), names it on the 'Not named by any codeFile()' line, and an apply no longer sends that folder whole for it. --- .../commands/protocol/workspace-source-dir.ts | 42 ++++++++++++++++++- .../protocol/workspace-source-status.test.ts | 30 +++++++++++++ .../protocol/workspace-source-status.ts | 1 + 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index e44363ca12a..2a9551628ad 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -6,7 +6,9 @@ */ import { realpath, stat } from 'node:fs/promises' import { mapWithConcurrency } from '@sim/utils/concurrency' +import { compareStrings } from '@sim/utils/string' import type { WorkflowInput } from '@sim/workflow-code' +import { SOURCE_CODE_DIR, SOURCE_ENTRY_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' import { SOURCE_COMPILER_OPTIONS, typecheckCommand, @@ -77,6 +79,7 @@ import { TSCONFIG_FULL_FILE, TSCONFIG_MARKER, } from './source-tree' +import { codeFilePaths } from './workflow-dir' import { statDigest } from './workspace-source-blobs' /** Where the workspace declaration SDK is written: `@sim/workflow-sdk/workspace` resolves here. */ @@ -552,10 +555,46 @@ export interface WorkspaceSubmission { base?: ApplyWorkspaceSourceBody['base'] /** Files under a kind folder that are not sent: hidden names and unsupported characters. */ ignored: string[] + /** + * Files under a workflow folder's `functions/` that neither its `workflow.ts` nor its lock names: + * never sent, as a workflow directory's apply never sends them. + */ + unnamed: string[] /** The blobs this read hashed or reused a recorded digest for, with their stats. */ hashed: ReadonlyMap } +/** + * Drops from `files` each file under a workflow folder's `functions/` that neither the folder's + * `workflow.ts` nor its lock names ({@link codeFilePaths}, the rule a workflow directory's apply + * and status read), returning their paths: an apply never sends one, so a status never counts it + * as a change and the folder is not sent as edited for it. + */ +function dropUnnamedCodeFiles(files: Record): string[] { + const named = new Map>() + const unnamed: string[] = [] + for (const path of Object.keys(files)) { + if (kindOfPath(path) !== 'workflow') continue + const segments = path.split('/') + if (segments[2] !== SOURCE_CODE_DIR) continue + const folder = segments.slice(0, 2).join('/') + let sent = named.get(folder) + if (!sent) { + let lock: unknown = null + try { + lock = JSON.parse(files[`${folder}/${SOURCE_LOCK_FILE}`] ?? 'null') + } catch {} + sent = new Set(codeFilePaths(files[`${folder}/${SOURCE_ENTRY_FILE}`] ?? null, lock)) + named.set(folder, sent) + } + if (!sent.has(segments.slice(2).join('/'))) { + unnamed.push(path) + delete files[path] + } + } + return unnamed.sort(compareStrings) +} + /** * Drops from `files` the workflow folders whose every file is as `.sim/written.json` records it, * keeping what `unchangedFiles` sends for one (its base alone when it was read from @@ -647,6 +686,7 @@ export async function readWorkspaceDir( const text = texts[index] if (text !== null) files[path] = text }) + const unnamed = dropUnnamedCodeFiles(files) if (!options.whole) pruneUnchangedWorkflows(files, written, workspaceId) const root = await realpath(dir) const verified = new Set() @@ -681,7 +721,7 @@ export async function readWorkspaceDir( ) } } - return { kinds, files, blobs, ignored, hashed, ...(base ? { base } : {}) } + return { kinds, files, blobs, ignored, unnamed, hashed, ...(base ? { base } : {}) } } /** `.sim/skipped.json`, or `null` when nothing was skipped. */ diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts index 8088ed0bd14..2602cb51bdd 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts @@ -390,6 +390,36 @@ describe('workspaces source status', () => { }) }) + it('leaves out a function file no codeFile() names, as an apply never sends it: clean, exit 0, naming it', async () => { + write('workflows/ingest/functions/scratch.js', 'return 1\n') + mockRequest.mockResolvedValue(dryRun()) + + await run('status') + + expect(printed()).toMatchObject({ + state: 'clean', + local: [], + unnamed: ['workflows/ingest/functions/scratch.js'], + }) + expect(sentFiles()[0]['workflows/ingest/functions/scratch.js']).toBeUndefined() + expect(process.exitCode ?? 0).toBe(0) + }) + + it('still counts a new function file the lock names as a local change', async () => { + write( + 'workflows/ingest/.sim/lock.json', + JSON.stringify({ version: 1, blocks: { run: { id: 'b1', codeFile: 'functions/run.js' } } }) + ) + write('workflows/ingest/functions/run.js', 'return 1\n') + mockRequest.mockResolvedValue(dryRun()) + + await run('status') + + expect(printed()).not.toHaveProperty('unnamed') + expect(sentFiles()[0]['workflows/ingest/functions/run.js']).toBe('return 1\n') + expect(process.exitCode).toBe(11) + }) + it('sends every workflow folder whole to the plan read, so it compares each with the stored workflow', async () => { mockRequest.mockResolvedValue(dryRun()) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts index 37067e8a0e8..c24f774a1b6 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts @@ -397,6 +397,7 @@ export function attachWorkspaceSourceStatus(source: Command): void { server, ...(note ? { note } : {}), ...status, + ...(local.submission.unnamed.length > 0 ? { unnamed: local.submission.unnamed } : {}), plan, next: (login From b3a808077662767315900f4e0d37a44b297f23c4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:02:47 -0800 Subject: [PATCH 36/51] fix(workflow-code): refuse an unknown dropdown option only on a field the edit sets, and not on deployment-shaped options offline The server now passes the fields its value gate judges (changedSubBlockFields) into the shared refusal rule, so a stored out-of-options value written bare again on an edited block is no longer refused. The catalog marks options a registry function derives from deployment settings (optionsVary); the CLI's offline check keeps a value outside them a warning, since its cached catalog may have been built under other settings, and leaves the verdict to the server. --- apps/docs/openapi-v2-resources.json | 15 ++++++++++ .../lib/api/contracts/v2/source-catalog.ts | 6 ++++ .../apply-workflow-source.integration.ts | 27 +++++++++++++++++ .../source-pipeline/evaluate/finalize.ts | 11 ++++++- .../source-pipeline/evaluate/refusals.ts | 8 +++-- apps/sim/lib/workflows/code/apply/gates.ts | 17 +++++++++++ .../sim/lib/workflows/code/catalog/project.ts | 4 ++- .../src/commands/protocol/source-check.ts | 30 ++++++++++++++----- .../protocol/workflow-source-check.test.ts | 18 +++++++++++ packages/sim-cli/src/generated/v2-api.ts | 3 ++ packages/workflow-code/DESIGN.md | 8 ++++- packages/workflow-code/src/catalog/types.ts | 6 ++++ .../workflow-code/src/diagnostics/refusals.ts | 21 +++++++++---- .../workflow-sdk/test/fixtures/catalog.json | 9 ++++-- 14 files changed, 161 insertions(+), 22 deletions(-) diff --git a/apps/docs/openapi-v2-resources.json b/apps/docs/openapi-v2-resources.json index 09a62bdb1e0..06b645fde37 100644 --- a/apps/docs/openapi-v2-resources.json +++ b/apps/docs/openapi-v2-resources.json @@ -32568,6 +32568,11 @@ } ] }, + "optionsVary": { + "description": "The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.", + "type": "boolean", + "const": true + }, "hidden": { "description": "Hidden from the editor.", "type": "boolean" @@ -33643,6 +33648,11 @@ } ] }, + "optionsVary": { + "description": "The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.", + "type": "boolean", + "const": true + }, "hidden": { "description": "Hidden from the editor.", "type": "boolean" @@ -34491,6 +34501,11 @@ } ] }, + "optionsVary": { + "description": "The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.", + "type": "boolean", + "const": true + }, "hidden": { "description": "Hidden from the editor.", "type": "boolean" diff --git a/apps/sim/lib/api/contracts/v2/source-catalog.ts b/apps/sim/lib/api/contracts/v2/source-catalog.ts index b157471e323..78091e623d3 100644 --- a/apps/sim/lib/api/contracts/v2/source-catalog.ts +++ b/apps/sim/lib/api/contracts/v2/source-catalog.ts @@ -168,6 +168,12 @@ const catalogSubBlockSchema = z ]) .optional() .describe('Selectable options.'), + optionsVary: z + .literal(true) + .optional() + .describe( + 'The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.' + ), hidden: z.boolean().optional().describe('Hidden from the editor.'), readOnly: z.boolean().optional().describe('Display-only.'), password: z.boolean().optional().describe('Holds a secret.'), diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index bfe4f1cfc8e..bb54bb17350 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -1567,6 +1567,33 @@ describe('source apply edge cases', () => { } ) + check( + "an edited block's stored out-of-options value written bare again is not refused: the edit does not set that field", + async () => { + const start = seedBlock('start_trigger', 'Start') + const score = seedBlock('function', 'Score', { + code: ['code', 'return 1'], + language: ['dropdown', 'cobol'], + }) + const workflowId = await seedGraph([start, score]) + const source = await getInlineSource(workflowId) + + const result = await apply( + source, + edit( + edit(source.files, "language: raw('cobol')", "language: 'cobol'"), + 'return 1', + 'return 2' + ) + ) + + expect(result.applied).toBe(true) + expect(result.diagnostics.map((d) => d.code)).not.toContain('E_APPLY_INVALID_SUBBLOCK_FIELD') + const after = await snapshot(workflowId) + expect(after.state?.blocks[score.id]?.subBlocks.language?.value).toBe('cobol') + } + ) + check( 'a value the field normalizes is stored normalized, and untouched blocks keep their bytes', async () => { diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts index 62b726fd821..9ac87868483 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts @@ -28,6 +28,7 @@ import type { } from '@/lib/workflows/application/source-pipeline/evaluate/types' import { foreignResourceRefusals } from '@/lib/workflows/application/source-resource-scope' import { normalizeWorkflowVariables } from '@/lib/workflows/application/workflow-variables' +import { changedSubBlockFields } from '@/lib/workflows/code/apply/gates' import { mergeSourceMetaFromLock } from '@/lib/workflows/code/source-lock' import type { WorkflowLintReport, @@ -189,7 +190,15 @@ export async function finalizePlan( const changedIds = new Set([...graphDiff.added, ...graphDiff.modified.map((m) => m.id)]) const changedKeys = new Set([...changedIds].flatMap((id) => keyById.get(id) ?? [])) const keyOfId = (id: string) => keyById.get(id) - const compileDiagnostics = editedCompileFindings(refusals, changedKeys) + const changedFields = new Map( + [...changedSubBlockFields(storedGraph, { blocks: gated.blocks, edges }, graphDiff)].flatMap( + ([id, fields]) => { + const key = keyById.get(id) + return key === undefined ? [] : [[key, fields] as const] + } + ) + ) + const compileDiagnostics = editedCompileFindings(refusals, changedKeys, changedFields) const compiledLint = withoutCompiledReferences( withoutPendingReferences(lint, pending), compileDiagnostics, diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts index d0e97ebff3c..3cfa38064d1 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts @@ -359,14 +359,18 @@ export async function sourceRefusals( /** * The compile's findings, each on a block this write changes (or whose code it edits) refused as - * the shared edited-block rule says, so a warning the author introduced cannot be written. + * the shared edited-block rule says, so a warning the author introduced cannot be written. A + * field finding is refused only on a field the write sets (`changedFields`, by block key), the + * fields the value gate judges. */ export function editedCompileFindings( refusals: Pick, - changedKeys: ReadonlySet + changedKeys: ReadonlySet, + changedFields: ReadonlyMap> ): LocatedDiagnostic[] { return refuseEditedFindings(refusals.compileFindings, { changed: (key) => changedKeys.has(key), codeEdited: refusals.codeEdited, + paramChanged: (key, param) => changedFields.get(key)?.has(param) === true, }) } diff --git a/apps/sim/lib/workflows/code/apply/gates.ts b/apps/sim/lib/workflows/code/apply/gates.ts index cf220ef7ecd..3ef777b6116 100644 --- a/apps/sim/lib/workflows/code/apply/gates.ts +++ b/apps/sim/lib/workflows/code/apply/gates.ts @@ -190,6 +190,23 @@ function collectGatedBlocks(base: SourceGraph, target: SourceGraph, diff: GraphD return gated } +/** + * Per block id the diff adds or modifies, the subblock ids whose value it sets: the fields the + * value gate judges ({@link GatedBlock.changedFields}). + */ +export function changedSubBlockFields( + base: SourceGraph, + target: SourceGraph, + diff: GraphDiff +): Map> { + return new Map( + collectGatedBlocks(base, target, diff).map((entry) => [ + entry.block.id, + new Set(entry.changedFields), + ]) + ) +} + /** * The placement gates on a new or retyped block, judged against the target graph's other blocks so * replacing a trigger in one apply (remove the old, add the new) passes. Returns whether the diff --git a/apps/sim/lib/workflows/code/catalog/project.ts b/apps/sim/lib/workflows/code/catalog/project.ts index 2b049d269f4..99e52c9f4b7 100644 --- a/apps/sim/lib/workflows/code/catalog/project.ts +++ b/apps/sim/lib/workflows/code/catalog/project.ts @@ -349,7 +349,9 @@ function projectSubBlock(subBlock: SubBlockConfig, ctx: SubBlockContext): Catalo if (writable && typeof subBlock.value === 'function') { assign(projected, 'valueDefault', projectValueDefault(subBlock.value, subBlock.valueScope)) } - assign(projected, 'options', projectOptions(subBlock)) + const options = projectOptions(subBlock) + assign(projected, 'options', options) + if (typeof subBlock.options === 'function' && Array.isArray(options)) projected.optionsVary = true assign(projected, 'hidden', subBlock.hidden) assign(projected, 'readOnly', subBlock.readOnly) diff --git a/packages/sim-cli/src/commands/protocol/source-check.ts b/packages/sim-cli/src/commands/protocol/source-check.ts index ca192bffa07..ef7f4f045d4 100644 --- a/packages/sim-cli/src/commands/protocol/source-check.ts +++ b/packages/sim-cli/src/commands/protocol/source-check.ts @@ -9,6 +9,7 @@ import { isRecordLike, toRecord } from '@sim/utils/object' import { type BlockCatalog, type CanonicalWorkflow, + catalogBlock, codeFrame, codePointLabel, compileWorkflow, @@ -390,7 +391,7 @@ async function compileWorkflowDir( ...(storedEdges ? { storedUnreadEdges: storedEdges } : {}), ...(envNames ? { envNames, envDeclaredIn: KIND_DIRS.secret } : {}), }) - return { lock, files, compiled, storedBlocks, printedWith } + return { lock, files, compiled, storedBlocks, printedWith, catalog } } /** @@ -437,11 +438,8 @@ export async function checkWorkflowDir( catalog: BlockCatalog, options: { declarations?: WorkspaceDeclarations; strict: StrictMode; prefix?: string } ): Promise<{ findings: CheckFinding[]; entry: string; unlisted: UnlistedCounts }> { - const { lock, files, compiled, storedBlocks, printedWith } = await compileWorkflowDir( - dir, - catalog, - options.declarations - ) + const compiledDir = await compileWorkflowDir(dir, catalog, options.declarations) + const { lock, files, compiled, storedBlocks, printedWith } = compiledDir const entryText = files[SOURCE_ENTRY_FILE] const locate = (block: string, param: string, names: string | undefined) => { const prefix = encodeAnchor({ kind: 'param', block, param: '' }) @@ -552,7 +550,23 @@ export async function checkWorkflowDir( const head = /^(?:\$ref\.|<)([A-Za-z0-9_]+)/.exec(text)?.[1] return head !== undefined && removedKeys.has(head) } - const refusable = (finding: (typeof compiled.diagnostics)[number]) => !mayBeRenamed(finding) + /** + * A dropdown value outside options that vary with deployment settings (`optionsVary`): the + * cached catalog may have been built under other settings than the server's gate runs with, so + * the check keeps the warning and leaves the verdict to the server. + */ + const variesByDeployment = (finding: (typeof compiled.diagnostics)[number]) => { + const { blockKey, param } = finding + if (finding.code !== 'W_UNKNOWN_OPTION' || blockKey === undefined || param === undefined) { + return false + } + const type = compiled.workflow ? getOwn(compiled.workflow.blocks, blockKey)?.type : undefined + const block = type === undefined ? undefined : catalogBlock(compiledDir.catalog, type) + return block?.subBlocks.some((sub) => sub.id === param && sub.optionsVary === true) === true + } + const exempt = (finding: (typeof compiled.diagnostics)[number]) => + mayBeRenamed(finding) || variesByDeployment(finding) + const refusable = (finding: (typeof compiled.diagnostics)[number]) => !exempt(finding) const refused = [ ...refuseEditedFindings( { @@ -561,7 +575,7 @@ export async function checkWorkflowDir( }, { changed, codeEdited } ), - ...compiled.diagnostics.filter(mayBeRenamed), + ...compiled.diagnostics.filter(exempt), ] const findings = [ ...refused.filter((diagnostic) => reported(diagnostic.code, diagnostic.blockKey)), diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index 9dda4da57e6..5d31ebd1c90 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -1036,6 +1036,24 @@ export default workflow('W', {}, () => { expect(process.exitCode).toBe(1) }) + it('keeps a value outside options that vary by deployment a warning, as the cached catalog may predate the settings the server checks with', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace( + ' start.to(done)', + " const run = fn('Run', { language: 'shell', code: 'echo hi' })\n start.to(done)\n done.to(run)" + ).replace('import { response,', 'import { fn, response,') + ) + writeLock([]) + + await checkIn(dir, '--no-strict') + + expect(findingLines()).toMatch(/warning W_UNKNOWN_OPTION: language has no option shell/) + expect(result()).toMatchObject({ errors: 0 }) + expect(process.exitCode).toBeUndefined() + }) + it('passes what a get prints from stored state the server keeps: a loop error edge, a cycle, an unwired branch', async () => { cacheCatalog() writeFileSync(join(dir, 'workflow.ts'), STORED_SHAPES) diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 74a90c48863..47f56e72e70 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -9110,6 +9110,7 @@ export type GetSourceCatalogResponse = { | { opaque: true } + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean @@ -9347,6 +9348,7 @@ export type GetSourceCatalogResponse = { | { opaque: true } + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean @@ -9537,6 +9539,7 @@ export type GetSourceCatalogResponse = { | { opaque: true } + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index c4d1e892b43..df8378eeb68 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -344,7 +344,13 @@ Compile also refuses or flags source the runtime would silently mishandle: input the unified Start passes through); a format with references inside is unread; - a dropdown value outside its static options is `W_UNKNOWN_OPTION`, a `model` the catalog does not list `W_UNKNOWN_MODEL` (free text stays valid), and an unknown `operation` - `E_UNKNOWN_OPERATION` unless the stored block already holds it; + `E_UNKNOWN_OPERATION` unless the stored block already holds it. An apply refuses + `W_UNKNOWN_OPTION` (`E_APPLY_INVALID_SUBBLOCK_FIELD`, `diagnostics/refusals.ts`) only on a field + the edit sets, the fields the server's value gate judges; the CLI's offline check, which holds no + stored values, counts every field of a changed block. Options a registry function derives from + deployment settings are marked `optionsVary` in the catalog: the offline check keeps a value + outside them a warning, since its cached catalog may have been built under other settings, and + leaves the verdict to the server, whose catalog and gate run in one process; - a new block of a legacy or deprecated type is `W_LEGACY_BLOCK_TYPE`, naming its replacement. ## Materialize diff --git a/packages/workflow-code/src/catalog/types.ts b/packages/workflow-code/src/catalog/types.ts index e4773d090d6..e45f7c1c2fe 100644 --- a/packages/workflow-code/src/catalog/types.ts +++ b/packages/workflow-code/src/catalog/types.ts @@ -143,6 +143,12 @@ export interface CatalogSubBlock { * function the projection could not evaluate. */ options?: CatalogOption[] | { dynamic: true } | OpaqueMarker + /** + * The static `options` are a registry function of deployment settings (a feature or a flag), + * evaluated where this catalog was built: the server's field gate evaluates it again at write + * time, so a catalog built elsewhere, or before the settings changed, may list other options. + */ + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index f55bc0d203a..2e75eb56aef 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -77,6 +77,12 @@ export interface EditedBlocks { * code before and after may compare that alone. */ codeEdited(blockKey: string, file: string): boolean + /** + * Whether the edit sets field `param` of the changed block `blockKey`: the server's field gate + * judges only those. A caller that cannot tell (an offline check holds no stored values) leaves + * it out, and every field of a changed block counts as set. + */ + paramChanged?(blockKey: string, param: string): boolean } /** @@ -88,10 +94,11 @@ const CHANGED_BLOCK_REFUSALS: ReadonlyMap = new Map([ ]) /** - * In `workflow.ts` on a block the edit changed: a dropdown value none of its options names, which - * the server's field gate refuses. A value the stored block already holds outside its options - * prints as `raw()`, which the compile does not check, so only a value the author wrote is - * refused. `operation` is left out: an unknown one is already an error, and its warning means the + * In `workflow.ts` on a field the edit set ({@link EditedBlocks.paramChanged}) of a block it + * changed: a dropdown value none of its options names, which the server's field gate refuses. A + * value the stored block already holds outside its options prints as `raw()`, which the compile + * does not check, and one written bare again is not a field the edit set, so only a value the + * author changed is refused. `operation` is left out: an unknown one is already an error, and its warning means the * stored block holds a retired operation, which the gate keeps. */ const PARAM_REFUSALS: ReadonlyMap = new Map([ @@ -178,8 +185,10 @@ function refusalOf(finding: LocatedDiagnostic, edit: EditedBlocks): EditRefusalC } const onParam = PARAM_REFUSALS.get(finding.code) if (onParam) { - if (!inEntryFile || finding.param === 'operation') return undefined - return edit.changed(blockKey) ? onParam : undefined + const { param } = finding + if (!inEntryFile || param === undefined || param === 'operation') return undefined + if (!edit.changed(blockKey)) return undefined + return (edit.paramChanged?.(blockKey, param) ?? true) ? onParam : undefined } if (inEntryFile) { const inEntry = ENTRY_REFUSALS.get(finding.code) diff --git a/packages/workflow-sdk/test/fixtures/catalog.json b/packages/workflow-sdk/test/fixtures/catalog.json index d486709ef08..79f16b768d0 100644 --- a/packages/workflow-sdk/test/fixtures/catalog.json +++ b/packages/workflow-sdk/test/fixtures/catalog.json @@ -1096,7 +1096,8 @@ "id": "bedrock/cohere.command-r-v1:0", "label": "bedrock/cohere.command-r-v1:0" } - ] + ], + "optionsVary": true }, { "id": "vertexCredential", @@ -6890,7 +6891,8 @@ "id": "python", "label": "Python" } - ] + ], + "optionsVary": true }, { "id": "code", @@ -12142,7 +12144,8 @@ "id": "bedrock/cohere.command-r-v1:0", "label": "bedrock/cohere.command-r-v1:0" } - ] + ], + "optionsVary": true }, { "id": "vertexCredential", From dac8326502d8c5e2d4a32f89561c6d45c9d676d5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:03:50 -0800 Subject: [PATCH 37/51] fix(workspace-source): match a re-reported finding by block key, keeping the block id when it names no key The write-time finding dedup dropped blockId from the key outright, so two findings that differed only in the block they named by id merged into one. --- .../application/apply-workspace-source.ts | 17 ++++++------- .../application/findings.test.ts | 25 +++++++++++++++++++ .../workspace-source/application/findings.ts | 14 +++++++++++ 3 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 apps/sim/lib/workspace-source/application/findings.test.ts diff --git a/apps/sim/lib/workspace-source/application/apply-workspace-source.ts b/apps/sim/lib/workspace-source/application/apply-workspace-source.ts index 3603b2028cc..ae8924f956d 100644 --- a/apps/sim/lib/workspace-source/application/apply-workspace-source.ts +++ b/apps/sim/lib/workspace-source/application/apply-workspace-source.ts @@ -2,7 +2,6 @@ import { AuditAction, AuditResourceType, withAuditCorrelation } from '@sim/audit import { createLogger } from '@sim/logger' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { stableStringify } from '@sim/workflow-code' import { APPLY_ORDER, type FlatDiagnostic, @@ -20,7 +19,7 @@ import { refusalDiagnostic, WorkspaceSourceRejectedError, } from '@/lib/workspace-source/application/errors' -import { finding, GET_HINT } from '@/lib/workspace-source/application/findings' +import { finding, findingIdentity, GET_HINT } from '@/lib/workspace-source/application/findings' import { claimKey } from '@/lib/workspace-source/application/keys' import { workspaceSourceOperations } from '@/lib/workspace-source/application/operations' import { @@ -397,23 +396,23 @@ async function writeAll( base: WorkspaceBaseV1, result: ApplyWorkspaceSourceResult ): Promise { - // A write re-reports what its plan already found (a workflow's commit runs its pipeline - // again): each finding is listed once, with the ids the write gave it. - const findingKey = (diagnostic: WorkspaceSourceDiagnostic) => - stableStringify({ ...diagnostic, blockId: undefined }) - const reported = new Map(result.diagnostics.map((diagnostic, at) => [findingKey(diagnostic), at])) + // A write re-reports what its plan already found: each finding is listed once, with the ids + // the write gave it. + const reported = new Map( + result.diagnostics.map((diagnostic, at) => [findingIdentity(diagnostic), at]) + ) for (const change of writeOrder(changes)) { try { await lease.confirm() const written = await write(change, writeCtx) Object.assign(result.files, written.text) for (const diagnostic of written.diagnostics ?? []) { - const at = reported.get(findingKey(diagnostic)) + const at = reported.get(findingIdentity(diagnostic)) if (at !== undefined) { result.diagnostics[at] = diagnostic continue } - reported.set(findingKey(diagnostic), result.diagnostics.length) + reported.set(findingIdentity(diagnostic), result.diagnostics.length) result.diagnostics.push(diagnostic) } result.applied.push(summarize(change, written.id)) diff --git a/apps/sim/lib/workspace-source/application/findings.test.ts b/apps/sim/lib/workspace-source/application/findings.test.ts new file mode 100644 index 00000000000..ecd7d41a267 --- /dev/null +++ b/apps/sim/lib/workspace-source/application/findings.test.ts @@ -0,0 +1,25 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { findingIdentity } from '@/lib/workspace-source/application/findings' + +const UNKNOWN_ENV = { + code: 'W_ENV_UNKNOWN', + severity: 'warning', + message: 'env.QA_NOT_SET is not set in this workspace.', +} as const + +describe('findingIdentity', () => { + it('matches a write finding to its plan finding by block key, whatever id the write gave the block', () => { + expect(findingIdentity({ ...UNKNOWN_ENV, blockKey: 'save', blockId: 'blk_new' })).toBe( + findingIdentity({ ...UNKNOWN_ENV, blockKey: 'save' }) + ) + }) + + it('keeps apart findings that name different blocks only by id', () => { + expect(findingIdentity({ ...UNKNOWN_ENV, blockId: 'blk_a' })).not.toBe( + findingIdentity({ ...UNKNOWN_ENV, blockId: 'blk_b' }) + ) + }) +}) diff --git a/apps/sim/lib/workspace-source/application/findings.ts b/apps/sim/lib/workspace-source/application/findings.ts index e1f2f47af61..531cdd18796 100644 --- a/apps/sim/lib/workspace-source/application/findings.ts +++ b/apps/sim/lib/workspace-source/application/findings.ts @@ -2,6 +2,7 @@ * The findings every kind reports the same way, so the plan reads alike across kinds. */ +import { stableStringify } from '@sim/workflow-code' import { diagnostic } from '@sim/workflow-code/diagnostics' import { type BaseResource, @@ -24,6 +25,19 @@ import type { WorkspaceSourceDiagnostic, } from '@/lib/workspace-source/application/types' +/** + * What makes two findings the same one, for a write that re-reports what its plan found (a + * workflow's commit runs its pipeline again) with the block ids the write gave it: a finding + * naming its block by key matches on the key, whatever id it carries, and one naming its block + * only by id keeps the id, so two such findings on different blocks never merge. + */ +export function findingIdentity(diagnostic: WorkspaceSourceDiagnostic): string { + return stableStringify({ + ...diagnostic, + blockId: diagnostic.blockKey === undefined ? diagnostic.blockId : undefined, + }) +} + /** * One workspace source finding, its severity fixed by the code's prefix. Every finding the app * reports with a workspace code goes through here, so a misspelled code fails the type-check; From 948b2330ed98686eaa70e3d891c999482ac38e11 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:05:11 -0800 Subject: [PATCH 38/51] fix(workflows): accept only a whole loop iterations or parallel count of at least 1 on the ops path containerCount took whatever Number() accepted, so "1e3", "0x10", "2.5", "-1" and "0" (and the numbers -1 and 2.5) were stored silently. They are now refused as input validation errors, the bound source compile holds a count to. --- ...y-workflow-operations-edits.integration.ts | 24 +++++++++++++++++++ apps/sim/lib/workflows/editing/operations.ts | 19 +++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts index 43560d445ba..ac5b0a3ae4e 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts @@ -196,4 +196,28 @@ describe('a loop or parallel count given as a string', () => { }) expect(Object.keys((await stored(workflowId)).loops)).toEqual([]) }) + + it.each(['1e3', '0x10', '2.5', '-1', '0', 2.5, -1])( + 'refuses %j, which is no whole number of at least 1', + async (iterations) => { + const workflowId = await seedWorkflow([block(generateId(), 'starter', 'Start')]) + + await expect( + apply( + workflowId, + [ + { + operation_type: 'add', + block_id: generateId(), + params: { type: 'loop', name: 'Repeat', inputs: { loopType: 'for', iterations } }, + }, + ], + true + ) + ).rejects.toMatchObject({ + droppedInputs: [expect.objectContaining({ field: 'iterations', value: iterations })], + }) + expect(Object.keys((await stored(workflowId)).loops)).toEqual([]) + } + ) }) diff --git a/apps/sim/lib/workflows/editing/operations.ts b/apps/sim/lib/workflows/editing/operations.ts index 907ebadad14..ff3cda25282 100644 --- a/apps/sim/lib/workflows/editing/operations.ts +++ b/apps/sim/lib/workflows/editing/operations.ts @@ -83,9 +83,10 @@ function renameBlock(blocks: Record, blockId: string, name: } /** - * A loop's `iterations` or a parallel's `count` as stored: a number as given, a numeric string as - * its number (the way a slider input coerces), and anything else refused as an input validation - * error, keeping `current`, since the loader would otherwise reset a non-number to its default. + * A loop's `iterations` or a parallel's `count` as stored: a whole number of at least 1, given as + * a number or as a string of digits (the way a number input coerces), the bound source compile + * holds a count to. Anything else (`0`, `-1`, `2.5`, `"1e3"`, `"0x10"`) is refused as an input + * validation error, keeping `current`, since the loader would otherwise reset it to its default. */ function containerCount( block: BlockState, @@ -94,15 +95,19 @@ function containerCount( validationErrors: ValidationError[], current?: unknown ): unknown { - if (typeof value === 'number') return value - const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : Number.NaN - if (Number.isFinite(parsed)) return parsed + const parsed = + typeof value === 'number' + ? value + : typeof value === 'string' && /^\s*\d+\s*$/.test(value) + ? Number(value) + : Number.NaN + if (Number.isSafeInteger(parsed) && parsed >= 1) return parsed validationErrors.push({ blockId: block.id, blockType: block.type, field, value, - error: `Invalid ${field} "${String(value)}" for ${block.type} "${block.name}" - must be a number`, + error: `Invalid ${field} "${String(value)}" for ${block.type} "${block.name}" - must be a whole number of at least 1`, }) return current } From fc3de063955bfa8957635e983d7c899311a2bb38 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:06:14 -0800 Subject: [PATCH 39/51] fix(workflow-code): keep omitted compile findings as one counted entry per code and block Reporter.omitted kept one object per warning past the caps. It now counts them per code and block, so what a compile keeps stays bounded by the blocks it reads; the CLI check sums the counts. --- .../src/commands/protocol/source-check.ts | 10 ++++--- .../workflow-code/src/compile/hostile.test.ts | 14 ++++++++++ packages/workflow-code/src/compile/index.ts | 5 +++- packages/workflow-code/src/compile/report.ts | 28 ++++++++++++++----- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-check.ts b/packages/sim-cli/src/commands/protocol/source-check.ts index ef7f4f045d4..9410b109456 100644 --- a/packages/sim-cli/src/commands/protocol/source-check.ts +++ b/packages/sim-cli/src/commands/protocol/source-check.ts @@ -595,13 +595,15 @@ export async function checkWorkflowDir( // refusal raised is listed already. const unlisted: UnlistedCounts = { errors: 0, warnings: 0 } const edit = { changed, codeEdited } - for (const { code, blockKey } of [ + for (const { code, blockKey, count } of [ ...compiled.omitted, - ...compiled.withheld.filter((finding) => !refusable(finding) || !refusesEdited(finding, edit)), + ...compiled.withheld + .filter((finding) => !refusable(finding) || !refusesEdited(finding, edit)) + .map(({ code, blockKey }) => ({ code, blockKey, count: 1 })), ]) { if (!reported(code, blockKey)) continue - if (fails(code, blockKey)) unlisted.errors++ - else unlisted.warnings++ + if (fails(code, blockKey)) unlisted.errors += count + else unlisted.warnings += count } return { findings, entry: entryText, unlisted } } diff --git a/packages/workflow-code/src/compile/hostile.test.ts b/packages/workflow-code/src/compile/hostile.test.ts index a8e7c91e488..ee8e63b80be 100644 --- a/packages/workflow-code/src/compile/hostile.test.ts +++ b/packages/workflow-code/src/compile/hostile.test.ts @@ -307,6 +307,20 @@ describe('hostile sources produce diagnostics only', () => { expect(computed).toBe(FINDING_LIMITS.warningsPerCode + 1) }) + it('keeps one omitted entry per code and block, counting every warning past the caps', () => { + const reporter = new Reporter() + reporter.addFile('w.ts', 'x') + const at = { file: 'w.ts', start: 0, end: 1 } + const total = FINDING_LIMITS.perSeverity + 5_000 + for (let i = 0; i < total; i++) { + reporter.report('W_LITERAL_SECRET', `x${i}`, at, { blockKey: i % 2 === 0 ? 'a' : 'b' }) + } + + expect(reporter.omitted).toHaveLength(2) + const listed = reporter.diagnostics.filter((d) => d.code === 'W_LITERAL_SECRET').length + expect(reporter.omitted.reduce((sum, entry) => sum + entry.count, 0)).toBe(total - listed) + }) + it('keeps infos from crowding out a warning', () => { const reporter = new Reporter() reporter.addFile('w.ts', 'x') diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 96b9028b7f6..dce5a31729c 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -115,7 +115,10 @@ export interface CompileResult { * what `refuseEditedFindings` reads beside `diagnostics`, so a cap never hides a refusal. */ withheld: LocatedDiagnostic[] - /** Every other warning the caps left out, by code and block, so a caller counts every finding. */ + /** + * Every other warning the caps left out, counted per code and block, so a caller counts every + * finding. + */ omitted: OmittedFinding[] /** Where each anchor sits, to map later server findings back to lines. */ spans: Partial> diff --git a/packages/workflow-code/src/compile/report.ts b/packages/workflow-code/src/compile/report.ts index 3c4a6a9d393..69b711eb267 100644 --- a/packages/workflow-code/src/compile/report.ts +++ b/packages/workflow-code/src/compile/report.ts @@ -109,10 +109,14 @@ export class Reporter { */ readonly withheld: LocatedDiagnostic[] = [] /** - * Every other warning a cap left out, by code and block only (no text, span or frame is built - * for one), so a caller can still count every finding. + * Every other warning a cap left out, counted by code and block (no text, span or frame is built + * for one), so a caller can still count every finding while what is kept stays one entry per + * code and block however many warnings come. */ - readonly omitted: OmittedFinding[] = [] + get omitted(): OmittedFinding[] { + return [...this.omittedCounts.values()] + } + private readonly omittedCounts = new Map() /** The code, block and file of each refusable warning listed or withheld. */ private readonly refusableHeld = new Set() private readonly indexes = new Map() @@ -216,9 +220,17 @@ export class Reporter { ): void { const held = refusableKey(code, at, extras) if (held === undefined || this.refusableHeld.has(held)) { - this.omitted.push( - extras.blockKey === undefined ? { code } : { code, blockKey: extras.blockKey } - ) + const key = `${code}\u0000${extras.blockKey ?? ''}` + const counted = this.omittedCounts.get(key) + if (counted) counted.count++ + else { + this.omittedCounts.set( + key, + extras.blockKey === undefined + ? { code, count: 1 } + : { code, blockKey: extras.blockKey, count: 1 } + ) + } return } this.refusableHeld.add(held) @@ -296,10 +308,12 @@ export class Reporter { } } -/** A warning a compile's caps left out of every list, as {@link Reporter.omitted} keeps it. */ +/** The warnings of one code and block a compile's caps left out of every list ({@link Reporter.omitted}). */ export interface OmittedFinding { code: CompileDiagnosticCode blockKey?: BlockKey + /** How many warnings of this code and block were left out. */ + count: number } /** The code, block and file a refusable warning is withheld by; none for any other finding. */ From 8db86afa518a1d64b96adeb8c9b3f26f2438f670 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:07:11 -0800 Subject: [PATCH 40/51] test(workspace-source): scope the racing authorize spy to its own workspace and tool The spy edited its tool on any delete authorize call, so a test that timed out with the spy still installed could edit the tool of whichever test ran next. --- .../application/tools-skills.integration.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/tools-skills.integration.ts b/apps/sim/lib/workspace-source/application/tools-skills.integration.ts index ea3d93b5496..289010517dd 100644 --- a/apps/sim/lib/workspace-source/application/tools-skills.integration.ts +++ b/apps/sim/lib/workspace-source/application/tools-skills.integration.ts @@ -10,7 +10,7 @@ import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' import { generateId } from '@sim/utils/id' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' -import { eq, inArray, sql } from 'drizzle-orm' +import { and, eq, inArray, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, vi } from 'vitest' import { createWorkspaceCustomToolUseCase, @@ -820,13 +820,17 @@ describe('workspace source: custom tools and skills', () => { const created = await apply(workspaceId, TOOLS, { kinds: ['custom-tool'] }) const toolId = created.base.resources['custom-tool:lookup']?.id ?? '' const authorize = deleteWorkspaceCustomToolUseCase.authorize + // Only this test's delete races: a call for another workspace (a test still running after its + // timeout) passes through untouched. const racing = vi .spyOn(deleteWorkspaceCustomToolUseCase, 'authorize') .mockImplementation(async (args) => { - await db - .update(customTools) - .set({ code: 'return "edited in the UI"' }) - .where(eq(customTools.id, toolId)) + if (args.input.workspaceId === workspaceId && args.input.toolId === toolId) { + await db + .update(customTools) + .set({ code: 'return "edited in the UI"' }) + .where(and(eq(customTools.id, toolId), eq(customTools.workspaceId, workspaceId))) + } return authorize.call(deleteWorkspaceCustomToolUseCase, args) }) From fdd05a9bca8006b955a5711ebce989cac19942d2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:09:45 -0800 Subject: [PATCH 41/51] fix(workspace-source): charge a workspace dry run to the plan budget, not the write budget A workspace apply with ?dryRun=true writes nothing and runs what a plan runs, yet drew one write unit per workflow folder from the 40-token bucket real applies share, so the preview of a 61-workflow apply left the apply waiting about two minutes. The route's new sourceApply policy charges a dry run to the source-plan bucket the status plan route uses and a real apply to the write bucket, each still per folder sent with its files and capped at the smaller bucket. --- .../[workspaceId]/source/route.test.ts | 44 ++++++++++++++++--- .../workspaces/[workspaceId]/source/route.ts | 8 ++-- .../lib/api/server/routes/v2-json-route.ts | 32 +++++++++++++- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts index 8838546e7d2..c9710866aa6 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts @@ -41,17 +41,17 @@ const auth = { keyType: 'workspace' as const, } -function apply(query = '') { +function apply( + query = '', + files: Record = { + 'workflows/a/workflow.ts': "// @sim-sdk 1\nexport default workflow('A', () => {})\n", + } +) { return POST( new NextRequest(`http://localhost:3000/api/v2/workspaces/${WORKSPACE_ID}/source${query}`, { method: 'POST', headers: { 'x-api-key': 'key', 'content-type': 'application/json' }, - body: JSON.stringify({ - files: { - 'workflows/a/workflow.ts': "// @sim-sdk 1\nexport default workflow('A', () => {})\n", - }, - kinds: ['workflow'], - }), + body: JSON.stringify({ files, kinds: ['workflow'] }), }), { params: Promise.resolve({ workspaceId: WORKSPACE_ID }) } ) @@ -76,6 +76,36 @@ describe('POST /api/v2/workspaces/{workspaceId}/source', () => { v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) }) + it('charges a dry run, which writes nothing, to the plan bucket per folder sent, and a real apply to the write bucket', async () => { + mocks.apply.mockResolvedValue({ + workspaceId: WORKSPACE_ID, + dryRun: true, + plan: [], + applied: [], + needBlobs: [], + files: {}, + base: { version: 1, workspaceId: WORKSPACE_ID, resources: {} }, + diagnostics: [], + }) + const folders = Object.fromEntries( + Array.from({ length: 61 }, (_, index) => [ + `workflows/w${index}/workflow.ts`, + "// @sim-sdk 1\nexport default workflow('W', () => {})\n", + ]) + ) + const charges = () => + v2RouteMocks.operationRate.mock.calls + .filter(([key]) => String(key).endsWith(':api-key:key-1')) + .map(([key, , cost]) => `${String(key).split(':')[1]} ${cost}`) + + expect((await apply('?dryRun=true', folders)).status).toBe(200) + expect(charges()).toEqual(['source.plan 1', 'source.plan 39']) + + v2RouteMocks.operationRate.mockClear() + expect((await apply('', folders)).status).toBe(200) + expect(charges()).toEqual(['workflows.source.write 1', 'workflows.source.write 39']) + }) + it('answers a plan refused before any write with at most 1000 findings, and one that counts the rest', async () => { mocks.apply.mockRejectedValue(new WorkspaceSourceRejectedError(folderErrors(1_500))) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts index 4afda03230c..dd95b4209ea 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts @@ -43,14 +43,16 @@ export const GET = defineV2JsonRoute({ * POST /api/v2/workspaces/[workspaceId]/source — applies a workspace source directory. Any error * in the plan writes nothing (400, 409 stale, 423 locked, 413 over a limit); a dry run answers * 200 with its findings. A write that fails mid-apply answers 200 with `failed` and what was - * written before it. Draws on the source-write budget (see `workspaceApplyRateCost`); a server - * already running as many applies as it allows answers 429. + * written before it. A real apply draws on the source-write budget and a dry run, which writes + * nothing, on the source-plan budget the status plan route uses (`v2RateLimits.sourceApply`), each + * charged per workflow folder sent with its files (`workspaceApplyRateCost`); a server already + * running as many applies as it allows answers 429. */ export const POST = defineV2JsonRoute({ contract: v2ApplyWorkspaceSourceContract, auth: v2ApiKeyAuth, operation: workspaceSourceOperations.apply, - rateLimit: v2RateLimits.sourceWrite, + rateLimit: v2RateLimits.sourceApply, errorPolicy: v2WorkspaceSourceErrorPolicy, parseOptions: { maxBodyBytes: WORKSPACE_SOURCE_REQUEST_MAX_BYTES }, mapInput: ({ params, query, body }) => ({ diff --git a/apps/sim/lib/api/server/routes/v2-json-route.ts b/apps/sim/lib/api/server/routes/v2-json-route.ts index dee705ef039..502c86a20ca 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.ts @@ -4,6 +4,7 @@ import { setRequestAuth } from '@sim/logger' import { LRUCache } from 'lru-cache' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' +import { booleanQueryFlagSchema } from '@/lib/api/contracts/primitives' import { recordRateLimitSnapshot } from '@/lib/api/server/rate-limit-context' import { type CopilotRouteUseCase, @@ -164,7 +165,8 @@ const VALIDATED_BODY_CACHE_MAX = 4 * The budget for writing workflow source (apply, dry run, and create from source), per subject * and on every plan. Each request parses, compiles, canonicalizes and diffs a body of up to * `SOURCE_REQUEST_MAX_BYTES`, so it costs far more than the row reads the generic budget is sized - * for; a dry run costs nearly as much as an apply, so it is charged the same. + * for; a dry run costs nearly as much as an apply, so it is charged the same amount (a workspace + * dry run on the plan bucket instead, see `sourceApply`). */ const V2_SOURCE_WRITE_RATE_LIMIT: TokenBucketConfig = { maxTokens: 40, @@ -195,6 +197,14 @@ const V2_SOURCE_PLAN_RATE_LIMIT: TokenBucketConfig = { refillIntervalMs: 60_000, } +/** Whether `request` asks for a dry run (`?dryRun=true`), as the source apply contract reads it. */ +function isDryRunRequest(request: NextRequest): boolean { + const flag = booleanQueryFlagSchema.safeParse( + request.nextUrl.searchParams.get('dryRun') ?? undefined + ) + return flag.success && flag.data +} + /** * Staging the binary files of a workspace source apply: each request carries up to 24 MB, so a * 1 GB directory stages in about 45 requests. Its own bucket, so staging never spends the @@ -243,6 +253,26 @@ export const v2RateLimits = { return enforceV2Buckets(request, auth, 'source.plan', V2_SOURCE_PLAN_RATE_LIMIT, cost) }, } satisfies V2RateLimitPolicy, + /** + * A source apply route that also serves dry runs (`?dryRun=true`): a dry run writes nothing and + * runs what a plan runs, so it draws on the {@link V2_SOURCE_PLAN_RATE_LIMIT} bucket as the plan + * routes do, and only a real apply spends the {@link V2_SOURCE_WRITE_RATE_LIMIT} bucket applies + * need. Capped at the smaller bucket, so the largest request fits either. + */ + sourceApply: { + maxCost: Math.min(V2_SOURCE_WRITE_RATE_LIMIT.maxTokens, V2_SOURCE_PLAN_RATE_LIMIT.maxTokens), + async enforce(request, auth, _operation, cost = 1) { + return isDryRunRequest(request) + ? enforceV2Buckets(request, auth, 'source.plan', V2_SOURCE_PLAN_RATE_LIMIT, cost) + : enforceV2Buckets( + request, + auth, + 'workflows.source.write', + V2_SOURCE_WRITE_RATE_LIMIT, + cost + ) + }, + } satisfies V2RateLimitPolicy, /** {@link V2_SOURCE_BLOB_STAGE_RATE_LIMIT}, shared by staging and asking what is staged. */ sourceBlobStage: { async enforce(request, auth) { From 2f78263d24bd31db21965c1c1252a37b3247b6a2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:11:32 -0800 Subject: [PATCH 42/51] fix(workspace-source): type the block fields findingIdentity reads off a workflow finding --- apps/sim/lib/workspace-source/application/findings.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/findings.ts b/apps/sim/lib/workspace-source/application/findings.ts index 531cdd18796..8b967624682 100644 --- a/apps/sim/lib/workspace-source/application/findings.ts +++ b/apps/sim/lib/workspace-source/application/findings.ts @@ -27,11 +27,14 @@ import type { /** * What makes two findings the same one, for a write that re-reports what its plan found (a - * workflow's commit runs its pipeline again) with the block ids the write gave it: a finding + * workflow's commit runs its pipeline again) with the block ids the write gave it (a workflow + * folder's findings carry the pipeline's `blockKey` and `blockId` beside the flat fields): a finding * naming its block by key matches on the key, whatever id it carries, and one naming its block * only by id keeps the id, so two such findings on different blocks never merge. */ -export function findingIdentity(diagnostic: WorkspaceSourceDiagnostic): string { +export function findingIdentity( + diagnostic: WorkspaceSourceDiagnostic & { blockKey?: string; blockId?: string } +): string { return stableStringify({ ...diagnostic, blockId: diagnostic.blockKey === undefined ? diagnostic.blockId : undefined, From fedc8b387ea841f96be65af00fbe2d93e689d0e5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 15:29:33 -0800 Subject: [PATCH 43/51] chore(audits): shrink the non-null assertion baseline after the column strip lost its `!` --- scripts/check-explicit-any.baseline.json | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 82f2ac9d67a..21836fa61fb 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1576,7 +1576,6 @@ "apps/sim/lib/slack-search/assistant-stream.ts": 2, "apps/sim/lib/table/column-keys.test.ts": 4, "apps/sim/lib/table/column-types/registry.server.ts": 7, - "apps/sim/lib/table/columns/service.ts": 1, "apps/sim/lib/table/dispatcher.ts": 2, "apps/sim/lib/table/jobs/service.test.ts": 1, "apps/sim/lib/table/query-builder/predicate.ts": 1, From 6a10f64344e644bc061dc9b5f020d3ed9689da79 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:01:52 -0800 Subject: [PATCH 44/51] fix(sim-cli): keep a deployment-shaped option warning unraised under the default strict mode The optionsVary exemption skipped the edit refusal but the strict raise still turned W_UNKNOWN_OPTION into an error on a new block. Listed, withheld and omitted findings now all skip the strict raise when the option varies by deployment. --- .../src/commands/protocol/source-check.ts | 34 ++++++++++++++----- .../protocol/workflow-source-check.test.ts | 4 +-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-check.ts b/packages/sim-cli/src/commands/protocol/source-check.ts index 9410b109456..ff3c7202c2d 100644 --- a/packages/sim-cli/src/commands/protocol/source-check.ts +++ b/packages/sim-cli/src/commands/protocol/source-check.ts @@ -555,14 +555,18 @@ export async function checkWorkflowDir( * cached catalog may have been built under other settings than the server's gate runs with, so * the check keeps the warning and leaves the verdict to the server. */ - const variesByDeployment = (finding: (typeof compiled.diagnostics)[number]) => { + const variesByDeployment = (finding: { code: string; blockKey?: string; param?: string }) => { const { blockKey, param } = finding if (finding.code !== 'W_UNKNOWN_OPTION' || blockKey === undefined || param === undefined) { return false } + return optionsVaryOn(blockKey, (id) => id === param) + } + /** Whether a field of block `blockKey` that `matches` takes options varying by deployment. */ + const optionsVaryOn = (blockKey: string, matches: (subBlockId: string) => boolean) => { const type = compiled.workflow ? getOwn(compiled.workflow.blocks, blockKey)?.type : undefined const block = type === undefined ? undefined : catalogBlock(compiledDir.catalog, type) - return block?.subBlocks.some((sub) => sub.id === param && sub.optionsVary === true) === true + return block?.subBlocks.some((sub) => matches(sub.id) && sub.optionsVary === true) === true } const exempt = (finding: (typeof compiled.diagnostics)[number]) => mayBeRenamed(finding) || variesByDeployment(finding) @@ -580,8 +584,9 @@ export async function checkWorkflowDir( const findings = [ ...refused.filter((diagnostic) => reported(diagnostic.code, diagnostic.blockKey)), ...declared, - ].map(({ code, severity, message, hint, frame, span, blockKey }): CheckFinding => { - const raised = severity === 'warning' && fails(code, blockKey) + ].map((finding): CheckFinding => { + const { code, severity, message, hint, frame, span, blockKey } = finding + const raised = severity === 'warning' && !variesByDeployment(finding) && fails(code, blockKey) return { code, severity: raised ? 'error' : severity, @@ -595,14 +600,27 @@ export async function checkWorkflowDir( // refusal raised is listed already. const unlisted: UnlistedCounts = { errors: 0, warnings: 0 } const edit = { changed, codeEdited } - for (const { code, blockKey, count } of [ - ...compiled.omitted, + // An omitted finding names no field, so one on a block with any deployment-shaped field stays a + // warning. + for (const { code, blockKey, count, varies } of [ + ...compiled.omitted.map((omitted) => ({ + ...omitted, + varies: + omitted.code === 'W_UNKNOWN_OPTION' && + omitted.blockKey !== undefined && + optionsVaryOn(omitted.blockKey, () => true), + })), ...compiled.withheld .filter((finding) => !refusable(finding) || !refusesEdited(finding, edit)) - .map(({ code, blockKey }) => ({ code, blockKey, count: 1 })), + .map((finding) => ({ + code: finding.code, + blockKey: finding.blockKey, + count: 1, + varies: variesByDeployment(finding), + })), ]) { if (!reported(code, blockKey)) continue - if (fails(code, blockKey)) unlisted.errors += count + if (!varies && fails(code, blockKey)) unlisted.errors += count else unlisted.warnings += count } return { findings, entry: entryText, unlisted } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index 5d31ebd1c90..44ecfd54303 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -1036,7 +1036,7 @@ export default workflow('W', {}, () => { expect(process.exitCode).toBe(1) }) - it('keeps a value outside options that vary by deployment a warning, as the cached catalog may predate the settings the server checks with', async () => { + it('keeps a value outside options that vary by deployment a warning on a new block under the default strict mode, as the cached catalog may predate the settings the server checks with', async () => { cacheCatalog() writeFileSync( join(dir, 'workflow.ts'), @@ -1047,7 +1047,7 @@ export default workflow('W', {}, () => { ) writeLock([]) - await checkIn(dir, '--no-strict') + await checkIn(dir) expect(findingLines()).toMatch(/warning W_UNKNOWN_OPTION: language has no option shell/) expect(result()).toMatchObject({ errors: 0 }) From a985ac499588b07e791d1828ca5d80af2ad9fb83 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:01:53 -0800 Subject: [PATCH 45/51] fix(workflows): refuse a falsy loop iterations or parallel count on add and nested edit The add and nested-edit paths guarded on a truthy value, so 0, false, null and '' skipped containerCount and saved the default. They now check presence, as the top-level edit does. --- .../lib/workflows/editing/operations.test.ts | 68 +++++++++++++++++++ apps/sim/lib/workflows/editing/operations.ts | 8 +-- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/workflows/editing/operations.test.ts b/apps/sim/lib/workflows/editing/operations.test.ts index cf8c4f02e4e..9643e1801e8 100644 --- a/apps/sim/lib/workflows/editing/operations.test.ts +++ b/apps/sim/lib/workflows/editing/operations.test.ts @@ -606,6 +606,74 @@ describe('handleEditOperation container inputs', () => { }) }) +describe('a falsy loop or parallel count', () => { + it.each([0, false, null, ''])('refuses iterations %j on an added loop', (iterations) => { + const { state, validationErrors } = applyOperationsToWorkflowState(makeLoopWorkflow(), [ + { + operation_type: 'add', + block_id: 'loop-2', + params: { type: 'loop', name: 'Loop 2', inputs: { loopType: 'for', iterations } }, + }, + ]) + + expect(validationErrors).toEqual([ + expect.objectContaining({ blockType: 'loop', field: 'iterations', value: iterations }), + ]) + const added = Object.values(state.blocks).find((block) => block.name === 'Loop 2') + expect(added?.data?.count).toBeUndefined() + }) + + it('refuses count 0 on an added parallel', () => { + const { validationErrors } = applyOperationsToWorkflowState(makeLoopWorkflow(), [ + { + operation_type: 'add', + block_id: 'parallel-2', + params: { + type: 'parallel', + name: 'Parallel 2', + inputs: { parallelType: 'count', count: 0 }, + }, + }, + ]) + + expect(validationErrors).toEqual([ + expect.objectContaining({ blockType: 'parallel', field: 'count', value: 0 }), + ]) + }) + + it('refuses iterations 0 on a loop merged in as an existing nested child, keeping its count', () => { + const workflow = makeParallelWorkflow() + const blocks: Record = workflow.blocks + blocks['inner-loop'] = { + id: 'inner-loop', + type: 'loop', + name: 'Inner', + position: { x: 0, y: 0 }, + enabled: true, + subBlocks: {}, + outputs: {}, + data: { parentId: 'loop-1', extent: 'parent', loopType: 'for', count: 4 }, + } + + const { state, validationErrors } = applyOperationsToWorkflowState(workflow, [ + { + operation_type: 'edit', + block_id: 'loop-1', + params: { + nestedNodes: { + inner: { type: 'loop', name: 'Inner', inputs: { loopType: 'for', iterations: 0 } }, + }, + }, + }, + ]) + + expect(validationErrors).toContainEqual( + expect.objectContaining({ blockId: 'inner-loop', field: 'iterations', value: 0 }) + ) + expect(state.blocks['inner-loop'].data.count).toBe(4) + }) +}) + describe('handleEditOperation nestedNodes merge', () => { it('preserves existing child block IDs when editing a loop with nestedNodes', () => { const workflow = makeLoopWorkflow() diff --git a/apps/sim/lib/workflows/editing/operations.ts b/apps/sim/lib/workflows/editing/operations.ts index ff3cda25282..fb3ccd57de9 100644 --- a/apps/sim/lib/workflows/editing/operations.ts +++ b/apps/sim/lib/workflows/editing/operations.ts @@ -131,7 +131,7 @@ function applyLoopOrParallelContainerData( ...(loopType === 'forEach' && params.inputs?.collection && { collection: params.inputs.collection }), ...(loopType === 'for' && - params.inputs?.iterations && { + params.inputs?.iterations !== undefined && { count: containerCount( block, 'iterations', @@ -156,7 +156,7 @@ function applyLoopOrParallelContainerData( ...(parallelType === 'collection' && params.inputs?.collection && { collection: params.inputs.collection }), ...(parallelType === 'count' && - params.inputs?.count && { + params.inputs?.count !== undefined && { count: containerCount( block, 'count', @@ -250,7 +250,7 @@ function updateLoopOrParallelContainerData( } } const effectiveLoopType = params.inputs?.loopType ?? block.data.loopType ?? 'for' - if (params.inputs?.iterations && effectiveLoopType === 'for') { + if (params.inputs?.iterations !== undefined && effectiveLoopType === 'for') { block.data.count = containerCount( block, 'iterations', @@ -281,7 +281,7 @@ function updateLoopOrParallelContainerData( } } const effectiveParallelType = params.inputs?.parallelType ?? block.data.parallelType ?? 'count' - if (params.inputs?.count && effectiveParallelType === 'count') { + if (params.inputs?.count !== undefined && effectiveParallelType === 'count') { block.data.count = containerCount( block, 'count', From eb42a62892c1b4bbc00d4fb5901ac7a091d28b79 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:02:48 -0800 Subject: [PATCH 46/51] fix(workflow-code): find a falsy-valued variable's readers by resolved block, so a reader renamed in the same edit still ties the rename --- .../src/materialize/index.test.ts | 34 +++++++++++++++++++ .../workflow-code/src/materialize/index.ts | 26 +++++++++----- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 5c1df6fdebc..519285156ef 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -1037,6 +1037,40 @@ describe('materialize: identity', () => { } ) + it('keeps the id of a falsy-valued variable renamed in the same edit as the block that reads it', () => { + const pulled = workflowState( + [ + blockState('a1', 'function', 'Read', { code: 'return + 1' }), + blockState('a2', 'function', 'Other', { code: 'return 2' }), + ], + { variables: { 'var-1': { id: 'var-1', name: 'Retries', type: 'number', value: 0 } } } + ) + const lock = lockFor(pulled) + const target = edit(canonical(pulled, lock).workflow, (draft) => { + draft.variables = { + attempts: { key: 'attempts', name: 'Attempts', type: 'number', value: 0 }, + } + draft.blocks.reader = { + ...draft.blocks.read, + key: 'reader', + name: 'Reader', + params: { + ...draft.blocks.read.params, + code: { kind: 'value', value: 'return + 1' }, + }, + } + draft.blocks = omit(draft.blocks, ['read']) + }) + + const result = run(target, lock, { + prev: pulled, + compiledIdents: { reader: lock.blocks.read.ident }, + }) + + expect(result.identity.blocks.reader.id).toBe('a1') + expect(Object.keys(result.state.variables ?? {})).toEqual(['var-1']) + }) + it('leaves an inputFormat id two fields share out of the identity items', () => { const prev = workflowState([ blockState('start', 'start_trigger', 'Start', { diff --git a/packages/workflow-code/src/materialize/index.ts b/packages/workflow-code/src/materialize/index.ts index 86349e5be13..f353768b423 100644 --- a/packages/workflow-code/src/materialize/index.ts +++ b/packages/workflow-code/src/materialize/index.ts @@ -47,7 +47,8 @@ function writeVariables( lock: WorkflowLockV1 | null, prev: WorkflowState | undefined, prevCanonical: CanonicalizeResult | null, - hooks: MaterializeHooks + hooks: MaterializeHooks, + keyByPrevKey: ReadonlyMap ): { variables: Record; idByKey: Map } { const prevVariables = prev?.variables ?? {} const prevKeyById = invertRecord(prevCanonical?.identity.variables ?? {}) @@ -73,7 +74,7 @@ function writeVariables( used.add(known) knownIds.set(key, known) } - const renamed = renamedVariable(workflow, knownIds, prevCanonical, used) + const renamed = renamedVariable(workflow, knownIds, prevCanonical, used, keyByPrevKey) if (renamed) knownIds.set(renamed.key, renamed.id) const variables: Record = {} @@ -110,7 +111,8 @@ function renamedVariable( workflow: CanonicalWorkflow, knownIds: ReadonlyMap, prevCanonical: CanonicalizeResult | null, - used: ReadonlySet + used: ReadonlySet, + keyByPrevKey: ReadonlyMap ): { key: VarKey; id: string } | null { if (!prevCanonical) return null const renamed = soleRename( @@ -127,7 +129,7 @@ function renamedVariable( former.type === variable.type && (sharesEvidence(former, variable, ['value']) || (deepEqual(former.value, variable.value) && - referencesFollow(prevCanonical.workflow, workflow, formerKey, key))) + referencesFollow(prevCanonical.workflow, workflow, formerKey, key, keyByPrevKey))) ) } ) @@ -137,19 +139,22 @@ function renamedVariable( /** * Whether every block that read variable `formerKey` now reads `key` instead: the evidence a * rename carries when the value itself says nothing (`0`, `false`, `''`), as it does for counters - * and flags. + * and flags. Each reader is found by the block it resolved to (`keyByPrevKey`), so a reader + * renamed in the same edit still counts. */ function referencesFollow( prev: CanonicalWorkflow, workflow: CanonicalWorkflow, formerKey: VarKey, - key: VarKey + key: VarKey, + keyByPrevKey: ReadonlyMap ): boolean { const readers = Object.values(prev.blocks).filter((block) => referencesVariable(block, formerKey)) return ( readers.length > 0 && readers.every((block) => { - const now = getOwn(workflow.blocks, block.key) + const current = keyByPrevKey.get(block.key) + const now = current === undefined ? undefined : getOwn(workflow.blocks, current) return now !== undefined && referencesVariable(now, key) }) ) @@ -278,12 +283,17 @@ export function materialize( ) const idByKey = new Map([...resolution.blocks].map(([key, block]) => [key, block.id])) + const keyByPrevKey = new Map() + for (const [key, block] of resolution.blocks) { + if (block.prevKey !== null) keyByPrevKey.set(block.prevKey, key) + } const { variables, idByKey: variableIdByKey } = writeVariables( workflow, lock, prev, prevCanonical, - hooks + hooks, + keyByPrevKey ) const context: MaterializeContext = { catalog, From de0ba6957f1d65fa5d9db43f2676724170a8d966 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:03:55 -0800 Subject: [PATCH 47/51] fix(sim-cli): status leaves a recorded function file no codeFile() names out of deletions while it is on disk --- .../protocol/workflow-source-status.test.ts | 24 +++++++++++++++++++ .../protocol/workflow-source-status.ts | 7 ++++-- .../protocol/workspace-source-status.test.ts | 24 +++++++++++++++++++ .../protocol/workspace-source-status.ts | 4 +++- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts index 22cc3d3861b..649fcdbb96e 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts @@ -264,6 +264,30 @@ describe('workflows source status', () => { expect(printed()).toMatchObject({ files: [] }) }) + it('does not report a recorded function file as deleted once no codeFile() names it but it is still on disk', async () => { + write( + root, + 'workflow.ts', + ENTRY.replace(" fn('Score', { code: codeFile('./functions/score.js') })\n", '') + ) + write(root, '.sim/lock.json', JSON.stringify({ ...LOCK, blocks: {} })) + mockRequest.mockResolvedValue(dryRun([], 1)) + + await run(root, 'status') + + expect(printed()).toMatchObject({ + local: [ + { + kind: 'workflow', + key: WORKFLOW_ID, + change: 'edited', + files: [{ path: 'workflow.ts', change: 'edited' }], + }, + ], + unnamed: ['functions/score.js'], + }) + }) + it('reports drift when the workflow changed on the server and nothing here, exit 12', async () => { mockRequest.mockResolvedValue(dryRun([STALE])) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts index 826ce9b7740..a64e84ad222 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts @@ -87,13 +87,16 @@ async function readLocalWorkflow(dir: string): Promise { const current = Object.fromEntries( Object.entries(texts).map(([path, text]) => [path, digest(text)]) ) + const unnamed = listed.filter((path) => !sent.has(path)) + // A recorded file still on disk that nothing names is left out on both sides, not deleted. + const ignored = new Set(unnamed) return { base, workflowId, written, texts, - changed: changedFiles(current, written.digests, () => true), - unnamed: listed.filter((path) => !sent.has(path)), + changed: changedFiles(current, written.digests, (path) => !ignored.has(path)), + unnamed, } } diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts index 2602cb51bdd..c68d63d979a 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts @@ -405,6 +405,30 @@ describe('workspaces source status', () => { expect(process.exitCode ?? 0).toBe(0) }) + it('does not report a recorded function file as deleted once no codeFile() names it but it is still on disk', async () => { + write( + '.sim/written.json', + JSON.stringify({ + version: 1, + files: { + ...Object.fromEntries(Object.entries(GOT).map(([path, text]) => [path, digest(text)])), + 'workflows/ingest/functions/scratch.js': digest('return 1\n'), + }, + }) + ) + write('workflows/ingest/functions/scratch.js', 'return 1\n') + mockRequest.mockResolvedValue(dryRun()) + + await run('status') + + expect(printed()).toMatchObject({ + state: 'clean', + local: [], + unnamed: ['workflows/ingest/functions/scratch.js'], + }) + expect(process.exitCode ?? 0).toBe(0) + }) + it('still counts a new function file the lock names as a local change', async () => { write( 'workflows/ingest/.sim/lock.json', diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts index c24f774a1b6..99e91ec8614 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts @@ -116,9 +116,11 @@ async function readLocalWorkspace( const current: Record = {} for (const [path, text] of Object.entries(submission.files)) current[path] = digest(text) for (const blob of submission.blobs) current[blob.path] = blob.sha256 + // A recorded function file still on disk that no codeFile() names is left out, not deleted. + const unnamed = new Set(submission.unnamed) const changed = changedFiles(current, written.digests, (path) => { const kind = kindOfPath(path) - return kind !== null && (!only || only.includes(kind)) + return kind !== null && (!only || only.includes(kind)) && !unnamed.has(path) }) return { submission, written, changed, readFrom } } From 6be621e82e0823fe2e5fb726ef3b7ba9dcee5f4c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:04:36 -0800 Subject: [PATCH 48/51] fix(workflows): write a renamed block's new name into references literally, not as a replace pattern --- apps/sim/lib/workflows/editing/operations.test.ts | 13 +++++++++++++ apps/sim/lib/workflows/references/block-rename.ts | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/workflows/editing/operations.test.ts b/apps/sim/lib/workflows/editing/operations.test.ts index 9643e1801e8..8b010461bc0 100644 --- a/apps/sim/lib/workflows/editing/operations.test.ts +++ b/apps/sim/lib/workflows/editing/operations.test.ts @@ -674,6 +674,19 @@ describe('a falsy loop or parallel count', () => { }) }) +describe('a block rename', () => { + it('writes a new name holding `$` replacement patterns into references literally', () => { + const workflow = makeLoopWorkflow() + workflow.blocks['agent-1'].subBlocks.systemPrompt.value = 'Use ' + + const { state } = applyOperationsToWorkflowState(workflow, [ + { operation_type: 'edit', block_id: 'condition-1', params: { name: "Cond$&$'" } }, + ]) + + expect(state.blocks['agent-1'].subBlocks.systemPrompt.value).toBe("Use ") + }) +}) + describe('handleEditOperation nestedNodes merge', () => { it('preserves existing child block IDs when editing a loop with nestedNodes', () => { const workflow = makeLoopWorkflow() diff --git a/apps/sim/lib/workflows/references/block-rename.ts b/apps/sim/lib/workflows/references/block-rename.ts index 09e91fe877a..09accb96ceb 100644 --- a/apps/sim/lib/workflows/references/block-rename.ts +++ b/apps/sim/lib/workflows/references/block-rename.ts @@ -28,7 +28,8 @@ export function rewriteBlockNameReferences( * backtracking pattern that pins the event loop on a sub-kilobyte input. */ const regex = new RegExp(`<${escapeRegExp(oldName)}\\.`, 'g') - updatedValue = updatedValue.replace(regex, `<${newName}.`) + // A callback, so `$&` or `$'` in a new name is written literally, not as a pattern. + updatedValue = updatedValue.replace(regex, () => `<${newName}.`) }) return updatedValue } From 00e5a07a0fea0dc93cd9368ca30ae5153af19d4f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:05:39 -0800 Subject: [PATCH 49/51] fix(workspace-source): scope a taken workflow name to its folder, and hint a taken file name be renamed locally --- .../application/errors.test.ts | 24 +++++++++++++++++++ .../workspace-source/application/errors.ts | 8 ++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/errors.test.ts b/apps/sim/lib/workspace-source/application/errors.test.ts index 65816ad1de5..07817945138 100644 --- a/apps/sim/lib/workspace-source/application/errors.test.ts +++ b/apps/sim/lib/workspace-source/application/errors.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it } from 'vitest' import { OrchestrationError } from '@/lib/core/orchestration/types' import { TableConflictError } from '@/lib/table/service' +import { FileConflictError } from '@/lib/uploads/contexts/workspace/file-manager' +import { WorkflowSourceNameTakenError } from '@/lib/workflows/application/source-error' import { refusalDiagnostic, WorkspaceSourceRejectedError, @@ -16,6 +18,28 @@ describe('refusalDiagnostic', () => { ) }) + it('scopes a taken workflow name to its folder, where the uniqueness check runs', () => { + const finding = refusalDiagnostic(new WorkflowSourceNameTakenError('Ingest'), { + kind: 'workflow', + key: 'ingest', + }) + + expect(finding.code).toBe('E_NAME_TAKEN') + expect(finding.message).not.toContain('workspace') + expect(finding.message).toContain('folder') + }) + + it('hints a taken file name be renamed locally, as a file has no declaration', () => { + const finding = refusalDiagnostic(new FileConflictError('notes.txt'), { + kind: 'file', + key: 'notes.txt', + }) + + expect(finding.code).toBe('E_NAME_TAKEN') + expect(finding.hint).not.toContain('declaration') + expect(finding.hint).toContain('locally') + }) + it('rethrows an outage, so the request answers 503 rather than blaming the caller', () => { const outage = new OrchestrationError('unavailable', 'The egress gateway is unreachable') expect(() => refusalDiagnostic(outage, { kind: 'table', key: 'leads' })).toThrow(outage) diff --git a/apps/sim/lib/workspace-source/application/errors.ts b/apps/sim/lib/workspace-source/application/errors.ts index 981688a2f9b..82ca6b4730b 100644 --- a/apps/sim/lib/workspace-source/application/errors.ts +++ b/apps/sim/lib/workspace-source/application/errors.ts @@ -121,9 +121,9 @@ export function refusalDiagnostic( const file = `${workflowFolder(resource.key)}/${SOURCE_ENTRY_FILE}` return finding( 'E_NAME_TAKEN', - `A workflow named ${JSON.stringify(error.workflowName)} already exists in this workspace.`, + `A workflow named ${JSON.stringify(error.workflowName)} already exists in the folder this workflow is saved to.`, { - hint: `Rename the workflow in ${file}, or rename or delete the one that holds the name in Sim.`, + hint: `Rename the workflow in ${file}, or rename, move or delete the one that holds the name in Sim.`, file, ...resource, } @@ -134,7 +134,9 @@ export function refusalDiagnostic( hint: error instanceof ResourceKeyTakenError ? 'Rename the key, or delete the resource that holds it.' - : 'Rename it in its declaration, or rename or delete the resource that holds the name in Sim.', + : resource.kind === 'file' + ? 'Rename it locally, or rename or delete the file that holds the name in Sim.' + : 'Rename it in its declaration, or rename or delete the resource that holds the name in Sim.', ...resource, }) } From d042838b76e6eea5ea23f7434ce5435740d1c676 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:06:17 -0800 Subject: [PATCH 50/51] test(workflows,workspace-source): assert the stored out-of-options value after apply, and the strip without polling --- .../apply-workflow-source.integration.ts | 4 +++- .../application/rows.integration.ts | 16 ++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index bb54bb17350..4a3abb65403 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -1562,8 +1562,10 @@ describe('source apply edge cases', () => { const result = await apply(source, edit(source.files, 'return 1', 'return 2')) - expect(source.files['workflow.ts']).toContain("language: raw('cobol')") expect(result.applied).toBe(true) + const stored = (await snapshot(workflowId)).state?.blocks[score.id] + expect(stored?.subBlocks.code?.value).toBe('return 2') + expect(stored?.subBlocks.language?.value).toBe('cobol') } ) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index ae0e734f316..f46f9ddca1d 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -15,7 +15,6 @@ import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/featur import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' import { tableTriggerMock, tableTriggerMockFns } from '@sim/testing/mocks/table-trigger.mock' -import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { compareStrings } from '@sim/utils/string' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' @@ -1192,16 +1191,13 @@ export default defineTable({ { base: read.base, allowDelete: true } ) expect(dropped.failed).toBeUndefined() - // Once the dropped column's cells are gone from every row, nothing is left to write them. + // The apply strips the dropped column's cells in its own transaction, so none are left. const table = await storedTable(workspaceId) - for (let waited = 0; waited < 50; waited++) { - const [{ holding }] = await db - .select({ holding: sql`count(*)::int` }) - .from(userTableRows) - .where(and(eq(userTableRows.tableId, table.id), sql`${userTableRows.data} ? ${tierId}`)) - if (holding === 0) break - await sleep(100) - } + const [{ holding }] = await db + .select({ holding: sql`count(*)::int` }) + .from(userTableRows) + .where(and(eq(userTableRows.tableId, table.id), sql`${userTableRows.data} ? ${tierId}`)) + expect(holding).toBe(0) const planned = await apply(workspaceId, dropped.files, { base: dropped.base, From d0789bcea89a034474834cdfdebc2afcdc771861 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 16:25:06 -0800 Subject: [PATCH 51/51] test(workflows): match the added loop by shape, which the loose state type allows --- apps/sim/lib/workflows/editing/operations.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/workflows/editing/operations.test.ts b/apps/sim/lib/workflows/editing/operations.test.ts index 8b010461bc0..1ac10156166 100644 --- a/apps/sim/lib/workflows/editing/operations.test.ts +++ b/apps/sim/lib/workflows/editing/operations.test.ts @@ -619,8 +619,12 @@ describe('a falsy loop or parallel count', () => { expect(validationErrors).toEqual([ expect.objectContaining({ blockType: 'loop', field: 'iterations', value: iterations }), ]) - const added = Object.values(state.blocks).find((block) => block.name === 'Loop 2') - expect(added?.data?.count).toBeUndefined() + expect(Object.values(state.blocks)).toContainEqual( + expect.objectContaining({ + name: 'Loop 2', + data: expect.not.objectContaining({ count: expect.anything() }), + }) + ) }) it('refuses count 0 on an added parallel', () => {