From a020ac87ff64e72927a1f983aceb40fc4ccde3df Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 14:24:53 -0800 Subject: [PATCH 01/59] fix(sim-cli): check a workflow against the catalog its get printed with A workflow holding a type the workspace catalog withholds (table_v2 behind a preview gate on staging) is printed and dry-run against its own per-workflow source catalog, but the get never cached that catalog, so the offline check compiled against the workspace catalog: table.insertRow became E_UNKNOWN_BUILDER and the branch wired to it W_BRANCH_UNWIRED, while the dry run was clean. The get now caches the per-workflow catalog beside the workspace one, and check and the tsconfig refresh use it when its hash is the one .sim/base.json records, else the workspace catalog. --- .../src/commands/protocol/source-tree.ts | 63 ++++++- .../protocol/workflow-source-check.ts | 18 +- .../workflow-source-roundtrip.test.ts | 160 +++++++++++++++++- 3 files changed, 229 insertions(+), 12 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-tree.ts b/packages/sim-cli/src/commands/protocol/source-tree.ts index bbc729c84df..36f11d5bc1c 100644 --- a/packages/sim-cli/src/commands/protocol/source-tree.ts +++ b/packages/sim-cli/src/commands/protocol/source-tree.ts @@ -6,10 +6,11 @@ */ import { createHash } from 'node:crypto' import { join } from 'node:path' -import { isRecordLike } from '@sim/utils/object' +import { isRecordLike, toRecord } from '@sim/utils/object' import { escapeRegExp } from '@sim/utils/string' import type { BlockCatalog } from '@sim/workflow-code' import { catalogBlock } from '@sim/workflow-code/catalog/view' +import { SOURCE_BASE_FILE } from '@sim/workflow-code/limits' import { COOKBOOK_EXAMPLES, CORE_DTS, @@ -34,6 +35,7 @@ import { SimApiError } from '../../http/client' import type { OperationClient } from '../../runtime/called-operations' import { cliVersion } from '../../version' import { + readSourceJson, readSourceText, SOURCE_SDK_DIR as SDK_DIR, SOURCE_AGENTS_FILE, @@ -87,14 +89,54 @@ function catalogCachePath(workspaceId: string): string { return join(configDir(), 'cache', 'source-catalog', `${workspaceId}.json`) } +/** + * Where a workflow's own source catalog is cached: the one a get fetches for a workflow holding a + * type the workspace catalog lacks (see {@link loadCatalog}), one file per workflow beside the + * workspace's. Callers gate both ids with `CACHE_KEY` first. + */ +function workflowCatalogCachePath(workspaceId: string, workflowId: string): string { + return join(configDir(), 'cache', 'source-catalog', workspaceId, `${workflowId}.json`) +} + +function readCachedCatalog(path: string): BlockCatalog | null { + const cached = readJsonFile(path, MAX_CATALOG_CACHE_BYTES) + return isBlockCatalog(cached) ? cached : null +} + /** * The cached catalog, or `null`. Standalone runs only: an embedded run has no local disk of its * own to cache on. The file is read without following a link and under a size cap. */ export function cachedCatalog(workspaceId: string): BlockCatalog | null { if (embedStore.getStore() || !CACHE_KEY.test(workspaceId)) return null - const cached = readJsonFile(catalogCachePath(workspaceId), MAX_CATALOG_CACHE_BYTES) - return isBlockCatalog(cached) ? cached : null + return readCachedCatalog(catalogCachePath(workspaceId)) +} + +/** + * The cached catalog a source directory was printed against, or `null`: the workflow's own + * catalog when `.sim/base.json` names a workflow whose cached catalog has the hash the base + * records (a workflow holding a type the workspace catalog lacks, which only that catalog + * declares), else the workspace's ({@link cachedCatalog}). An offline check compiles against it, + * so it agrees with the dry run, which compiles against the same per-workflow catalog. + */ +export async function cachedSourceCatalog( + dir: string, + workspaceId: string +): Promise { + const base = toRecord(await readSourceJson(dir, SOURCE_BASE_FILE)) + const { workflowId, catalogHash } = base + if ( + !embedStore.getStore() && + CACHE_KEY.test(workspaceId) && + typeof workflowId === 'string' && + CACHE_KEY.test(workflowId) && + base.workspaceId === workspaceId && + typeof catalogHash === 'string' + ) { + const own = readCachedCatalog(workflowCatalogCachePath(workspaceId, workflowId)) + if (own?.hash === catalogHash) return own + } + return cachedCatalog(workspaceId) } /** The catalog a `getSourceCatalog` answer carries. */ @@ -115,7 +157,8 @@ function writeCachedCatalog(workspaceId: string, catalog: BlockCatalog): void { * The workspace's block catalog, revalidated by ETag against the per-workspace cache: a `304` * reuses it, anything else replaces it. A workflow can hold a type the workspace no longer offers * (a legacy version, a block an allowlist now withholds); only then is that workflow's compile - * catalog fetched instead, uncached, so its types still resolve. + * catalog fetched instead, so its types still resolve, and cached per workflow for the offline + * check ({@link cachedSourceCatalog}). */ export async function loadCatalog( client: OperationClient<'getSourceCatalog'>, @@ -143,12 +186,20 @@ export async function loadCatalog( ) { return catalog } - return catalogFrom( + const own = catalogFrom( await client.request<{ data: unknown }>('getSourceCatalog', { params: { workspaceId: source.workspaceId }, query: { workflowId: source.workflowId }, }) ) + if ( + !embedStore.getStore() && + CACHE_KEY.test(source.workspaceId) && + CACHE_KEY.test(source.workflowId) + ) { + writeJsonFile(workflowCatalogCachePath(source.workspaceId, source.workflowId), own) + } + return own } /** @@ -450,7 +501,7 @@ export async function refreshTsconfigs( use: SourceTreeUse, options: { incremental?: boolean } = {} ): Promise { - const catalog = cachedCatalog(workspaceId) + const catalog = await cachedSourceCatalog(dir, workspaceId) if (!catalog) return const [tsconfig, full, workspaceTypes] = await Promise.all([ readSourceText(dir, TSCONFIG_FILE), diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index 743b8593d81..4542ff7f03a 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -26,6 +26,7 @@ import { printProtocolResult } from './result' import { readSourceJson, readSourceText, workspaceFolderOf } from './source-dir' import { cachedCatalog, + cachedSourceCatalog, type LocalDeclarations, refreshTsconfigs, refreshWorkspaceTypes, @@ -492,9 +493,12 @@ export function reportCheck( }) } -/** The cached catalog of `workspaceId`, or a refusal naming `get`, the command that caches one. */ -export function requireCachedCatalog(workspaceId: string, get: string): BlockCatalog { - const catalog = cachedCatalog(workspaceId) +/** The catalog a cache read found, or a refusal naming `get`, the command that caches one. */ +function cachedOrRefuse( + catalog: BlockCatalog | null, + workspaceId: string, + get: string +): BlockCatalog { if (catalog) return catalog throw new SimApiError( `No block catalog is cached for workspace ${workspaceId}. Run ${get} once, then check again.`, @@ -503,6 +507,11 @@ export function requireCachedCatalog(workspaceId: string, get: string): BlockCat ) } +/** The cached catalog of `workspaceId`, or a refusal naming `get`, the command that caches one. */ +export function requireCachedCatalog(workspaceId: string, get: string): BlockCatalog { + return cachedOrRefuse(cachedCatalog(workspaceId), workspaceId, get) +} + /** * `sim workflows source check [--dir ]`: compiles a source directory locally with the bundled * compiler and the cached catalog, so an agent's edit loop gets syntax and builder findings in @@ -541,7 +550,8 @@ export function attachWorkflowSourceCheck(source: Command): void { 0 ) } - const catalog = requireCachedCatalog( + const catalog = cachedOrRefuse( + await cachedSourceCatalog(options.dir, workspaceId), workspaceId, workspace ? `\`sim workspaces source get --dir ${workspace.root}\`` diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 8a48a6bdb02..33c04aad8d9 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -186,7 +186,7 @@ const MANIFEST = { /** What the mocked server answers; each test replaces the routes it exercises. */ interface Server { source: () => Response - catalog: (headers: Headers) => Response + catalog: (headers: Headers, url: URL) => Response manifest: () => Response apply: () => Response create: () => Response @@ -233,7 +233,7 @@ beforeEach(() => { if (url.pathname === `/api/v2/workspaces/${WORKSPACE_ID}/workflows/source/catalog`) { const headers = new Headers(init?.headers) catalogRequests.push(headers) - return server.catalog(headers) + return server.catalog(headers, url) } if (url.pathname === `/api/v2/workspaces/${WORKSPACE_ID}/workflows/source/resources`) { return server.manifest() @@ -853,3 +853,159 @@ describe('reading the directory', () => { expect(applyBodies[0].files).toEqual({ 'workflow.ts': WORKFLOW_TS }) }) }) + +/** + * The block types the workspace catalog withholds from the caller (an unrevealed preview, an + * allowlist) while a stored workflow still holds them: only that workflow's own source catalog, + * which the get fetches with `workflowId`, declares them, and the dry run compiles against it. + */ +const WITHHELD_TYPES = new Set(['table_v2', 'knowledge']) + +/** {@link CATALOG} as the workspace catalog of a caller it withholds {@link WITHHELD_TYPES} from. */ +const WORKSPACE_CATALOG = { + ...CATALOG, + hash: 'workspace-catalog', + blocks: Object.fromEntries( + Object.entries(CATALOG.blocks).filter(([type]) => !WITHHELD_TYPES.has(type)) + ), + builders: Object.fromEntries( + Object.entries(CATALOG.builders as Record).filter( + ([, builder]) => !WITHHELD_TYPES.has(builder.type) + ) + ), +} + +/** Every builder path of a withheld type, in catalog order. */ +const WITHHELD_BUILDERS = Object.entries(CATALOG.builders as Record) + .filter(([, builder]) => WITHHELD_TYPES.has(builder.type)) + .map(([path]) => path) + +/** A ticket router as a get prints it: a condition whose else branch writes a table row. */ +const TICKET_ROUTER = `// @sim-sdk 1 +import { + condition, + js, + knowledge, + knowledgeBase, + response, + t, + table, + trigger, + workflow, +} from '@sim/workflow-sdk' + +export default workflow('Ticket Router', {}, () => { + const start = trigger.start('Start', { inputFormat: { subject: 'string', priority: 'string' } }) + const isUrgent = condition('Is Urgent', { if: js\`\${start.priority} === 'high'\`, else: true }) + const policy = knowledge.search('Policy', { + knowledgeBaseId: knowledgeBase('policies'), + query: start.subject, + }) + const logTicket = table.insertRow('Log Ticket', { + tableId: table('tickets'), + data: { subject: start.subject, status: 'open' }, + }) + const upsertTicket = table.upsertRow('Upsert Ticket', { + tableId: table('tickets'), + data: { subject: start.subject, status: 'open' }, + }) + const recent = table.queryRows('Recent Tickets', { tableId: table('tickets'), limit: 10 }) + const done = response('Done', { body: { rowId: t\`\${logTicket.row.id}\`, rows: recent.rows } }) + + start.to(isUrgent) + isUrgent.if.to(policy) + isUrgent.else.to(logTicket) + logTicket.to(upsertTicket) + upsertTicket.to(recent) + recent.to(done) +}) +` + +/** One block of every withheld builder, each with no params, as an edit could add them. */ +const EVERY_WITHHELD_BUILDER = [ + '// @sim-sdk 1', + "import { knowledge, table, trigger, workflow } from '@sim/workflow-sdk'", + '', + "export default workflow('Every Builder', {}, () => {", + ...WITHHELD_BUILDERS.map((path, index) => ` const b${index} = ${path}('Block ${index}', {})`), + '})', + '', +].join('\n') + +const check = (...flags: string[]) => sim('workflows', 'source', 'check', '--dir', dir, ...flags) + +function checked(): { errors: number; warnings: number; diagnostics: unknown[] } { + return JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') +} + +describe('checking a workflow holding a type the workspace catalog withholds', () => { + afterEach(() => { + process.exitCode = undefined + }) + + beforeEach(() => { + server.catalog = (_, url) => + json({ data: url.searchParams.has('workflowId') ? CATALOG : WORKSPACE_CATALOG }) + }) + + function serve(source: string, catalogHash: string = CATALOG.hash): void { + const { data } = sourceResponse( + { 'workflow.ts': source }, + { version: 1, blocks: {}, variables: {} } + ) + server.source = () => + json({ + data: { + ...data, + base: { ...data.base, catalogHash }, + blockTypes: ['start_trigger', 'condition', 'response', ...WITHHELD_TYPES], + }, + }) + } + + it('passes the unedited source of a get, with the else branch wired to the table write', async () => { + serve(TICKET_ROUTER) + await get() + + await check() + + expect(checked()).toMatchObject({ errors: 0, warnings: 0, diagnostics: [] }) + expect(process.exitCode).toBeUndefined() + }) + + it('finds what the dry run finds for every builder only the workflow catalog declares', async () => { + serve(EVERY_WITHHELD_BUILDER) + await get() + const dryRun = compileWorkflow({ + entry: { path: 'workflow.ts', text: EVERY_WITHHELD_BUILDER }, + files: new Map(), + catalog: CATALOG, + sdkMajor: CATALOG.sdkMajor, + }).diagnostics.map(({ code, message, span }) => ({ code, message, line: span?.line })) + + await check('--no-strict') + + expect(WITHHELD_BUILDERS).toEqual( + expect.arrayContaining(['table.insertRow', 'knowledge.search']) + ) + expect(dryRun.map(({ code }) => code)).not.toContain('E_UNKNOWN_BUILDER') + expect( + (checked().diagnostics as Array<{ code: string; message: string; line?: number }>).map( + ({ code, message, line }) => ({ code, message, line }) + ) + ).toEqual(dryRun) + }) + + it('checks against the workspace catalog again once a later get no longer needs its own', async () => { + serve(TICKET_ROUTER) + await get() + const revealed = { ...CATALOG, hash: 'revealed' } + server.catalog = () => json({ data: revealed }) + serve(TICKET_ROUTER, revealed.hash) + await get() + + await check() + + expect(checked()).toMatchObject({ errors: 0, catalogHash: revealed.hash }) + }) +}) From 307744ab665f9a2261a8292045bf34f88f6ddc2b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 14:29:44 -0800 Subject: [PATCH 02/59] fix(workflow-code): offline check refuses a cycle the edit adds between bound blocks Offline (storedUnread), compile took any edge between two blocks the lock binds as one the stored workflow may hold, so a cycle or container error edge an edit added between known blocks was only W_EDGE_CYCLE / W_ERROR_PORT_STORED and check exited 0 even with --strict, while the dry run refused it with E_EDGE_CYCLE. A get, apply or create now compiles the printed source and records its edges by lock key with the base fingerprint in .sim/stored-edges.json (a CLI-only record). Check passes them as storedUnreadEdges, and compile counts an edge between bound blocks as possibly stored only when it is one of them, mapping a block renamed in source to its vacated lock key, so a printed cycle stays a warning and a new one is the server's error. Without a record for the current base (an older directory, a workspace directory) the old rule holds. CompileResult gains edges, the accepted edges even when the compile fails elsewhere (a printed stored() secret). --- .../src/commands/protocol/source-dir.ts | 17 +++- .../src/commands/protocol/source-tree.ts | 19 ++++ .../src/commands/protocol/stored-edges.ts | 75 ++++++++++++++ .../protocol/workflow-source-apply.ts | 5 +- .../protocol/workflow-source-check.ts | 7 +- .../commands/protocol/workflow-source-get.ts | 9 +- .../workflow-source-roundtrip.test.ts | 97 +++++++++++++++++++ packages/workflow-code/DESIGN.md | 7 +- packages/workflow-code/src/compile/edges.ts | 31 ++++-- packages/workflow-code/src/compile/index.ts | 46 +++++++-- .../workflow-code/src/diagnostics/codes.ts | 3 +- packages/workflow-code/src/index.ts | 2 +- 12 files changed, 292 insertions(+), 26 deletions(-) create mode 100644 packages/sim-cli/src/commands/protocol/stored-edges.ts diff --git a/packages/sim-cli/src/commands/protocol/source-dir.ts b/packages/sim-cli/src/commands/protocol/source-dir.ts index cd9c8561523..cd329da970a 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.ts @@ -63,6 +63,12 @@ export const SOURCE_GITIGNORE_FILE = '.gitignore' * edit from a file it may replace. The CLI's own record: never sent to the server. */ export const SOURCE_WRITTEN_FILE = '.sim/written.json' +/** + * The edges of the workflow a get or apply printed, by lock key, with the base fingerprint they + * belong to, so an offline check tells an edge the stored workflow holds from one the source + * adds. The CLI's own record: never sent to the server. + */ +export const SOURCE_STORED_EDGES_FILE = '.sim/stored-edges.json' /** * Where a workspace get keeps the blobs it verified until they move into place. A get that fails * leaves them there, so the next one downloads only the rest; the one that completes removes the @@ -283,12 +289,17 @@ export interface SourceDirChange { } /** - * The files that record what the directory was read from: each workflow's lock, base and - * `.sim/written.json`, and the workspace base. They move into place only after every other file, + * The files that record what the directory was read from: each workflow's lock, base, + * `.sim/written.json` and stored edges, and the workspace base. They move into place only after every other file, * so a write cut short leaves the old records beside whatever text did land: the next apply is * refused as stale, never sent as if the old text were current. */ -const RECORD_FILES = [SOURCE_BASE_FILE, SOURCE_LOCK_FILE, SOURCE_WRITTEN_FILE] +const RECORD_FILES = [ + SOURCE_BASE_FILE, + SOURCE_LOCK_FILE, + SOURCE_WRITTEN_FILE, + SOURCE_STORED_EDGES_FILE, +] function isRecordFile(path: string): boolean { return RECORD_FILES.some((file) => path === file || path.endsWith(`/${file}`)) diff --git a/packages/sim-cli/src/commands/protocol/source-tree.ts b/packages/sim-cli/src/commands/protocol/source-tree.ts index 36f11d5bc1c..870b14c0f0b 100644 --- a/packages/sim-cli/src/commands/protocol/source-tree.ts +++ b/packages/sim-cli/src/commands/protocol/source-tree.ts @@ -139,6 +139,25 @@ export async function cachedSourceCatalog( return cachedCatalog(workspaceId) } +/** + * The cached catalog a printed source's base names by hash (`.sim/base.json` of a get, apply or + * create answer): the workflow's own or the workspace's, or `null` when neither cached catalog + * has that hash. + */ +export function cachedCatalogOfBase(base: unknown): BlockCatalog | null { + const { workspaceId, workflowId, catalogHash } = toRecord(base) + if (embedStore.getStore() || typeof workspaceId !== 'string' || !CACHE_KEY.test(workspaceId)) { + return null + } + const candidates = [ + typeof workflowId === 'string' && CACHE_KEY.test(workflowId) + ? readCachedCatalog(workflowCatalogCachePath(workspaceId, workflowId)) + : null, + cachedCatalog(workspaceId), + ] + return candidates.find((catalog) => catalog !== null && catalog.hash === catalogHash) ?? null +} + /** The catalog a `getSourceCatalog` answer carries. */ function catalogFrom(response: { data: unknown }): BlockCatalog { if (!isBlockCatalog(response.data)) { diff --git a/packages/sim-cli/src/commands/protocol/stored-edges.ts b/packages/sim-cli/src/commands/protocol/stored-edges.ts new file mode 100644 index 00000000000..dfc4fa6010a --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/stored-edges.ts @@ -0,0 +1,75 @@ +import { isRecordLike, toRecord } from '@sim/utils/object' +import { + type BlockCatalog, + compileWorkflow, + parseWorkflowLock, + type StoredEdgeRef, +} from '@sim/workflow-code' +import { SOURCE_BASE_FILE, SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' +import { readSourceJson, SOURCE_STORED_EDGES_FILE } from './source-dir' + +/** A printed source as a get, apply or create answers it. */ +interface PrintedAnswer { + files: Record + lock: unknown + base: unknown +} + +/** + * The `.sim/stored-edges.json` write for a printed source: its edges, compiled against `catalog` + * (the one it was printed with), tagged with its base fingerprint. `null` when the answer has no + * fingerprint or entry, or nothing to compile it with; the record left on disk then names an + * older fingerprint, which a check ignores. + */ +export function storedEdgesWrite( + answer: PrintedAnswer, + catalog: BlockCatalog | null +): readonly [string, string] | null { + const { fingerprint } = toRecord(answer.base) + const entry = answer.files[SOURCE_ENTRY_FILE] + const { lock } = parseWorkflowLock(answer.lock) + if (!catalog || typeof fingerprint !== 'string' || entry === undefined || !lock) return null + const { edges } = compileWorkflow({ + entry: { path: SOURCE_ENTRY_FILE, text: entry }, + files: new Map(Object.entries(answer.files).filter(([path]) => path !== SOURCE_ENTRY_FILE)), + catalog, + sdkMajor: catalog.sdkMajor, + knownKeys: new Set(Object.keys(lock.blocks)), + storedUnread: lock, + }) + return [ + SOURCE_STORED_EDGES_FILE, + `${JSON.stringify({ version: 1, fingerprint, edges }, null, 2)}\n`, + ] +} + +function isStoredEdge(value: unknown): value is StoredEdgeRef { + return ( + isRecordLike(value) && + typeof value.from === 'string' && + typeof value.port === 'string' && + typeof value.to === 'string' + ) +} + +/** + * The stored edges recorded for the base `dir` holds now, or `undefined` when none are: no + * record, one an older CLI or another base wrote, or a malformed one. + */ +export async function readStoredEdges(dir: string): Promise { + const [record, base] = await Promise.all([ + readSourceJson(dir, SOURCE_STORED_EDGES_FILE), + readSourceJson(dir, SOURCE_BASE_FILE), + ]) + const { version, fingerprint, edges } = toRecord(record) + if ( + version !== 1 || + typeof fingerprint !== 'string' || + fingerprint !== toRecord(base).fingerprint || + !Array.isArray(edges) || + !edges.every(isStoredEdge) + ) { + return undefined + } + return edges +} diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 69b7f704ce1..6171027c47f 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -28,6 +28,7 @@ import { withWorkspaceFolderLock, writeBack, } from './source-dir' +import { cachedCatalogOfBase } from './source-tree' import { confirm, fresh, @@ -37,6 +38,7 @@ import { requireConsent, resultDiagnostics, } from './source-ui' +import { storedEdgesWrite } from './stored-edges' import { refreshWorkflowTsconfigs, writeSourceDirectory, @@ -316,7 +318,8 @@ export function attachWorkflowSourceApply(source: Command): void { params: { workflowId }, body: { files, lock, base, provenance: { client: 'cli' } }, save: async (data) => { - await writeBack(options.dir, data, lock) + const edges = storedEdgesWrite(data, cachedCatalogOfBase(data.base)) + await writeBack(options.dir, data, lock, { write: edges ? [edges] : [] }) await refreshWorkflowTsconfigs(options.dir, data) }, ifAnswerLost: `It may have been saved. Run \`sim workflows source apply ${workflowId} --dir ${options.dir} --dry-run\`: exit 5 (stale) means the draft changed since this directory was read, by this apply or by someone else. Run \`sim workflows source get ${workflowId} --dir \` and compare it with ${options.dir}: if it holds your edit, \`sim workflows source get ${workflowId} --dir ${options.dir} --force\` brings this directory up to date; if not, reapply your edit in the fresh directory. Any other outcome means it was not saved: apply again.`, diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index 4542ff7f03a..fdeff539d9c 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -32,6 +32,7 @@ import { refreshWorkspaceTypes, } from './source-tree' import { printNewDiagnostics, resultDiagnostics } from './source-ui' +import { readStoredEdges } from './stored-edges' import { readSourceFiles } from './workflow-source-apply' import { WORKFLOW_TREE } from './workflow-source-get' import { @@ -356,7 +357,10 @@ async function compileWorkflowDir( declarations?: WorkspaceDeclarations ) { const { lock } = parseWorkflowLock(await readSourceJson(dir, SOURCE_LOCK_FILE)) - const files = await readSourceFiles(dir, lock ?? null) + const [files, storedEdges] = await Promise.all([ + readSourceFiles(dir, lock ?? null), + readStoredEdges(dir), + ]) const envNames = declarations?.envNames const compiled = compileWorkflow({ entry: { path: SOURCE_ENTRY_FILE, text: files[SOURCE_ENTRY_FILE] }, @@ -365,6 +369,7 @@ async function compileWorkflowDir( sdkMajor: catalog.sdkMajor, knownKeys: new Set(Object.keys(lock?.blocks ?? {})), storedUnread: lock ?? undefined, + ...(storedEdges ? { storedUnreadEdges: storedEdges } : {}), ...(envNames ? { envNames, envDeclaredIn: KIND_DIRS.secret } : {}), }) return { lock, files, compiled } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index 72cb9ad3022..7cf27307ed5 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -39,6 +39,7 @@ import { type WorkspaceTypes, withSourceExitCode, } from './source-tree' +import { storedEdgesWrite } from './stored-edges' const DIR_FLAG = 'dir' const FORCE_FLAG = 'force' @@ -155,7 +156,13 @@ async function writeSourceTree( blockTypes, }), ]) - const written = await writeBack(dir, source, previousLock, tree.change) + const edges = storedEdgesWrite(source, catalog) + const written = await writeBack( + dir, + source, + previousLock, + edges ? { ...tree.change, write: [...tree.change.write, edges] } : tree.change + ) return finishTree(dir, written, workspaceTypes, tree.ownsTsconfig) } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 33c04aad8d9..6b0516c690b 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -1009,3 +1009,100 @@ describe('checking a workflow holding a type the workspace catalog withholds', ( expect(checked()).toMatchObject({ errors: 0, catalogHash: revealed.hash }) }) }) + +/** A get's source holding a stored cycle between A and B, every block bound by the lock. */ +const STORED_CYCLE = `// @sim-sdk 1 +import { api, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('W', {}, () => { + const start = trigger.start('Start', {}) + const a = api('A', { url: 'https://example.com', method: 'GET' }) + const b = api('B', { url: 'https://example.com', method: 'GET' }) + const c = api('C', { url: 'https://example.com', method: 'GET' }) + + start.to(a) + a.to(b) + b.to(a) + b.to(c) +}) +` + +const STORED_CYCLE_LOCK = { + version: 1, + blocks: Object.fromEntries( + ['start', 'a', 'b', 'c'].map((key, order) => [ + key, + { id: `id-${key}`, parent: null, ident: key, order }, + ]) + ), + variables: {}, +} + +describe('checking wiring between blocks the lock binds', () => { + beforeEach(() => { + server.source = () => + json({ + data: { + ...sourceResponse({ 'workflow.ts': STORED_CYCLE }, STORED_CYCLE_LOCK).data, + blockTypes: ['start_trigger', 'api'], + }, + }) + }) + + afterEach(() => { + process.exitCode = undefined + }) + + const check = (...flags: string[]) => sim('workflows', 'source', 'check', '--dir', dir, ...flags) + + function codes(): string[] { + const printed = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') + return (printed.diagnostics as Array<{ code: string }>).map(({ code }) => code) + } + + function edit(change: (source: string) => string): void { + writeFileSync(join(dir, 'workflow.ts'), change(read('workflow.ts'))) + } + + it('refuses a cycle the edit closes between bound blocks, as the dry run does', async () => { + await get() + edit((source) => source.replace(' b.to(c)\n', ' b.to(c)\n c.to(b)\n')) + + await check('--strict') + + expect(codes()).toEqual(['E_EDGE_CYCLE']) + expect(process.exitCode).toBe(1) + }) + + it('keeps the cycle the get printed a warning', async () => { + await get() + + await check('--strict') + + expect(codes()).toEqual(['W_EDGE_CYCLE']) + expect(process.exitCode).toBeUndefined() + }) + + it('keeps the printed cycle a warning through a block renamed in source', async () => { + await get() + edit((source) => source.replace("api('B',", "api('Bee',")) + + await check('--strict') + + expect(codes()).toEqual(['W_EDGE_CYCLE']) + expect(process.exitCode).toBeUndefined() + }) + + it('records the edges an apply printed, for the base it saves', async () => { + await get() + server.apply = () => + json(appliedResponse({ 'workflow.ts': STORED_CYCLE }, { lock: STORED_CYCLE_LOCK })) + await apply() + edit((source) => source.replace(' b.to(c)\n', ' b.to(c)\n c.to(b)\n')) + + await check() + + expect(JSON.parse(read('.sim/stored-edges.json')).fingerprint).toBe(SECOND) + expect(codes()).toEqual(['E_EDGE_CYCLE']) + }) +}) diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index 4088972298c..59db30c4aad 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -310,8 +310,8 @@ print compiles with zero errors. ## Compile -`compileWorkflow({ entry, files, catalog, sdkMajor, limits?, knownKeys?, stored?, storedUnread? })` → `{ workflow, -diagnostics, spans, hints, sdkMajor }`. It parses with Babel and interprets the AST statically; it +`compileWorkflow({ entry, files, catalog, sdkMajor, limits?, knownKeys?, stored?, storedUnread?, +storedUnreadEdges? })` → `{ workflow, diagnostics, spans, hints, sdkMajor, envReferences, edges }`. It parses with Babel and interprets the AST statically; it never executes the file. `workflow` is `null` iff any diagnostic is an error. It accepts exactly what the printer emits plus the conveniences SYNTAX.md lists, checks names, scope, wiring and catalog fields, fills defaults, resolves `stored()` from `CompileInput.stored`, and returns `spans` @@ -326,7 +326,8 @@ Compile also refuses or flags source the runtime would silently mishandle: of edges the stored workflow holds is `W_EDGE_CYCLE`, so stored state keeps round-tripping (`storedUnread`, the CLI's offline check, which has the lock but not the stored workflow, takes every edge between two blocks its lock binds, by key or by ident for a block renamed in source, - as one it may hold; the same goes for `W_ERROR_PORT_STORED`); + as one it may hold, or with `storedUnreadEdges`, the edges the get printed by lock key, only + those; the same goes for `W_ERROR_PORT_STORED`); - a reference into an agent whose `responseFormat` declares its fields names a declared field or a static output (`W_UNKNOWN_OUTPUT_PATH` otherwise); a format with references inside is unread; - a dropdown value outside its static options is `W_UNKNOWN_OPTION`, a `model` the catalog does diff --git a/packages/workflow-code/src/compile/edges.ts b/packages/workflow-code/src/compile/edges.ts index a7c6110de1e..15e29283d77 100644 --- a/packages/workflow-code/src/compile/edges.ts +++ b/packages/workflow-code/src/compile/edges.ts @@ -51,6 +51,12 @@ function portForHandle(handle: string, from: BlockDecl): Port { /** Implicit branch keys of a never-persisted branch list. */ const IMPLICIT_BRANCHES = { condition: ['if', 'else'], router: ['route0'] } as const +/** What `EdgeCollector` knows of a stored workflow it continues but cannot read. */ +interface UnreadBound { + lockKeys: ReadonlyMap + edges?: ReadonlySet +} + /** Pass 2 of wiring: validates each edge a wiring statement denotes and collects accepted ones. */ export class EdgeCollector { readonly edges: CanonicalEdge[] = [] @@ -65,10 +71,12 @@ export class EdgeCollector { private readonly ctx: CompileContext, private readonly blocks: ReadonlyMap, /** - * Under `CompileInput.storedUnread`, the source keys of the blocks its lock binds, renamed - * ones included; `undefined` when the compile holds the stored workflow or continues none. + * Under `CompileInput.storedUnread`, the lock key of each source block its lock binds, by + * source key (a renamed block's is its vacated lock key), and with + * `CompileInput.storedUnreadEdges` the tuple keys of the stored edges by lock key; + * `undefined` when the compile holds the stored workflow or continues none. */ - private readonly unreadBound: ReadonlySet | undefined + private readonly unreadBound: UnreadBound | undefined ) { for (const decl of ctx.decls) { if (decl.duplicate) continue @@ -129,7 +137,7 @@ export class EdgeCollector { const stored = (this.ctx.stored?.edges ?? []).some( (edge) => edge.from === from.key && edge.to === to.key && edge.port.kind === 'error' ) - const unread = !stored && this.maybeStored(from.key, to.key) + const unread = !stored && this.maybeStored(from.key, port, to.key) if (!stored && !unread) { this.error('E_ERROR_PORT_UNSUPPORTED', `${from.name} has no error port.`, node, { hint: containerKindOf(from) @@ -237,7 +245,7 @@ export class EdgeCollector { return ( edge && !storedTuples.has(edgeTupleKey(edge.from, portKey(edge.port), edge.to)) && - !this.maybeStored(edge.from, edge.to) + !this.maybeStored(edge.from, edge.port, edge.to) ) }) const candidates = fresh.length > 0 ? fresh : inside @@ -304,11 +312,16 @@ export class EdgeCollector { } /** - * Whether an edge between `from` and `to` may be one the stored workflow holds although this - * compile cannot read it (`CompileInput.storedUnread`): both ends are blocks its lock binds. + * Whether an edge may be one the stored workflow holds although this compile cannot read it + * (`CompileInput.storedUnread`): both ends are blocks its lock binds, and with + * `CompileInput.storedUnreadEdges` the stored workflow held this edge between them. */ - private maybeStored(from: BlockKey, to: BlockKey): boolean { - return this.unreadBound?.has(from) === true && this.unreadBound.has(to) + private maybeStored(from: BlockKey, port: Port, to: BlockKey): boolean { + const lockFrom = this.unreadBound?.lockKeys.get(from) + const lockTo = this.unreadBound?.lockKeys.get(to) + if (lockFrom === undefined || lockTo === undefined) return false + const stored = this.unreadBound?.edges + return stored === undefined || stored.has(edgeTupleKey(lockFrom, portKey(port), lockTo)) } /** diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 1677338661d..3ea902e54c4 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -6,7 +6,7 @@ import { VARIABLE_TYPES, type VariableType, } from '@sim/workflow-types/workflow' -import { encodeAnchor } from '../anchors' +import { edgeTupleKey, encodeAnchor, portKey } from '../anchors' import { literalText } from '../babel' import { compareEdges } from '../canonicalize/edges' import type { BlockCatalog } from '../catalog/types' @@ -21,6 +21,7 @@ import type { CanonicalBlock, CanonicalVariable, CanonicalWorkflow, + PortKey, VarKey, } from '../model' import { setOwn } from '../records' @@ -73,6 +74,14 @@ export interface CompileInput { * workflow holds it, not an error. Ignored when `stored` is given. */ storedUnread?: WorkflowLockV1 + /** + * With `storedUnread`, the edges the stored workflow held when that lock was written, by lock + * key (a get's printed source compiled: `CompileResult.edges`). An edge between bound blocks + * then counts as possibly stored only when it is one of these, so a cycle or an error edge the + * source adds between bound blocks is the error the server reports. Absent, any such edge may + * be stored. + */ + storedUnreadEdges?: ReadonlyArray /** * The env variable names a run resolves (the workspace's plus the caller's personal ones). * `env.NAME` outside it compiles with `W_ENV_UNKNOWN`; absent, no name is checked. @@ -102,6 +111,19 @@ export interface CompileResult { * workspace apply checks against the secrets its directory declares. */ envReferences: string[] + /** + * Every edge the wiring denotes that passed validation, by block key, whether or not the + * compile failed elsewhere: what an offline check records of a get's printed source as + * `CompileInput.storedUnreadEdges`. + */ + edges: StoredEdgeRef[] +} + +/** One edge as `CompileInput.storedUnreadEdges` names it: block keys and the port's `portKey`. */ +export interface StoredEdgeRef { + from: BlockKey + port: PortKey + to: BlockKey } const WORKFLOW_OPTIONS = ['description', 'variables'] @@ -387,6 +409,7 @@ function emptyResult(reporter: Reporter, sdkMajor: number | null): CompileResult hints: { blocks: {} }, sdkMajor, envReferences: [], + edges: [], } } @@ -622,11 +645,21 @@ function compileEntry( const edges = new EdgeCollector( ctx, compiled, - unread && - new Set([ - ...Object.keys(unread.blocks), - ...lockIdentRenames(hints, unread, (key) => Object.hasOwn(unread.blocks, key)).values(), - ]) + unread && { + lockKeys: new Map([ + ...Object.keys(unread.blocks).map((key) => [key, key] as const), + ...[...lockIdentRenames(hints, unread, (key) => Object.hasOwn(unread.blocks, key))].map( + ([lockKey, key]) => [key, lockKey] as const + ), + ]), + ...(input.storedUnreadEdges + ? { + edges: new Set( + input.storedUnreadEdges.map((edge) => edgeTupleKey(edge.from, edge.port, edge.to)) + ), + } + : {}), + } ) for (const entry of ctx.wiring) edges.statement(entry) for (const implied of ctx.implied) edges.implied(implied.container, implied.child, implied.body) @@ -667,5 +700,6 @@ function compileEntry( hints, sdkMajor: parsed.sdkMajor, envReferences: [...ctx.envReferences].sort(compareStrings), + edges: sortedEdges.map((edge) => ({ from: edge.from, port: portKey(edge.port), to: edge.to })), } } diff --git a/packages/workflow-code/src/diagnostics/codes.ts b/packages/workflow-code/src/diagnostics/codes.ts index d4f6574686a..9b40b36e773 100644 --- a/packages/workflow-code/src/diagnostics/codes.ts +++ b/packages/workflow-code/src/diagnostics/codes.ts @@ -127,7 +127,8 @@ const COMPILE_EDGE_CODES = [ /** * `.onError()` on a block with no error port (a loop or parallel) that the stored workflow * already wires: kept so it round-trips; source never adds one. Unread (`storedUnread`), any such - * edge between two bound blocks, which an apply keeps only if the stored workflow holds it. + * edge between two bound blocks (with `storedUnreadEdges`, one the get printed), which an apply + * keeps only if the stored workflow holds it. */ 'W_ERROR_PORT_STORED', /** `x.key.to()` where `key` is not a branch of `x`. */ diff --git a/packages/workflow-code/src/index.ts b/packages/workflow-code/src/index.ts index 07445c505fa..e56652be09b 100644 --- a/packages/workflow-code/src/index.ts +++ b/packages/workflow-code/src/index.ts @@ -36,7 +36,7 @@ export { fromCodeFile, isIdentifierName, } from './code-files' -export { type CompileResult, compileWorkflow } from './compile' +export { type CompileResult, compileWorkflow, type StoredEdgeRef } from './compile' export { codePointLabel, scriptName } from './compile/characters' export { didYouMean } from './compile/hints' export { From 671bf3d4a1555b61c9c1a03e75374614bdd685eb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 14:35:59 -0800 Subject: [PATCH 03/59] fix(sim-cli): cache source catalogs by content hash Each catalog is cached once at source-catalog/.json (verified against its hash on read), and each workspace records its current catalog's hash in source-catalog/workspaces/.json for ETag revalidation. Check and apply resolve the catalog by the hash .sim/base.json records, and a get reuses a cached workflow catalog of that hash instead of fetching it again, so the cache grows with distinct catalogs rather than with workflows. The per-workspace and per-workflow copies an older CLI wrote are removed on the next catalog read. --- .../src/commands/protocol/source-tree.ts | 192 ++++++++++-------- .../protocol/workflow-source-apply.test.ts | 7 +- .../protocol/workflow-source-apply.ts | 8 +- .../protocol/workflow-source-check.test.ts | 7 +- .../protocol/workflow-source-check.ts | 2 +- .../commands/protocol/workflow-source-get.ts | 1 + .../workflow-source-roundtrip.test.ts | 29 ++- .../protocol/workspace-source-check.test.ts | 11 +- packages/sim-cli/src/test/catalog-cache.ts | 20 ++ 9 files changed, 164 insertions(+), 113 deletions(-) create mode 100644 packages/sim-cli/src/test/catalog-cache.ts diff --git a/packages/sim-cli/src/commands/protocol/source-tree.ts b/packages/sim-cli/src/commands/protocol/source-tree.ts index 870b14c0f0b..9ef93cb14dc 100644 --- a/packages/sim-cli/src/commands/protocol/source-tree.ts +++ b/packages/sim-cli/src/commands/protocol/source-tree.ts @@ -5,6 +5,7 @@ * the agent guide and the LF lines. */ import { createHash } from 'node:crypto' +import { rmSync } from 'node:fs' import { join } from 'node:path' import { isRecordLike, toRecord } from '@sim/utils/object' import { escapeRegExp } from '@sim/utils/string' @@ -77,85 +78,61 @@ function isBlockCatalog(value: unknown): value is BlockCatalog { ) } -/** Workspace ids are uuids; anything else is never used to name a cache file. */ +/** Workspace ids are uuids and catalog hashes hex; anything else never names a cache file. */ const CACHE_KEY = /^[\w-]+$/ +/** The CLI's catalog cache: every catalog by its hash, and each workspace's current hash. */ +function catalogCacheDir(): string { + return join(configDir(), 'cache', 'source-catalog') +} + /** - * Where the workspace's catalog is cached: one file per workspace in the CLI's config directory, - * so every `source get` in the workspace (whatever the workflow or output directory) revalidates - * the same copy. Callers gate the id with `CACHE_KEY` first. + * Where a catalog is cached: by its content hash, so the workspace catalog and every per-workflow + * catalog share one file per distinct catalog, however many workflows and workspaces name it. + * Callers gate the hash with `CACHE_KEY` first. */ -function catalogCachePath(workspaceId: string): string { - return join(configDir(), 'cache', 'source-catalog', `${workspaceId}.json`) +function catalogCachePath(hash: string): string { + return join(catalogCacheDir(), `${hash}.json`) } /** - * Where a workflow's own source catalog is cached: the one a get fetches for a workflow holding a - * type the workspace catalog lacks (see {@link loadCatalog}), one file per workflow beside the - * workspace's. Callers gate both ids with `CACHE_KEY` first. + * Where the hash of a workspace's catalog is recorded, so a get revalidates it by ETag and an + * offline command finds it. Callers gate the id with `CACHE_KEY` first. */ -function workflowCatalogCachePath(workspaceId: string, workflowId: string): string { - return join(configDir(), 'cache', 'source-catalog', workspaceId, `${workflowId}.json`) +function workspaceCatalogPointerPath(workspaceId: string): string { + return join(catalogCacheDir(), 'workspaces', `${workspaceId}.json`) } -function readCachedCatalog(path: string): BlockCatalog | null { - const cached = readJsonFile(path, MAX_CATALOG_CACHE_BYTES) - return isBlockCatalog(cached) ? cached : null -} +/** A pointer is one hash; a larger file was not written by this CLI. */ +const MAX_POINTER_BYTES = 4096 /** - * The cached catalog, or `null`. Standalone runs only: an embedded run has no local disk of its - * own to cache on. The file is read without following a link and under a size cap. + * The cached catalog of `hash`, or `null`. Standalone runs only: an embedded run has no local disk + * of its own to cache on. The file is read without following a link and under a size cap, and + * kept only when the catalog in it names the hash its file does. */ -export function cachedCatalog(workspaceId: string): BlockCatalog | null { - if (embedStore.getStore() || !CACHE_KEY.test(workspaceId)) return null - return readCachedCatalog(catalogCachePath(workspaceId)) +export function cachedCatalogByHash(hash: unknown): BlockCatalog | null { + if (embedStore.getStore() || typeof hash !== 'string' || !CACHE_KEY.test(hash)) return null + const cached = readJsonFile(catalogCachePath(hash), MAX_CATALOG_CACHE_BYTES) + return isBlockCatalog(cached) && cached.hash === hash ? cached : null } -/** - * The cached catalog a source directory was printed against, or `null`: the workflow's own - * catalog when `.sim/base.json` names a workflow whose cached catalog has the hash the base - * records (a workflow holding a type the workspace catalog lacks, which only that catalog - * declares), else the workspace's ({@link cachedCatalog}). An offline check compiles against it, - * so it agrees with the dry run, which compiles against the same per-workflow catalog. - */ -export async function cachedSourceCatalog( - dir: string, - workspaceId: string -): Promise { - const base = toRecord(await readSourceJson(dir, SOURCE_BASE_FILE)) - const { workflowId, catalogHash } = base - if ( - !embedStore.getStore() && - CACHE_KEY.test(workspaceId) && - typeof workflowId === 'string' && - CACHE_KEY.test(workflowId) && - base.workspaceId === workspaceId && - typeof catalogHash === 'string' - ) { - const own = readCachedCatalog(workflowCatalogCachePath(workspaceId, workflowId)) - if (own?.hash === catalogHash) return own - } - return cachedCatalog(workspaceId) +/** The cached catalog of the workspace, by the hash its last catalog read recorded, or `null`. */ +export function cachedCatalog(workspaceId: string): BlockCatalog | null { + if (embedStore.getStore() || !CACHE_KEY.test(workspaceId)) return null + const pointer = readJsonFile(workspaceCatalogPointerPath(workspaceId), MAX_POINTER_BYTES) + return isRecordLike(pointer) ? cachedCatalogByHash(pointer.hash) : null } /** - * The cached catalog a printed source's base names by hash (`.sim/base.json` of a get, apply or - * create answer): the workflow's own or the workspace's, or `null` when neither cached catalog - * has that hash. + * The cached catalog a workflow source directory was printed against, by the hash its + * `.sim/base.json` records (the workspace's, or the workflow's own for a workflow holding a type + * the workspace catalog lacks), or `null` when the directory has no base or that catalog is not + * cached. An offline check compiles against it, so it agrees with the dry run, which compiles + * against the same catalog. */ -export function cachedCatalogOfBase(base: unknown): BlockCatalog | null { - const { workspaceId, workflowId, catalogHash } = toRecord(base) - if (embedStore.getStore() || typeof workspaceId !== 'string' || !CACHE_KEY.test(workspaceId)) { - return null - } - const candidates = [ - typeof workflowId === 'string' && CACHE_KEY.test(workflowId) - ? readCachedCatalog(workflowCatalogCachePath(workspaceId, workflowId)) - : null, - cachedCatalog(workspaceId), - ] - return candidates.find((catalog) => catalog !== null && catalog.hash === catalogHash) ?? null +export async function cachedSourceCatalog(dir: string): Promise { + return cachedCatalogByHash(toRecord(await readSourceJson(dir, SOURCE_BASE_FILE)).catalogHash) } /** The catalog a `getSourceCatalog` answer carries. */ @@ -166,23 +143,71 @@ function catalogFrom(response: { data: unknown }): BlockCatalog { return response.data } -/** Replaces the cached catalog atomically; a failed write only costs the next get a download. */ -function writeCachedCatalog(workspaceId: string, catalog: BlockCatalog): void { +/** + * Caches `catalog` by its hash, replacing the file atomically; a failed write only costs the next + * get a download. An unchanged catalog is not rewritten. + */ +function writeCachedCatalog(catalog: BlockCatalog): void { + if (embedStore.getStore() || !CACHE_KEY.test(catalog.hash)) return + if (cachedCatalogByHash(catalog.hash)) return + writeJsonFile(catalogCachePath(catalog.hash), catalog) +} + +/** + * Records `hash` as the workspace's catalog, dropping the per-workspace copy and the + * per-workflow copies an older CLI cached under the workspace id. + */ +function writeWorkspaceCatalogPointer(workspaceId: string, hash: string): void { if (embedStore.getStore() || !CACHE_KEY.test(workspaceId)) return - writeJsonFile(catalogCachePath(workspaceId), catalog) + writeJsonFile(workspaceCatalogPointerPath(workspaceId), { hash }) + for (const legacy of [`${workspaceId}.json`, workspaceId]) { + if (legacy === `${hash}.json`) continue + try { + rmSync(join(catalogCacheDir(), legacy), { recursive: true, force: true }) + } catch {} + } } /** - * The workspace's block catalog, revalidated by ETag against the per-workspace cache: a `304` - * reuses it, anything else replaces it. A workflow can hold a type the workspace no longer offers - * (a legacy version, a block an allowlist now withholds); only then is that workflow's compile - * catalog fetched instead, so its types still resolve, and cached per workflow for the offline - * check ({@link cachedSourceCatalog}). + * The catalog of the workflow `workflowId` (the workspace catalog plus the types it stores that the + * workspace catalog lacks), fetched and cached by hash; `knownHash`, a hash the workflow's catalog + * may have, is sent as its ETag, so a catalog already cached answers `304` with no body. + */ +export async function loadWorkflowCatalog( + client: OperationClient<'getSourceCatalog'>, + workspaceId: string, + workflowId: string, + knownHash?: string +): Promise { + const known = cachedCatalogByHash(knownHash) + try { + const catalog = catalogFrom( + await client.request<{ data: unknown }>('getSourceCatalog', { + params: { workspaceId }, + query: { workflowId }, + ...(known ? { headers: { 'if-none-match': `"${known.hash}"` } } : {}), + }) + ) + writeCachedCatalog(catalog) + return catalog + } catch (error) { + if (!known || !(error instanceof SimApiError) || error.status !== 304) throw error + return known + } +} + +/** + * The workspace's block catalog, revalidated by ETag against the cached one: a `304` reuses it, + * anything else replaces it. A workflow can hold a type the workspace no longer offers (a legacy + * version, a block an allowlist now withholds); only then is that workflow's own catalog used + * instead, so its types still resolve: the one cached under the hash its base records + * (`catalogHash`), else fetched ({@link loadWorkflowCatalog}). */ export async function loadCatalog( client: OperationClient<'getSourceCatalog'>, source: Pick & { workflowId?: string + catalogHash?: string } ): Promise { const cached = cachedCatalog(source.workspaceId) @@ -194,31 +219,18 @@ export async function loadCatalog( ...(cached ? { headers: { 'if-none-match': `"${cached.hash}"` } } : {}), }) ) - writeCachedCatalog(source.workspaceId, catalog) + writeCachedCatalog(catalog) } catch (error) { if (!cached || !(error instanceof SimApiError) || error.status !== 304) throw error catalog = cached } - if ( - source.workflowId === undefined || - source.blockTypes.every((type) => catalogBlock(catalog, type) !== undefined) - ) { - return catalog - } - const own = catalogFrom( - await client.request<{ data: unknown }>('getSourceCatalog', { - params: { workspaceId: source.workspaceId }, - query: { workflowId: source.workflowId }, - }) - ) - if ( - !embedStore.getStore() && - CACHE_KEY.test(source.workspaceId) && - CACHE_KEY.test(source.workflowId) - ) { - writeJsonFile(workflowCatalogCachePath(source.workspaceId, source.workflowId), own) - } - return own + writeWorkspaceCatalogPointer(source.workspaceId, catalog.hash) + const declaresAll = (candidate: BlockCatalog) => + source.blockTypes.every((type) => catalogBlock(candidate, type) !== undefined) + if (source.workflowId === undefined || declaresAll(catalog)) return catalog + const printedWith = cachedCatalogByHash(source.catalogHash) + if (printedWith && declaresAll(printedWith)) return printedWith + return loadWorkflowCatalog(client, source.workspaceId, source.workflowId) } /** @@ -520,7 +532,7 @@ export async function refreshTsconfigs( use: SourceTreeUse, options: { incremental?: boolean } = {} ): Promise { - const catalog = await cachedSourceCatalog(dir, workspaceId) + const catalog = (await cachedSourceCatalog(dir)) ?? cachedCatalog(workspaceId) if (!catalog) return const [tsconfig, full, workspaceTypes] = await Promise.all([ readSourceText(dir, TSCONFIG_FILE), diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index ec73df948bb..ddcf921d563 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -7,6 +7,7 @@ import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' import { buildGeneratedCommands } from '../../runtime/build' +import { cacheCatalogText } from '../../test/catalog-cache' import { contextMockFns, contextMockState } from '../../test/context-mock' import { digest } from './source-dir' import { TSCONFIG_MARKER } from './source-tree' @@ -203,11 +204,7 @@ describe('sim workflows source apply', () => { it('points tsconfig.json at a block the applied source added, from the cached catalog', async () => { setTerminal(false) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) - mkdirSync(join(dir, 'config/cache/source-catalog'), { recursive: true }) - writeFileSync( - join(dir, `config/cache/source-catalog/${WORKSPACE_ID}.json`), - readFileSync(FIXTURE_CATALOG, 'utf8') - ) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, readFileSync(FIXTURE_CATALOG, 'utf8')) writeFileSync(join(dir, 'tsconfig.json'), `${TSCONFIG_MARKER}\n{}\n`) const stored = outcome() stored.data.files = { 'workflow.ts': `${ENTRY}// slack.sendMessage('Post', {})\n` } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 6171027c47f..26c2abae0e9 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -1,4 +1,5 @@ import { mapWithConcurrency } from '@sim/utils/concurrency' +import { toRecord } from '@sim/utils/object' import { parseSourceBase, parseWorkflowLock, type WorkflowLockV1 } from '@sim/workflow-code' import { codeFileReferences } from '@sim/workflow-code/code-files/references' import { @@ -28,7 +29,7 @@ import { withWorkspaceFolderLock, writeBack, } from './source-dir' -import { cachedCatalogOfBase } from './source-tree' +import { cachedCatalogByHash } from './source-tree' import { confirm, fresh, @@ -318,7 +319,10 @@ export function attachWorkflowSourceApply(source: Command): void { params: { workflowId }, body: { files, lock, base, provenance: { client: 'cli' } }, save: async (data) => { - const edges = storedEdgesWrite(data, cachedCatalogOfBase(data.base)) + const edges = storedEdgesWrite( + data, + cachedCatalogByHash(toRecord(data.base).catalogHash) + ) await writeBack(options.dir, data, lock, { write: edges ? [edges] : [] }) await refreshWorkflowTsconfigs(options.dir, data) }, diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index c597f47ff97..261d7f85fc6 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { SimApiError } from '../../http/client' +import { cacheCatalogText } from '../../test/catalog-cache' import { contextMockFns } from '../../test/context-mock' import { attachWorkflowSourceCheck } from './workflow-source-check' @@ -33,11 +34,7 @@ async function check(): Promise { } function cacheCatalog(): void { - mkdirSync(join(dir, 'config/cache/source-catalog'), { recursive: true }) - writeFileSync( - join(dir, `config/cache/source-catalog/${WORKSPACE_ID}.json`), - readFileSync(FIXTURE_CATALOG, 'utf8') - ) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, readFileSync(FIXTURE_CATALOG, 'utf8')) } function result(): Record { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index fdeff539d9c..e041cdd5605 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -556,7 +556,7 @@ export function attachWorkflowSourceCheck(source: Command): void { ) } const catalog = cachedOrRefuse( - await cachedSourceCatalog(options.dir, workspaceId), + (await cachedSourceCatalog(options.dir)) ?? cachedCatalog(workspaceId), workspaceId, workspace ? `\`sim workspaces source get --dir ${workspace.root}\`` diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index 7cf27307ed5..768ca8fc5fe 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -223,6 +223,7 @@ export async function writeSourceDirectory( loadCatalog(client, { workspaceId: source.workspaceId, workflowId: source.workflowId, + catalogHash: source.base.catalogHash, blockTypes: [...blockTypes], }), loadWorkspaceTypes(client, source.workspaceId), diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 6b0516c690b..870b79dbfc6 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -527,8 +527,12 @@ describe('catalog and declarations', () => { }) it('heals a torn catalog cache by fetching the catalog whole and rewriting it', async () => { - const cache = join(root, 'config/cache/source-catalog', `${WORKSPACE_ID}.json`) - mkdirSync(join(cache, '..'), { recursive: true }) + const cache = join(root, 'config/cache/source-catalog', `${CATALOG.hash}.json`) + mkdirSync(join(cache, '../workspaces'), { recursive: true }) + writeFileSync( + join(cache, '../workspaces', `${WORKSPACE_ID}.json`), + JSON.stringify({ hash: CATALOG.hash }) + ) writeFileSync(cache, '{"hash":"torn","catalog":{"blo') await get() @@ -996,6 +1000,27 @@ describe('checking a workflow holding a type the workspace catalog withholds', ( ).toEqual(dryRun) }) + it('caches each catalog once by its hash, fetching the workflow catalog only while uncached', async () => { + const cacheDir = join(root, 'config/cache/source-catalog') + mkdirSync(join(cacheDir, WORKSPACE_ID), { recursive: true }) + writeFileSync(join(cacheDir, `${WORKSPACE_ID}.json`), '{}') + serve(TICKET_ROUTER) + const queries: string[] = [] + const answer = server.catalog + server.catalog = (headers, url) => { + queries.push(url.searchParams.get('workflowId') ?? 'workspace') + return answer(headers, url) + } + + await get() + await get() + + expect(readdirSync(cacheDir).sort()).toEqual( + [`${CATALOG.hash}.json`, `${WORKSPACE_CATALOG.hash}.json`, 'workspaces'].sort() + ) + expect(queries.filter((query) => query === WORKFLOW_ID)).toHaveLength(1) + }) + it('checks against the workspace catalog again once a later get no longer needs its own', async () => { serve(TICKET_ROUTER) await get() diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts index 969ade98a8f..2db74283680 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' +import { cacheCatalogText } from '../../test/catalog-cache' import { contextMockFns } from '../../test/context-mock' import { attachWorkspaceSourceCheck } from './workspace-source-check' @@ -74,10 +75,7 @@ beforeEach(() => { '', ].join('\n') ) - writeFile( - `config/cache/source-catalog/${WORKSPACE_ID}.json`, - readFileSync(FIXTURE_CATALOG, 'utf8') - ) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, readFileSync(FIXTURE_CATALOG, 'utf8')) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_other', output: 'json' }) contextMockFns.mockRequest.mockReset() @@ -155,10 +153,7 @@ describe('sim workspaces source check', () => { }) it('in a fresh clone (no cached catalog), downloads the catalog once and checks', async () => { - const catalog = readFileSync( - join(dir, `config/cache/source-catalog/${WORKSPACE_ID}.json`), - 'utf8' - ) + const catalog = readFileSync(FIXTURE_CATALOG, 'utf8') rmSync(join(dir, 'config'), { recursive: true }) writeFile('workflows/first/workflow.ts', insertWorkflow()) contextMockFns.mockRequest.mockResolvedValue({ data: JSON.parse(catalog) }) diff --git a/packages/sim-cli/src/test/catalog-cache.ts b/packages/sim-cli/src/test/catalog-cache.ts new file mode 100644 index 00000000000..07f01dab955 --- /dev/null +++ b/packages/sim-cli/src/test/catalog-cache.ts @@ -0,0 +1,20 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +/** + * Caches catalog JSON `text` in config directory `configDir` as the CLI does: the catalog under its + * hash, and `workspaceId`'s pointer to it unless `workspaceId` is `null`. + */ +export function cacheCatalogText( + configDir: string, + workspaceId: string | null, + text: string +): void { + const { hash } = JSON.parse(text) as { hash: string } + const root = join(configDir, 'cache/source-catalog') + mkdirSync(join(root, 'workspaces'), { recursive: true }) + writeFileSync(join(root, `${hash}.json`), text) + if (workspaceId !== null) { + writeFileSync(join(root, 'workspaces', `${workspaceId}.json`), JSON.stringify({ hash })) + } +} From 610203986aa27af03a8dcb190c563481a0a73804 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 14:42:11 -0800 Subject: [PATCH 04/59] fix(sim-cli): check each workspace workflow against the catalog its base names A workspace get or apply now fetches, once per distinct hash, the catalog of each workflow folder whose base names one other than the workspace catalog (a workflow holding a type the workspace withholds) and caches it by hash; the offline workspace check compiles each folder against the catalog its base names. Each workflow folder also gets its .sim/stored-edges.json, so an edit's new cycle between bound blocks is E_EDGE_CYCLE in a workspace directory too. The record is a CLI file: never sent, not listed as ignored, and removed with its workflow. --- apps/docs/content/docs/cli/reference.mdx | 2 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- .../src/commands/protocol/stored-edges.ts | 79 +++++++++- .../protocol/workflow-source-check.ts | 12 +- .../protocol/workspace-source-apply.ts | 20 ++- .../commands/protocol/workspace-source-dir.ts | 8 +- .../protocol/workspace-source-get.test.ts | 140 ++++++++++++++++++ .../commands/protocol/workspace-source-get.ts | 9 ++ 8 files changed, 258 insertions(+), 14 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index c66a09de33b..153ec878e47 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -9952,7 +9952,7 @@ sim workspaces source apply [options] Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. -API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource). +API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). ### sim workspaces source check diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 3cbe8c01746..12b9dcf1f8e 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -762,7 +762,7 @@ sim workspaces source apply [options] Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. -API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource). +API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). ### Compile every workflow of a workspace source directory locally, without the server diff --git a/packages/sim-cli/src/commands/protocol/stored-edges.ts b/packages/sim-cli/src/commands/protocol/stored-edges.ts index dfc4fa6010a..888c553e1b8 100644 --- a/packages/sim-cli/src/commands/protocol/stored-edges.ts +++ b/packages/sim-cli/src/commands/protocol/stored-edges.ts @@ -4,9 +4,13 @@ import { compileWorkflow, parseWorkflowLock, type StoredEdgeRef, + setOwn, } from '@sim/workflow-code' -import { SOURCE_BASE_FILE, SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' +import { SOURCE_BASE_FILE, SOURCE_ENTRY_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' +import { KIND_DIRS } from '@sim/workspace-code' +import type { OperationClient } from '../../runtime/called-operations' import { readSourceJson, SOURCE_STORED_EDGES_FILE } from './source-dir' +import { cachedCatalogByHash, loadWorkflowCatalog } from './source-tree' /** A printed source as a get, apply or create answers it. */ interface PrintedAnswer { @@ -23,7 +27,8 @@ interface PrintedAnswer { */ export function storedEdgesWrite( answer: PrintedAnswer, - catalog: BlockCatalog | null + catalog: BlockCatalog | null, + folder = '' ): readonly [string, string] | null { const { fingerprint } = toRecord(answer.base) const entry = answer.files[SOURCE_ENTRY_FILE] @@ -38,11 +43,70 @@ export function storedEdgesWrite( storedUnread: lock, }) return [ - SOURCE_STORED_EDGES_FILE, + `${folder}${SOURCE_STORED_EDGES_FILE}`, `${JSON.stringify({ version: 1, fingerprint, edges }, null, 2)}\n`, ] } +/** The workflow folders a workspace answer's files hold, by key, each as a printed source. */ +function printedWorkflowFolders(files: Record): Map { + const folders = new Map() + const prefix = `${KIND_DIRS.workflow}/` + for (const [path, text] of Object.entries(files)) { + if (!path.startsWith(prefix)) continue + const slash = path.indexOf('/', prefix.length) + if (slash < 0) continue + const key = path.slice(prefix.length, slash) + const relative = path.slice(slash + 1) + const folder = folders.get(key) ?? { files: {}, lock: undefined, base: undefined } + if (relative === SOURCE_LOCK_FILE) folder.lock = parseJson(text) + else if (relative === SOURCE_BASE_FILE) folder.base = parseJson(text) + else setOwn(folder.files, relative, text) + folders.set(key, folder) + } + return folders +} + +function parseJson(text: string): unknown { + try { + return JSON.parse(text) + } catch { + return undefined + } +} + +/** + * The `.sim/stored-edges.json` write of each workflow folder a workspace get or apply answered + * with, each compiled against the catalog its base names. A workflow holding a type the workspace + * catalog lacks was printed with its own catalog: one not cached yet is fetched, once per distinct + * hash, and cached, so an offline check compiles that folder against it too. + */ +export async function workflowFolderRecords( + client: OperationClient<'getSourceCatalog'>, + workspaceId: string, + workspaceCatalog: BlockCatalog, + files: Record +): Promise> { + const folders = printedWorkflowFolders(files) + const uncached = new Map() + for (const { base } of folders.values()) { + const { workflowId, catalogHash } = toRecord(base) + if (typeof workflowId !== 'string' || typeof catalogHash !== 'string') continue + if (catalogHash === workspaceCatalog.hash || cachedCatalogByHash(catalogHash)) continue + if (!uncached.has(catalogHash)) uncached.set(catalogHash, workflowId) + } + for (const workflowId of uncached.values()) { + await loadWorkflowCatalog(client, workspaceId, workflowId) + } + return [...folders].flatMap(([key, folder]) => { + const { catalogHash } = toRecord(folder.base) + const catalog = + catalogHash === workspaceCatalog.hash ? workspaceCatalog : cachedCatalogByHash(catalogHash) + const write = storedEdgesWrite(folder, catalog, `${KIND_DIRS.workflow}/${key}/`) + return write ? [write] : [] + }) +} + function isStoredEdge(value: unknown): value is StoredEdgeRef { return ( isRecordLike(value) && @@ -73,3 +137,12 @@ export async function readStoredEdges(dir: string): Promise): string[] { + return [...paths].flatMap((path) => + path.startsWith(`${KIND_DIRS.workflow}/`) && path.endsWith(`/${SOURCE_BASE_FILE}`) + ? [`${path.slice(0, -SOURCE_BASE_FILE.length)}${SOURCE_STORED_EDGES_FILE}`] + : [] + ) +} diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index e041cdd5605..e02b9c08fdb 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -348,19 +348,23 @@ function declarationFindings( } /** - * Compiles the workflow source directory `dir` against `catalog`, checking env names against the - * names `secrets.ts` declares when `declarations` hold them. + * Compiles the workflow source directory `dir` against the catalog its base names when that one is + * cached (a workflow holding a type the workspace catalog lacks was printed with its own), else + * `catalog`, checking env names against the names `secrets.ts` declares when `declarations` hold + * them. */ async function compileWorkflowDir( dir: string, - catalog: BlockCatalog, + workspaceCatalog: BlockCatalog, declarations?: WorkspaceDeclarations ) { const { lock } = parseWorkflowLock(await readSourceJson(dir, SOURCE_LOCK_FILE)) - const [files, storedEdges] = await Promise.all([ + const [files, storedEdges, printedWith] = await Promise.all([ readSourceFiles(dir, lock ?? null), readStoredEdges(dir), + cachedSourceCatalog(dir), ]) + const catalog = printedWith ?? workspaceCatalog const envNames = declarations?.envNames const compiled = compileWorkflow({ entry: { path: SOURCE_ENTRY_FILE, text: files[SOURCE_ENTRY_FILE] }, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 46da2dec2a5..6fdd523985c 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -38,6 +38,7 @@ import { writtenFile, writtenRecord, } from './source-dir' +import { cachedCatalog } from './source-tree' import { confirm, printNewDiagnostics, @@ -46,6 +47,7 @@ import { requireConsent, resultDiagnostics, } from './source-ui' +import { removedWorkflowRecords, workflowFolderRecords } from './stored-edges' import { appliedBlobDigests, assertBlobsFetched, @@ -298,13 +300,15 @@ function droppedResourceFiles( * their digests in `.sim/written.json` (`written` is what it held), beside the digests of the * blobs the server now holds. The function files a written workflow no longer uses, and the files * a reprinted resource no longer has, are removed from disk and the record; the record also drops - * the files of resources the apply deleted ({@link deletedRecordedPaths}). + * the files of resources the apply deleted ({@link deletedRecordedPaths}). `records` are the + * stored-edges records of the workflows it reprinted. */ async function saveApplied( dir: string, data: ApplyData, submission: WorkspaceSubmission, - written: Written + written: Written, + records: ReadonlyArray ): Promise { assertWorkspacePaths(Object.keys(data.files)) const orphaned = [ @@ -324,8 +328,9 @@ async function saveApplied( ...Object.entries(data.files), [WORKSPACE_BASE_FILE, `${JSON.stringify(data.base, null, 2)}\n`], [SOURCE_WRITTEN_FILE, writtenFile(record)], + ...records, ], - remove: orphaned, + remove: [...orphaned, ...removedWorkflowRecords(deleted)], }) } @@ -410,6 +415,7 @@ export function attachWorkspaceSourceApply(source: Command): void { 'findMissingWorkspaceSourceBlobs', 'stageWorkspaceSourceBlobs', 'getWorkspaceSource', + 'getSourceCatalog', ]) command .requiredOption('--dir ', 'Workspace source directory written by `workspaces source get`') @@ -551,7 +557,13 @@ export function attachWorkspaceSourceApply(source: Command): void { writing = false printNewDiagnostics(data.diagnostics, printed) // A promotion leaves the directory bound to the workspace it was read from. - if (!options.to) await saveApplied(options.dir, data, submission, written) + if (!options.to) { + const workspaceCatalog = cachedCatalog(workspaceId) + const records = workspaceCatalog + ? await workflowFolderRecords(client, workspaceId, workspaceCatalog, data.files) + : [] + await saveApplied(options.dir, data, submission, written, records) + } const held = data.plan.filter((change) => change.held) if (held.length > 0) { printError( diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index 73f5263129e..b07edac2b31 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -57,6 +57,7 @@ import { SOURCE_DOWNLOADS_DIR, SOURCE_SDK_DIR, SOURCE_STAGING_PREFIX, + SOURCE_STORED_EDGES_FILE, type StatDigest, type Written, } from './source-dir' @@ -428,6 +429,11 @@ export function assertWorkspacePaths(paths: Iterable): void { } } +/** A workflow folder's CLI-only record (`.sim/stored-edges.json`): never sent, never ignored. */ +function isWorkflowRecord(kind: WorkspaceKind, path: string): boolean { + return kind === 'workflow' && path.endsWith(`/${SOURCE_STORED_EDGES_FILE}`) +} + /** * The folders (and root files) of `kinds`, each walked once: `present`, the kinds whose folder * holds any file (its `.gitkeep` marker included) or whose root file exists; `paths`, their source @@ -452,7 +458,7 @@ export async function kindFolders( if (files.length > 0) present.push(kind) for (const path of files) { if (isWorkspaceSourcePath(path)) paths.push(path) - else if (path !== keepFile(kind)) ignored.push(path) + else if (path !== keepFile(kind) && !isWorkflowRecord(kind, path)) ignored.push(path) } } return { present, paths, ignored } diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts index e25e1a229b0..fb633377108 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts @@ -21,6 +21,7 @@ import { SimApiError } from '../../http/client' import { contextMockFns, contextMockState } from '../../test/context-mock' import { runWorkflowSourceCheck, runWorkspaceSource } from '../../test/workspace-source-command' import { blobBytes, blobDigests } from './workspace-source-blobs' +import { kindFolders } from './workspace-source-dir' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -897,3 +898,142 @@ describe('workspaces source get', () => { expect(request).not.toHaveBeenCalled() }) }) + +/** Block types the workspace catalog withholds while a stored workflow holds them. */ +const WITHHELD_TYPES = new Set(['table_v2', 'knowledge']) + +/** {@link CATALOG} as the workspace catalog of a caller it withholds {@link WITHHELD_TYPES} from. */ +const WORKSPACE_CATALOG = { + ...CATALOG, + hash: 'workspace-catalog', + blocks: Object.fromEntries( + Object.entries(CATALOG.blocks).filter(([type]) => !WITHHELD_TYPES.has(type)) + ), + builders: Object.fromEntries( + Object.entries(CATALOG.builders as Record).filter( + ([, builder]) => !WITHHELD_TYPES.has(builder.type) + ) + ), +} + +const ROUTER = `// @sim-sdk 1 +import { condition, js, knowledge, knowledgeBase, table, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('Router', {}, () => { + const start = trigger.start('Start', { inputFormat: { subject: 'string', priority: 'string' } }) + const isUrgent = condition('Is Urgent', { if: js\`\${start.priority} === 'high'\`, else: true }) + const policy = knowledge.search('Policy', { + knowledgeBaseId: knowledgeBase('policies'), + query: start.subject, + }) + const logTicket = table.insertRow('Log Ticket', { + tableId: table('tickets'), + data: { subject: start.subject }, + }) + + start.to(isUrgent) + isUrgent.if.to(policy) + isUrgent.else.to(logTicket) +}) +` + +/** A stored cycle between A and B, every block bound by the lock. */ +const CYCLE = `// @sim-sdk 1 +import { api, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('Cycle', {}, () => { + const start = trigger.start('Start', {}) + const a = api('A', { url: 'https://example.com', method: 'GET' }) + const b = api('B', { url: 'https://example.com', method: 'GET' }) + const c = api('C', { url: 'https://example.com', method: 'GET' }) + + start.to(a) + a.to(b) + b.to(a) + b.to(c) +}) +` + +function lockOf(keys: readonly string[]): string { + return JSON.stringify({ + version: 1, + blocks: Object.fromEntries( + keys.map((key, order) => [key, { id: `id-${key}`, parent: null, ident: key, order }]) + ), + variables: {}, + }) +} + +/** + * A workspace answer holding workflow `key` (`source`, bound by a lock on `keys`) printed with + * the catalog of hash `catalogHash`; the catalog route answers the full catalog for a workflow and + * the withholding one for the workspace. + */ +function serveWorkflow(key: string, source: string, keys: readonly string[], catalogHash: string) { + const files = { + [`workflows/${key}/workflow.ts`]: source, + [`workflows/${key}/.sim/lock.json`]: lockOf(keys), + [`workflows/${key}/.sim/base.json`]: JSON.stringify({ + version: 1, + workflowId: `wf-${key}`, + workspaceId: 'ws_local', + fingerprint: `sf1:${key}`, + catalogHash, + }), + } + serve(files, { + kinds: ['workflow'], + resources: { [`workflow:${key}`]: { id: `wf-${key}`, fingerprint: `rf1:${key}` } }, + }) + const answer = request.getMockImplementation() + request.mockImplementation( + async (path: string, options?: { query?: { workflowId?: string } }) => { + if (path !== CATALOG_PATH) return answer?.(path, options) + return { data: options?.query?.workflowId ? CATALOG : WORKSPACE_CATALOG } + } + ) +} + +function checked(): { errors: number; diagnostics: Array<{ code: string }> } { + return JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') +} + +describe('workspaces source check after a get', () => { + afterEach(() => { + process.exitCode = undefined + }) + + it('checks a workflow holding a type the workspace catalog withholds against its own catalog', async () => { + serveWorkflow('router', ROUTER, ['start', 'isurgent', 'policy', 'logticket'], CATALOG.hash) + await get() + + await runWorkspaceSource('check', dir) + + expect(checked()).toMatchObject({ errors: 0, diagnostics: [] }) + expect(existsSync(join(dir, 'workflows/router/.sim/stored-edges.json'))).toBe(true) + expect((await kindFolders(dir, ['workflow'])).ignored).toEqual([]) + expect( + request.mock.calls.filter( + ([path, options]) => path === CATALOG_PATH && options?.query?.workflowId === 'wf-router' + ) + ).toHaveLength(1) + }) + + it('refuses a cycle an edit closes between bound blocks of a workflow folder', async () => { + serveWorkflow('cycle', CYCLE, ['start', 'a', 'b', 'c'], WORKSPACE_CATALOG.hash) + await get() + await runWorkspaceSource('check', dir, '--strict') + expect(checked().diagnostics.map(({ code }) => code)).toEqual(['W_EDGE_CYCLE']) + expect(process.exitCode).toBeUndefined() + const entry = join(dir, 'workflows/cycle/workflow.ts') + writeFileSync( + entry, + readFileSync(entry, 'utf8').replace(' b.to(c)\n', ' b.to(c)\n c.to(b)\n') + ) + + await runWorkspaceSource('check', dir, '--strict') + + expect(checked().diagnostics.map(({ code }) => code)).toEqual(['E_EDGE_CYCLE']) + expect(process.exitCode).toBe(1) + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index 7d869ceb9e8..2c25ff88f9e 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -45,6 +45,7 @@ import { sourceTreeChange, withSourceExitCode, } from './source-tree' +import { removedWorkflowRecords, workflowFolderRecords } from './stored-edges' import { assertListedBlobs, blobDigests, @@ -206,6 +207,12 @@ async function writeWorkspaceSource( loadWorkspaceTypes(client, source.workspaceId, localDeclarations(source.files)), ]) assertSdkMajor(catalog) + const workflowRecords = await workflowFolderRecords( + client, + source.workspaceId, + catalog, + source.files + ) // A get that leaves the workflows unread keeps the blocks those on disk type-check against. const tree = await sourceTreeChange( dir, @@ -276,6 +283,7 @@ async function writeWorkspaceSource( [WORKSPACE_BASE_FILE, `${JSON.stringify(base, null, 2)}\n`], [SOURCE_WRITTEN_FILE, writtenFile(record)], ...(skipped ? [[WORKSPACE_SKIPPED_FILE, skipped] as const] : []), + ...workflowRecords, ...tree.change.write, ] await updateSourceDir(dir, { @@ -285,6 +293,7 @@ async function writeWorkspaceSource( ), remove: [ ...gone, + ...removedWorkflowRecords(gone), ...(skipped ? [] : [WORKSPACE_SKIPPED_FILE]), ...(tree.change.remove ?? []), ], From cc0cdd02bcc001a6bf40627b06253cb0a963d71c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 16:01:54 -0800 Subject: [PATCH 05/59] fix(workflow-code): warn on a reference to an input the Start trigger does not declare --- packages/workflow-code/DESIGN.md | 6 +- packages/workflow-code/src/compile/SYNTAX.md | 5 +- .../workflow-code/src/compile/checks.test.ts | 42 ++++++++++ packages/workflow-code/src/compile/context.ts | 11 +-- packages/workflow-code/src/compile/index.ts | 6 +- packages/workflow-code/src/compile/values.ts | 84 ++++++++++++++++--- packages/workflow-code/src/start-inputs.ts | 16 +++- 7 files changed, 141 insertions(+), 29 deletions(-) diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index 59db30c4aad..f7aaa16a25f 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -328,8 +328,10 @@ Compile also refuses or flags source the runtime would silently mishandle: every edge between two blocks its lock binds, by key or by ident for a block renamed in source, as one it may hold, or with `storedUnreadEdges`, the edges the get printed by lock key, only those; the same goes for `W_ERROR_PORT_STORED`); -- a reference into an agent whose `responseFormat` declares its fields names a declared field or - a static output (`W_UNKNOWN_OUTPUT_PATH` otherwise); a format with references inside is unread; +- a reference into an agent whose `responseFormat` declares its fields, or into a Start, API or + Input trigger (whose `inputFormat` declares its inputs), names a declared field or a static + output (`W_UNKNOWN_OUTPUT_PATH` otherwise; a warning, since a caller may pass an undeclared + input the unified Start passes through); a format with references inside is unread; - a dropdown value outside its static options is `W_UNKNOWN_OPTION`, a `model` the catalog does not list `W_UNKNOWN_MODEL` (free text stays valid), and an unknown `operation` `E_UNKNOWN_OPERATION` unless the stored block already holds it; diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index cb7c733d100..752f9cd7d1e 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -190,8 +190,9 @@ reordered tool never takes another tool's secret. - `blk.a.b` → ``, `blk['x-y']` → ``, `blk.items[0]` → ``, `blk.items['0']` → ``. Paths are typed from the block's outputs; a path no output has is `W_UNKNOWN_OUTPUT_PATH`. For an agent with a `responseFormat`, its declared fields count - beside the static outputs (skipped when the format holds references); other dynamic outputs - skip the check. + beside the static outputs (skipped when the format holds references); for a Start, API or Input + trigger its declared `inputFormat` inputs do, so `start.nope`, or a reference to an input since + removed, warns at the reference; other dynamic outputs skip the check. - A ref to a block declared later is `W_FORWARD_REF`; it still resolves. A ref into a sibling subflow's child is `E_OUT_OF_SCOPE`. An unknown identifier is `E_UNKNOWN_IDENTIFIER` with a did-you-mean. Wiring resolves its identifiers the same way. diff --git a/packages/workflow-code/src/compile/checks.test.ts b/packages/workflow-code/src/compile/checks.test.ts index c4f9f781a67..d342923b928 100644 --- a/packages/workflow-code/src/compile/checks.test.ts +++ b/packages/workflow-code/src/compile/checks.test.ts @@ -104,6 +104,48 @@ describe('cycles', () => { }) }) +describe("references into a start trigger's declared inputs", () => { + const START_WITH = (inputs: string) => + `const start = trigger.start('Start', { inputFormat: ${inputs} })` + const unknown = (inputs: string, ref: string, extra = '') => + findings( + run( + [ + START_WITH(inputs), + `const next = api('Next', { url: t\`https://x/\${${ref}}\` }${extra})`, + 'start.to(next)', + ].join('\n') + ), + 'W_UNKNOWN_OUTPUT_PATH' + ) + + it('warns on an input the start does not declare, at the reference', () => { + const [finding] = unknown("{ label: 'string' }", 'start.nope') + expect(finding?.message).toBe('Start declares no input nope: its inputFormat declares label.') + expect(finding?.span).toMatchObject({ file: 'workflow.ts', line: 6 }) + }) + + it('warns on an input that was removed while a reference still reads it', () => { + const [finding] = unknown('{}', 'start.label') + expect(finding?.message).toBe('Start declares no input label: its inputFormat declares none.') + expect(finding?.hint).toBe( + "Add label to start's inputFormat, or reference an input it declares." + ) + }) + + it('suggests a declared input an unknown one is close to', () => { + expect(unknown("{ label: 'string' }", 'start.lable')[0]?.hint).toBe('Did you mean label?') + }) + + it.each([ + ['a declared input', 'start.label'], + ['a path under a declared input', 'start.label.length'], + ['a static output', 'start.input'], + ])('accepts %s', (_label, ref) => { + expect(unknown("{ label: 'string' }", ref)).toEqual([]) + }) +}) + describe('references into an agent responseFormat', () => { const FORMAT = "responseFormat: { name: 'classify', schema: { type: 'object', properties: { priority: { type: 'string' }, score: { type: 'number' } } } }" diff --git a/packages/workflow-code/src/compile/context.ts b/packages/workflow-code/src/compile/context.ts index e628bf928d5..2e5079e2f9f 100644 --- a/packages/workflow-code/src/compile/context.ts +++ b/packages/workflow-code/src/compile/context.ts @@ -159,12 +159,13 @@ export interface CompileContext { */ jsonFields: JsonFieldEntry[] /** - * References into blocks whose outputs come from a `responseFormat` param, checked once every - * block is compiled, since the format is JSON field text written last (`checkResponseFormatRefs`). + * References into blocks whose outputs a param declares (`responseFormat`, an input trigger's + * `inputFormat`), checked once every block is compiled, since the format is JSON field text + * written last (`checkDynamicOutputRefs`). */ - responseFormatRefs: ResponseFormatRef[] + dynamicOutputRefs: DynamicOutputRef[] /** `x.error` references, checked once the edges are known (`checkErrorOutputRefs`). */ - errorOutputRefs: Array> + errorOutputRefs: Array> /** * Branch keys already refused with an error, by block key. Wiring and branch meta that name one * stay quiet instead of adding `E_UNKNOWN_BRANCH` for the same mistake. @@ -178,7 +179,7 @@ export interface CompileContext { } /** A reference path into a block whose outputs its `responseFormat` declares. */ -interface ResponseFormatRef { +interface DynamicOutputRef { at: Range target: BlockDecl segments: RefSegment[] diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 3ea902e54c4..6810aedac00 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -37,8 +37,8 @@ import { ErrorLimitReached, Reporter } from './report' import { checkSuspiciousName, collectBody, collisionSuffix } from './scope' import { checkCodeFileRefs, + checkDynamicOutputRefs, checkErrorOutputRefs, - checkResponseFormatRefs, compileJsonLiteral, objectEntries, } from './values' @@ -502,7 +502,7 @@ function compileEntry( envDeclaredIn: input.envDeclaredIn, envReferences: new Set(), jsonFields: [], - responseFormatRefs: [], + dynamicOutputRefs: [], errorOutputRefs: [], refusedBranches: new Map(), valueConsts: new Set(), @@ -639,7 +639,7 @@ function compileEntry( } } - checkResponseFormatRefs(ctx, (key) => compiled.get(key)) + checkDynamicOutputRefs(ctx, (key) => compiled.get(key)) const unread = input.stored === undefined ? input.storedUnread : undefined const edges = new EdgeCollector( diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index e1e4e1e625c..2f319a4da9f 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -47,6 +47,7 @@ import { } from '../model' import { getOwn, setOwn } from '../records' import { isTemplateTag, SDK_RESOURCE_CALLS, type SdkCoreExport, TEMPLATE_TAGS } from '../sdk' +import { inputFormatFields } from '../start-inputs' import { type BlockDecl, type CompileContext, @@ -700,8 +701,12 @@ function checkOutputPath( return } const block = catalogBlockOf(target) - if (block?.dynamicOutputs?.source === 'responseFormat') { - ctx.responseFormatRefs.push({ at, target, segments, blockKey }) + const dynamicSource = block?.dynamicOutputs?.source + if ( + dynamicSource === 'responseFormat' || + (dynamicSource === 'inputFormat' && block && DECLARED_INPUT_TRIGGERS.has(block.type)) + ) { + ctx.dynamicOutputRefs.push({ at, target, segments, blockKey }) return } // Table rows only refine the element type of `rows`/`row`; the outputs themselves are static. @@ -750,23 +755,59 @@ export function checkErrorOutputRefs(ctx: CompileContext, erroring: ReadonlySet< } /** - * Checks each reference into a block whose outputs a `responseFormat` param declares, once every - * block is compiled: the first path segment must be a field the format declares or one of the - * block's static outputs (`W_UNKNOWN_OUTPUT_PATH` otherwise). A format the compiler cannot read - * (not JSON, or holding references) leaves every path allowed; an absent one leaves the static - * outputs. + * The input triggers whose outputs are exactly their declared inputs plus static outputs, as the + * editor's tag picker lists them (`getEffectiveBlockOutputs`). The legacy starter's outputs + * depend on its mode, so it is left out. */ -export function checkResponseFormatRefs( +const DECLARED_INPUT_TRIGGERS: ReadonlySet = new Set([ + 'start_trigger', + 'api_trigger', + 'input_trigger', +]) + +/** The unified Start's run-metadata output, present while its `runMetadata` switch is on. */ +const START_RUN_METADATA = 'metadata' + +/** + * The output fields a block's dynamic-outputs param declares: an agent's `responseFormat` fields, + * or an input trigger's `inputFormat` inputs (plus the unified Start's `metadata` while run + * metadata is on). `null` when the param cannot be read statically. + */ +function declaredOutputFields( + source: 'responseFormat' | 'inputFormat', + compiled: CanonicalBlock, + param: string +): string[] | null { + if (source === 'responseFormat') return responseFormatFields(getOwn(compiled.params, param)) + const inputs = inputFormatFields(getOwn(compiled.params, param)) + if (inputs === null) return null + const names = inputs.map((input) => input.name) + const metadata = getOwn(compiled.params, 'runMetadata') + return metadata?.kind === 'value' && metadata.value === true + ? [...names, START_RUN_METADATA] + : names +} + +/** + * Checks each reference into a block whose outputs a param declares (an agent's + * `responseFormat`, an input trigger's `inputFormat`), once every block is compiled: the first + * path segment must be a declared field or one of the block's static outputs + * (`W_UNKNOWN_OUTPUT_PATH` otherwise). A warning, as for any unknown output path: a caller may + * still pass an undeclared input, which the unified Start passes through. A param the compiler + * cannot read (not JSON, or holding references) leaves every path allowed; an absent one leaves + * the static outputs. + */ +export function checkDynamicOutputRefs( ctx: CompileContext, blockOf: (key: BlockKey) => CanonicalBlock | undefined ): void { - for (const { at, target, segments, blockKey } of ctx.responseFormatRefs) { + for (const { at, target, segments, blockKey } of ctx.dynamicOutputRefs) { const block = catalogBlockOf(target) const compiled = blockOf(target.key) const dynamic = block?.dynamicOutputs if (!block || !compiled || !dynamic) continue - const param = getOwn(compiled.params, dynamic.param) - const fields = responseFormatFields(param) + if (dynamic.source !== 'responseFormat' && dynamic.source !== 'inputFormat') continue + const fields = declaredOutputFields(dynamic.source, compiled, dynamic.param) const [first] = segments if (fields === null || first?.kind !== 'name' || fields.includes(first.name)) continue const builder = target.builder.kind === 'catalog' ? target.builder.builder : undefined @@ -777,11 +818,28 @@ export function checkResponseFormatRefs( continue } const known = [...fields, ...Object.keys(outputs)] + const path = formatStoredPath(segments) + if (dynamic.source === 'inputFormat') { + ctx.reporter.report( + 'W_UNKNOWN_OUTPUT_PATH', + fields.length > 0 + ? `${target.name} declares no input ${first.name}: its inputFormat declares ${fields.join(', ')}.` + : `${target.name} declares no input ${first.name}: its inputFormat declares none.`, + at, + { + hint: () => + didYouMean(first.name, known) ?? + `Add ${first.name} to ${target.ident ?? target.name}'s inputFormat, or reference an input it declares.`, + blockKey, + } + ) + continue + } ctx.reporter.report( 'W_UNKNOWN_OUTPUT_PATH', fields.length > 0 - ? `${target.name} has no output ${formatStoredPath(segments)}: its responseFormat declares ${fields.join(', ')}.` - : `${target.name} has no output ${formatStoredPath(segments)}, and declares no responseFormat field of that name.`, + ? `${target.name} has no output ${path}: its responseFormat declares ${fields.join(', ')}.` + : `${target.name} has no output ${path}, and declares no responseFormat field of that name.`, at, { hint: () => diff --git a/packages/workflow-code/src/start-inputs.ts b/packages/workflow-code/src/start-inputs.ts index dfb621fdfb1..362c62bce76 100644 --- a/packages/workflow-code/src/start-inputs.ts +++ b/packages/workflow-code/src/start-inputs.ts @@ -3,7 +3,7 @@ * workflow call's `inputMapping` fills and a run's input carries. */ import { isRecordLike } from '@sim/utils/object' -import type { CanonicalWorkflow } from './model' +import type { CanonicalParam, CanonicalWorkflow } from './model' /** * The trigger types whose `inputFormat` defines a workflow's inputs. Webhook and schedule @@ -32,9 +32,17 @@ export function startInputs(workflow: CanonicalWorkflow): WorkflowInput[] | null INPUT_TRIGGER_TYPES.some((type) => type === block.type) ) if (!start) return null - const format = start.params.inputFormat - if (format?.kind !== 'value' || !Array.isArray(format.value)) return [] - return format.value.flatMap((field) => { + return inputFormatFields(start.params.inputFormat) ?? [] +} + +/** + * The fields an `inputFormat` param declares, in order: `[]` when it is absent, `null` when its + * value is not a list (a reference or other text), so nothing can be said about its fields. + */ +export function inputFormatFields(param: CanonicalParam | undefined): WorkflowInput[] | null { + if (param === undefined) return [] + if (param.kind !== 'value' || !Array.isArray(param.value)) return null + return param.value.flatMap((field) => { if (!isRecordLike(field) || typeof field.name !== 'string' || field.name === '') return [] return [{ name: field.name, type: typeof field.type === 'string' ? field.type : 'string' }] }) From c53d4b29a3652ef06db71d6edd031a2a0ce89939 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 16:02:50 -0800 Subject: [PATCH 06/59] fix(sim-cli): write all output before exiting when stdout is a pipe A result past the 64 KiB a pipe buffers was cut short with exit 0: Bun drops console writes once process.stdout exists, and process.exit (every error path) ended the process before Node or Bun drained the pipe. Lines go through the stream under Bun, refused writes are tracked until they drain, and the terminal entrypoint is the only place that exits, after flushing; exitCli now throws CliExit for it. --- packages/sim-cli/src/commands/secrets.test.ts | 5 +- packages/sim-cli/src/commands/telemetry.ts | 7 +- packages/sim-cli/src/embed-context.ts | 12 +- packages/sim-cli/src/embed.ts | 4 +- packages/sim-cli/src/output/io.ts | 61 +++++++- packages/sim-cli/src/terminal-output.test.ts | 145 ++++++++++++++++++ packages/sim-cli/src/terminal.ts | 29 ++-- 7 files changed, 236 insertions(+), 27 deletions(-) create mode 100644 packages/sim-cli/src/terminal-output.test.ts diff --git a/packages/sim-cli/src/commands/secrets.test.ts b/packages/sim-cli/src/commands/secrets.test.ts index dcfa206109f..4b2b413a835 100644 --- a/packages/sim-cli/src/commands/secrets.test.ts +++ b/packages/sim-cli/src/commands/secrets.test.ts @@ -156,13 +156,10 @@ describe('secrets set cancellation', () => { it('exits 130 when the user aborts the prompt', async () => { mockPromptSecret.mockRejectedValue(new MockCancelled()) - vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { - throw new Error(`exit ${code}`) - }) as never) await expect( program().parseAsync(['node', 'sim', 'secrets', 'set', 'K', '--scope', 'workspace']) - ).rejects.toThrow('exit 130') + ).rejects.toMatchObject({ code: 130 }) expect(mockRequest).not.toHaveBeenCalled() }) }) diff --git a/packages/sim-cli/src/commands/telemetry.ts b/packages/sim-cli/src/commands/telemetry.ts index f30b31418e3..0897968fbdb 100644 --- a/packages/sim-cli/src/commands/telemetry.ts +++ b/packages/sim-cli/src/commands/telemetry.ts @@ -1,4 +1,5 @@ import { Command } from 'commander' +import { printLine } from '#sim-cli/output/io' import { builtInIngestTarget, DO_NOT_TRACK_VARIABLE, @@ -40,7 +41,7 @@ function currentStatus(): TelemetryStatus { */ function setEnabled(enabled: boolean): void { writeTelemetryState({ ...loadTelemetryState(), enabled }) - console.log(describe(currentStatus())) + printLine(describe(currentStatus())) } export function telemetryCommand(): Command { @@ -54,8 +55,8 @@ export function telemetryCommand(): Command { 'Telemetry commands always print text. DO_NOT_TRACK and SIM_TELEMETRY_DISABLED override the saved preference; a build without a reporting destination remains off. See the usage-data guide for the data collected.' ) .action(() => { - console.log(describe(currentStatus())) - console.log(`Learn more: ${USAGE_DATA_DOCS_URL}`) + printLine(describe(currentStatus())) + printLine(`Learn more: ${USAGE_DATA_DOCS_URL}`) }) telemetry diff --git a/packages/sim-cli/src/embed-context.ts b/packages/sim-cli/src/embed-context.ts index 812b8895f76..df7999d046e 100644 --- a/packages/sim-cli/src/embed-context.ts +++ b/packages/sim-cli/src/embed-context.ts @@ -75,8 +75,11 @@ export function setSoftExitCode(code: number): void { export const embedStore = new AsyncLocalStorage() -/** Thrown in place of process.exit inside an embedded run. */ -export class EmbeddedExit extends Error { +/** + * Thrown in place of `process.exit`: an embedded run returns its code to the host, and the + * terminal CLI exits with it once its output is written out. + */ +export class CliExit extends Error { constructor(readonly code: number) { super(`CLI exited with code ${code}`) } @@ -106,8 +109,7 @@ export function embeddedProfile(): ResolvedProfile | null { } } -/** Exits the terminal CLI without terminating an embedding host. */ +/** Ends the command with `code`, without terminating an embedding host. */ export function exitCli(code: number): never { - if (embedStore.getStore()) throw new EmbeddedExit(code) - return process.exit(code) + throw new CliExit(code) } diff --git a/packages/sim-cli/src/embed.ts b/packages/sim-cli/src/embed.ts index 984c44f4b1e..be26c306f55 100644 --- a/packages/sim-cli/src/embed.ts +++ b/packages/sim-cli/src/embed.ts @@ -2,9 +2,9 @@ import { getErrorMessage } from '@sim/utils/errors' import { type Command, CommanderError, type OutputConfiguration } from 'commander' import { ProfileConfigError } from './config/index' import { + CliExit, type EmbedContext, type EmbeddedCliIdentity, - EmbeddedExit, type EmbeddedFileSnapshot, embedStore, } from './embed-context' @@ -173,7 +173,7 @@ export async function runEmbeddedCli( /** Mirrors the installed CLI's top-level error handling (src/index.ts), minus process.exit. */ function renderEmbeddedError(ctx: EmbedContext, error: unknown): number { if (error instanceof EmbeddedOutputLimitError) return 1 - if (error instanceof EmbeddedExit) return error.code + if (error instanceof CliExit) return error.code if (error instanceof CommanderError) { // commander's CommanderError from exitOverride: usage/parse errors already // printed through the (captured) output; help/version exit 0. diff --git a/packages/sim-cli/src/output/io.ts b/packages/sim-cli/src/output/io.ts index 33c2724bf1b..76a90041f4f 100644 --- a/packages/sim-cli/src/output/io.ts +++ b/packages/sim-cli/src/output/io.ts @@ -1,29 +1,86 @@ import { format } from 'node:util' import { embedStore } from '../embed-context' +/** + * Bun writes `console.log` to a piped stdout outside `process.stdout`, and once that stream + * exists (anything reading `process.stdout.isTTY` creates it) the console's writes past the + * pipe buffer are dropped when the process ends. The stream's own writes are kept, so under + * Bun every line goes through it. + */ +const writeLinesThroughStream = process.versions.bun !== undefined + +/** + * A drain still owed per standard stream: set when a write is refused (`false`), settled by + * `drain`, or by `error`/`close` when the reader went away. Bun reports `writableLength` as `0` + * while it still holds the refused bytes, so this is the only signal it gives. + */ +const pendingDrains = new Map>() + +function writeStream(stream: NodeJS.WriteStream, chunk: string | Uint8Array): boolean { + const accepted = stream.write(chunk) + if (!accepted && !pendingDrains.has(stream)) { + pendingDrains.set( + stream, + new Promise((resolve) => { + const settle = () => { + stream.off('drain', settle) + stream.off('error', settle) + stream.off('close', settle) + pendingDrains.delete(stream) + resolve() + } + stream.once('drain', settle) + stream.once('error', settle) + stream.once('close', settle) + }) + ) + } + return accepted +} + /** Only CLI-owned output enters the invocation result; host logs retain their normal sinks. */ export function printLine(...args: unknown[]): void { const context = embedStore.getStore() if (context) context.stdout.write(`${format(...args)}\n`) + else if (writeLinesThroughStream) writeStream(process.stdout, `${format(...args)}\n`) else console.log(...args) } export function printError(...args: unknown[]): void { const context = embedStore.getStore() if (context) context.stderr.write(`${format(...args)}\n`) + else if (writeLinesThroughStream) writeStream(process.stderr, `${format(...args)}\n`) else console.error(...args) } export function writeStdout(chunk: string | Uint8Array): boolean { const context = embedStore.getStore() - if (!context) return process.stdout.write(chunk) + if (!context) return writeStream(process.stdout, chunk) context.stdout.write(chunk) return true } export function writeStderr(chunk: string | Uint8Array): boolean { const context = embedStore.getStore() - if (!context) return process.stderr.write(chunk) + if (!context) return writeStream(process.stderr, chunk) context.stderr.write(chunk) return true } + +/** Resolves once `stream` has handed everything written to it to the OS. */ +async function drained(stream: NodeJS.WriteStream): Promise { + await pendingDrains.get(stream) + if (stream.writableLength === 0 || stream.destroyed) return + await new Promise((resolve) => { + stream.write('', () => resolve()) + }) +} + +/** + * Waits until stdout and stderr have written out what the CLI printed. `process.exit` ends the + * process at once, and a pipe holds only 64 KiB, so exiting without this cuts a large result + * short for whatever reads it (`sim … | jq`) while the exit status still says success. + */ +export async function flushOutput(): Promise { + await Promise.all([drained(process.stdout), drained(process.stderr)]) +} diff --git a/packages/sim-cli/src/terminal-output.test.ts b/packages/sim-cli/src/terminal-output.test.ts new file mode 100644 index 00000000000..82ebcd4e3de --- /dev/null +++ b/packages/sim-cli/src/terminal-output.test.ts @@ -0,0 +1,145 @@ +import { execFileSync, spawn } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type Server } from 'node:http' +import { createRequire } from 'node:module' +import type { AddressInfo } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +/** + * Output past the 64 KiB a pipe buffers is where an exit that does not wait + * for stdout loses data, so every case here writes megabytes into a pipe whose + * reader starts late. + */ +const ROWS = 20_000 +const READER_DELAY_MS = 400 + +interface PipedRun { + exitCode: number | null + stdout: string + stderr: string +} + +/** Spawns `command`, holds its stdout pipe unread for a moment, then drains it. */ +function runPiped(command: string, args: string[], env: NodeJS.ProcessEnv): Promise { + const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }) + let stdout = '' + let stderr = '' + child.stdout.setEncoding('utf8').pause() + setTimeout(() => child.stdout.resume(), READER_DELAY_MS) + child.stdout.on('data', (chunk: string) => { + stdout += chunk + }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk + }) + return new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (exitCode) => resolve({ exitCode, stdout, stderr })) + }) +} + +describe('stdout piped to a slow reader', () => { + let dir: string + let server: Server + let endpoint: string + const rows = Array.from({ length: ROWS }, (_, index) => ({ + id: `row_${index}`, + data: { name: 'x'.repeat(50), index }, + })) + + beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'sim-cli-pipe-')) + server = createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1') + const limit = Number(url.searchParams.get('limit') ?? 100) + const start = Number(url.searchParams.get('cursor') ?? 0) + const next = start + limit < rows.length ? String(start + limit) : null + response.setHeader('content-type', 'application/json') + response.end(JSON.stringify({ data: rows.slice(start, start + limit), nextCursor: next })) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + }) + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())) + rmSync(dir, { recursive: true, force: true }) + }) + + function cliEnv(): NodeJS.ProcessEnv { + return { + ...process.env, + SIM_CONFIG_DIR: join(dir, 'config'), + SIM_ENDPOINT: endpoint, + SIM_API_KEY: 'sk-test', + SIM_WORKSPACE: 'ws_1', + SIM_NO_UPDATE_CHECK: '1', + SIM_CLI_TELEMETRY_DISABLED: '1', + NO_PROXY: '127.0.0.1', + } + } + + it('writes every row of a long list before exiting 0', async () => { + const entrypoint = join(import.meta.dirname, 'index.ts') + const list = (output: string) => + runPiped( + 'bun', + [entrypoint, '--output', output, 'tables', 'rows', 'list', 'tbl_1', '--limit', '0'], + cliEnv() + ) + + const json = await list('json') + expect(json.stderr).toBe('') + expect(json.exitCode).toBe(0) + expect(json.stdout.length).toBeGreaterThan(1024 * 1024) + expect(JSON.parse(json.stdout).data).toHaveLength(ROWS) + + const table = await list('table') + expect(table.exitCode).toBe(0) + expect(table.stdout.trimEnd().split('\n').length).toBeGreaterThan(ROWS) + }, 60_000) + + /** + * The failing command writes its output first and its error after, the order + * a failed `workflows run` uses, and exits with the error's code. + */ + function failingCommandSource(terminalModule: string): string { + return ` + import { Command, runTerminalCli, printLine, SimApiError } from ${JSON.stringify(terminalModule)} + const program = new Command('sim') + program.command('fail').action(() => { + printLine('y'.repeat(${2 * 1024 * 1024})) + throw new SimApiError('the run failed', 0, 'RUN_FAILED', undefined, 5) + }) + process.argv = [process.argv[0], 'sim', 'fail'] + await runTerminalCli(program) + ` + } + + it('drains stdout before exiting with an error code, under Bun and Node', async () => { + const harness = join(dir, 'harness.ts') + writeFileSync( + harness, + `export { Command } from ${JSON.stringify(createRequire(import.meta.url).resolve('commander'))} +export { runTerminalCli } from ${JSON.stringify(join(import.meta.dirname, 'terminal.ts'))} +export { printLine } from ${JSON.stringify(join(import.meta.dirname, 'output/io.ts'))} +export { SimApiError } from ${JSON.stringify(join(import.meta.dirname, 'http/client.ts'))} +` + ) + const bundle = join(dir, 'harness.mjs') + execFileSync('bun', ['build', harness, '--target=node', '--format=esm', '--outfile', bundle], { + stdio: 'pipe', + }) + const script = join(dir, 'fail.mjs') + writeFileSync(script, failingCommandSource(bundle)) + + for (const runtime of ['bun', process.execPath]) { + const run = await runPiped(runtime, [script], cliEnv()) + expect(run.exitCode).toBe(5) + expect(run.stdout.length).toBe(2 * 1024 * 1024 + 1) + expect(run.stderr).toContain('the run failed') + } + }, 60_000) +}) diff --git a/packages/sim-cli/src/terminal.ts b/packages/sim-cli/src/terminal.ts index d5d8f217d43..1bd094917ed 100644 --- a/packages/sim-cli/src/terminal.ts +++ b/packages/sim-cli/src/terminal.ts @@ -4,12 +4,14 @@ import { styles } from '#sim-cli/output/presentation' import { CliUpdateError } from '#sim-cli/update/install' import { ProfileConfigError } from './config/index' import { profileFrom } from './context' +import { CliExit } from './embed-context' import { formatApiErrorDetails, isRequestTimeout, RAISE_TIMEOUT_HINT, SimApiError, } from './http/client' +import { flushOutput, printError, writeStderr } from './output/io' import { sanitize } from './output/render' import { buildProgram } from './program' import { createCommandTelemetry } from './telemetry/index' @@ -18,11 +20,12 @@ import { createCommandTelemetry } from './telemetry/index' * Prints the one-line explanation for an error the CLI understands and returns * the exit code it deserves, or `null` for an error it does not: that is a bug * in the CLI, and hiding it behind a friendly message would make it - * unreportable, so the caller lets it keep its stack trace. + * unreportable, so the caller prints it with its stack trace. */ function explainFailure(error: unknown, program: Command): number | null { + if (error instanceof CliExit) return error.code if (error instanceof ProfileConfigError || error instanceof CliUpdateError) { - console.error(styles().red(`Error: ${sanitize(error.message)}`)) + printError(styles().red(`Error: ${sanitize(error.message)}`)) return 1 } // `AbortSignal.timeout` keeps firing after `fetch` resolves, so a bound that @@ -30,7 +33,7 @@ function explainFailure(error: unknown, program: Command): number | null { // surfaces here rather than inside the client. A user's own Ctrl-C raises // `AbortError` instead, which is deliberately left alone. if (isRequestTimeout(error)) { - console.error(styles().red(`Error: the request timed out. ${RAISE_TIMEOUT_HINT}`)) + printError(styles().red(`Error: the request timed out. ${RAISE_TIMEOUT_HINT}`)) return 1 } if (error instanceof SimApiError) { @@ -49,17 +52,17 @@ function explainFailure(error: unknown, program: Command): number | null { ...(error.requestId ? { requestId: error.requestId } : {}), }, } - process.stderr.write(output === 'json' ? `${JSON.stringify(payload)}\n` : dump(payload)) + writeStderr(output === 'json' ? `${JSON.stringify(payload)}\n` : dump(payload)) return error.exitCode } - console.error(styles().red(`Error: ${sanitize(error.message)}`)) - if (error.code) console.error(styles().dim(` code: ${sanitize(error.code)}`)) + printError(styles().red(`Error: ${sanitize(error.message)}`)) + if (error.code) printError(styles().dim(` code: ${sanitize(error.code)}`)) if (error.details !== undefined) { for (const line of formatApiErrorDetails(error.details)) { - console.error(styles().dim(sanitize(line))) + printError(styles().dim(sanitize(line))) } } - if (error.requestId) console.error(styles().dim(` request id: ${sanitize(error.requestId)}`)) + if (error.requestId) printError(styles().dim(` request id: ${sanitize(error.requestId)}`)) return error.exitCode } return null @@ -69,6 +72,7 @@ function explainFailure(error: unknown, program: Command): number | null { * Anything the CLI can explain prints as one line and exits with its code. A * failure is reported here, where its class and code are known; every other * way the process ends is reported from the exit listener telemetry installs. + * Every exit waits for stdout and stderr to drain first (see {@link flushOutput}). */ export async function runTerminalCli(suppliedProgram?: Command): Promise { const telemetry = suppliedProgram ? undefined : createCommandTelemetry() @@ -78,8 +82,11 @@ export async function runTerminalCli(suppliedProgram?: Command): Promise { await program.parseAsync(process.argv) } catch (error) { const exitCode = explainFailure(error, program) - telemetry?.complete({ exitCode: exitCode ?? 1, error }) - if (exitCode === null) throw error - process.exit(exitCode) + telemetry?.complete( + error instanceof CliExit ? { exitCode: error.code } : { exitCode: exitCode ?? 1, error } + ) + if (exitCode === null) printError(error) + await flushOutput() + process.exit(exitCode ?? 1) } } From 2fc27f34e3419c0770e7b75e8cb3444788912377 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 16:05:57 -0800 Subject: [PATCH 07/59] fix(providers): send tool ids providers reject as provider-safe wire names A custom tool's canonical id is custom_, and the title is free text, so an agent using a tool titled with a space failed on Anthropic (tools.0.custom.name must match ^[a-zA-Z0-9_-]{1,128}$). The wire id now replaces every character outside that alphabet with _, aliasing on collision; the canonical id keeps the title, which execution resolves by, and the existing wire-to-canonical map projects calls back. --- apps/sim/providers/tool-identity.test.ts | 41 ++++++++++++++++++++++++ apps/sim/providers/tool-identity.ts | 26 +++++++++++---- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/apps/sim/providers/tool-identity.test.ts b/apps/sim/providers/tool-identity.test.ts index a8b12b4e54a..e2fc55f17cb 100644 --- a/apps/sim/providers/tool-identity.test.ts +++ b/apps/sim/providers/tool-identity.test.ts @@ -171,4 +171,45 @@ describe('provider tool identities', () => { expect(output.toolCalls?.list[0].name).toBe('gmail_send') expect(output.providerTiming?.timeSegments?.[0].name).toBe('gmail_send') }) + + it('gives a custom tool titled with spaces and unicode a provider-safe name that maps back', () => { + const PROVIDER_NAME = /^[a-zA-Z0-9_-]{1,64}$/ + const spaced = providerTool('custom_Get Weather', 'a') + const unicode = providerTool('custom_Météo du jour ☀', 'b') + + const identities = assignProviderToolIdentities([spaced, unicode]) + + expect(spaced.id).toMatch(PROVIDER_NAME) + expect(unicode.id).toMatch(PROVIDER_NAME) + expect(identities.toolIdByWireId.get(spaced.id)).toBe('custom_Get Weather') + expect(identities.toolIdByWireId.get(unicode.id)).toBe('custom_Météo du jour ☀') + + const response: ProviderResponse = { + content: 'done', + model: 'test-model', + toolCalls: [{ name: spaced.id, arguments: {} }], + } + projectProviderResponseToolIdentities(response, identities) + expect(response.toolCalls?.[0].name).toBe('custom_Get Weather') + }) + + it('keeps custom tools whose safe names collide distinct, and a safe id unchanged', () => { + const tools = [ + providerTool('custom_Get Weather', 'a'), + providerTool('custom_Get/Weather', 'b'), + providerTool('custom_Get_Weather', 'c'), + ] + + const identities = assignProviderToolIdentities(tools) + const wireIds = tools.map((tool) => tool.id) + + expect(new Set(wireIds).size).toBe(3) + expect(tools[2].id).toBe('custom_Get_Weather') + expect(identities.toolIdByWireId.get(wireIds[0])).toBe('custom_Get Weather') + expect(identities.toolIdByWireId.get(wireIds[1])).toBe('custom_Get/Weather') + expect(identities.toolIdByWireId.has('custom_Get_Weather')).toBe(false) + + assignProviderToolIdentities(tools) + expect(tools.map((tool) => tool.id)).toEqual(wireIds) + }) }) diff --git a/apps/sim/providers/tool-identity.ts b/apps/sim/providers/tool-identity.ts index 28b53e98fdc..be9cb7392cb 100644 --- a/apps/sim/providers/tool-identity.ts +++ b/apps/sim/providers/tool-identity.ts @@ -4,6 +4,17 @@ import type { ProviderResponse, ProviderToolConfig } from '@/providers/types' const MAX_PROVIDER_TOOL_ID_LENGTH = 64 const PROVIDER_ALIAS_MARKER = '__sim_' +/** Characters outside every provider's function-name alphabet (Anthropic: `^[a-zA-Z0-9_-]{1,128}$`). */ +const PROVIDER_UNSAFE_TOOL_ID_CHARS = /[^a-zA-Z0-9_-]/g + +/** + * The id with every character no provider accepts in a function name replaced by `_`. A custom + * tool's canonical id carries its display title (`custom_Get Weather`), which execution resolves by + * title, so only the wire id is made safe and the canonical id keeps the title. + */ +function toProviderSafeToolId(toolId: string): string { + return toolId.replace(PROVIDER_UNSAFE_TOOL_ID_CHARS, '_') +} export interface ProviderToolIdentityMap { /** Provider-only tool id to the canonical registry id used for execution and observability. */ @@ -18,10 +29,12 @@ function buildProviderAlias(toolId: string, occurrence: number, attempt: number) } /** - * Gives duplicate configured tools deterministic provider-safe wire ids. + * Gives configured tools deterministic provider-safe wire ids. * - * The first occurrence and every already-unique tool keep their existing id for backwards - * compatibility. Later occurrences receive opaque ordinal aliases; resource and credential ids + * The first occurrence and every already-unique tool with a provider-safe id keep it for backwards + * compatibility. An id with characters no provider accepts (a custom tool titled `Get Weather`) is + * sent with them replaced by `_`, and when that collides with another tool's id it is aliased like + * a later occurrence. Later occurrences receive opaque ordinal aliases; resource and credential ids * never enter the provider-visible name. Tool objects are updated in place so their instance-bound * params and secret provenance remain attached to the exact object selected by provider adapters. */ @@ -46,11 +59,12 @@ export function assignProviderToolIdentities< const occurrence = (occurrences.get(canonicalId) ?? 0) + 1 occurrences.set(canonicalId, occurrence) - let wireId = canonicalId - if (usedWireIds.has(wireId)) { + const safeId = toProviderSafeToolId(canonicalId) + let wireId = safeId + if (usedWireIds.has(wireId) || (safeId !== canonicalId && reservedIds.has(safeId))) { let attempt = 0 do { - wireId = buildProviderAlias(canonicalId, occurrence, attempt) + wireId = buildProviderAlias(safeId, occurrence, attempt) attempt += 1 } while (reservedIds.has(wireId) || usedWireIds.has(wireId)) } From dbf5567825225bfc6cc73238ab21d33b4897a7aa Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:07:51 -0800 Subject: [PATCH 08/59] fix(workflows): refuse placing a block type hidden from the caller The source catalog hides an unrevealed preview block or one the block-visibility kill switch names, but the write paths only checked the allowlist and deployment availability, so source apply (generic block() form) and editing operations placed a kill-switched table_v2 for a non-admin. A VISIBLE_TYPE placement gate in BLOCK_GATES now refuses a new placement or type change to a type isHiddenUnder hides, for both writers; state replace refuses a hidden type the stored graph does not already hold on that block. Placed blocks of a hidden type stay editable. --- .../block-visibility-writes.integration.ts | 402 ++++++++++++++++++ .../application/replace-workflow-state.ts | 37 +- apps/sim/lib/workflows/editing/gates.ts | 27 +- 3 files changed, 463 insertions(+), 3 deletions(-) create mode 100644 apps/sim/lib/workflows/application/block-visibility-writes.integration.ts diff --git a/apps/sim/lib/workflows/application/block-visibility-writes.integration.ts b/apps/sim/lib/workflows/application/block-visibility-writes.integration.ts new file mode 100644 index 00000000000..d42554fe47a --- /dev/null +++ b/apps/sim/lib/workflows/application/block-visibility-writes.integration.ts @@ -0,0 +1,402 @@ +/** + * Every write that places a block, against real PostgreSQL, under a `block-visibility` document + * that kill-switches `api` for everyone but platform admins and one allowlisted user. A caller who + * cannot see a type cannot add it (source apply, editing operations, state replace), a block + * already of that type stays editable, and a caller the rule reveals it to adds it. The document + * goes through the real evaluation and the admin flag through the real user row; only the + * AppConfig fetch is replaced. + */ +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { db } from '@sim/db' +import { member, organization, permissions, user, workflow, workspace } from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { envFlagsMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { saveWorkflowToNormalizedTables } from '@sim/workflow-persistence/save' +import type { BlockState } from '@sim/workflow-types/workflow' +import { eq, inArray } from 'drizzle-orm' +import { afterAll, beforeAll, expect, it, vi } from 'vitest' + +const { mockFetchProfile } = vi.hoisted(() => ({ mockFetchProfile: vi.fn() })) + +vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) +vi.mock('@/lib/core/config/appconfig', () => ({ fetchAppConfigProfile: mockFetchProfile })) +vi.mock('@/lib/realtime/notify', () => realtimeNotifyMock) + +import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' +import { applyWorkflowOperations } from '@/lib/workflows/application/apply-workflow-operations' +import { applyWorkflowSource } from '@/lib/workflows/application/apply-workflow-source' +import { readWorkflowSource } from '@/lib/workflows/application/read-workflow-source' +import { replaceWorkflowState } from '@/lib/workflows/application/replace-workflow-state' +import { WorkflowSourceRejectedError } from '@/lib/workflows/application/source-error' +import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' + +const organizationId = generateId() +const adminId = generateId() +const memberId = generateId() +const revealedId = generateId() +const workspaceId = generateId() +const userIds = [adminId, memberId, revealedId] +const admin = createSessionPrincipal({ userId: adminId, sessionId: generateId() }) +const hiddenFrom = createSessionPrincipal({ userId: memberId, sessionId: generateId() }) +const revealedTo = createSessionPrincipal({ userId: revealedId, sessionId: generateId() }) + +type Principal = typeof admin + +const VISIBILITY_DOCUMENT = { + api: { enabled: false, adminEnabled: true, userIds: [revealedId] }, +} + +const checks: { name: string; status: string; durationMs: number; error?: string }[] = [] + +function check(name: string, run: () => Promise<void>) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +function seedBlock( + type: string, + name: string, + subBlocks: Record<string, [type: string, value: unknown]> = {} +): BlockState { + return { + id: generateId(), + type, + name, + position: { x: 0, y: 0 }, + subBlocks: Object.fromEntries( + Object.entries(subBlocks).map(([key, [subType, value]]) => [ + key, + { id: key, type: subType, value }, + ]) + ) as BlockState['subBlocks'], + outputs: {}, + enabled: true, + } +} + +function seedApi(url: string): BlockState { + return seedBlock('api', 'Fetch', { + url: ['short-input', url], + method: ['dropdown', 'GET'], + }) +} + +/** A workflow whose first block feeds every other block. */ +async function seedGraph(blocks: BlockState[]): Promise<string> { + const workflowId = generateId() + const now = new Date() + await db.insert(workflow).values({ + id: workflowId, + userId: adminId, + workspaceId, + name: `Block visibility ${workflowId}`, + lastSynced: now, + createdAt: now, + updatedAt: now, + }) + const [first, ...rest] = blocks + const saved = await saveWorkflowToNormalizedTables(workflowId, { + blocks: Object.fromEntries(blocks.map((block) => [block.id, block])), + edges: rest.map((block) => ({ id: generateId(), source: first.id, target: block.id })), + loops: {}, + parallels: {}, + }) + if (!saved.success) throw new Error(`Seeding the workflow failed: ${saved.error}`) + return workflowId +} + +function seedPlainGraph(): Promise<string> { + return seedGraph([ + seedBlock('start_trigger', 'Start'), + seedBlock('function', 'Score', { code: ['code', 'return 1'] }), + ]) +} + +function getSource(principal: Principal, workflowId: string) { + return withPermissionGroupScope(() => + readWorkflowSource.execute({ principal, input: { workflowId, functionCode: 'inline' } }) + ) +} + +function applySource( + principal: Principal, + source: Awaited<ReturnType<typeof getSource>>, + files: Record<string, string> +) { + return withPermissionGroupScope(() => + applyWorkflowSource.execute({ + principal, + input: { workflowId: source.workflowId, files, lock: source.lock, base: source.base }, + }) + ) +} + +/** The source with a generic `api` block declared just before the workflow body closes. */ +function withAddedApi(files: Record<string, string>): Record<string, string> { + const text = files['workflow.ts'].replace(/import \{ /, 'import { block, ') + const close = text.lastIndexOf('})') + expect(close, 'workflow.ts closes its workflow body').toBeGreaterThan(0) + const added = + " block('api', 'Added fetch', { url: 'https://example.com/added', method: 'GET' })\n" + return { ...files, 'workflow.ts': `${text.slice(0, close)}${added}${text.slice(close)}` } +} + +function edit(files: Record<string, string>, from: string, to: string): Record<string, string> { + const text = files['workflow.ts'] + expect(text.split(from), `workflow.ts holds "${from}" once`).toHaveLength(2) + return { ...files, 'workflow.ts': text.replace(from, to) } +} + +function addApiOperation(principal: Principal, workflowId: string) { + return withPermissionGroupScope(() => + applyWorkflowOperations.execute({ + principal, + input: { + workflowId, + layout: 'none', + operations: [ + { + operation_type: 'add', + block_id: generateId(), + params: { + type: 'api', + name: 'Added fetch', + inputs: { url: 'https://example.com/added', method: 'GET' }, + }, + }, + ], + }, + }) + ) +} + +function replaceState( + principal: Principal, + workflowId: string, + stored: NonNullable<Awaited<ReturnType<typeof loadWorkflowFromNormalizedTables>>>, + blocks: Record<string, BlockState> +) { + return withPermissionGroupScope(() => + replaceWorkflowState.execute({ + principal, + input: { workflowId, blocks, edges: stored.edges }, + }) + ) +} + +async function storedTypes(workflowId: string): Promise<string[]> { + const stored = await loadWorkflowFromNormalizedTables(workflowId) + return Object.values(stored?.blocks ?? {}).map((block) => block.type) +} + +async function loadStored(workflowId: string) { + const stored = await loadWorkflowFromNormalizedTables(workflowId) + if (!stored) throw new Error('The seeded workflow has no stored graph') + return stored +} + +beforeAll(async () => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation( + async (flag: string) => flag === 'workflow-code' + ) + setEnvFlags({ isAppConfigEnabled: true }) + mockFetchProfile.mockImplementation( + async (ids: { profile: string }, parse: (doc: unknown) => unknown) => + parse(ids.profile === 'block-visibility' ? VISIBILITY_DOCUMENT : {}) + ) + const now = new Date() + await db.insert(user).values( + userIds.map((id) => ({ + id, + name: 'Block visibility fixture', + email: `${id}@block-visibility.invalid`, + emailVerified: true, + role: id === adminId ? 'admin' : 'user', + createdAt: now, + updatedAt: now, + })) + ) + await db.insert(organization).values({ + id: organizationId, + name: 'Block visibility fixture', + slug: organizationId, + createdAt: now, + updatedAt: now, + }) + await db.insert(member).values( + userIds.map((userId) => ({ + id: generateId(), + userId, + organizationId, + role: userId === adminId ? 'owner' : 'member', + createdAt: now, + })) + ) + await insertWorkspaceFixture(db, [ + { + id: workspaceId, + name: 'Block visibility fixture', + ownerId: adminId, + billedAccountUserId: adminId, + organizationId, + workspaceMode: 'organization' as const, + }, + ]) + await db.insert(permissions).values( + userIds.map((userId) => ({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: userId === adminId ? ('admin' as const) : ('write' as const), + })) + ) +}) + +afterAll(async () => { + const reportPath = + process.env.BLOCK_VISIBILITY_WRITES_REPORT_PATH ?? + resolve('test-results/block-visibility-writes.json') + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ checks }, null, 2)) + resetEnvFlagsMock() + await deleteWorkspaceFixture(db, eq(workspace.organizationId, organizationId)) + await db.delete(organization).where(eq(organization.id, organizationId)) + await db.delete(user).where(inArray(user.id, userIds)) +}) + +check('source apply refuses adding a kill-switched type the caller cannot see', async () => { + const workflowId = await seedPlainGraph() + const source = await getSource(hiddenFrom, workflowId) + const before = await loadWorkflowFromNormalizedTables(workflowId) + + const refused = await applySource(hiddenFrom, source, withAddedApi(source.files)).then( + () => null, + (error: unknown) => error + ) + + expect(refused).toBeInstanceOf(WorkflowSourceRejectedError) + expect((refused as WorkflowSourceRejectedError).diagnostics).toContainEqual( + expect.objectContaining({ + code: 'E_APPLY_BLOCK_NOT_ALLOWED', + message: expect.stringContaining('Block type "api" is not available to you'), + }) + ) + expect(await loadWorkflowFromNormalizedTables(workflowId)).toEqual(before) +}) + +check('editing operations withhold adding a kill-switched type the caller cannot see', async () => { + const workflowId = await seedPlainGraph() + + const result = await addApiOperation(hiddenFrom, workflowId) + + expect(result.skipped).toContainEqual( + expect.objectContaining({ + type: 'block_not_allowed', + reason: expect.stringContaining('Block type "api" is not available to you'), + }) + ) + expect(await storedTypes(workflowId)).not.toContain('api') +}) + +check( + 'state replace refuses a graph adding a kill-switched type the caller cannot see', + async () => { + const workflowId = await seedPlainGraph() + const stored = await loadStored(workflowId) + const added = seedApi('https://example.com/added') + + await expect( + replaceState(hiddenFrom, workflowId, stored, { ...stored.blocks, [added.id]: added }) + ).rejects.toMatchObject({ + code: 'forbidden', + message: expect.stringContaining('Block type "api" is not available to you'), + }) + expect(await storedTypes(workflowId)).not.toContain('api') + } +) + +check('a placed block of a hidden type stays editable on every write path', async () => { + const placed = seedApi('https://example.com/before') + const workflowId = await seedGraph([seedBlock('start_trigger', 'Start'), placed]) + + const source = await getSource(hiddenFrom, workflowId) + const applied = await applySource( + hiddenFrom, + source, + edit(source.files, 'https://example.com/before', 'https://example.com/source') + ) + expect(applied.applied).toBe(true) + + const operated = await withPermissionGroupScope(() => + applyWorkflowOperations.execute({ + principal: hiddenFrom, + input: { + workflowId, + layout: 'none', + operations: [ + { + operation_type: 'edit', + block_id: placed.id, + params: { type: 'api', inputs: { url: 'https://example.com/operations' } }, + }, + ], + }, + }) + ) + expect(operated.skipped).toEqual([]) + expect((await loadStored(workflowId)).blocks[placed.id]?.subBlocks.url?.value).toBe( + 'https://example.com/operations' + ) + + const stored = await loadStored(workflowId) + const replaced = structuredClone(stored.blocks) + replaced[placed.id].subBlocks.url.value = 'https://example.com/state' + await replaceState(hiddenFrom, workflowId, stored, replaced) + expect((await loadStored(workflowId)).blocks[placed.id]?.subBlocks.url?.value).toBe( + 'https://example.com/state' + ) +}) + +check('a platform admin the rule reveals the type to adds it through source', async () => { + const workflowId = await seedPlainGraph() + const source = await getSource(admin, workflowId) + + const result = await applySource(admin, source, withAddedApi(source.files)) + + expect(result.applied).toBe(true) + expect(await storedTypes(workflowId)).toContain('api') +}) + +check('an allowlisted user adds the type through operations and state replace', async () => { + const workflowId = await seedPlainGraph() + + const result = await addApiOperation(revealedTo, workflowId) + expect(result.skipped).toEqual([]) + expect(await storedTypes(workflowId)).toContain('api') + + const stored = await loadStored(workflowId) + const added = seedApi('https://example.com/second') + await replaceState(revealedTo, workflowId, stored, { ...stored.blocks, [added.id]: added }) + expect((await storedTypes(workflowId)).filter((type) => type === 'api')).toHaveLength(2) +}) diff --git a/apps/sim/lib/workflows/application/replace-workflow-state.ts b/apps/sim/lib/workflows/application/replace-workflow-state.ts index 4bcafb107b9..2fd779f4487 100644 --- a/apps/sim/lib/workflows/application/replace-workflow-state.ts +++ b/apps/sim/lib/workflows/application/replace-workflow-state.ts @@ -10,6 +10,7 @@ import { createLogger } from '@sim/logger' import { evaluateSubBlockCondition } from '@sim/workflow-code/catalog/canonical-index' import type { BlockState, WorkflowState } from '@sim/workflow-types/workflow' import { principalAuditSource } from '@/lib/core/application' +import { getBlockVisibility } from '@/lib/core/config/block-visibility' import { OrchestrationError } from '@/lib/core/orchestration/types' import { notifyWorkflowUpdated } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' @@ -36,6 +37,7 @@ import { validateToolBindingAuthoring, } from '@/lib/workflows/tool-input/authoring' import { getBlock } from '@/blocks/registry' +import { isHiddenUnder } from '@/blocks/visibility/context' const logger = createLogger('ReplaceWorkflowState') @@ -61,6 +63,36 @@ function humanSubjectUserId(principal: Principal): string | null { } } +/** + * Refuses a graph that places a block type the acting user cannot see (an unrevealed `preview` + * block or a kill-switched one): on a block the stored graph lacks, or one it holds with another + * type. The editing operations and source apply refuse the same placement through their + * visibility gate; a block the stored graph already holds with that type stays writable, so a + * workflow keeps saving after its type is hidden. The stored graph is read only when the body + * names a hidden type at all. + */ +async function assertNoHiddenPlacement( + blocks: Record<string, BlockState>, + context: { workflowId: string; workspaceOrganizationId: string | null }, + subjectUserId: string | null, + previous: Awaited<ReturnType<typeof loadWorkflowFromNormalizedTables>> | undefined +): Promise<void> { + const visibility = await getBlockVisibility({ + ...(subjectUserId ? { userId: subjectUserId } : {}), + ...(context.workspaceOrganizationId ? { orgId: context.workspaceOrganizationId } : {}), + }) + const hidden = Object.values(blocks).filter((block) => { + const config = getBlock(block.type) + return config !== undefined && isHiddenUnder(visibility, config) + }) + if (hidden.length === 0) return + const stored = previous ?? (await loadWorkflowFromNormalizedTables(context.workflowId)) + const placed = hidden.find((block) => stored?.blocks[block.id]?.type !== block.type) + if (placed) { + throw new OrchestrationError('forbidden', `Block type "${placed.type}" is not available to you`) + } +} + export interface ReplaceWorkflowStateInput { workflowId: string assertedWorkspaceId?: string @@ -177,6 +209,9 @@ export const replaceWorkflowState = defineAuthorizedWorkflowUseCase({ } } + const subjectUserId = humanSubjectUserId(principal) + await assertNoHiddenPlacement(blocks, context, subjectUserId, previous) + const graph = { blocks, edges: sanitized.edges as WorkflowState['edges'], @@ -189,8 +224,6 @@ export const replaceWorkflowState = defineAuthorizedWorkflowUseCase({ * the reference pass is skipped for them rather than resolved against the * billing owner. See {@link buildWorkflowLintReport}. */ - const subjectUserId = humanSubjectUserId(principal) - const lint = await buildWorkflowLintReport(graph, { workflowId: context.workflowId, workspaceId: context.workspaceId, diff --git a/apps/sim/lib/workflows/editing/gates.ts b/apps/sim/lib/workflows/editing/gates.ts index 3bca52b11d6..134549d4aa5 100644 --- a/apps/sim/lib/workflows/editing/gates.ts +++ b/apps/sim/lib/workflows/editing/gates.ts @@ -15,7 +15,7 @@ import type { SkippedItem, SkippedItemType } from '@/lib/workflows/editing/types import { isBlockTypeAllowed } from '@/lib/workflows/editing/validation' import { TriggerUtils } from '@/lib/workflows/triggers/triggers' import { getBlock } from '@/blocks/registry' -import { overlayVisibility } from '@/blocks/visibility/context' +import { isHiddenUnder, overlayVisibility } from '@/blocks/visibility/context' /** * The per-user gates on a block write, as one list every writer runs: the editing engine's @@ -157,6 +157,30 @@ const AVAILABLE_TYPE: PlacementGate<'block_not_allowed'> = { }, } +/** + * Refuses placing a type the acting user cannot see: an unrevealed `preview` block or one the + * `block-visibility` kill switch names, by the catalog's own predicate ({@link isHiddenUnder}), so a + * write cannot add a block the catalog hides. Placed instances keep executing and stay editable, so + * a block keeping its own type is not refused. Reads the visibility in scope: outside + * `withBlockVisibility` preview blocks stay hidden and nothing is kill-switched. + */ +const VISIBLE_TYPE: PlacementGate<'block_not_allowed'> = { + subject: 'placement', + skipTypes: ['block_not_allowed'], + check: ({ blockType, previousType, blockId, operationType, outcome, details }) => { + if (blockType === previousType) return null + const block = getBlock(blockType) + if (!block || !isHiddenUnder(overlayVisibility(), block)) return null + return { + type: 'block_not_allowed', + operationType, + blockId, + reason: `Block type "${blockType}" is not available to you - ${outcome}`, + details: { requestedType: blockType, ...details }, + } + }, +} + /** * Refuses placing a sunset (`legacy` / `deprecated`) type and names its successor. Placed * instances keep executing and stay editable, so a block keeping its own type is not refused. @@ -373,6 +397,7 @@ const PERMITTED_TOOL: ToolEntryGate<'tool_not_allowed'> = { export const BLOCK_GATES: readonly BlockGate[] = [ REGISTERED_TYPE, AVAILABLE_TYPE, + VISIBLE_TYPE, ACTIVE_TYPE, SINGLE_TRIGGER, SINGLE_INSTANCE, From 22a0de24aa704140d4da072791f90986f2c04391 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:06:03 -0800 Subject: [PATCH 09/59] fix(sim-cli): finish an in-flight source apply at Ctrl-C before exiting An interrupt while workflows/workspaces source apply (or create) was committing exited 130 at once: the server still stored the write, and the directory, never saved, was stranded as a conflict on both sides. The send and the write-back now run with interrupts deferred: the first Ctrl-C waits (up to 60s) for the answer, saves it, then ends by the signal; a second one, or the bound, stops at once and prints that it may have been written and how to recover. Staging and lock cleanup handlers wait for the deferral. --- .../src/commands/protocol/files-get.ts | 5 + .../protocol/source-apply-interrupt.test.ts | 158 ++++++++++++++++++ .../src/commands/protocol/source-dir.test.ts | 56 +++++-- .../src/commands/protocol/source-dir.ts | 19 +-- .../protocol/workflow-source-apply.ts | 26 ++- .../protocol/workspace-source-apply.ts | 42 ++--- packages/sim-cli/src/terminal/interrupts.ts | 76 +++++++++ 7 files changed, 323 insertions(+), 59 deletions(-) create mode 100644 packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts create mode 100644 packages/sim-cli/src/terminal/interrupts.ts diff --git a/packages/sim-cli/src/commands/protocol/files-get.ts b/packages/sim-cli/src/commands/protocol/files-get.ts index c6b35a34ae9..c587ea7b245 100644 --- a/packages/sim-cli/src/commands/protocol/files-get.ts +++ b/packages/sim-cli/src/commands/protocol/files-get.ts @@ -6,6 +6,7 @@ import { Readable, type Writable } from 'node:stream' import { pipeline } from 'node:stream/promises' import type { Command } from 'commander' import { writeStdout } from '#sim-cli/output/io' +import { interruptsDeferred } from '#sim-cli/terminal/interrupts' import { embedStore } from '../../embed-context' import { V2_OPERATIONS } from '../../generated/v2-api' import { @@ -125,6 +126,9 @@ function reRaise(signal: NodeJS.Signals): void { * The removal is synchronous because the termination that follows gives an * async `rm` no turn to run. * + * An interrupt held by {@link interruptsDeferred} is left alone: the deferral + * raises it again when its write is done, and the cleanup runs then. + * * Exported for its own test: driving it through a real interrupt would take the * test runner down with it. */ @@ -134,6 +138,7 @@ export function removeStagingOnSignal( ): () => void { const installed = STAGE_SIGNALS.map((signal) => { const onSignal = () => { + if (interruptsDeferred()) return process.off(signal, onSignal) const directory = stagingDirectory() if (directory) { diff --git a/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts new file mode 100644 index 00000000000..b3a2fd47a1e --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts @@ -0,0 +1,158 @@ +import { spawn } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import type { AddressInfo } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +const WORKFLOW_ID = '00000000-0000-4000-8000-00000000000a' +const WORKSPACE_ID = 'ws_local' +const BASE = { + workflowId: WORKFLOW_ID, + workspaceId: WORKSPACE_ID, + fingerprint: 'sf1:a', + catalogHash: 'h', +} +const LOCK = { version: 1, blocks: {}, variables: {} } +const ENTRY = "// @sim-sdk 1\nexport default workflow('W', () => {})\n" +/** How long the server holds the committing request before it answers. */ +const COMMIT_DELAY_MS = 1_500 + +function applied() { + return { + data: { + workflowId: WORKFLOW_ID, + workspaceId: WORKSPACE_ID, + dryRun: false, + applied: true, + diff: { + noop: false, + blocks: { added: [{ id: 'b-1', key: 'start' }], removed: [], renamed: [], modified: [] }, + edges: { added: 1, removed: 0 }, + variables: { added: [], removed: [], modified: [] }, + }, + diagnostics: [], + files: { 'workflow.ts': `${ENTRY}// stored\n` }, + lock: LOCK, + base: { ...BASE, fingerprint: 'sf1:b' }, + mintedIds: {}, + notes: [], + needsRedeployment: true, + }, + } +} + +interface InterruptedRun { + code: number | null + signal: NodeJS.Signals | null + stderr: string +} + +let dir: string +let server: Server +let endpoint: string +/** Resolves the moment the server receives the committing request. */ +let onCommit: () => void + +beforeEach(async () => { + dir = mkdtempSync(join(tmpdir(), 'sim-source-interrupt-')) + mkdirSync(join(dir, 'src')) + mkdirSync(join(dir, 'src/.sim')) + writeFileSync(join(dir, 'src/workflow.ts'), ENTRY) + writeFileSync(join(dir, 'src/.sim/lock.json'), JSON.stringify(LOCK)) + writeFileSync(join(dir, 'src/.sim/base.json'), JSON.stringify(BASE)) + server = createServer((request: IncomingMessage, response: ServerResponse) => { + request.resume() + request.on('end', () => { + onCommit() + setTimeout(() => { + response.setHeader('content-type', 'application/json') + response.end(JSON.stringify(applied())) + }, COMMIT_DELAY_MS) + }) + }) + await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve)) + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}` +}) + +afterEach(async () => { + server.closeAllConnections() + await new Promise<void>((resolve) => server.close(() => resolve())) + rmSync(dir, { recursive: true, force: true }) +}) + +/** + * Runs `sim workflows source apply --yes` as a real process and sends it SIGINT `interrupts` + * times once the server holds its commit, 100 ms apart. + */ +function applyInterrupted(interrupts: number): Promise<InterruptedRun> { + const child = spawn( + 'bun', + [ + join(import.meta.dirname, '../../index.ts'), + '--output', + 'json', + 'workflows', + 'source', + 'apply', + '--yes', + '--dir', + join(dir, 'src'), + ], + { + env: { + ...process.env, + SIM_CONFIG_DIR: join(dir, 'config'), + SIM_ENDPOINT: endpoint, + SIM_API_KEY: 'sk-test', + SIM_WORKSPACE: WORKSPACE_ID, + SIM_NO_UPDATE_CHECK: '1', + SIM_CLI_TELEMETRY_DISABLED: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + } + ) + let stderr = '' + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk + }) + child.stdout.resume() + onCommit = () => { + for (let index = 0; index < interrupts; index += 1) { + setTimeout(() => child.kill('SIGINT'), 100 * (index + 1)) + } + } + return new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (code, signal) => resolve({ code, signal, stderr })) + }) +} + +function savedFingerprint(): string { + return JSON.parse(readFileSync(join(dir, 'src/.sim/base.json'), 'utf8')).fingerprint +} + +describe('Ctrl-C while a workflow apply is committing', () => { + it('waits for the answer, saves it into the directory, then ends by the interrupt', async () => { + const run = await applyInterrupted(1) + + expect(run.signal ?? run.code).toSatisfy( + (status: unknown) => status === 'SIGINT' || status === 130 + ) + expect(run.stderr).toContain('the apply was already sent') + expect(savedFingerprint()).toBe('sf1:b') + expect(readFileSync(join(dir, 'src/workflow.ts'), 'utf8')).toContain('// stored') + }, 30_000) + + it('stops at a second Ctrl-C and says the apply may have been saved and how to recover', async () => { + const run = await applyInterrupted(2) + + expect(run.signal ?? run.code).toSatisfy( + (status: unknown) => status === 'SIGINT' || status === 130 + ) + expect(run.stderr).toContain('Stopped before the apply answered. It may have been saved.') + expect(run.stderr).toContain(`sim workflows source get ${WORKFLOW_ID}`) + expect(savedFingerprint()).toBe('sf1:a') + }, 30_000) +}) diff --git a/packages/sim-cli/src/commands/protocol/source-dir.test.ts b/packages/sim-cli/src/commands/protocol/source-dir.test.ts index f53b24a849e..0f01c8f1ade 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.test.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.test.ts @@ -150,27 +150,55 @@ describe('withSourceDirLock', () => { expect(existsSync(join(dir, SOURCE_DIR_LOCK))).toBe(false) }) - it('prints what the interrupted command may have done, then ends with the signal', () => { - const script = ` + /** + * The shape of `workspaces source apply`: the commit and its write-back run deferred inside the + * directory lock, whose own interrupt handler must wait for them too. + */ + function deferredWriteScript(interrupts: number): string { + return ` + import { writeFileSync } from 'node:fs' + import { join } from 'node:path' + import { setTimeout as sleep } from 'node:timers/promises' import { withSourceDirLock } from ${JSON.stringify(join(import.meta.dirname, 'source-dir.ts'))} - let sending = false - await withSourceDirLock( - process.env.SOURCE_DIR, - async () => { - sending = true - process.kill(process.pid, 'SIGINT') - await new Promise(() => {}) - }, - { interruptNote: () => (sending ? 'It may have been written.' : null) } - ) + import { deferInterrupts } from ${JSON.stringify(join(import.meta.dirname, '../../terminal/interrupts.ts'))} + await withSourceDirLock(process.env.SOURCE_DIR, async () => { + await deferInterrupts( + async () => { + for (let sent = 0; sent < ${interrupts}; sent += 1) { + process.kill(process.pid, 'SIGINT') + await sleep(50) + } + await sleep(300) + writeFileSync(join(process.env.SOURCE_DIR, 'saved'), 'yes') + }, + { waiting: 'Waiting for the write.', abandoned: 'Stopped before the write answered.' } + ) + await new Promise(() => {}) + }) ` - const interrupted = spawnSync('bun', ['-e', script], { + } + + it('finishes a deferred write at an interrupt, then releases the lock and ends with the signal', () => { + const interrupted = spawnSync('bun', ['-e', deferredWriteScript(1)], { + env: { ...process.env, SOURCE_DIR: dir }, + timeout: 20_000, + }) + + expect(interrupted.signal).toBe('SIGINT') + expect(interrupted.stderr.toString()).toContain('Waiting for the write.') + expect(existsSync(join(dir, 'saved'))).toBe(true) + expect(existsSync(join(dir, SOURCE_DIR_LOCK))).toBe(false) + }) + + it('abandons a deferred write at a second interrupt, saying so, and releases the lock', () => { + const interrupted = spawnSync('bun', ['-e', deferredWriteScript(2)], { env: { ...process.env, SOURCE_DIR: dir }, timeout: 20_000, }) expect(interrupted.signal).toBe('SIGINT') - expect(interrupted.stderr.toString()).toContain('It may have been written.') + expect(interrupted.stderr.toString()).toContain('Stopped before the write answered.') + expect(existsSync(join(dir, 'saved'))).toBe(false) expect(existsSync(join(dir, SOURCE_DIR_LOCK))).toBe(false) }) diff --git a/packages/sim-cli/src/commands/protocol/source-dir.ts b/packages/sim-cli/src/commands/protocol/source-dir.ts index cd329da970a..ac8e87306ed 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.ts @@ -37,7 +37,6 @@ import { SOURCE_LOCK_FILE, } from '@sim/workflow-code/limits' import { fingerprintOf, KIND_DIRS } from '@sim/workspace-code' -import { printError } from '#sim-cli/output/io' import { embedStore } from '../../embed-context' import { SOURCE_BUSY_EXIT_CODE } from '../../exit-codes' import { SimApiError } from '../../http/client' @@ -835,14 +834,7 @@ export async function takeOverStaleLock(lock: string, seen: LockSighting): Promi export async function withSourceDirLock<T>( dir: string, run: () => Promise<T>, - options: { - readOnly?: boolean - /** - * What to print when an interrupt (Ctrl-C) ends the command: read when it arrives, so a - * command can name what a write in flight may have done; `null` prints nothing. - */ - interruptNote?: () => string | null - } = {} + options: { readOnly?: boolean } = {} ): Promise<T> { if (embedStore.getStore()) return run() const absent = async (path: string) => @@ -899,14 +891,7 @@ export async function withSourceDirLock<T>( const owner: LockOwner = { pid: process.pid, host: hostname() } // An interrupt (Ctrl-C) releases the lock too: a lock left behind is taken over only by a // command on the same host, so one left on a folder shared with a container would stay. - const disposeSignalRelease = removeStagingOnSignal( - () => lock, - (signal) => { - const note = options.interruptNote?.() - if (note) printError(note) - process.kill(process.pid, signal) - } - ) + const disposeSignalRelease = removeStagingOnSignal(() => lock) try { await writeFile(join(lock, LOCK_OWNER_FILE), JSON.stringify(owner)) return await run() diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 26c2abae0e9..5611058a379 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -10,6 +10,7 @@ import { } from '@sim/workflow-code/limits' import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' +import { DEFERRED_INTERRUPT_BOUND_MS, deferInterrupts } from '#sim-cli/terminal/interrupts' import { setSoftExitCode } from '../../embed-context' import type { ApplyWorkflowSourceBody, @@ -205,7 +206,9 @@ type SourceWriteOperation = 'applyWorkflowSource' | 'createWorkflowFromSource' /** * The shared write flow of `apply` and `create`: refuse a write without consent when no terminal * can ask, preview and ask on a terminal, send, print the diagnostics, and after a successful - * write save it into the directory with the caller's `save`. + * write save it into the directory with the caller's `save`. From the send through the save an + * interrupt waits for them ({@link deferInterrupts}), so Ctrl-C never strands a stored write the + * directory did not record. */ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { options: SourceWriteOptions @@ -258,12 +261,21 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { } } - const data = await send(options.dryRun === true) - printFindings(data, printed) - const refused = data.diagnostics.some((diagnostic) => diagnostic.severity === 'error') - if (refused) setSoftExitCode(refusedExitCode(data.diagnostics)) - - if (!data.dryRun && !refused && (data.applied || !creating)) await args.save(data) + const verb = creating ? 'create' : 'apply' + const sendAndSave = async (): Promise<SourceApplyData> => { + const data = await send(options.dryRun === true) + printFindings(data, printed) + const refused = data.diagnostics.some((diagnostic) => diagnostic.severity === 'error') + if (refused) setSoftExitCode(refusedExitCode(data.diagnostics)) + if (!data.dryRun && !refused && (data.applied || !creating)) await args.save(data) + return data + } + const data = options.dryRun + ? await sendAndSave() + : await deferInterrupts(sendAndSave, { + waiting: `Interrupted: the ${verb} was already sent. Waiting up to ${DEFERRED_INTERRUPT_BOUND_MS / 1000}s for its answer and saving it into ${options.dir}, so the directory matches the workflow; press Ctrl-C again to stop now.`, + abandoned: `Stopped before the ${verb} answered. ${args.ifAnswerLost}`, + }) printProtocolResult(args.output, { ...summarize(data, creating), diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 6fdd523985c..e4248f05c55 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -15,6 +15,7 @@ import { kindOfPath, WORKSPACE_BASE_FILE } from '@sim/workspace-code/layout' import { promotionSubmission } from '@sim/workspace-code/promote' import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' +import { DEFERRED_INTERRUPT_BOUND_MS, deferInterrupts } from '#sim-cli/terminal/interrupts' import { setSoftExitCode } from '../../embed-context' import { SOURCE_PARTIAL_EXIT_CODE } from '../../exit-codes' import type { @@ -497,8 +498,6 @@ export function attachWorkspaceSourceApply(source: Command): void { ) } - // Set while the apply write is in flight, so an interrupt names what it may have done. - let writing = false await withSourceDirLock( options.dir, async () => { @@ -552,18 +551,25 @@ export function attachWorkspaceSourceApply(source: Command): void { // Without a preview, stage the blobs the base does not hold as they are. await stage(submission, preview ? new Set(preview.needBlobs) : blobsToStage(submission)) - writing = true - const data = await send(submission, false) - writing = false - printNewDiagnostics(data.diagnostics, printed) - // A promotion leaves the directory bound to the workspace it was read from. - if (!options.to) { - const workspaceCatalog = cachedCatalog(workspaceId) - const records = workspaceCatalog - ? await workflowFolderRecords(client, workspaceId, workspaceCatalog, data.files) - : [] - await saveApplied(options.dir, data, submission, written, records) - } + const data = await deferInterrupts( + async () => { + const answer = await send(submission, false) + printNewDiagnostics(answer.diagnostics, printed) + // A promotion leaves the directory bound to the workspace it was read from. + if (!options.to) { + const workspaceCatalog = cachedCatalog(workspaceId) + const records = workspaceCatalog + ? await workflowFolderRecords(client, workspaceId, workspaceCatalog, answer.files) + : [] + await saveApplied(options.dir, answer, submission, written, records) + } + return answer + }, + { + waiting: `Interrupted: the apply was already sent. Waiting up to ${DEFERRED_INTERRUPT_BOUND_MS / 1000}s for its answer and saving it into ${options.dir}, so the directory matches the workspace; press Ctrl-C again to stop now.`, + abandoned: `Stopped before the apply answered. ${answerLostHint(options.dir)}`, + } + ) const held = data.plan.filter((change) => change.held) if (held.length > 0) { printError( @@ -580,13 +586,7 @@ export function attachWorkspaceSourceApply(source: Command): void { ...resultDiagnostics(profile.output, data.diagnostics), }) }, - { - readOnly: options.dryRun === true, - interruptNote: () => - writing - ? `Interrupted while the apply was being sent. ${answerLostHint(options.dir)}` - : null, - } + { readOnly: options.dryRun === true } ) }) } diff --git a/packages/sim-cli/src/terminal/interrupts.ts b/packages/sim-cli/src/terminal/interrupts.ts new file mode 100644 index 00000000000..de14710f2b3 --- /dev/null +++ b/packages/sim-cli/src/terminal/interrupts.ts @@ -0,0 +1,76 @@ +import { embedStore } from '#sim-cli/embed-context' +import { printError } from '#sim-cli/output/io' + +const INTERRUPT_SIGNALS: readonly NodeJS.Signals[] = ['SIGINT', 'SIGTERM'] + +/** How long an interrupt waits for a deferred write before the command stops without it. */ +export const DEFERRED_INTERRUPT_BOUND_MS = 60_000 + +interface Deferral { + pending: NodeJS.Signals | null +} + +let active: Deferral | null = null + +/** + * Whether an interrupt arriving now is held by {@link deferInterrupts}. A signal handler that + * cleans up and ends the process returns without acting while this holds: the deferral raises + * the signal again once its write is done (or abandoned), and the handler runs then. + */ +export function interruptsDeferred(): boolean { + return active !== null +} + +export interface DeferInterruptsOptions { + /** Printed at the first interrupt: what the command finishes before it stops. */ + waiting: string + /** Printed when the command stops without finishing: a second interrupt, or the bound elapsed. */ + abandoned: string + boundMs?: number +} + +/** + * Runs `run`, a write the server may already be committing followed by saving its answer + * locally, with Ctrl-C (and `SIGTERM`) held until it ends: stopping in between leaves a write the + * server made that the directory never recorded. The first interrupt prints `waiting`; when + * `run` settles the process ends by that signal (exit 130 for Ctrl-C). A second interrupt, or + * `boundMs` passing, stops at once and prints `abandoned`, which names how to recover. + */ +export async function deferInterrupts<T>( + run: () => Promise<T>, + options: DeferInterruptsOptions +): Promise<T> { + if (embedStore.getStore() || active) return run() + const deferral: Deferral = { pending: null } + let bound: ReturnType<typeof setTimeout> | undefined + + const release = () => { + for (const signal of INTERRUPT_SIGNALS) process.off(signal, onSignal) + clearTimeout(bound) + if (active === deferral) active = null + } + const abandon = (signal: NodeJS.Signals) => { + release() + printError(options.abandoned) + process.kill(process.pid, signal) + } + function onSignal(signal: NodeJS.Signals) { + if (deferral.pending) { + abandon(signal) + return + } + deferral.pending = signal + printError(options.waiting) + bound = setTimeout(() => abandon(signal), options.boundMs ?? DEFERRED_INTERRUPT_BOUND_MS) + } + + active = deferral + for (const signal of INTERRUPT_SIGNALS) process.on(signal, onSignal) + try { + return await run() + } finally { + const held = active === deferral ? deferral.pending : null + release() + if (held) process.kill(process.pid, held) + } +} From 29b9d1ac3deafc57dc767a6e1289a096f3c0030a Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:08:30 -0800 Subject: [PATCH 10/59] fix(sim-cli): a source create targets the workspace its starter was typed for The starter wrote no record of its workspace and printed a create command without --workspace, so the create silently went to the profile's default workspace. The starter now writes .sim/starter.json, prints the create with --workspace, and a create into another workspace is refused (SOURCE_OTHER_WORKSPACE) unless --workspace names it; the record goes once the workflow is created. --- .../src/commands/protocol/source-dir.ts | 12 ++++ .../protocol/workflow-source-apply.test.ts | 72 +++++++++++++++++++ .../protocol/workflow-source-apply.ts | 38 +++++++++- .../commands/protocol/workflow-source-get.ts | 12 +++- 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-dir.ts b/packages/sim-cli/src/commands/protocol/source-dir.ts index ac8e87306ed..08169a37d5c 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.ts @@ -62,6 +62,12 @@ export const SOURCE_GITIGNORE_FILE = '.gitignore' * edit from a file it may replace. The CLI's own record: never sent to the server. */ export const SOURCE_WRITTEN_FILE = '.sim/written.json' +/** + * The workspace a starter written by `workflows source create` was typed for, so the create that + * follows targets that workspace. The CLI's own record: never sent, and removed once the workflow + * exists. + */ +export const SOURCE_STARTER_FILE = '.sim/starter.json' /** * The edges of the workflow a get or apply printed, by lock key, with the base fingerprint they * belong to, so an offline check tells an edge the stored workflow holds from one the source @@ -175,6 +181,12 @@ export async function readSourceJson(dir: string, path: string): Promise<unknown } } +/** The workspace {@link SOURCE_STARTER_FILE} names, or `null` when the directory holds no starter. */ +export async function readStarterWorkspace(dir: string): Promise<string | null> { + const workspaceId = toRecord(await readSourceJson(dir, SOURCE_STARTER_FILE)).workspaceId + return typeof workspaceId === 'string' ? workspaceId : null +} + /** * A line file (`.gitattributes`, `.gitignore`) with `lines` appended where missing, or `null` * when it has them all. diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index ddcf921d563..11197e9669d 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -752,6 +752,78 @@ describe('sim workflows source create on a directory with no workflow.ts', () => expect(printed.files).toBeGreaterThan(0) }) + /** Answers the starter's catalog and manifest reads, and a create with a stored workflow. */ + function answerStarterAndCreate() { + request.mockImplementation(async (path: string, init: { query?: { dryRun?: boolean } }) => { + if (path.endsWith('/workflows/source/catalog')) return { data: CATALOG } + if (path.endsWith('/workflows/source/resources')) return MANIFEST + if (path.endsWith('/workflows/source')) return outcome({ dryRun: init.query?.dryRun }) + throw new Error(`unexpected request ${path}`) + }) + } + + function createCalls(): string[] { + return request.mock.calls + .map((call) => call[0] as string) + .filter((path) => path.endsWith('/workflows/source')) + } + + it('records the workspace the starter was typed for and names it in the next step', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + const empty = join(dir, 'new') + answerStarterAndCreate() + + await create('--dir', empty) + + expect(JSON.parse(readFileSync(join(empty, '.sim/starter.json'), 'utf8'))).toEqual({ + workspaceId: WORKSPACE_ID, + }) + expect(stderrLines().join('\n')).toContain( + `sim workflows source create --dir ${empty} --workspace ${WORKSPACE_ID} --yes` + ) + }) + + it('refuses to create a starter in a profile workspace other than its own, creating nothing', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + const starter = join(dir, 'new') + answerStarterAndCreate() + await create('--dir', starter) + contextMockState.workspaceId = 'ws_other' + contextMockState.workspaceSource = 'config' + + try { + await expect(create('--dir', starter, '--yes')).rejects.toMatchObject({ + code: 'SOURCE_OTHER_WORKSPACE', + message: expect.stringContaining(`--workspace ${WORKSPACE_ID}`), + }) + expect(createCalls()).toEqual([]) + } finally { + contextMockState.workspaceId = WORKSPACE_ID + } + }) + + it('creates a starter where --workspace names, and drops the starter record once created', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + const starter = join(dir, 'new') + answerStarterAndCreate() + await create('--dir', starter) + contextMockState.workspaceId = 'ws_other' + contextMockState.workspaceSource = 'flag' + + try { + await create('--dir', starter, '--yes') + expect(createCalls()).toEqual(['/api/v2/workspaces/ws_other/workflows/source']) + expect(existsSync(join(starter, '.sim/starter.json'))).toBe(false) + expect(existsSync(join(starter, '.sim/base.json'))).toBe(true) + } finally { + contextMockState.workspaceId = WORKSPACE_ID + contextMockState.workspaceSource = 'config' + } + }) + it.each(['--dry-run', '--yes'])('refuses %s instead of scaffolding', async (flag) => { setTerminal(false) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 5611058a379..9c7334baedf 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -24,8 +24,11 @@ import { FILE_CONCURRENCY, lockCodeFiles, readSourceText, + readStarterWorkspace, SOURCE_AGENTS_FILE, + SOURCE_STARTER_FILE, sourceRelativePath, + updateSourceDir, WORKSPACE_FOLDER_BUSY_HELP, withWorkspaceFolderLock, writeBack, @@ -378,9 +381,10 @@ export function attachWorkflowSourceCreate(source: Command): void { ) } const { client, profile } = connectCreate() - const tree = await writeSourceScaffold(client, options.dir, client.requireWorkspace()) + const workspaceId = client.requireWorkspace() + const tree = await writeSourceScaffold(client, options.dir, workspaceId) printError( - `${options.dir} had no ${ENTRY_FILE}, so a starter, the SDK types, tsconfig.json, .sim/AGENTS.md and the examples in .sim/examples/ were written and nothing was created. Edit ${ENTRY_FILE}, then run \`sim workflows source create --dir ${options.dir} --yes\`.` + `${options.dir} had no ${ENTRY_FILE}, so a starter, the SDK types for workspace ${workspaceId}, tsconfig.json, .sim/AGENTS.md and the examples in .sim/examples/ were written and nothing was created. Edit ${ENTRY_FILE}, then run \`${createCommand(options.dir, workspaceId)} --yes\`.` ) printProtocolResult(profile.output, { created: false, scaffolded: true, ...tree }) return @@ -388,6 +392,11 @@ export function attachWorkflowSourceCreate(source: Command): void { requireConsent(options, 'creates a workflow', 'create') const { client, profile } = connectCreate() const workspaceId = client.requireWorkspace() + await assertStarterWorkspace( + options.dir, + workspaceId, + profile.sources.workspaceId === 'flag' + ) const files = await readSourceFiles(options.dir, null) await runSourceWrite<'createWorkflowFromSource'>({ options, @@ -400,6 +409,7 @@ export function attachWorkflowSourceCreate(source: Command): void { ifAnswerLost: `It may have been created. Run this command again: a SOURCE_NAME_TAKEN refusal means it was, so find its id with \`sim workflows list\` and run \`sim workflows source get <workflowId> --dir ${options.dir} --force\`.`, save: async (data) => { await writeSourceDirectory(client, options.dir, data, []) + await updateSourceDir(options.dir, { write: [], remove: [SOURCE_STARTER_FILE] }) }, body: { files, @@ -412,6 +422,30 @@ export function attachWorkflowSourceCreate(source: Command): void { ) } +/** The create command for `dir` in `workspaceId`, as the next step a starter prints. */ +function createCommand(dir: string, workspaceId: string): string { + return `sim workflows source create --dir ${dir} --workspace ${workspaceId}` +} + +/** + * Refuses a create into another workspace than the one a starter's types were written for + * ({@link SOURCE_STARTER_FILE}), unless `--workspace` names it: the profile's default workspace + * silently differing from the starter's sent the workflow somewhere its author never looked. + */ +async function assertStarterWorkspace( + dir: string, + workspaceId: string, + explicit: boolean +): Promise<void> { + const starterWorkspace = await readStarterWorkspace(dir) + if (starterWorkspace === null || starterWorkspace === workspaceId || explicit) return + throw new SimApiError( + `${dir} holds a starter written for workspace ${starterWorkspace}, but this profile targets ${workspaceId}; nothing was created. Run \`${createCommand(dir, starterWorkspace)}\` to create it where its types came from, or pass --workspace ${workspaceId} to create it there.`, + 0, + 'SOURCE_OTHER_WORKSPACE' + ) +} + /** A create that did not apply stored nothing, so its provisional workflow id names no workflow. */ function summarize(data: SourceApplyData, created: boolean) { return { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index 768ca8fc5fe..116ff93456c 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -19,6 +19,7 @@ import { readSourceJson, SOURCE_SDK_DIR as SDK_DIR, SOURCE_EXAMPLES_DIR, + SOURCE_STARTER_FILE, updateSourceDir, WORKSPACE_FOLDER_BUSY_HELP, withWorkspaceFolderLock, @@ -91,7 +92,7 @@ An apply that deletes blocks (a declaration removed from \`${SOURCE_ENTRY_FILE}\ Lines starting \`note:\` explain something the write did differently than the source says; read them. -A new workflow: \`sim workflows source create --dir <new dir>\` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, these types, \`tsconfig.json\` and this guide, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, run step 2 and 3, then \`sim workflows source create --dir <dir> --dry-run\` and \`sim workflows source create --dir <dir> --yes\`: it prints the new workflow id and writes the lock files. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir <copy> --name <new name> --yes\`. +A new workflow: \`sim workflows source create --dir <new dir> --workspace <workspaceId>\` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, the types of that workspace, \`tsconfig.json\` and this guide, records the workspace in \`${SOURCE_STARTER_FILE}\`, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, run step 2 and 3, then \`sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run\` and the same with \`--yes\` (the starter prints both with its workspace filled in): it prints the new workflow id and writes the lock files. A create into another workspace than the starter's is refused (\`SOURCE_OTHER_WORKSPACE\`) unless \`--workspace\` names it. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir <copy> --workspace <workspaceId> --name <new name> --yes\`. ## File shape @@ -187,7 +188,8 @@ export default workflow('New workflow', { description: 'Describe what this workf /** * Starts a new source directory: a starter `workflow.ts` plus the tree a get writes (the * declarations for the workspace catalog and its resources, `tsconfig.json`, the agent guide), - * with no lock or base, since no workflow exists yet. Only for a directory with no `workflow.ts`. + * with no lock or base, since no workflow exists yet, and {@link SOURCE_STARTER_FILE} naming the + * workspace those declarations are for. Only for a directory with no `workflow.ts`. */ export async function writeSourceScaffold( client: OperationClient<'getSourceCatalog' | 'getSourceResources'>, @@ -202,7 +204,11 @@ export async function writeSourceScaffold( const tree = await sourceTreeChange(dir, catalog, workspaceTypes, WORKFLOW_TREE, { sources: [STARTER_SOURCE], }) - const write = [[SOURCE_ENTRY_FILE, STARTER_SOURCE] as const, ...tree.change.write] + const write = [ + [SOURCE_ENTRY_FILE, STARTER_SOURCE] as const, + [SOURCE_STARTER_FILE, `${JSON.stringify({ workspaceId }, null, 2)}\n`] as const, + ...tree.change.write, + ] await updateSourceDir(dir, { ...tree.change, write }) return finishTree(dir, write.length, workspaceTypes, tree.ownsTsconfig) } From 5ee4bb3da88ec114a3b0b55479a2121514257f61 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:16:40 -0800 Subject: [PATCH 11/59] fix(sim-cli): exit 13 from a status without login, and clearer cancels and hints - workflows/workspaces source status without a login exited 0 (or 11): statusExitCode now returns 13 when the server was not checked for that. - Ctrl-D at a confirmation or login prompt hung (Node) or printed Bun's AbortError stack: askLine resolves null when input ends, so the prompt cancels cleanly (exit 1). - A refusal in a display format no longer points at details.diagnostics, a JSON field: it says each finding is printed above. - status, diff and apply on a starter no workflow was created from name the create to run (with its workspace), not a get. - A key the server refuses from SIM_API_KEY (or --api-key) says to fix or unset it, since sim login cannot override it. --- packages/sim-cli/src/commands/auth.test.ts | 3 + packages/sim-cli/src/commands/auth.ts | 37 ++++---- .../src/commands/protocol/source-status.ts | 14 ++- .../src/commands/protocol/source-ui.ts | 23 +++-- .../protocol/workflow-source-apply.test.ts | 87 ++++++++++++++++++- .../protocol/workflow-source-apply.ts | 17 +++- .../protocol/workflow-source-status.test.ts | 27 +++++- .../protocol/workflow-source-status.ts | 2 +- .../protocol/workspace-source-status.test.ts | 1 + .../protocol/workspace-source-status.ts | 2 +- packages/sim-cli/src/http/client.test.ts | 23 ++++- packages/sim-cli/src/http/client.ts | 18 +++- packages/sim-cli/src/terminal/line-input.ts | 34 ++++++++ 13 files changed, 241 insertions(+), 47 deletions(-) create mode 100644 packages/sim-cli/src/terminal/line-input.ts diff --git a/packages/sim-cli/src/commands/auth.test.ts b/packages/sim-cli/src/commands/auth.test.ts index 700c1e39ad8..c19a122dcff 100644 --- a/packages/sim-cli/src/commands/auth.test.ts +++ b/packages/sim-cli/src/commands/auth.test.ts @@ -218,6 +218,7 @@ describe('login command', () => { mocks.createInterface.mockReturnValue({ question: vi.fn(async () => 'yes'), close: vi.fn(), + once: vi.fn(), }) vi.spyOn(console, 'log').mockImplementation(() => {}) }) @@ -269,6 +270,7 @@ describe('login command', () => { mocks.createInterface.mockReturnValue({ question: vi.fn(async () => 'no'), close: vi.fn(), + once: vi.fn(), }) await login() @@ -463,6 +465,7 @@ describe('profiles command', () => { mocks.createInterface.mockReturnValue({ question: vi.fn(async () => '1'), close: vi.fn(), + once: vi.fn(), }) vi.spyOn(console, 'log').mockImplementation(() => {}) }) diff --git a/packages/sim-cli/src/commands/auth.ts b/packages/sim-cli/src/commands/auth.ts index fb10fa017b1..0641916a2bd 100644 --- a/packages/sim-cli/src/commands/auth.ts +++ b/packages/sim-cli/src/commands/auth.ts @@ -1,11 +1,11 @@ import { spawn } from 'node:child_process' import { randomBytes } from 'node:crypto' -import { createInterface } from 'node:readline/promises' import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' import { Command, Option } from 'commander' import { printLine } from '#sim-cli/output/io' import { styles } from '#sim-cli/output/presentation' +import { askLine } from '#sim-cli/terminal/line-input' import { buildApprovalUrl, createAuthRequest, @@ -120,15 +120,14 @@ async function confirmProfileOverwrite(profileName: string): Promise<boolean> { ) } - const prompt = createInterface({ input: process.stdin, output: process.stderr }) - try { - const answer = await prompt.question( + const answer = ( + await askLine( `Profile "${redact(profileName)}" already exists. Replace its login and defaults? (y/N) ` ) - return answer.trim().toLowerCase() === 'y' || answer.trim().toLowerCase() === 'yes' - } finally { - prompt.close() - } + ) + ?.trim() + .toLowerCase() + return answer === 'y' || answer === 'yes' } function selectedProfileName(command: Command): string { @@ -309,20 +308,16 @@ async function chooseWorkspace( printLine(` ${index + 1}) ${safeOneLine(workspace.name)} (${workspace.id})`) } - const prompt = createInterface({ input: process.stdin, output: process.stderr }) - try { - const answer = await prompt.question(`Choose a workspace [1-${workspaces.length}]: `) - const selected = Number(answer.trim()) - if (!Number.isInteger(selected) || selected < 1 || selected > workspaces.length) { - throw new SimApiError( - `Invalid workspace selection "${safeOneLine(answer)}". Choose a number from 1 to ${workspaces.length}.`, - 0 - ) - } - return workspaces[selected - 1] - } finally { - prompt.close() + const answer = await askLine(`Choose a workspace [1-${workspaces.length}]: `) + if (answer === null) throw new SimApiError('No workspace was chosen.', 0, 'CANCELLED') + const selected = Number(answer.trim()) + if (!Number.isInteger(selected) || selected < 1 || selected > workspaces.length) { + throw new SimApiError( + `Invalid workspace selection "${safeOneLine(answer)}". Choose a number from 1 to ${workspaces.length}.`, + 0 + ) } + return workspaces[selected - 1] } function addProfileCommand(): Command { diff --git a/packages/sim-cli/src/commands/protocol/source-status.ts b/packages/sim-cli/src/commands/protocol/source-status.ts index 96e0d092ea6..51b2b8776c5 100644 --- a/packages/sim-cli/src/commands/protocol/source-status.ts +++ b/packages/sim-cli/src/commands/protocol/source-status.ts @@ -12,6 +12,7 @@ import { createTwoFilesPatch, FILE_HEADERS_ONLY } from 'diff' import type { OutputFormat } from '../../config/index' import { AUTH_REQUIRED_CODE, + AUTH_REQUIRED_EXIT_CODE, SOURCE_DRIFT_EXIT_CODE, SOURCE_LOCAL_CHANGES_EXIT_CODE, SOURCE_TOO_LARGE_EXIT_CODE, @@ -47,8 +48,9 @@ read); ${SOURCE_LOCAL_CHANGES_EXIT_CODE} local changes only (applying writes the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; ${SOURCE_TOO_LARGE_EXIT_CODE} the source is too large for the server to read in one dry run; 1 the command -failed. A server that cannot be reached is not a failure: the status shows the local changes only -and says so (exit 0 or ${SOURCE_LOCAL_CHANGES_EXIT_CODE}).` +failed; ${AUTH_REQUIRED_EXIT_CODE} not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the +local changes only, then run \`sim login\`. A server that cannot be reached is not a failure: the +status shows the local changes only and says so (exit 0 or ${SOURCE_LOCAL_CHANGES_EXIT_CODE}).` /** What `diff` compares, and its exit codes. */ export const DIFF_HELP = ` @@ -60,8 +62,12 @@ file or folder (and in a workspace directory by kind:key, such as table:leads). differences were shown, whether or not there were any; ${SOURCE_TOO_LARGE_EXIT_CODE} the source is too large for the server to read; 1 the command failed.` -/** The exit status of a status report: drift or conflict, else local changes, else clean. */ -export function statusExitCode(state: SourceStatus['state']): number { +/** + * The exit status of a status report: no login (`login` holds the command to run), else drift or + * conflict, else local changes, else clean. + */ +export function statusExitCode(state: SourceStatus['state'], login: string | null): number { + if (login) return AUTH_REQUIRED_EXIT_CODE if (state === 'conflict' || state === 'drift') return SOURCE_DRIFT_EXIT_CODE return state === 'local' ? SOURCE_LOCAL_CHANGES_EXIT_CODE : 0 } diff --git a/packages/sim-cli/src/commands/protocol/source-ui.ts b/packages/sim-cli/src/commands/protocol/source-ui.ts index a9ec4068f05..6c7a896a845 100644 --- a/packages/sim-cli/src/commands/protocol/source-ui.ts +++ b/packages/sim-cli/src/commands/protocol/source-ui.ts @@ -3,11 +3,11 @@ * findings (each with its location, frame and hint), the confirmation prompt, the refusal of a * write nobody confirmed, and an API refusal re-thrown with the exit status a script branches on. */ -import { createInterface } from 'node:readline/promises' import { isRecordLike, toRecord } from '@sim/utils/object' import { SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' import { type FlatDiagnostic, type RefusalClass, refusalClass } from '@sim/workspace-code' import { printError } from '#sim-cli/output/io' +import { askLine } from '#sim-cli/terminal/line-input' import type { OutputFormat } from '../../config/index' import { embedStore } from '../../embed-context' import { @@ -146,13 +146,8 @@ function canPrompt(): boolean { /** Asks `question` on the terminal; anything but `y` or `yes` is no. */ export async function confirm(question: string): Promise<boolean> { - const prompt = createInterface({ input: process.stdin, output: process.stderr }) - try { - const answer = (await prompt.question(`${question} (y/N) `)).trim().toLowerCase() - return answer === 'y' || answer === 'yes' - } finally { - prompt.close() - } + const answer = (await askLine(`${question} (y/N) `))?.trim().toLowerCase() + return answer === 'y' || answer === 'yes' } /** The exit status of each `409` the server tells apart by `details.code`; any other exits 1. */ @@ -185,6 +180,12 @@ function answerLost(error: SimApiError): boolean { return noAnswer(error, LOST_ANSWER_STATUSES) || error.code === REQUEST_CANCELLED } +/** + * The server's pointer to the findings in the error document (`See details.diagnostics.`): a + * display format prints them instead, so the sentence names a JSON field the reader never sees. + */ +const DETAILS_REFERENCE = / See details\.diagnostics\.$/ + /** * Re-throws an API refusal after printing its diagnostics, so each shows its source line, with * the exit status a script branches on. In a machine `format` the error document keeps the @@ -198,6 +199,10 @@ export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost? const diagnostics = details.diagnostics const printed = isDiagnosticList(diagnostics) if (printed) printDiagnostics(diagnostics) + const machine = format === 'json' || format === 'yaml' + const message = machine + ? error.message + : error.message.replace(DETAILS_REFERENCE, printed ? ' Each finding is printed above.' : '') const nameTaken = error.status === 409 && details.code === 'SOURCE_NAME_TAKEN' const hint = nameTaken ? 'Pass --name to create it under another name, or apply to that workflow.' @@ -205,7 +210,7 @@ export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost? ? ifAnswerLost : null throw new SimApiError( - hint ? `${error.message} ${hint}` : error.message, + hint ? `${message} ${hint}` : message, error.status, error.code, printed ? refusalDetails(format, details, diagnostics) : error.details, diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index 11197e9669d..5bd13c68794 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -13,12 +13,26 @@ import { digest } from './source-dir' import { TSCONFIG_MARKER } from './source-tree' import { attachWorkflowSourceApply, attachWorkflowSourceCreate } from './workflow-source-apply' -const prompt = vi.hoisted(() => ({ question: vi.fn() })) +const prompt = vi.hoisted(() => ({ + question: vi.fn(), + /** The open readline interface, to end its input as Ctrl-D does. */ + open: null as import('node:events').EventEmitter | null, +})) vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) -vi.mock('node:readline/promises', () => ({ - createInterface: () => ({ question: prompt.question, close: () => {} }), -})) +vi.mock('node:readline/promises', async () => { + const { EventEmitter } = await import('node:events') + return { + createInterface: () => { + const readline = Object.assign(new EventEmitter(), { + question: prompt.question, + close: () => readline.emit('close'), + }) + prompt.open = readline + return readline + }, + } +}) const { mockRequest: request } = contextMockFns @@ -254,6 +268,71 @@ describe('sim workflows source apply', () => { expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(ENTRY) }) + it.each([ + [ + 'Node closes the prompt', + () => { + queueMicrotask(() => prompt.open?.emit('close')) + return new Promise(() => {}) + }, + ], + [ + 'Bun aborts the question', + async () => { + throw Object.assign(new Error('The operation was aborted.'), { name: 'AbortError' }) + }, + ], + ])( + 'at a terminal, applies nothing when input ends at the prompt (%s)', + async (_runtime, answer) => { + setTerminal(true) + request.mockResolvedValue(outcome({ dryRun: true })) + prompt.question.mockImplementation(answer) + + await expect(apply(WORKFLOW_ID, '--dir', dir)).rejects.toMatchObject({ + code: 'CANCELLED', + exitCode: 1, + message: 'Nothing was applied.', + }) + + expect(dryRunFlags()).toEqual([true]) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(ENTRY) + } + ) + + it.each([ + [ + 'table', + 'The workflow source has 1 error(s); nothing was written. Each finding is printed above.', + ], + ['json', 'The workflow source has 1 error(s); nothing was written. See details.diagnostics.'], + ] as const)('names where the findings of a refusal are in %s output', async (output, message) => { + contextMockState.output = output + setTerminal(false) + request.mockRejectedValue( + new SimApiError( + 'The workflow source has 1 error(s); nothing was written. See details.diagnostics.', + 400, + 'BAD_REQUEST', + { + code: 'SOURCE_REJECTED', + diagnostics: [ + { + code: 'E_PARSE', + severity: 'error', + message: 'Unexpected token.', + file: 'workflow.ts', + line: 2, + column: 1, + }, + ], + } + ) + ) + + await expect(apply(WORKFLOW_ID, '--dir', dir, '--yes')).rejects.toMatchObject({ message }) + }) + it('at a terminal, does not ask when the dry run already found errors', async () => { setTerminal(true) request.mockResolvedValue( diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 9c7334baedf..ffd6b3ec12d 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -78,11 +78,22 @@ interface SourceWriteOptions { allowDelete?: boolean } -/** The text of a file the directory must hold, or a refusal that says how to write it. */ +/** + * The text of a file the directory must hold, or a refusal that says how to write it: `hint`, or + * for a starter no workflow was created from yet, the create that makes it one. + */ async function requireText(directory: string, path: string, hint: string): Promise<string> { const text = await readSourceText(directory, path) - if (text === null) throw new SimApiError(`${directory} has no ${path}. ${hint}`, 0) - return text + if (text !== null) return text + const starterWorkspace = await readStarterWorkspace(directory) + throw new SimApiError( + `${directory} has no ${path}. ${ + starterWorkspace === null + ? hint + : `It holds a starter no workflow was created from yet: run \`${createCommand(directory, starterWorkspace)} --dry-run\`, then the same with --yes.` + }`, + 0 + ) } async function readJson(directory: string, path: string, hint: string): Promise<unknown> { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts index a59cb309b46..acaeec0bbcf 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts @@ -115,6 +115,20 @@ afterEach(() => { }) describe('workflows source status', () => { + it.each(['status', 'diff'] as const)( + 'on a starter no workflow was created from, %s names the create to run, not a get', + async (command) => { + const starter = join(root, 'starter') + write(starter, 'workflow.ts', GOT['workflow.ts']) + write(starter, '.sim/starter.json', JSON.stringify({ workspaceId: WORKSPACE_ID })) + + await expect(run(starter, command)).rejects.toThrow( + `It holds a starter no workflow was created from yet: run \`sim workflows source create --dir ${starter} --workspace ${WORKSPACE_ID} --dry-run\`` + ) + expect(mockRequest).not.toHaveBeenCalled() + } + ) + it('is clean, exit 0, with nothing changed on either side', async () => { mockRequest.mockResolvedValue(dryRun()) @@ -241,7 +255,18 @@ describe('workflows source status', () => { // A dry run would only exit 13: the login is the next step. next: { command: 'sim login --profile default' }, }) - expect(process.exitCode).toBe(11) + expect(process.exitCode).toBe(13) + }) + + it('exits 13 without a login even when nothing changed here, like every other command', async () => { + mockRequest.mockRejectedValue( + new SimApiError('Not logged in on profile "default".', 0, 'AUTH_REQUIRED', undefined, 13) + ) + + await run(root, 'status') + + expect(printed()).toMatchObject({ server: 'unreachable', state: 'clean' }) + expect(process.exitCode).toBe(13) }) it('fails on a cancelled request instead of reporting the server unreachable', async () => { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts index 100d8160830..33b84981d49 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts @@ -213,7 +213,7 @@ export function attachWorkflowSourceStatus(source: Command): void { }), } printStatusReport(output, `Workflow ${workflowId} in ${options.dir}`, report, planLine) - setSoftExitCode(statusExitCode(status.state)) + setSoftExitCode(statusExitCode(status.state, login)) }) } diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts index a2e9cc02570..ace1cee4fea 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts @@ -541,6 +541,7 @@ describe('workspaces source status', () => { server: 'unreachable', next: { command: 'sim login --profile default' }, }) + expect(process.exitCode).toBe(13) }) it('fails rather than hides a server refusal that is not about reaching it', async () => { diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts index 44ee49eb403..21bf1d09faf 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts @@ -392,7 +392,7 @@ export function attachWorkspaceSourceStatus(source: Command): void { report, planLine ) - setSoftExitCode(statusExitCode(status.state)) + setSoftExitCode(statusExitCode(status.state, login)) }, READ_ONLY ) diff --git a/packages/sim-cli/src/http/client.test.ts b/packages/sim-cli/src/http/client.test.ts index 68eba1e667e..85b7312a37d 100644 --- a/packages/sim-cli/src/http/client.test.ts +++ b/packages/sim-cli/src/http/client.test.ts @@ -264,11 +264,32 @@ describe('workspace-key refusals', () => { await expect(client().request('/api/v2/secrets')).rejects.toMatchObject({ message: - 'Workspace API key cannot perform this operation — this operation does not support workspace API keys; use an OAuth login or personal API key: sim login --profile default', + 'Workspace API key cannot perform this operation — this operation does not support workspace API keys; use an OAuth login or personal API key: the key comes from SIM_API_KEY, which overrides any login: fix it, or unset it and run sim login --profile default', code: 'FORBIDDEN', }) }) + it('names SIM_API_KEY, not a login that it would override, when the server refuses that key', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response(JSON.stringify({ error: { message: 'Invalid API key' } }), { + status: 401, + headers: { 'content-type': 'application/json' }, + }) + ) + ) + + const refused = client().request('/api/v2/meta') + + await expect(refused).rejects.toMatchObject({ + message: + 'Invalid API key — the key comes from SIM_API_KEY, which overrides any login: fix it, or unset it and run sim login --profile default', + exitCode: 13, + }) + }) + it('invents no remedy for other forbidden codes', async () => { vi.stubGlobal( 'fetch', diff --git a/packages/sim-cli/src/http/client.ts b/packages/sim-cli/src/http/client.ts index 53a2b9a229e..d3fbdf5e618 100644 --- a/packages/sim-cli/src/http/client.ts +++ b/packages/sim-cli/src/http/client.ts @@ -799,7 +799,7 @@ export class SimClient { const error = response.status === 401 ? new SimApiError( - `${refused.message} — run: sim login --profile ${this.profile.authProfile}`, + `${refused.message} — ${this.credentialRemedy()}`, refused.status, refused.code ?? AUTH_REQUIRED_CODE, refused.details, @@ -809,7 +809,7 @@ export class SimClient { : refused error.retryAfterMs = parseRetryAfter(response.headers.get('retry-after'), 300_000) if (namesKeyScopeRefusal(error)) { - error.message = `${error.message} — this operation does not support workspace API keys; use an OAuth login or personal API key: sim login --profile ${this.profile.authProfile}` + error.message = `${error.message} — this operation does not support workspace API keys; use an OAuth login or personal API key: ${this.credentialRemedy()}` } if (response.status === 429) throw rateLimitedError(error, rateLimitResends) throw error @@ -826,6 +826,20 @@ export class SimClient { * endpoint. A `Location` that is missing or unparseable still has to produce a * sentence — the redirect is the finding either way. */ + /** + * What to do about a credential the server refused, by where it came from: `sim login` replaces + * a stored login, but a key from `SIM_API_KEY` or `--api-key` overrides any login, so logging in + * again would change nothing. + */ + private credentialRemedy(): string { + const login = `sim login --profile ${this.profile.authProfile}` + if (this.profile.sources.credential === 'env') { + return `the key comes from SIM_API_KEY, which overrides any login: fix it, or unset it and run ${login}` + } + if (this.profile.sources.credential === 'flag') return 'pass a valid key with --api-key' + return `run: ${login}` + } + private toRedirectError(url: string, path: string, response: Response): SimApiError { const requestId = responseRequestId(response) const location = response.headers.get('location')?.trim() diff --git a/packages/sim-cli/src/terminal/line-input.ts b/packages/sim-cli/src/terminal/line-input.ts new file mode 100644 index 00000000000..991864b55ca --- /dev/null +++ b/packages/sim-cli/src/terminal/line-input.ts @@ -0,0 +1,34 @@ +import { createInterface } from 'node:readline/promises' +import { writeStderr } from '#sim-cli/output/io' + +/** + * Whether a pending question failed because its input ended: Bun rejects it with an + * `AbortError` at Ctrl-D, where Node leaves it pending and closes the interface instead. + */ +function isInputEnded(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === 'AbortError' || ('code' in error && error.code === 'ABORT_ERR')) + ) +} + +/** + * Asks `question` on stderr and resolves the line typed, or `null` when the input ends first + * (Ctrl-D, or a stdin closed under the prompt), so a caller cancels cleanly instead of hanging + * or printing a runtime's stack trace. + */ +export async function askLine(question: string): Promise<string | null> { + const prompt = createInterface({ input: process.stdin, output: process.stderr }) + const ended = new Promise<null>((resolve) => prompt.once('close', () => resolve(null))) + try { + const answer = await Promise.race([prompt.question(question), ended]) + if (answer === null) writeStderr('\n') + return answer + } catch (error) { + if (!isInputEnded(error)) throw error + writeStderr('\n') + return null + } finally { + prompt.close() + } +} From 7915adecd6c62094f0fd51b8a10e46be768328a4 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:21:43 -0800 Subject: [PATCH 12/59] docs(sim-cli): list every source exit code, create's --workspace, and tsc as a step The directory guides and the workflow SDK README listed only some exit codes (no 8, 10-12 or 130, and the README no 13). Both guides now print one table built from exit-codes.ts, the README carries the same rows, and the apply/create help and the scripting page add 13 and 130 and that a locked block exits 1. The guides and README make tsc a firm step after every edit and pass --workspace to create. CLI reference pages regenerated. --- apps/docs/content/docs/cli/reference.mdx | 10 +- apps/docs/content/docs/cli/scripting.mdx | 17 +++- apps/docs/content/docs/cli/workflows.mdx | 6 +- apps/docs/content/docs/cli/workspaces.mdx | 4 +- apps/docs/content/docs/workflows/code.mdx | 2 +- packages/sim-cli/README.md | 11 ++- .../protocol/source-exit-codes.test.ts | 35 +++++++ .../commands/protocol/source-exit-codes.ts | 94 +++++++++++++++++++ .../protocol/workflow-source-apply.ts | 23 +++-- .../commands/protocol/workflow-source-get.ts | 7 +- .../protocol/workspace-source-apply.ts | 7 +- .../commands/protocol/workspace-source-dir.ts | 7 +- packages/workflow-sdk/README.md | 36 ++++--- 13 files changed, 216 insertions(+), 43 deletions(-) create mode 100644 packages/sim-cli/src/commands/protocol/source-exit-codes.test.ts create mode 100644 packages/sim-cli/src/commands/protocol/source-exit-codes.ts diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 153ec878e47..ea0a5625766 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8580,7 +8580,7 @@ sim workflows source apply [workflowId] [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). @@ -8606,7 +8606,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources). @@ -8656,7 +8656,7 @@ sim workflows source status [options] </CommandTable> -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). @@ -9950,7 +9950,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). @@ -10005,7 +10005,7 @@ sim workspaces source status [options] </CommandTable> -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/scripting.mdx b/apps/docs/content/docs/cli/scripting.mdx index e3e8540d3b0..7edbc6ed28b 100644 --- a/apps/docs/content/docs/cli/scripting.mdx +++ b/apps/docs/content/docs/cli/scripting.mdx @@ -170,10 +170,21 @@ The CLI resends a request the API answers `429` only when sending it twice is sa | `0` | Applied or created, or a dry run with no errors | | `1` | The source has errors (each printed as `file:line:column` with its frame and hint), or command/request failure | | `5` | The workflow changed since the directory was read (`409 SOURCE_STALE`): get it again, reapply the edit, and retry | -| `6` | The workflow is locked (`423`) | +| `6` | The workflow or its folder is locked (`423`, `E_APPLY_LOCKED`); a locked block exits `1` (`E_APPLY_LOCKED_BLOCK`) | | `7` | The source is over the size limits (`413`) | | `9` | Still rate limited (`429`) after the CLI's retries; nothing was written | | `10` | `--dir` is a workflow folder of a workspace directory another `sim` command is using (`SOURCE_DIR_BUSY`); nothing was written: run it again once the other finished | +| `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; nothing was sent | +| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check | + +| Code | `sim workflows source status` or `sim workspaces source status` | +| --- | --- | +| `0` | Clean: nothing to apply, and the server changed nothing since the directory was read | +| `1` | Command/request failure | +| `7` | The source is too large for the server to read in one dry run | +| `11` | Local changes only: applying writes them | +| `12` | The server changed since the directory was read (drift, or a conflict with local changes) | +| `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; the local changes are still shown | | Code | `sim workflows source check` or `sim workspaces source check` | | --- | --- | @@ -192,11 +203,13 @@ The CLI resends a request the API answers `429` only when sending it twice is sa | `0` | Applied, or a dry run with no errors | | `1` | The source has errors, changes were held back for a secret or credential the workspace cannot satisfy, or command/request failure | | `5` | A resource changed since the directory was read (`409 SOURCE_STALE`): get it again into a fresh directory, reapply the edit, and retry | -| `6` | A resource is locked (`423`) | +| `6` | A resource (a workflow or its folder, a table) is locked (`423`); a locked block exits `1` (`E_APPLY_LOCKED_BLOCK`) | | `7` | The directory is over the size limits (`413`) | | `8` | The apply stopped part-way: what was written is saved into the directory; apply again | | `9` | Still rate limited (`429`) after the CLI's retries; nothing was written | | `10` | Busy, retry: another apply is writing the workspace (`409 SOURCE_BUSY`), or another `sim workspaces source` command is using the same directory (`SOURCE_DIR_BUSY`); nothing was written: run the command again once the other finished | +| `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; nothing was sent | +| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check | Both waits default to 3600 seconds; `--wait-timeout 0` waits indefinitely. `SIM_TIMEOUT_SECONDS` independently bounds one API request. A timeout does not cancel the underlying run or operation. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 318297758b1..eafc3758d94 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1337,7 +1337,7 @@ sim workflows source status [options] </CommandTable> -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). @@ -1361,7 +1361,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources). @@ -1394,7 +1394,7 @@ sim workflows source apply [workflowId] [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 12b9dcf1f8e..89641bd048d 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -733,7 +733,7 @@ sim workspaces source status [options] </CommandTable> -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. @@ -760,7 +760,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index 8366500512f..eca8b571710 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -90,7 +90,7 @@ Nothing in the file runs on your machine. Sim reads it statically, checks it aga A save that removes a block (its declaration is gone from `workflow.ts`) is refused with `E_APPLY_BLOCK_DELETE`, naming the blocks. Pass `--allow-delete` when the deletion is intended. -To create a new workflow, run `sim workflows source create --dir <dir>` on an empty directory: without `--dry-run` or `--yes` it creates nothing and writes a starter `workflow.ts` with the type declarations, `tsconfig.json` and `.sim/AGENTS.md`. Edit the starter, check it, then run `sim workflows source create --dir <dir> --dry-run` and `--yes`. Until the workflow is created the directory has no `.sim/base.json`, so `status`, `diff` and `apply` refuse it. The file starts with the `// @sim-sdk 1` line and default-exports one `workflow('Name', {}, () => { ... })`. The command prints the new workflow ID and writes the same type declarations, `tsconfig.json`, and `.sim/` files that `get` writes, so you can continue editing with `apply`. If another workflow already has that name, the command exits `1`; pass `--name` to choose another. +To create a new workflow, run `sim workflows source create --dir <dir> --workspace <workspaceId>` on an empty directory: without `--dry-run` or `--yes` it creates nothing and writes a starter `workflow.ts` with that workspace's type declarations, `tsconfig.json` and `.sim/AGENTS.md`, and records the workspace in `.sim/starter.json`. Edit the starter, check it, then run `sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run` and `--yes` (the starter prints both). A create into another workspace than the starter's is refused (`SOURCE_OTHER_WORKSPACE`) unless `--workspace` names it. Until the workflow is created the directory has no `.sim/base.json`, so `status`, `diff` and `apply` refuse it. The file starts with the `// @sim-sdk 1` line and default-exports one `workflow('Name', {}, () => { ... })`. The command prints the new workflow ID and writes the same type declarations, `tsconfig.json`, and `.sim/` files that `get` writes, so you can continue editing with `apply`. If another workflow already has that name, the command exits `1`; pass `--name` to choose another. Keep the `.sim/` folder the commands write. `.sim/lock.json` ties each block in the file to the stored block, so renaming a block keeps the same block, with its webhook URL, schedules and run history. Changing its name string keeps it through the lock; renaming the variable too keeps it when the block's type, parent and connections are unchanged, and the dry run reports that as `W_MAT_INFERRED_RENAME`. Renaming two blocks of the same type in the same place in one save is ambiguous, so do those one save at a time. `.sim/base.json` records the version you read. diff --git a/packages/sim-cli/README.md b/packages/sim-cli/README.md index 016e5b84b0f..be28a0b24b9 100644 --- a/packages/sim-cli/README.md +++ b/packages/sim-cli/README.md @@ -322,8 +322,11 @@ Success exits 0 and most failures exit 1. Two failures any command can hit have `401` (run `sim login`), and 9 when the API answered `429`. Every `GET`, and the source applies, creates and dry runs, are resent up to 6 times first, each wait the longer of `Retry-After` (up to 60 seconds) and a backoff of about 1 to 30 seconds; any other write exits 9 at once. Before this -release both exited 1. Command-specific codes: -[Exit codes](https://docs.sim.ai/cli/scripting#exit-codes). +release both exited 1. Ctrl-C exits 130; a source apply or create already sent is finished and +saved into its directory first. The `workflows source` and `workspaces source` commands also use +5 (stale), 6 (a locked workflow or folder; a locked block exits 1), 7 (too large), 8 (a workspace +apply stopped part-way), 10 (busy), 11 (status: local changes) and 12 (status: the server +changed). Command-specific codes: [Exit codes](https://docs.sim.ai/cli/scripting#exit-codes). ## Workflows and workspaces as code @@ -342,8 +345,8 @@ sim workflows source apply --dir ./support-triage --yes # save the draft `sim workspaces source get|check|status|diff|apply --dir <dir>` does the same for tables, knowledge bases, files, custom tools, skills, MCP servers, workflows and -deployments in one directory. `sim workflows source create --dir <dir>` starts a -new workflow. +deployments in one directory. `sim workflows source create --dir <dir> --workspace <id>` +starts a new workflow in that workspace. Coding agents (Claude Code, Cursor, Codex) start at `.sim/AGENTS.md`, which `get` writes into the directory. The loop is get, edit `workflow.ts`, check, status, diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.test.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.test.ts new file mode 100644 index 00000000000..e2d3979b4e8 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.test.ts @@ -0,0 +1,35 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import * as exitCodes from '../../exit-codes' +import { WORKFLOW_TREE } from './workflow-source-get' +import { WORKSPACE_TREE } from './workspace-source-dir' + +/** + * Every code a source command can exit with: each `*_EXIT_CODE` the CLI defines except the + * run-wait timeout (4), which no source command returns, plus 0, 1 and Ctrl-C's 130. + */ +const SOURCE_EXIT_CODES = [ + 0, + 1, + ...Object.entries(exitCodes) + .filter(([name]) => name.endsWith('_EXIT_CODE') && name !== 'STILL_RUNNING_EXIT_CODE') + .map(([, code]) => code as number), + 130, +].sort((a, b) => a - b) + +const WORKFLOW_SDK_README = readFileSync( + join(import.meta.dirname, '../../../../workflow-sdk/README.md'), + 'utf8' +) + +describe('the exit-code tables an agent or a reader relies on', () => { + it.each([ + ['the workflow directory guide', WORKFLOW_TREE.agents], + ['the workspace directory guide', WORKSPACE_TREE.agents], + ['the workflow SDK README', WORKFLOW_SDK_README], + ])('%s lists every code a source command exits with', (_name, text) => { + const listed = [...text.matchAll(/^\| (\d+) \|/gm)].map((match) => Number(match[1])) + expect(listed).toEqual(SOURCE_EXIT_CODES) + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts new file mode 100644 index 00000000000..9d210e3f79a --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts @@ -0,0 +1,94 @@ +import { + AUTH_REQUIRED_EXIT_CODE, + RATE_LIMITED_EXIT_CODE, + SOURCE_BUSY_EXIT_CODE, + SOURCE_DRIFT_EXIT_CODE, + SOURCE_LOCAL_CHANGES_EXIT_CODE, + SOURCE_LOCKED_EXIT_CODE, + SOURCE_PARTIAL_EXIT_CODE, + SOURCE_STALE_EXIT_CODE, + SOURCE_TOO_LARGE_EXIT_CODE, +} from '../../exit-codes' + +/** The status a shell reports for a command Ctrl-C ended (128 + SIGINT). */ +const INTERRUPTED_EXIT_CODE = 130 + +/** + * Every exit code of the `workflows source` and `workspaces source` commands, as the Markdown + * table both directory guides (`.sim/AGENTS.md`) print, built from the codes the CLI exits with so + * the guides cannot drift from them. `prefix` is the command group the guide is for. + */ +export function sourceExitCodesTable( + prefix: 'sim workflows source' | 'sim workspaces source' +): string { + const rows: Array<[number, string, string]> = [ + [0, 'Applied or created, a dry run or check with no errors, or a clean status', 'Go on'], + [ + 1, + 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed', + 'Fix what it printed and run it again', + ], + [ + SOURCE_STALE_EXIT_CODE, + 'Stale (`SOURCE_STALE`, or `E_APPLY_STALE` on a dry run): the server changed what this directory was read from, and nothing was written', + `\`${prefix} get --dir <fresh dir>\`, reapply your edit there, dry-run, apply`, + ], + [ + SOURCE_LOCKED_EXIT_CODE, + 'A workflow or its folder is locked (`E_APPLY_LOCKED`); a locked block exits 1 instead', + 'Ask someone to unlock it in the editor', + ], + [ + SOURCE_TOO_LARGE_EXIT_CODE, + 'The source is over the size limits (on a get, status or diff too)', + 'Split function code, or remove blocks or resources', + ], + [ + SOURCE_PARTIAL_EXIT_CODE, + '`sim workspaces source apply` stopped part-way: what was written is saved into the directory', + 'Apply again until it exits 0', + ], + [ + RATE_LIMITED_EXIT_CODE, + "Still rate limited (HTTP 429) after the CLI's retries (up to 6, at most a minute apart); nothing more was written", + 'Wait a minute and run the same command again', + ], + [ + SOURCE_BUSY_EXIT_CODE, + 'Busy: another apply is writing the workspace (`SOURCE_BUSY`), or another sim command is using this directory (`SOURCE_DIR_BUSY`); nothing was written', + 'Run it again once the other finished', + ], + [ + SOURCE_LOCAL_CHANGES_EXIT_CODE, + '`status`: only your edits, which applying writes', + 'Dry-run, then apply', + ], + [ + SOURCE_DRIFT_EXIT_CODE, + '`status`: the server changed since the get (drift, or a conflict with your edits)', + 'Follow the `Next:` line it prints', + ], + [ + AUTH_REQUIRED_EXIT_CODE, + 'Not logged in, the login expired, or the server refused the credential (`AUTH_REQUIRED`); nothing was sent', + '`sim login`, or fix or unset `SIM_API_KEY` when the key comes from it; then the same command again', + ], + [ + INTERRUPTED_EXIT_CODE, + 'Interrupted (Ctrl-C). An apply or create already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written', + `\`${prefix} status --dir .\``, + ], + ] + return [ + '| Code | Meaning | Do |', + '|---|---|---|', + ...rows.map(([code, meaning, next]) => `| ${code} | ${meaning} | ${next} |`), + ].join('\n') +} + +/** The exit codes every source write (apply, create) shares, appended to its help. */ +export const SOURCE_WRITE_EXIT_CODES_HELP = ` +Exit ${AUTH_REQUIRED_EXIT_CODE}: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. +Run \`sim login\`, or fix or unset SIM_API_KEY when the key comes from it. Exit ${INTERRUPTED_EXIT_CODE}: interrupted +(Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C +stops at once and prints that it may have been written and how to check.` diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index ffd6b3ec12d..a2d68e9cc86 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -33,6 +33,7 @@ import { withWorkspaceFolderLock, writeBack, } from './source-dir' +import { SOURCE_WRITE_EXIT_CODES_HELP } from './source-exit-codes' import { cachedCatalogByHash } from './source-tree' import { confirm, @@ -57,15 +58,17 @@ const APPLY_EXIT_CODES_HELP = ` Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); -6 the workflow or its folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after -the CLI's retries, so nothing was written: wait a minute and run it again.` +6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, +E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's +retries, so nothing was written: wait a minute and run it again.` const CREATE_EXIT_CODES_HELP = ` Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken -(SOURCE_NAME_TAKEN: pass --name), or the command failed; 6 the folder is locked; 7 the source -is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a -minute and run it again.` +(SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace +(SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the +source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: +wait a minute and run it again.` type SourceApplyData = ApplyWorkflowSourceResponse['data'] @@ -326,7 +329,10 @@ export function attachWorkflowSourceApply(source: Command): void { ) .allowExcessArguments(false) .description('Apply a workflow source directory to a workflow draft') - .addHelpText('after', `${APPLY_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}`) + .addHelpText( + 'after', + `${APPLY_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}${SOURCE_WRITE_EXIT_CODES_HELP}` + ) .action(async (named: string | undefined, options: SourceWriteOptions) => withWorkspaceFolderLock(options.dir, async () => { requireConsent(options, 'writes the workflow draft', 'apply') @@ -381,7 +387,10 @@ export function attachWorkflowSourceCreate(source: Command): void { .option('-y, --yes', 'Create without asking for confirmation') .allowExcessArguments(false) .description('Create a workflow from a workflow source directory') - .addHelpText('after', `${CREATE_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}`) + .addHelpText( + 'after', + `${CREATE_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}${SOURCE_WRITE_EXIT_CODES_HELP}` + ) .action(async (options: SourceWriteOptions & { name?: string; folder?: string }) => withWorkspaceFolderLock(options.dir, async () => { if ((await readSourceText(options.dir, ENTRY_FILE)) === null) { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index 116ff93456c..fd5c5a2d915 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -25,6 +25,7 @@ import { withWorkspaceFolderLock, writeBack, } from './source-dir' +import { sourceExitCodesTable } from './source-exit-codes' import { assertSdkMajor, finishTree, @@ -83,7 +84,7 @@ This directory is a Sim workflow written as TypeScript. The server compiles \`${ 1. Edit \`${SOURCE_ENTRY_FILE}\` (and \`functions/*\`). 2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. Fix and rerun until it exits 0. -3. \`${typecheckCommand()}\` type-checks with the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. +3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks with the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. 4. \`sim workflows source status --dir .\` before applying: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed it since (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\`: fix every error it prints, rerun until it exits 0. 5. \`sim workflows source apply --dir . --yes\`: saves the draft (never deploys). It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out), so read the file again before the next edit rather than editing from memory. 6. Exit 5 (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get <workflowId> --dir <fresh dir>\`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare it with this one. Only when it holds your edit, run \`sim workflows source get <workflowId> --dir . --force\`; otherwise reapply your edit in the fresh directory. @@ -119,7 +120,9 @@ The \`// @sim-sdk 1\` header must be the first line, and the file has exactly on ## Diagnostics and exit codes -Findings print as \`file:line:column severity CODE: message\`, then the code frame and a hint. \`E_*\` refuses the whole apply; \`W_*\` is saved as written and names a real problem. \`check\` prints the \`W_*\` codes step 2 lists as \`error\` on a block you added, since they fail it there; the apply still saves them as warnings. Exit 0 applied, 1 source errors (or a name another workflow holds), 5 stale, 6 workflow locked, 7 source too large (on a get too), 9 rate limited (HTTP 429) after the CLI's retries (up to 6, at most a minute apart): nothing was written, so wait a minute and run the same command again; 13 not logged in (\`AUTH_REQUIRED\`): run \`sim login\`, then the command again. +Findings print as \`file:line:column severity CODE: message\`, then the code frame and a hint. \`E_*\` refuses the whole apply; \`W_*\` is saved as written and names a real problem. \`check\` prints the \`W_*\` codes step 2 lists as \`error\` on a block you added, since they fail it there; the apply still saves them as warnings. Exit codes of every \`workflows source\` command: + +${sourceExitCodesTable('sim workflows source')} ## Rules diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index e4248f05c55..1315235984d 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -39,6 +39,7 @@ import { writtenFile, writtenRecord, } from './source-dir' +import { SOURCE_WRITE_EXIT_CODES_HELP } from './source-exit-codes' import { cachedCatalog } from './source-tree' import { confirm, @@ -85,8 +86,8 @@ another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace t to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read -(SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is -locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was +(SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a +workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a @@ -430,7 +431,7 @@ export function attachWorkspaceSourceApply(source: Command): void { ) .allowExcessArguments(false) .description('Apply a workspace source directory') - .addHelpText('after', EXIT_CODES_HELP) + .addHelpText('after', `${EXIT_CODES_HELP}${SOURCE_WRITE_EXIT_CODES_HELP}`) .action(async (options: ApplyOptions) => { requireConsent(options, 'writes the workspace', 'apply') const only = parseKinds(options.only) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index b07edac2b31..f07abccffb4 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -61,6 +61,7 @@ import { type StatDigest, type Written, } from './source-dir' +import { sourceExitCodesTable } from './source-exit-codes' import { type LocalDeclarations, type SourceTreeProfile, @@ -121,7 +122,7 @@ Secrets never appear in source: a workflow reads an environment variable as \`en 1. Edit the files. 2. \`sim workspaces source check --dir .\`: compiles every workflow offline with the server's compiler, against what this directory declares: an unknown resource key (skills included) or table column, a child workflow's inputs, and an env name \`${KIND_DIRS.secret}\` lacks (a warning); and checks each \`${KIND_DIRS.table}/<key>.rows.jsonl\` line against its table as the apply would. It fails (exit 1) on errors and on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block), regenerates \`.sim/sdk/workspace.d.ts\` and adds the blocks the workflows now use to \`tsconfig.json\`. In a workflow folder, \`sim workflows source check --dir .\` checks that one workflow the same way. Fix and rerun until it exits 0. -3. \`${typecheckCommand()}\` type-checks the files with the TypeScript release the declarations are built for. \`tsconfig.json\` includes only the blocks the workflows used at the last check or get, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). \`${SOURCE_SDK_DIR}/${SDK_GUIDE_FILE}\` lists every builder with its file and params type; \`grep\` it rather than reading the declarations. The dry run is the authority: a printed file you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. +3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks the files with the TypeScript release the declarations are built for. \`tsconfig.json\` includes only the blocks the workflows used at the last check or get, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). \`${SOURCE_SDK_DIR}/${SDK_GUIDE_FILE}\` lists every builder with its file and params type; \`grep\` it rather than reading the declarations. The dry run is the authority: a printed file you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. 4. \`sim workspaces source status --dir .\` before applying: your changes by resource (\`sim workspaces source diff --dir .\` shows the edits), what changed on the server since the get (a workflow edited in the editor included), the conflicts, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed since (\`--offline\`: skip the server). Then \`sim workspaces source apply --dir . --dry-run\`: shows the plan (\`+\` create, \`~\` update, \`=\` adopt an existing resource of the same name, \`-\` delete) and every error; fix and rerun until it exits 0. 5. \`sim workspaces source apply --dir . --yes\`: writes it and rewrites this directory from what was stored. @@ -159,7 +160,9 @@ Secrets never appear in source: a workflow reads an environment variable as \`en ## Exit codes -0 applied; 1 errors (each printed as file:line:column with a hint); 5 a resource changed on the server since this directory was read (get it again into a fresh directory, reapply the edit; right after an apply that ended without an answer, such as a timeout, that apply or someone else changed it: get into a fresh directory and compare, and only when it holds your edits run \`sim workspaces source get --dir . --force\`); 6 locked; 7 too large; 8 stopped part-way (a resource failed, or a later round stayed rate limited): what was written is saved here, so apply again, as often as it takes, until it exits 0; 9 rate limited (HTTP 429) after the CLI's retries (up to 6, at most a minute apart): nothing more was written, so wait a minute and run the same command again (a get keeps the binary files it downloaded and fetches only the rest); 10 another apply (a CI job, a teammate) is writing this workspace, or another command is using this directory: nothing was written, so wait for it to finish and run the command again. 13 not logged in (\`AUTH_REQUIRED\`): run \`sim login\`, then the command again. +${sourceExitCodesTable('sim workspaces source')} + +Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means that apply or someone else changed the workspace: get into a fresh directory and compare, and only when it holds your edits run \`sim workspaces source get --dir . --force\`. A get that exits 9 keeps the binary files it downloaded and fetches only the rest. Keep \`${WORKSPACE_BASE_FILE}\` and each workflow's \`.sim/\` files as written. ` diff --git a/packages/workflow-sdk/README.md b/packages/workflow-sdk/README.md index 23de41ae4a1..20a1ade672b 100644 --- a/packages/workflow-sdk/README.md +++ b/packages/workflow-sdk/README.md @@ -26,9 +26,11 @@ have the CLI confirm the directory holds that workflow. cannot (duplicate names, cycles, literal secrets). It exits 1 on errors and on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value or a literal secret in a block the lock does not bind yet (`--strict`: in every block; `--no-strict`: never on a - warning). It also adds the blocks the source now uses to `tsconfig.json`. Then - `bunx -p typescript@7.0.2 tsc -p ./wf` type-checks with the TypeScript release the - declarations are built for. `tsconfig.json` includes only the blocks the source used at the + warning). It also adds the blocks the source now uses to `tsconfig.json`. Then always run + `bunx -p typescript@7.0.2 tsc -p ./wf` after an edit and fix what it reports: it takes about a + second and catches what check and the dry run do not, such as a reference to a start input the + trigger does not declare. It type-checks with the TypeScript release the declarations are built + for. `tsconfig.json` includes only the blocks the source used at the last check, get or apply (fast); `tsconfig.full.json` includes every current block. In a workspace directory, `sim workspaces source check --dir .` checks every workflow against the directory's own secrets, credentials, resource keys and table columns, and @@ -43,11 +45,13 @@ have the CLI confirm the directory holds that workflow. source to find it), dry-run, apply. Keep the fresh `.sim/` files: `.sim/base.json` holds the fingerprint that makes the next apply safe. 6. **New workflow**: write `workflow.ts` in an empty directory and run - `sim workflows source create --dir ./wf [--name <name>] [--folder <path>] --yes`. It prints the + `sim workflows source create --dir ./wf --workspace <id> [--name <name>] [--folder <path>] --yes`. It prints the new workflow id and writes the full tree a `get` writes: the `.sim/sdk` types, `tsconfig.json`, `.sim/AGENTS.md` and the lock files. Continue with the loop above. Run on a directory without - `workflow.ts` and without `--dry-run` or `--yes`, `create --dir ./wf` writes a starter - `workflow.ts` and the types instead, and creates nothing. + `workflow.ts` and without `--dry-run` or `--yes`, `create --dir ./wf --workspace <id>` writes a + starter `workflow.ts` and that workspace's types instead, records the workspace in + `.sim/starter.json`, and creates nothing; the create that follows targets that workspace, and + one into another workspace is refused unless `--workspace` names it. `apply` and `create` ask for confirmation on a terminal; elsewhere pass `--yes` (without it they refuse with `CONFIRMATION_REQUIRED`). `--dry-run` never asks. @@ -99,14 +103,22 @@ Every code and its meaning: the "Diagnostics" table of `SYNTAX.md` in the Sim re ## Exit codes +Every `sim workflows source` and `sim workspaces source` command exits with one of these: + | Code | Meaning | Do | |---|---|---| -| 0 | Applied, or a clean dry run | Done | -| 1 | The source has errors, or the command failed | Fix the printed findings | -| 5 | `SOURCE_STALE`: the workflow changed since `get` | Get again, reapply the edit (step 5) | -| 6 | The workflow is locked | Ask someone to unlock it in the workflow builder | -| 7 | The source is too large | Split function code or remove blocks | -| 9 | Still rate limited (HTTP 429) after the CLI's retries; nothing was written | Wait a minute and run the same command again | +| 0 | Applied or created, a dry run or check with no errors, or a clean status | Done | +| 1 | The source has errors, a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed | Fix what it printed | +| 5 | `SOURCE_STALE`: the workflow changed since `get`, and nothing was written | Get again, reapply the edit (step 5) | +| 6 | The workflow or its folder is locked (`E_APPLY_LOCKED`) | Ask someone to unlock it in the workflow builder | +| 7 | The source is too large (on a get, status or diff too) | Split function code or remove blocks | +| 8 | `sim workspaces source apply` stopped part-way; what was written is saved into the directory | Apply again until it exits 0 | +| 9 | Still rate limited (HTTP 429) after the CLI's retries; nothing more was written | Wait a minute and run the same command again | +| 10 | Busy: another apply is writing the workspace, or another `sim` command is using the directory; nothing was written | Run it again once the other finished | +| 11 | `status`: only your local edits | Dry-run, then apply | +| 12 | `status`: the server changed since the get (drift, or a conflict with your edits) | Follow the `Next:` line it prints | +| 13 | Not logged in, or the server refused the credential (`AUTH_REQUIRED`); nothing was sent | `sim login`, or fix or unset `SIM_API_KEY` | +| 130 | Interrupted (Ctrl-C). An apply or create already sent is finished and saved first; a second Ctrl-C stops at once and says it may have been written | `sim workflows source status --dir ./wf` | ## Write declarations, not programs From 74d80ea3ae938a32b5b6b445bc4d0293d884134a Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 15:57:06 -0800 Subject: [PATCH 13/59] fix(workspace-source): stage blobs in bodies the proxy forwards whole Next.js truncates a proxied /api body past 10 MiB, so a 24 MB staging batch reached the route as a broken multipart prefix (400, never converging). Batches now carry at most 8 MB framing included, a file up to the 10 MB blob limit goes alone, the route caps the body below the proxy ceiling and answers 413 PAYLOAD_TOO_LARGE naming the limit, and the CLI maps that 413 to the upload it refused and the fix. Server hashing keeps its own 24 MB budget. --- apps/docs/openapi-v2-workflows.json | 2 +- .../[workspaceId]/source/blobs/route.test.ts | 85 +++++++++++++++++++ .../[workspaceId]/source/blobs/route.ts | 19 +++-- .../contracts/v2/openapi/workspace-source.ts | 7 +- .../lib/api/contracts/v2/workspace-source.ts | 2 +- .../application/cross-kind.integration.ts | 35 ++++++-- .../application/files.integration.ts | 2 +- .../application/hash-budget.ts | 2 +- .../application/stage-blobs.ts | 7 +- .../workspace-source.test-helpers.ts | 34 +++++--- .../protocol/workspace-source-apply.test.ts | 30 +++++++ .../protocol/workspace-source-blobs.ts | 42 +++++++-- .../protocol/workspace-source-staging.test.ts | 80 +++++++++++++++++ packages/workspace-code/DESIGN.md | 4 +- packages/workspace-code/src/limits.ts | 46 ++++++++-- 15 files changed, 348 insertions(+), 49 deletions(-) create mode 100644 apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts create mode 100644 packages/sim-cli/src/commands/protocol/workspace-source-staging.test.ts diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 847540cbd2a..8bdfbfac91f 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -8986,7 +8986,7 @@ "post": { "operationId": "stageWorkspaceSourceBlobs", "summary": "Stage workspace source blobs", - "description": "Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most 1000 files and 24000000 bytes per request, 10000000 bytes per file. A part whose bytes hash to another SHA-256 refuses the whole request (`400`), nothing staged. Staged files are scoped to the workspace and the caller, capped at 20000 files and 4000000000 bytes (`413` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:write`.", + "description": "Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most 1000 files and 8000000 bytes per request, 10000000 bytes per file; a larger file goes in a request of its own. A body over 10001024 bytes answers `413` (`PAYLOAD_TOO_LARGE`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (`400`), nothing staged. Staged files are scoped to the workspace and the caller, capped at 20000 files and 4000000000 bytes (`413` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:write`.", "x-sim-operation": "workspaces.source.blobs.stage", "x-oauth-scope": "api:write", "tags": ["Workflows"], diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts new file mode 100644 index 00000000000..10c48131826 --- /dev/null +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts @@ -0,0 +1,85 @@ +import { createHash } from 'node:crypto' +import { + V2_OPERATION_RATE_LIMIT_ALLOWED, + V2_PREAUTH_RATE_LIMIT_ALLOWED, + v2ApiKeyAuthModuleMock, + v2RateLimiterModuleMock, + v2RouteMocks, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ admit: vi.fn(), stage: vi.fn() })) + +vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) +vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) +vi.mock('@/lib/workspace-source/application/stage-blobs', () => ({ + admitSourceBlobStage: { + operation: { id: 'workspaces.source.blobs.stage' }, + execute: mocks.admit, + }, + stageSourceBlobs: { operation: { id: 'workspaces.source.blobs.stage' }, execute: mocks.stage }, +})) + +import { POST } from '@/app/api/v2/workspaces/[workspaceId]/source/blobs/route' + +const WORKSPACE_ID = '11111111-2222-4333-8444-555555555555' + +const auth = { + principal: { kind: 'personal_api_key' as const, userId: 'user-1', keyId: 'key-2' }, + rateLimitSubjectIds: ['api-key:key-2'] as const, + rateLimitSubscription: null, + keyType: 'personal' as const, +} + +const context = { params: Promise.resolve({ workspaceId: WORKSPACE_ID }) } + +/** A staging request carrying one part per buffer, as the CLI builds it. */ +async function stagingRequest(buffers: Buffer[]): Promise<NextRequest> { + const form = new FormData() + for (const bytes of buffers) { + const sha256 = createHash('sha256').update(bytes).digest('hex') + form.append(sha256, new Blob([new Uint8Array(bytes)]), sha256) + } + const encoded = new Response(form) + const body = Buffer.from(await encoded.arrayBuffer()) + return new NextRequest(`http://localhost:3000/api/v2/workspaces/${WORKSPACE_ID}/source/blobs`, { + method: 'POST', + headers: { + 'x-api-key': 'key', + 'content-type': encoded.headers.get('content-type') ?? '', + 'content-length': String(body.byteLength), + }, + body, + }) +} + +describe('POST /api/v2/workspaces/{workspaceId}/source/blobs', () => { + beforeEach(() => { + v2RouteMocks.authenticate.mockResolvedValue(auth) + v2RouteMocks.preauthRate.mockResolvedValue(V2_PREAUTH_RATE_LIMIT_ALLOWED) + v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) + mocks.admit.mockReset().mockResolvedValue({ workspaceId: WORKSPACE_ID }) + mocks.stage.mockReset().mockResolvedValue({ workspaceId: WORKSPACE_ID, staged: [] }) + }) + + it('answers a body the proxy would cut off with 413 PAYLOAD_TOO_LARGE naming the limit, before reading it', async () => { + const response = await POST( + await stagingRequest([Buffer.alloc(6_000_000, 1), Buffer.alloc(6_000_000, 2)]), + context + ) + + expect(response.status).toBe(413) + const { error } = await response.json() + expect(error.code).toBe('PAYLOAD_TOO_LARGE') + expect(error.message).toMatch(/at most \d+ bytes; this one carries \d+/) + expect(mocks.stage).not.toHaveBeenCalled() + }) + + it('stages a single file of the largest size a directory holds', async () => { + const response = await POST(await stagingRequest([Buffer.alloc(10_000_000, 3)]), context) + + expect(response.status).toBe(200) + expect(mocks.stage.mock.calls[0][0].input.blobs).toHaveLength(1) + }) +}) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts index 6c8845c5a0e..2a172009630 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts @@ -1,4 +1,7 @@ -import { WORKSPACE_SOURCE_STAGE_MAX_BYTES } from '@sim/workspace-code/limits' +import { + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' import { v2StageWorkspaceSourceBlobsContract } from '@/lib/api/contracts/v2/workspace-source' import { defineV2BodyLifecycleRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -23,8 +26,9 @@ const SHA256 = /^[0-9a-f]{64}$/ /** * POST /api/v2/workspaces/[workspaceId]/source/blobs — stages the binary files of a workspace * source apply: a multipart form with one file part per blob, named by its SHA-256. The caller's - * write access and staging room are checked before the body is read; a part whose bytes hash to - * another SHA-256 refuses the batch. Draws on its own staging budget, not the source-write one. + * write access and staging room are checked before the body is read; a body over + * `WORKSPACE_SOURCE_STAGE_MAX_BYTES` (a declared length, or one the proxy truncated at its ceiling) + * answers `413 PAYLOAD_TOO_LARGE`, and a part whose bytes hash to another SHA-256 refuses the batch. Draws on its own staging budget, not the source-write one. */ export const POST = defineV2BodyLifecycleRoute({ contract: v2StageWorkspaceSourceBlobsContract, @@ -44,7 +48,12 @@ export const POST = defineV2BodyLifecycleRoute({ label: 'workspace source staging body', }) } catch (error) { - if (isPayloadSizeLimitError(error)) throw error + if (isPayloadSizeLimitError(error)) { + throw new OrchestrationError( + 'payload_too_large', + `A staging request body carries at most ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes${error.observedBytes === undefined ? '' : `; this one carries ${error.observedBytes}`}. Send fewer files per request, a file over ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes in a request of its own.` + ) + } if (isMultipartFieldValidationError(error)) { throw new OrchestrationError('validation', error.message) } @@ -61,7 +70,7 @@ export const POST = defineV2BodyLifecycleRoute({ blobs.push({ sha256: name, content: await readFileToBufferWithLimit(value, { - maxBytes: WORKSPACE_SOURCE_STAGE_MAX_BYTES, + maxBytes: WORKSPACE_SOURCE_LIMITS.maxBlobBytes, label: 'workspace source blob', }), }) diff --git a/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts b/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts index f1b12e7500a..b970de08982 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts @@ -1,4 +1,7 @@ -import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' +import { + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' import { documentedSchema, RATE_LIMIT_HEADERS, @@ -192,7 +195,7 @@ export const workspaceSourceOpenApiRoutes = [ applicationOperation: workspaceSourceOperations.stageBlobs, operationId: 'stageWorkspaceSourceBlobs', summary: 'Stage workspace source blobs', - description: `Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, ${WORKSPACE_SOURCE_LIMITS.maxBlobBytes} bytes per file. A part whose bytes hash to another SHA-256 refuses the whole request (\`400\`), nothing staged. Staged files are scoped to the workspace and the caller, capped at ${WORKSPACE_SOURCE_LIMITS.maxStagedBlobs} files and ${WORKSPACE_SOURCE_LIMITS.maxStagedBytes} bytes (\`413\` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns \`403\` while workflows as code is not enabled.`, + description: `Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, ${WORKSPACE_SOURCE_LIMITS.maxBlobBytes} bytes per file; a larger file goes in a request of its own. A body over ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes answers \`413\` (\`PAYLOAD_TOO_LARGE\`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (\`400\`), nothing staged. Staged files are scoped to the workspace and the caller, capped at ${WORKSPACE_SOURCE_LIMITS.maxStagedBlobs} files and ${WORKSPACE_SOURCE_LIMITS.maxStagedBytes} bytes (\`413\` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns \`403\` while workflows as code is not enabled.`, tags: ['Workflows'], errors: [...RESOURCE_MUTATION_ERRORS, 'PayloadTooLarge'], success: { description: 'The binary files staged.', headers: RATE_LIMIT_HEADERS }, diff --git a/apps/sim/lib/api/contracts/v2/workspace-source.ts b/apps/sim/lib/api/contracts/v2/workspace-source.ts index f21adf6277e..4a6199e535a 100644 --- a/apps/sim/lib/api/contracts/v2/workspace-source.ts +++ b/apps/sim/lib/api/contracts/v2/workspace-source.ts @@ -354,7 +354,7 @@ export const v2StageWorkspaceSourceBlobsFormSchema = z z.file().max(WORKSPACE_SOURCE_LIMITS.maxBlobBytes).describe('The bytes of one binary file.') ) .describe( - `One file part per binary file, named by its SHA-256: at most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request.` + `One file part per binary file, named by its SHA-256: at most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, a larger file alone.` ) /** `POST /api/v2/workspaces/{workspaceId}/source/blobs`: stage binary files for an apply. */ diff --git a/apps/sim/lib/workspace-source/application/cross-kind.integration.ts b/apps/sim/lib/workspace-source/application/cross-kind.integration.ts index 78388601d77..3ec6127beb6 100644 --- a/apps/sim/lib/workspace-source/application/cross-kind.integration.ts +++ b/apps/sim/lib/workspace-source/application/cross-kind.integration.ts @@ -84,7 +84,10 @@ import { applyWorkspaceSource, } from '@/lib/workspace-source/application/apply-workspace-source' import { readWorkspaceSource } from '@/lib/workspace-source/application/read-workspace-source' -import { findMissingSourceBlobs } from '@/lib/workspace-source/application/stage-blobs' +import { + findMissingSourceBlobs, + stageSourceBlobs, +} from '@/lib/workspace-source/application/stage-blobs' import { blobManifest, codes, @@ -1009,7 +1012,7 @@ export default workflow('Big', {}, () => { check('workspace files and knowledge documents hash within one per-request budget', async () => { const workspaceId = await newWorkspace() - const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest / 2.5) + const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxHashBytesPerRequest / 2.5) for (const name of ['a.txt', 'b.txt']) { await uploadWorkspaceFile( workspaceId, @@ -1053,10 +1056,17 @@ export default workflow('Big', {}, () => { () => true ) - const refused = await stageBlobsAs( - owner, - workspaceId, - blobs.map((blob) => ({ sha256: blob.sha256, content: blob.content ?? '' })) + const refused = await withPermissionGroupScope(() => + stageSourceBlobs.execute({ + principal: owner, + input: { + workspaceId, + blobs: blobs.map((blob) => ({ + sha256: blob.sha256, + content: Buffer.from(blob.content ?? '', 'base64'), + })), + }, + }) ).then( () => null, (error: unknown) => error @@ -1070,6 +1080,19 @@ export default workflow('Big', {}, () => { }) ) expect(missing.missing).toHaveLength(3) + + const [alone] = blobManifest( + { 'files/alone.bin': Buffer.alloc(WORKSPACE_SOURCE_LIMITS.maxBlobBytes, 'd') }, + () => true + ) + await stageBlobsAs(owner, workspaceId, [{ sha256: alone.sha256, content: alone.content ?? '' }]) + const stillMissing = await withPermissionGroupScope(() => + findMissingSourceBlobs.execute({ + principal: owner, + input: { workspaceId, blobs: [{ sha256: alone.sha256, size: alone.size }] }, + }) + ) + expect(stillMissing.missing).toEqual([]) }) check('a plan already refused authorizes no change', async () => { diff --git a/apps/sim/lib/workspace-source/application/files.integration.ts b/apps/sim/lib/workspace-source/application/files.integration.ts index 256337c7be5..b049c8cce33 100644 --- a/apps/sim/lib/workspace-source/application/files.integration.ts +++ b/apps/sim/lib/workspace-source/application/files.integration.ts @@ -775,7 +775,7 @@ describe('workspace source files', () => { 'server hashing stops at the per-request budget: a get lists the rest without a hash, and a plan refuses them rather than creating', async () => { const workspaceId = await newWorkspace() - const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest / 2.5) + const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxHashBytesPerRequest / 2.5) const directory: Record<string, Buffer> = {} for (const name of ['a.bin', 'b.bin', 'c.bin']) { const bytes = Buffer.alloc(size, name) diff --git a/apps/sim/lib/workspace-source/application/hash-budget.ts b/apps/sim/lib/workspace-source/application/hash-budget.ts index 922d95fedf4..fa4e147af5a 100644 --- a/apps/sim/lib/workspace-source/application/hash-budget.ts +++ b/apps/sim/lib/workspace-source/application/hash-budget.ts @@ -18,7 +18,7 @@ export interface HashBudget { /** A full hashing budget for one request. */ export function newHashBudget(): HashBudget { - return { bytes: WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest, count: MAX_HASHED_PER_REQUEST } + return { bytes: WORKSPACE_SOURCE_LIMITS.maxHashBytesPerRequest, count: MAX_HASHED_PER_REQUEST } } /** Downloads one plan or get runs at once while hashing. */ diff --git a/apps/sim/lib/workspace-source/application/stage-blobs.ts b/apps/sim/lib/workspace-source/application/stage-blobs.ts index 736d41cbe8f..f3a50d06bb2 100644 --- a/apps/sim/lib/workspace-source/application/stage-blobs.ts +++ b/apps/sim/lib/workspace-source/application/stage-blobs.ts @@ -71,7 +71,8 @@ export interface StageSourceBlobsResult { } /** - * Stages a batch of blobs: refuses one over `maxBlobBytes`, a batch over the request caps, and any + * Stages a batch of blobs: refuses one over `maxBlobBytes`, a batch over the request caps (a single + * blob may fill its request alone), and any * blob whose bytes do not hash to the SHA-256 it was named by (nothing of the batch is staged), * then stores the rest within the scope's caps. Staging a blob already staged only renews it. */ @@ -108,10 +109,10 @@ export const stageSourceBlobs = defineAuthorizedWorkspaceUseCase({ ) } } - if (total > maxBlobBytesPerRequest) { + if (input.blobs.length > 1 && total > maxBlobBytesPerRequest) { throw new OrchestrationError( 'payload_too_large', - `A staging request carries at most ${maxBlobBytesPerRequest} bytes; this one carries ${total}.` + `A staging request of more than one file carries at most ${maxBlobBytesPerRequest} bytes; this one carries ${total}. Send a larger file in a request of its own.` ) } const staged = await stageBlobs( diff --git a/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts b/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts index 05a4713fb6c..62a331e7e94 100644 --- a/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts +++ b/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts @@ -13,6 +13,7 @@ import { sha256Hex } from '@sim/security/hash' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import type { WorkspaceKind } from '@sim/workspace-code' +import { stageBatches } from '@sim/workspace-code/limits' import { inArray, sql } from 'drizzle-orm' import { expect, it } from 'vitest' import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' @@ -124,25 +125,30 @@ export type SourceSubmission = Omit< principal?: Principal } -/** Stages `blobs` (base64 bytes by SHA-256) in `workspaceId` as `principal`, as the CLI does. */ +/** + * Stages `blobs` (base64 bytes by SHA-256) in `workspaceId` as `principal`, as the CLI does: in + * the requests `stageBatches` splits them into. + */ export async function stageBlobsAs( principal: Principal, workspaceId: string, blobs: ReadonlyArray<{ sha256: string; content: string }> ) { - if (blobs.length === 0) return - await withPermissionGroupScope(() => - stageSourceBlobs.execute({ - principal, - input: { - workspaceId, - blobs: blobs.map((blob) => ({ - sha256: blob.sha256, - content: Buffer.from(blob.content, 'base64'), - })), - }, - }) - ) + const decoded = blobs.map((blob) => { + const content = Buffer.from(blob.content, 'base64') + return { sha256: blob.sha256, content, size: content.byteLength } + }) + for (const batch of stageBatches(decoded)) { + await withPermissionGroupScope(() => + stageSourceBlobs.execute({ + principal, + input: { + workspaceId, + blobs: batch.map(({ sha256, content }) => ({ sha256, content })), + }, + }) + ) + } } /** diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 6fb1071a98d..4be48aa466b 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -647,6 +647,36 @@ describe('workspaces source apply', () => { expect(request).not.toHaveBeenCalled() }) + it('a staging request the server refuses as too large applies nothing, names the upload and the fix, and exits 7', async () => { + write('files/a.bin', Buffer.alloc(3_000, 1)) + mockRequest.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path.endsWith('/source/blobs')) { + throw new SimApiError('Request body is too large', 413, 'PAYLOAD_TOO_LARGE') + } + return path.includes('/source/blobs') ? stagingAnswer(path, options) : request(path, options) + }) + + const refused = await apply('--yes').then( + () => null, + (error: unknown) => error + ) + + expect(refused).toMatchObject({ status: 413, code: 'PAYLOAD_TOO_LARGE', exitCode: 7 }) + expect((refused as SimApiError).message).toContain('files/a.bin') + expect((refused as SimApiError).message).toContain('sim update') + expect(request).not.toHaveBeenCalled() + }) + + it('uploads two files that would overflow one request body in two requests', async () => { + write('files/a.bin', Buffer.alloc(6_000_000, 1)) + write('files/b.bin', Buffer.alloc(6_000_000, 2)) + request.mockResolvedValueOnce(answer()) + + await apply('--yes') + + expect(staging.uploaded.map((batch) => batch.length)).toEqual([1, 1]) + }) + describe('rate limiting', () => { const limited = () => new SimApiError('API rate limit exceeded', 429, 'RATE_LIMITED', undefined, 9) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts b/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts index b309a864756..8ef45a4ecfd 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts @@ -29,7 +29,11 @@ import { KIND_LAYOUTS, } from '@sim/workspace-code/kinds/registry' import { isWorkspaceSourcePath, kindOfPath } from '@sim/workspace-code/layout' -import { stageBatches, WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' +import { + stageBatches, + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' import type { FindMissingWorkspaceSourceBlobsResponse, GetWorkspaceSourceResponse, @@ -470,7 +474,7 @@ type StagingClient = OperationClient< /** * Stages the bytes of the blobs at `paths` for an apply to `workspaceId`: asks which of their * hashes the server does not hold staged for this caller, then uploads only those, in requests of - * at most `maxBlobBytesPerRequest`. A blob staged by an earlier, interrupted run is not sent + * at most `maxBlobBytesPerRequest` (a larger blob alone). A blob staged by an earlier, interrupted run is not sent * again. A file whose bytes no longer hash to what the directory read is refused before it is * sent. Returns how many blobs and bytes were uploaded. */ @@ -516,11 +520,35 @@ export async function stageBlobs( form.append(blob.sha256, new Blob([new Uint8Array(content)]), blob.sha256) bytes += content.byteLength } - await client.request<StageWorkspaceSourceBlobsResponse>('stageWorkspaceSourceBlobs', { - params: { workspaceId }, - form, - retryRateLimited: true, - }) + try { + await client.request<StageWorkspaceSourceBlobsResponse>('stageWorkspaceSourceBlobs', { + params: { workspaceId }, + form, + retryRateLimited: true, + }) + } catch (error) { + throw stagingTooLarge(error, batch) + } } return { count: sending.length, bytes } } + +/** + * A staging request the server refused as too large (`413`), re-raised naming what it carried and + * what to do: nothing was applied, and since this CLI batches within the limits it shares with the + * server, a refusal means the server runs other limits (another version, or a proxy in front of + * it with a smaller body limit). Any other error passes through. + */ +function stagingTooLarge(error: unknown, batch: readonly BlobEntry[]): unknown { + if (!(error instanceof SimApiError) || error.status !== 413) return error + const bytes = batch.reduce((sum, blob) => sum + blob.size, 0) + const largest = batch.reduce((top, blob) => (blob.size > top.size ? blob : top), batch[0]) + return new SimApiError( + `The server refused an upload of ${batch.length} binary file(s), ${bytes} bytes (largest ${largest.path}), as too large: ${error.message} Nothing was applied. Update the CLI (\`sim update\`) so it splits uploads the way this server accepts; if a proxy in front of the server limits request bodies, raise its limit above ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes.`, + error.status, + error.code, + error.details, + error.exitCode, + error.requestId + ) +} diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-staging.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-staging.test.ts new file mode 100644 index 00000000000..38360cb006a --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/workspace-source-staging.test.ts @@ -0,0 +1,80 @@ +import { createHash } from 'node:crypto' +import { + SOURCE_REQUEST_BODY_CEILING_BYTES, + STAGE_PART_FRAMING_BYTES, + stageBatches, + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' +import { describe, expect, it } from 'vitest' + +interface Sized { + size: number + fill: number +} + +/** The bytes of each staging request `blobs` split into, serialized as the CLI sends them. */ +async function requestBodies(blobs: Sized[]): Promise<number[]> { + const sizes: number[] = [] + for (const batch of stageBatches(blobs)) { + const form = new FormData() + for (const blob of batch) { + const bytes = Buffer.alloc(blob.size, blob.fill) + const sha256 = createHash('sha256').update(bytes).digest('hex') + form.append(sha256, new Blob([new Uint8Array(bytes)]), sha256) + } + sizes.push((await new Response(form).arrayBuffer()).byteLength) + } + return sizes +} + +const of = (count: number, size: number): Sized[] => + Array.from({ length: count }, (_, index) => ({ size, fill: index % 256 })) + +describe('staging requests', () => { + it.each([ + ['two 6 MB files', of(2, 6_000_000)], + ['two files just under the per-request cap together', of(2, 4_900_000)], + ['one file of the largest size a directory holds', of(1, WORKSPACE_SOURCE_LIMITS.maxBlobBytes)], + ['a large file between small ones', [...of(3, 1_000), ...of(1, 9_999_999), ...of(3, 1_000)]], + [ + 'as many tiny files as one request carries', + of(WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest, 1), + ], + ])( + '%s: every request body fits the route cap, which fits the proxy ceiling', + async (_, blobs) => { + const bodies = await requestBodies(blobs) + + for (const bytes of bodies) + expect(bytes).toBeLessThanOrEqual(WORKSPACE_SOURCE_STAGE_MAX_BYTES) + expect(WORKSPACE_SOURCE_STAGE_MAX_BYTES).toBeLessThan(SOURCE_REQUEST_BODY_CEILING_BYTES) + } + ) + + it('sends a file too large to share a request in one of its own, never dropping it', () => { + const blobs = [ + ...of(1, 3_000_000), + ...of(1, WORKSPACE_SOURCE_LIMITS.maxBlobBytes), + ...of(1, 10), + ] + + const batches = stageBatches(blobs) + + expect(batches.map((batch) => batch.map((blob) => blob.size))).toEqual([ + [3_000_000], + [WORKSPACE_SOURCE_LIMITS.maxBlobBytes], + [10], + ]) + }) + + it('budgets the framing of each part, so a full request of tiny files is split by count', () => { + const count = WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest + 1 + const perPart = Math.floor(WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest / 10) + + expect(stageBatches(of(count, 0)).map((batch) => batch.length)).toEqual([count - 1, 1]) + expect( + stageBatches(of(10, perPart - STAGE_PART_FRAMING_BYTES + 1)).map((batch) => batch.length) + ).toEqual([9, 1]) + }) +}) diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index 3ce623354cf..c7e19d5f7c6 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -267,7 +267,9 @@ the write is still caught only where the child use case takes a precondition (a **Staged blobs.** Blob bytes never travel in an apply. A client first asks which of the blobs it will send it has not staged (`POST /source/blobs/missing`, by SHA-256 and size), uploads only those (`POST /source/blobs`, multipart, one part per blob named by its hash, at most -`maxBlobBytesPerRequest` per request, `stageBatches` in `src/limits.ts`), then sends one apply +`maxBlobBytesPerRequest` per request framing included, a larger blob alone, `stageBatches` in +`src/limits.ts`; every body stays under the 10 MiB Next.js proxy ceiling, past which the route +would receive a truncated body, and the route answers a larger one `413 PAYLOAD_TOO_LARGE`), then sends one apply whose manifest names each blob by path, hash and size. The server checks every uploaded part against its hash before staging it, so a staged blob is verified bytes. The plan resolves the blobs its writes need against the staging area once, before any write: a missing one (never diff --git a/packages/workspace-code/src/limits.ts b/packages/workspace-code/src/limits.ts index 5c035593dde..c788d84709d 100644 --- a/packages/workspace-code/src/limits.ts +++ b/packages/workspace-code/src/limits.ts @@ -21,8 +21,14 @@ export const WORKSPACE_SOURCE_LIMITS = { maxDirectoryWorkflowBytes: 32_000_000, /** Bytes of one blob (a knowledge document or a workspace file). */ maxBlobBytes: 10_000_000, - /** Blob bytes one staging request carries (and one request may download to hash). */ - maxBlobBytesPerRequest: 24_000_000, + /** + * Blob bytes, multipart framing included, one staging request carries when it holds more than + * one blob; a blob up to `maxBlobBytes` goes in a request of its own. Either body stays under + * {@link SOURCE_REQUEST_BODY_CEILING_BYTES}. + */ + maxBlobBytesPerRequest: 8_000_000, + /** Stored bytes one request (a get, or one apply) may download to hash. */ + maxHashBytesPerRequest: 24_000_000, /** Blobs one staging request carries. */ maxBlobsPerRequest: 1_000, /** Blobs one caller may hold staged in one workspace at once. */ @@ -60,13 +66,38 @@ export const WORKSPACE_SOURCE_REQUEST_MAX_BYTES = WORKSPACE_SOURCE_LIMITS.maxWorkflowBytes) + 6_000_000 -/** Body cap for one staging request: its blobs plus multipart framing. */ +/** + * The most bytes a request body may carry and still reach a route whole: Next.js buffers each + * `/api` body for `apps/sim/proxy.ts` and silently truncates it past + * `experimental.proxyClientMaxBodySize` (its 10 MiB default; `PROXY_CLIENT_MAX_BODY_BYTES` in + * apps/sim/lib/api/server/validation.ts), so a larger staging body arrives as a broken multipart + * prefix. + */ +export const SOURCE_REQUEST_BODY_CEILING_BYTES = 10 * 1024 * 1024 + +/** + * Multipart framing budgeted per staged part: its boundary line and the headers naming its SHA-256 + * as field and file name (about 270 bytes as `FormData` serializes them). + */ +export const STAGE_PART_FRAMING_BYTES = 512 + +/** + * Body cap for one staging request: the larger of a full batch and one blob of `maxBlobBytes` + * alone, with its framing and the closing boundary. Below + * {@link SOURCE_REQUEST_BODY_CEILING_BYTES}, so a body the proxy truncated (whose declared or read + * length reaches the ceiling) is refused as too large rather than misread as malformed. + */ export const WORKSPACE_SOURCE_STAGE_MAX_BYTES = - WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest + 2_000_000 + Math.max( + WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest, + WORKSPACE_SOURCE_LIMITS.maxBlobBytes + STAGE_PART_FRAMING_BYTES + ) + STAGE_PART_FRAMING_BYTES /** * Splits `blobs` into staging requests, in order: each carries at most `maxBlobsPerRequest` blobs - * and `maxBlobBytesPerRequest` bytes, and always at least one blob, so every request progresses. + * and `maxBlobBytesPerRequest` bytes counting {@link STAGE_PART_FRAMING_BYTES} per blob, and always + * at least one blob, so every request progresses and a blob too large to share a request goes + * alone. */ export function stageBatches<T extends { size: number }>(blobs: readonly T[]): T[][] { const { maxBlobBytesPerRequest, maxBlobsPerRequest } = WORKSPACE_SOURCE_LIMITS @@ -74,16 +105,17 @@ export function stageBatches<T extends { size: number }>(blobs: readonly T[]): T let batch: T[] = [] let bytes = 0 for (const blob of blobs) { + const framed = blob.size + STAGE_PART_FRAMING_BYTES if ( batch.length > 0 && - (bytes + blob.size > maxBlobBytesPerRequest || batch.length >= maxBlobsPerRequest) + (bytes + framed > maxBlobBytesPerRequest || batch.length >= maxBlobsPerRequest) ) { batches.push(batch) batch = [] bytes = 0 } batch.push(blob) - bytes += blob.size + bytes += framed } if (batch.length > 0) batches.push(batch) return batches From b6fd91d5e29d167b065d7637eb10bcf1e409b8db Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:04:15 -0800 Subject: [PATCH 14/59] fix(workspace-source): compare a promotion with what the last one left in the target apply --to read the target's live base at promote time and used it as the three-way base, so a target edit since the last promotion could never look changed and was overwritten. Each promotion now records the target's base entries and promoted workflow folders in .sim/targets/<workspaceId>.json; the next promotion uses a recorded entry in place of the live one for the same id, sends a workflow folder unchanged since as its base alone, and an edited one with the recorded fingerprint. A target edit is kept when the directory left the resource alone and refused as stale (exit 5) when it changed it too. A resource without a record (the first promotion) takes the target as it stands; docs say so. --- apps/docs/content/docs/cli/reference.mdx | 2 +- apps/docs/content/docs/cli/scripting.mdx | 4 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- .../content/docs/workflows/workspace-code.mdx | 10 +- .../application/promote.integration.ts | 104 ++++++++- .../protocol/workspace-source-apply.test.ts | 40 ++++ .../protocol/workspace-source-apply.ts | 76 ++++++- .../commands/protocol/workspace-source-dir.ts | 2 +- packages/workspace-code/DESIGN.md | 16 +- packages/workspace-code/src/promote.test.ts | 103 +++++++++ packages/workspace-code/src/promote.ts | 214 ++++++++++++++++-- 11 files changed, 541 insertions(+), 32 deletions(-) create mode 100644 packages/workspace-code/src/promote.test.ts diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 153ec878e47..6b8a803671e 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -9946,7 +9946,7 @@ sim workspaces source apply [options] | `-y, --yes` | No | Apply without asking for confirmation. | | `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | | `--only <kinds>` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | -| `--to <workspaceId>` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place; the directory is not changed. | +| `--to <workspaceId>` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | </CommandTable> diff --git a/apps/docs/content/docs/cli/scripting.mdx b/apps/docs/content/docs/cli/scripting.mdx index e3e8540d3b0..6e4a7aed536 100644 --- a/apps/docs/content/docs/cli/scripting.mdx +++ b/apps/docs/content/docs/cli/scripting.mdx @@ -403,12 +403,14 @@ jobs: `check` runs first because it is offline and fast, and it regenerates `.sim/sdk/workspace.d.ts`, so the `tsc` step after it types resources the pull request adds. A workspace API key can apply tables, rows, knowledge bases, files, custom tools, MCP servers and credential bindings. Workflows, skills, secrets and deployments need a member's personal API key, and deployments need an admin. Apply rewrites files under the directory (each `.sim/base.json`, ids it allocated, values the server normalized), and the next apply compares with them, so the last step commits them back to the branch; `[skip ci]` keeps that commit from running the job again. Where the branch is protected, run `sim workspaces source get --dir workspace` on a new branch instead and open a pull request. After merging two branches that each applied, a dry run with no changes means the files match the workspace. Exit `5` means a resource changed since this branch's base: get the workspace into a fresh directory and carry the edit over. Exit `10` means another apply is writing the workspace (or another command is using the directory), so retry; the `concurrency` key keeps two runs of this job from overlapping. -To promote the same directory to a second workspace, add a step with `--to`: +To promote the same directory to a second workspace, add a step with `--to` before the step that commits the files back: ```bash sim workspaces source apply --dir workspace --to "$PROD_WORKSPACE_ID" --yes ``` +The promotion writes `workspace/.sim/targets/<workspaceId>.json`, what it left in that workspace. Committing it lets the next run tell a change made in that workspace since (kept, or exit `5` when the directory changed the same resource) from one to promote. + ## Related docs - [Output formats](/cli/output) diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 12b9dcf1f8e..83e1091373d 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -756,7 +756,7 @@ sim workspaces source apply [options] | `-y, --yes` | No | Apply without asking for confirmation. | | `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | | `--only <kinds>` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | -| `--to <workspaceId>` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place; the directory is not changed. | +| `--to <workspaceId>` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | </CommandTable> diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index bdbbc7e1df5..54cb4f379b9 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -90,7 +90,15 @@ sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --dry sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --yes ``` -Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`. The directory is not changed and stays bound to the workspace it was read from. Without `--to`, an apply refuses a directory read from another workspace. +Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`. The directory stays bound to the workspace it was read from. Without `--to`, an apply refuses a directory read from another workspace. + +Each promotion records what it left in the target in `.sim/targets/<workspaceId>.json`, and the next promotion compares with that record: + +- A resource changed in the target since the last promotion, such as a hotfix made in Sim, is kept when the directory left it alone. +- When the directory changed it too, the promotion is refused as stale and exits `5`. Carry the target's change into the source workspace, or undo it in the target, then promote again. +- The first promotion, and any resource the record does not hold (one created in the target since), updates the target to match the directory. Review the dry run first. + +Commit the record with the rest of the directory, so a promotion from CI or another clone compares with it. Deleting it makes the next promotion take the target as it stands. ## Git and CI diff --git a/apps/sim/lib/workspace-source/application/promote.integration.ts b/apps/sim/lib/workspace-source/application/promote.integration.ts index aa9a00df6de..e8d4e4b3c73 100644 --- a/apps/sim/lib/workspace-source/application/promote.integration.ts +++ b/apps/sim/lib/workspace-source/application/promote.integration.ts @@ -11,7 +11,11 @@ import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/featur import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' import { generateId } from '@sim/utils/id' -import { promotionSubmission } from '@sim/workspace-code/promote' +import { + type PromotionRecord, + promotionRecord, + promotionSubmission, +} from '@sim/workspace-code/promote' import { and, eq, inArray, isNull } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, vi } from 'vitest' import { @@ -94,6 +98,41 @@ async function promote(dev: Read, target: string, options: { allowDelete?: boole return runner.apply(target, { files: composed.files, base: composed.base, ...options }) } +/** + * Promotes `dev` to `target` as `apply --to` does with the target's promotion record, returning + * the apply's result and the record it leaves. + */ +async function promoteRecorded(dev: Read, target: string, record: PromotionRecord | undefined) { + const prod = await runner.get(target) + const composed = promotionSubmission({ + files: dev.files, + blobPaths: [], + kinds: dev.kinds, + target: prod, + record, + allowDelete: false, + }) + const result = await runner.apply(target, { files: composed.files, base: composed.base }) + return { + result, + record: promotionRecord({ + workspaceId: target, + previous: record, + kinds: dev.kinds, + files: dev.files, + submission: composed, + applied: result, + }), + } +} + +/** `dev` with `edits` applied to it, read back as the directory would hold it. */ +async function editDev(dev: string, edits: Record<string, string>) { + const read = await runner.get(dev) + await runner.apply(dev, { files: { ...read.files, ...edits }, base: read.base }) + return runner.get(dev) +} + async function blockIds(workspaceId: string) { const rows = await db .select({ id: workflowBlocks.id, name: workflowBlocks.name }) @@ -200,6 +239,69 @@ describe('workspace source promotion', () => { } ) + check( + 'a change made in the target since the last promotion is kept when the directory left it alone, and refused as stale when the directory changed it too', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: DEV }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + expect(first.result.failed).toBeUndefined() + + const prodRead = await runner.get(prod) + await runner.apply(prod, { + files: { + ...prodRead.files, + 'skills/tone.md': TONE.replace('Be brief.', 'Be brief, prod.'), + 'workflows/ingest/workflow.ts': ingest('Prod hotfix.'), + }, + base: prodRead.base, + }) + + const untouched = await promoteRecorded(await runner.get(dev), prod, first.record) + expect(untouched.result.applied).toEqual([]) + const kept = await runner.get(prod) + expect(kept.files['skills/tone.md']).toContain('Be brief, prod.') + expect(kept.files['workflows/ingest/workflow.ts']).toContain('Prod hotfix.') + + const edited = await editDev(dev, { + 'skills/tone.md': TONE.replace('Be brief.', 'Be very brief.'), + 'workflows/ingest/workflow.ts': ingest('Be terse.'), + }) + const stale = await refusal(promoteRecorded(edited, prod, untouched.record)) + expect(stale.stale).toBe(true) + expect(codes(stale.diagnostics)).toContain('E_APPLY_STALE') + const unchanged = await runner.get(prod) + expect(unchanged.files['skills/tone.md']).toContain('Be brief, prod.') + expect(unchanged.files['workflows/ingest/workflow.ts']).toContain('Prod hotfix.') + } + ) + + check( + 'with the record, a promotion that changed nothing in the target since converges, and the next edit updates it in place', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: DEV }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + const again = await promoteRecorded(await runner.get(dev), prod, first.record) + expect(again.result.applied).toEqual([]) + + const edited = await editDev(dev, { + 'skills/tone.md': TONE.replace('Be brief.', 'Be very brief.'), + 'workflows/ingest/workflow.ts': ingest('Be terse.'), + }) + const updated = await promoteRecorded(edited, prod, again.record) + expect( + updated.result.applied + .map((change) => `${change.action} ${change.kind}:${change.key}`) + .sort() + ).toEqual(['update skill:tone', 'update workflow:ingest']) + const settled = await promoteRecorded(edited, prod, updated.record) + expect(settled.result.applied).toEqual([]) + } + ) + check( "the target's own resources are kept, with a warning, unless the promotion allows deletes", async () => { diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 4be48aa466b..110ae4ac83e 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -289,6 +289,46 @@ describe('workspaces source apply', () => { expect(readFileSync(join(dir, 'tables/leads.ts'), 'utf8')).toBe(LEADS) }) + it('records what a promotion left in the target, and compares the next promotion with it rather than the target as read then', async () => { + const leads = (fingerprint: string) => ({ + id: 'tbl_prod', + fingerprint: `rf1:${fingerprint.repeat(64)}`, + }) + let live = { version: 1, workspaceId: 'ws_prod', resources: { 'table:leads': leads('a') } } + request.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path === '/api/v2/workspaces/ws_prod/source' && options.body === undefined) { + return { + data: { + workspaceId: 'ws_prod', + kinds: ['table'], + files: { 'tables/leads.ts': LEADS }, + blobs: [], + base: live, + skipped: [], + diagnostics: [], + }, + } + } + return answer({ workspaceId: 'ws_prod', base: live }) + }) + + await apply('--to', 'ws_prod', '--only', 'tables', '--dry-run') + expect(existsSync(join(dir, '.sim/targets/ws_prod.json'))).toBe(false) + await apply('--to', 'ws_prod', '--only', 'tables', '--yes') + expect(JSON.parse(readFileSync(join(dir, '.sim/targets/ws_prod.json'), 'utf8'))).toMatchObject({ + version: 1, + workspaceId: 'ws_prod', + resources: { 'table:leads': leads('a') }, + }) + + live = { ...live, resources: { 'table:leads': leads('b') } } + request.mockClear() + await apply('--to', 'ws_prod', '--only', 'tables', '--yes') + + const post = request.mock.calls.find(([, options]) => options.body !== undefined) + expect(post?.[1].body.base.resources['table:leads']).toEqual(leads('a')) + }) + it('counts the changes that delete data in a dry run, and says re-applying an older directory does not restore it', async () => { request.mockResolvedValue( answer({ diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 6fdd523985c..4bed6286492 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -12,7 +12,13 @@ import { } from '@sim/workspace-code' import { isBlobPath, KIND_LAYOUTS } from '@sim/workspace-code/kinds/registry' import { kindOfPath, WORKSPACE_BASE_FILE } from '@sim/workspace-code/layout' -import { promotionSubmission } from '@sim/workspace-code/promote' +import { + type PromotionRecord, + parsePromotionRecord, + promotionRecord, + promotionRecordPath, + promotionSubmission, +} from '@sim/workspace-code/promote' import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' import { setSoftExitCode } from '../../embed-context' @@ -364,11 +370,21 @@ function baseWorkspaceOf(base: unknown): string | null { return isRecordLike(base) && typeof base.workspaceId === 'string' ? base.workspaceId : null } +/** A promotion's submission, with what it needs to record the target once applied. */ +interface Promotion { + submission: WorkspaceSubmission + /** The directory's own text files, before the target's records replaced its workflows'. */ + files: Record<string, string> + /** The record the last promotion to this target left, if any. */ + record: PromotionRecord | undefined +} + /** * The submission that promotes `submission` (read for the target, so every workflow folder is * whole) to `target`: the target's get supplies its base and each workflow's lock and base, so - * its resources are bound by key and updated in place. Refuses a target the directory was read - * from, which a plain apply serves. + * its resources are bound by key and updated in place, and the target's promotion record (what + * the last promotion from this directory left there) is the base each resource it holds compares + * with. Refuses a target the directory was read from, which a plain apply serves. */ async function promoted( client: OperationClient<'getWorkspaceSource'>, @@ -376,7 +392,7 @@ async function promoted( submission: WorkspaceSubmission, target: string, allowDelete: boolean -): Promise<WorkspaceSubmission> { +): Promise<Promotion> { if (baseWorkspaceOf(submission.base) === target) { throw new SimApiError( `${dir} was read from workspace ${target}; apply it without --to.`, @@ -384,6 +400,10 @@ async function promoted( 'SOURCE_PROMOTE_SAME_WORKSPACE' ) } + const recordPath = promotionRecordPath(target) + const record = recordPath + ? parsePromotionRecord(await readSourceJson(dir, recordPath), target) + : undefined const { data } = await client.request<GetWorkspaceSourceResponse>('getWorkspaceSource', { params: { workspaceId: target }, query: { kinds: submission.kinds.join(',') }, @@ -397,9 +417,41 @@ async function promoted( blobPaths: submission.blobs.map((blob) => blob.path), kinds: submission.kinds, target: { files: data.files, base: targetBase }, + record, allowDelete, }) - return { ...submission, files: composed.files, base: composed.base } + return { + submission: { ...submission, files: composed.files, base: composed.base }, + files: submission.files, + record, + } +} + +/** + * Saves what a promotion applied to `target` as its promotion record in the directory, the base + * the next promotion there compares with. Nothing else in the directory changes. + */ +async function savePromotion( + dir: string, + target: string, + promotion: Promotion, + data: ApplyData +): Promise<void> { + const path = promotionRecordPath(target) + const base = parseWorkspaceBase(data.base).base + if (!path || !base) return + const record = promotionRecord({ + workspaceId: target, + previous: promotion.record, + kinds: promotion.submission.kinds, + files: promotion.files, + submission: { + files: promotion.submission.files, + base: parseWorkspaceBase(promotion.submission.base).base ?? base, + }, + applied: { base, files: data.files, plan: data.plan }, + }) + await updateSourceDir(dir, { write: [[path, `${JSON.stringify(record, null, 2)}\n`]] }) } /** @@ -425,7 +477,7 @@ export function attachWorkspaceSourceApply(source: Command): void { .option('--only <kinds>', `Apply only these kinds (${ONLY_KINDS_HELP})`) .option( '--to <workspaceId>', - "Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place; the directory is not changed" + "Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too" ) .allowExcessArguments(false) .description('Apply a workspace source directory') @@ -518,9 +570,10 @@ export function attachWorkspaceSourceApply(source: Command): void { ) } assertBlobsFetched(options.dir, read, written.digests) - const submission = options.to + const promotion = options.to ? await promoted(client, options.dir, read, options.to, options.allowDelete === true) - : read + : null + const submission = promotion?.submission ?? read if (submission.ignored.length > 0) { printError( `Not sent (a hidden name, or a character no workspace path holds): ${submission.ignored.join(', ')}` @@ -556,8 +609,11 @@ export function attachWorkspaceSourceApply(source: Command): void { const data = await send(submission, false) writing = false printNewDiagnostics(data.diagnostics, printed) - // A promotion leaves the directory bound to the workspace it was read from. - if (!options.to) { + // A promotion leaves the directory bound to the workspace it was read from, and records + // what it left in the target for the next promotion to compare with. + if (promotion && options.to) { + await savePromotion(options.dir, options.to, promotion, data) + } else { const workspaceCatalog = cachedCatalog(workspaceId) const records = workspaceCatalog ? await workflowFolderRecords(client, workspaceId, workspaceCatalog, data.files) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index b07edac2b31..728d6871713 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -146,7 +146,7 @@ Secrets never appear in source: a workflow reads an environment variable as \`en ## Promote to another workspace -\`sim workspaces source apply --dir . --to <workspaceId> --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name), and a change made in the target since is refused as stale (exit 5). Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w <workspaceId>\`) and connect its credentials, or the workflows using them are held back. This directory is not changed and stays bound to the workspace it was read from. Without \`--to\`, an apply refuses a directory read from another workspace (\`SOURCE_OTHER_WORKSPACE\`). +\`sim workspaces source apply --dir . --to <workspaceId> --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name). Each promote records what it left in the target in \`.sim/targets/<workspaceId>.json\` (commit it), and the next promote compares with that: a resource changed in the target since is kept when this directory left it alone, and refused as stale (exit 5) when this directory changed it too. A resource with no record (the first promote, or one created in the target since) is updated to match this directory; delete the record to take the target as it stands again. Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w <workspaceId>\`) and connect its credentials, or the workflows using them are held back. This directory is not changed and stays bound to the workspace it was read from. Without \`--to\`, an apply refuses a directory read from another workspace (\`SOURCE_OTHER_WORKSPACE\`). ## Good to know diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index c7e19d5f7c6..ac558e68f5e 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -313,9 +313,19 @@ each folder's `.sim/lock.json` and `.sim/base.json` replaced by the target's fol directory declares unless `--allow-delete`. The server sees an ordinary apply against its own base, so every rule holds unchanged: a resource bound under the key is updated in place (a workflow keeps its id and block ids through the target's lock), an unbound one of the same name is adopted, a -missing one created, a change in the target between the read and the apply is `E_APPLY_STALE`, and -secrets and credentials bind by name and key to the target's own. The answer is not saved: the -directory stays bound to the workspace it was read from. Without `--to`, the CLI refuses a +missing one created, and secrets and credentials bind by name and key to the target's own. The +directory stays bound to the workspace it was read from; the one thing a promotion saves is the +target's promotion record, `.sim/targets/<workspaceId>.json` (`promotionRecord`): the target's +base entries after the apply and, per workflow folder, the target workflow's fingerprint and a +digest of the folder as promoted. The next promotion composes with it: each base entry the record +holds for the same id stands in for the target's live one, so the server's three-way rule compares +with what the last promotion left (a target edit since is kept when the directory left the resource +alone, `E_APPLY_STALE` when it changed it too); a workflow folder whose digest is unchanged is sent +as its base alone (a no-op), and an edited one whole with the recorded fingerprint, so a target edit +since is the folder's 409. Without a record entry (the first promotion, or a resource created in +the target since) the live entry is the base and the target is updated to match the directory; +deleting the record re-takes the target as it stands. Read alone, the target's live base could +never look changed. Without `--to`, the CLI refuses a directory read from another workspace (`SOURCE_OTHER_WORKSPACE`); `--workspace <target>` instead applies it as a copy whose base the server ignores (`W_BASE_OTHER_WORKSPACE`), which only a first apply to an empty target survives (a second is `E_APPLY_NO_BASE` for everything it created). diff --git a/packages/workspace-code/src/promote.test.ts b/packages/workspace-code/src/promote.test.ts new file mode 100644 index 00000000000..ac47d5c83ac --- /dev/null +++ b/packages/workspace-code/src/promote.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest' +import type { WorkspaceBaseV1 } from './base' +import { + type PromotionRecord, + parsePromotionRecord, + promotionRecord, + promotionSubmission, +} from './promote' + +const folderBase = (workflowId: string, fingerprint: string) => + JSON.stringify({ version: 1, workflowId, workspaceId: 'ws_prod', fingerprint, catalogHash: 'c' }) + +const DIR = { + 'workflows/ingest/workflow.ts': '// @sim-sdk 1\n', + 'workflows/ingest/.sim/base.json': folderBase('wf_dev', 'sf1:dev'), +} + +const TARGET = { + files: { + 'workflows/ingest/workflow.ts': '// @sim-sdk 1\n// prod hotfix\n', + 'workflows/ingest/.sim/base.json': folderBase('wf_prod', 'sf1:hotfix'), + }, + base: { + version: 1, + workspaceId: 'ws_prod', + resources: { 'workflow:ingest': { id: 'wf_prod' } }, + } satisfies WorkspaceBaseV1, +} + +/** The record the first promotion of {@link DIR} leaves, with `plan` as the apply's. */ +function firstRecord(plan: Array<{ kind: string; key: string; action: string; held?: boolean }>) { + const submission = promotionSubmission({ + files: DIR, + blobPaths: [], + kinds: ['workflow'], + target: TARGET, + allowDelete: false, + }) + return promotionRecord({ + workspaceId: 'ws_prod', + previous: undefined, + kinds: ['workflow'], + files: DIR, + submission, + applied: { base: TARGET.base, files: {}, plan }, + }) +} + +describe('promotion records', () => { + it('does not record a workflow the apply held back, so the next promotion sends it again', () => { + const record = firstRecord([{ kind: 'workflow', key: 'ingest', action: 'update', held: true }]) + + expect(record.workflows).toEqual({}) + const next = promotionSubmission({ + files: DIR, + blobPaths: [], + kinds: ['workflow'], + target: TARGET, + record, + allowDelete: false, + }) + expect(next.files['workflows/ingest/workflow.ts']).toBe(DIR['workflows/ingest/workflow.ts']) + }) + + it('sends an unchanged folder as its base alone, and an edited one with the recorded fingerprint', () => { + const record = firstRecord([{ kind: 'workflow', key: 'ingest', action: 'noop' }]) + const promote = (files: Record<string, string>) => + promotionSubmission({ + files, + blobPaths: [], + kinds: ['workflow'], + target: { + ...TARGET, + files: { + ...TARGET.files, + 'workflows/ingest/.sim/base.json': folderBase('wf_prod', 'sf1:edited-in-prod'), + }, + }, + record, + allowDelete: false, + }) + + expect(Object.keys(promote(DIR).files)).toEqual(['workflows/ingest/.sim/base.json']) + const edited = promote({ ...DIR, 'workflows/ingest/workflow.ts': '// @sim-sdk 1\n// v2\n' }) + expect(JSON.parse(edited.files['workflows/ingest/.sim/base.json']).fingerprint).toBe( + 'sf1:hotfix' + ) + }) + + it('ignores a record for another target, or one this build cannot read', () => { + const record: PromotionRecord = { + version: 1, + workspaceId: 'ws_prod', + resources: {}, + workflows: {}, + } + + expect(parsePromotionRecord(record, 'ws_prod')).toEqual(record) + expect(parsePromotionRecord(record, 'ws_other')).toBeUndefined() + expect(parsePromotionRecord({ ...record, workflows: { ingest: 1 } }, 'ws_prod')).toBeUndefined() + expect(parsePromotionRecord({ ...record, version: 2 }, 'ws_prod')).toBeUndefined() + }) +}) diff --git a/packages/workspace-code/src/promote.ts b/packages/workspace-code/src/promote.ts index 2dd67473e81..4013c5dda91 100644 --- a/packages/workspace-code/src/promote.ts +++ b/packages/workspace-code/src/promote.ts @@ -3,13 +3,29 @@ * resources by key there. The target's own get answer supplies everything that records ids (the * root base and each workflow folder's lock and base), so the target is planned exactly as an * edit of its own checkout would be: a resource the target holds under the same key is updated - * in place (its ids, a workflow's block ids and so its webhook paths stay), a key it lacks is - * created or adopted by name, and a change made in the target since its get is stale. + * in place (its ids, a workflow's block ids and so its webhook paths stay), and a key it lacks is + * created or adopted by name. + * + * What the target held after the last promotion from this directory is its + * {@link PromotionRecord} (`.sim/targets/<workspaceId>.json`), and it is the base the three-way + * rule compares with: a resource changed in the target since then is kept when the directory left + * it alone and refused as stale when the directory changed it too. A resource the record does not + * hold for the target's id (the first promotion, or one created there since) takes the target as + * it stands. * * Client-safe and pure: the CLI composes the submission, the server sees an ordinary apply. */ +import { isRecordLike } from '@sim/utils/object' +import { compareStrings } from '@sim/utils/string' +import { parseSourceBase, sha256Hex, stableStringify } from '@sim/workflow-code' import { SOURCE_BASE_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' -import { parseResourceAddress, type WorkspaceBaseV1 } from './base' +import { + type BaseResource, + parseResourceAddress, + parseWorkspaceBase, + resourceAddress, + type WorkspaceBaseV1, +} from './base' import { isFileKind, KIND_DIRS, type WorkspaceKind } from './kind' import { fileKindGroupKey } from './kinds/entries' import { FILE_KIND_CODECS, KIND_LAYOUTS } from './kinds/registry' @@ -33,6 +49,8 @@ export interface PromotionInput { kinds: readonly WorkspaceKind[] /** The target's get answer for those kinds. */ target: { files: Readonly<Record<string, string>>; base: WorkspaceBaseV1 } + /** What the last promotion from this directory left in the target, when one is recorded. */ + record?: PromotionRecord /** * Whether the promotion may delete what the target holds and the directory does not declare. * Without it, those resources leave the base, so the apply keeps them with a warning instead of @@ -66,12 +84,105 @@ function declaredKeys( return new Set(read.decl ? Object.keys(read.decl) : []) } +/** + * `.sim/targets/<workspaceId>.json`: what one target workspace held after the last promotion from + * this directory, so the next promotion compares with it rather than with the target as it is + * read then (which would never look changed). + */ +export interface PromotionRecord { + version: 1 + /** The target workspace. */ + workspaceId: string + /** The target's root base entries after the last promotion, by resource address. */ + resources: WorkspaceBaseV1['resources'] + /** Each promoted workflow folder, by key. */ + workflows: Record<string, PromotedWorkflow> +} + +/** A workflow folder as the last promotion left it in the target. */ +interface PromotedWorkflow { + /** The target's workflow id. */ + workflowId: string + /** The target workflow's source fingerprint after the promotion (its folder base's). */ + fingerprint: string + /** {@link folderDigest} of the directory's folder as promoted. */ + digest: string +} + +/** The directory path of a target's promotion record. */ +export function promotionRecordPath(workspaceId: string): string | null { + return /^[A-Za-z0-9_-]{1,128}$/.test(workspaceId) ? `.sim/targets/${workspaceId}.json` : null +} + +/** The digest of a workflow folder's source files (its records excluded), by path and text. */ +function folderDigest(key: string, files: Readonly<Record<string, string>>): string { + const prefix = `${workflowFolder(key)}/` + const entries = Object.entries(files) + .filter(([path]) => path.startsWith(prefix) && !path.slice(prefix.length).startsWith('.sim/')) + .sort(([a], [b]) => compareStrings(a, b)) + return sha256Hex(stableStringify(entries)) +} + +/** A workflow folder base's text with `fingerprint` in place of its own, or `null` if unreadable. */ +function withFingerprint(text: string, fingerprint: string): string | null { + const parsed = parseSourceBase(text) + if (!('base' in parsed)) return null + return `${JSON.stringify({ ...parsed.base, fingerprint }, null, 2)}\n` +} + +/** The workflow a folder base's text names, or `null` if unreadable. */ +function folderWorkflow( + text: string | undefined +): { workflowId: string; fingerprint: string } | null { + if (text === undefined) return null + const parsed = parseSourceBase(text) + return 'base' in parsed + ? { workflowId: parsed.base.workflowId, fingerprint: parsed.base.fingerprint } + : null +} + +/** + * Reads a promotion record for `workspaceId`; `undefined` for anything else (no record, another + * target's, an unreadable one), which makes the promotion take the target as it stands. + */ +export function parsePromotionRecord( + input: unknown, + workspaceId: string +): PromotionRecord | undefined { + if (!isRecordLike(input) || input.version !== 1 || input.workspaceId !== workspaceId) { + return undefined + } + const base = parseWorkspaceBase({ version: 1, workspaceId, resources: input.resources }).base + if (!base || !isRecordLike(input.workflows)) return undefined + const workflows: Record<string, PromotedWorkflow> = {} + for (const [key, entry] of Object.entries(input.workflows)) { + if ( + !isRecordLike(entry) || + typeof entry.workflowId !== 'string' || + typeof entry.fingerprint !== 'string' || + typeof entry.digest !== 'string' + ) { + return undefined + } + workflows[key] = { + workflowId: entry.workflowId, + fingerprint: entry.fingerprint, + digest: entry.digest, + } + } + return { version: 1, workspaceId, resources: base.resources, workflows } +} + /** * The submission that promotes a directory to the target its get answer describes: the * directory's source with every workflow folder's lock and base replaced by the target's folder * of the same key (or dropped, so a key the target lacks creates a workflow), and the target's - * base. Values never travel: secrets and credentials are names and keys the target binds to its - * own. + * base, narrowed to what the directory declares unless deletes are allowed. With a + * {@link PromotionRecord}, each entry the record holds for the same id stands in for the target's + * own, and a workflow folder records the fingerprint the last promotion left: sent as its base + * alone (a no-op) when the directory has not changed it since, else whole, so a target edit since + * is refused as stale. Values never travel: secrets and credentials are names and keys the target + * binds to its own. */ export function promotionSubmission(input: PromotionInput): PromotionSubmission { const files: Record<string, string> = {} @@ -83,20 +194,97 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission input.kinds.map((kind) => [kind, declaredKeys(kind, files, paths)] as const) ) for (const key of declared.get('workflow') ?? []) { - for (const record of [SOURCE_LOCK_FILE, SOURCE_BASE_FILE]) { - const path = `${workflowFolder(key)}/${record}` - const text = input.target.files[path] - if (text !== undefined) files[path] = text + const folder = workflowFolder(key) + const basePath = `${folder}/${SOURCE_BASE_FILE}` + const lockPath = `${folder}/${SOURCE_LOCK_FILE}` + const targetBase = input.target.files[basePath] + const live = folderWorkflow(targetBase) + const promoted = input.record?.workflows[key] + if (targetBase === undefined || !live || promoted?.workflowId !== live.workflowId) { + for (const path of [basePath, lockPath]) { + const text = input.target.files[path] + if (text !== undefined) files[path] = text + } + continue + } + if (promoted.digest === folderDigest(key, input.files)) { + for (const path of Object.keys(files)) { + if (path.startsWith(`${folder}/`)) delete files[path] + } + files[basePath] = targetBase + continue } + files[basePath] = withFingerprint(targetBase, promoted.fingerprint) ?? targetBase + const lock = input.target.files[lockPath] + if (lock !== undefined) files[lockPath] = lock } const resources = Object.fromEntries( - Object.entries(input.target.base.resources).filter(([address]) => { + Object.entries(input.target.base.resources).flatMap(([address, live]) => { const parsed = parseResourceAddress(address) - if (!parsed) return false + if (!parsed) return [] const keys = declared.get(parsed.kind) - if (keys === undefined) return false - return input.allowDelete || keys.has(parsed.key) + if (keys === undefined || !(input.allowDelete || keys.has(parsed.key))) return [] + const recorded: BaseResource | undefined = + input.record?.resources[resourceAddress(parsed.kind, parsed.key)] + return [[address, recorded?.id === live.id ? recorded : live] as const] }) ) return { files, base: { ...input.target.base, resources } } } + +/** + * The record a promotion leaves for its target, from the apply's answer (`applied`: the target's + * base after it, the files it reprinted, among them each written workflow folder's base, and its + * plan). A resource's entry is the answer's base entry; a workflow folder's is what the apply + * wrote, or what was sent whole and planned as a no-op, else the `previous` entry (a folder sent + * as its base alone, or held back, or never reached). The record of a kind the promotion did not + * manage is kept. + */ +export function promotionRecord(input: { + workspaceId: string + previous: PromotionRecord | undefined + kinds: readonly WorkspaceKind[] + /** The directory's text files, as {@link PromotionInput} took them. */ + files: Readonly<Record<string, string>> + submission: PromotionSubmission + applied: { + base: WorkspaceBaseV1 + files: Readonly<Record<string, string>> + plan: ReadonlyArray<{ kind: string; key: string; action: string; held?: boolean }> + } +}): PromotionRecord { + const managed = new Set(input.kinds) + const resources: WorkspaceBaseV1['resources'] = {} + for (const [address, entry] of Object.entries(input.previous?.resources ?? {})) { + const parsed = parseResourceAddress(address) + if (parsed && !managed.has(parsed.kind)) { + resources[resourceAddress(parsed.kind, parsed.key)] = entry + } + } + Object.assign(resources, input.applied.base.resources) + const workflows: Record<string, PromotedWorkflow> = {} + if (!managed.has('workflow')) { + Object.assign(workflows, input.previous?.workflows) + return { version: 1, workspaceId: input.workspaceId, resources, workflows } + } + const paths = Object.keys(input.files).filter((path) => kindOfPath(path) === 'workflow') + for (const key of KIND_LAYOUTS.workflow.group(paths).groups.keys()) { + const folder = `${workflowFolder(key)}/` + const basePath = `${folder}${SOURCE_BASE_FILE}` + const change = input.applied.plan.find( + (planned) => planned.kind === 'workflow' && planned.key === key + ) + const sentWhole = Object.keys(input.submission.files).some( + (path) => path.startsWith(folder) && path !== basePath + ) + const left = + folderWorkflow(input.applied.files[basePath]) ?? + (sentWhole && change?.action === 'noop' && !change.held + ? folderWorkflow(input.submission.files[basePath]) + : null) + const previous = input.previous?.workflows[key] + if (left) workflows[key] = { ...left, digest: folderDigest(key, input.files) } + else if (previous) workflows[key] = previous + } + return { version: 1, workspaceId: input.workspaceId, resources, workflows } +} From 0de769104d8576dcadf7a47f657d5a8544fb038b Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:20:42 -0800 Subject: [PATCH 15/59] fix(workspace-source): persist table column keys so every read prints them A column key lived only in the directory's base, so a fresh get (or a promotion's read of the target) keyed each column by its name: `note: column.string({ name: 'note_text' })` came back as `note_text: column.string()`, against the documented promise that the key stays bound to the column. Every table write now records each declared column whose key is not its name in metadata.sourceColumnKeys (column id -> key), reads print base binding, then that record, then the name, and an apply that finds a column a read would key otherwise records the keys alone. --- apps/sim/lib/table/application/reshape.ts | 35 +++- apps/sim/lib/table/types.ts | 7 + .../application/kinds/table/adapter.ts | 156 +++++++++++++----- .../application/kinds/table/stored.ts | 27 ++- .../application/rows.integration.ts | 107 ++++++++++++ packages/workspace-code/DESIGN.md | 7 + 6 files changed, 283 insertions(+), 56 deletions(-) diff --git a/apps/sim/lib/table/application/reshape.ts b/apps/sim/lib/table/application/reshape.ts index a97cff3853f..edc08005051 100644 --- a/apps/sim/lib/table/application/reshape.ts +++ b/apps/sim/lib/table/application/reshape.ts @@ -80,6 +80,11 @@ export interface ReshapeTableInput { * reshape leaves the columns (`null` clears the mark); absent leaves it alone. */ sourceRows?: { mode: 'managed'; keyColumn: string } | null + /** + * Records each source column's key, by column name as the reshape leaves the columns + * (`metadata.sourceColumnKeys`, replaced whole); absent leaves the record alone. + */ + sourceColumnKeys?: Record<string, string> /** * The row version the caller read the rows at. Checked under the table's lock before anything * changes: a row written since refuses the reshape (`TableRowsChangedError`), so `rows` @@ -143,6 +148,20 @@ function withSourceRows( return sourceRows ? { ...rest, sourceRows } : rest } +/** Source column keys by column name, as `metadata.sourceColumnKeys` holds them: by column id. */ +function sourceColumnKeysById( + keysByName: Readonly<Record<string, string>>, + idByName: ReadonlyMap<string, string> +): Record<string, string> { + const byId: Record<string, string> = {} + for (const [name, key] of Object.entries(keysByName)) { + const id = idByName.get(name) + if (!id) throw new OrchestrationError('validation', `Column "${name}" does not exist`) + byId[id] = key + } + return byId +} + /** A column name no column of `table` holds, to park a column under during a rename cycle. */ function freeColumnName(table: TableDefinition): string { const taken = new Set(table.schema.columns.map((column) => column.name.toLowerCase())) @@ -376,11 +395,17 @@ export const reshapeTableUseCase = defineAuthorizedTableUseCase({ }) } const idByName = buildIdByName(table.schema) - if (input.sourceRows !== undefined) { - const metadata = withSourceRows( - table.metadata, - sourceRowsMark(input.sourceRows, idByName) - ) + if (input.sourceRows !== undefined || input.sourceColumnKeys !== undefined) { + let metadata: TableMetadata = table.metadata ?? {} + if (input.sourceRows !== undefined) { + metadata = withSourceRows(metadata, sourceRowsMark(input.sourceRows, idByName)) + } + if (input.sourceColumnKeys !== undefined) { + metadata = { + ...metadata, + sourceColumnKeys: sourceColumnKeysById(input.sourceColumnKeys, idByName), + } + } await tx.trx .update(userTableDefinitions) .set({ metadata }) diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 9e1abd5533c..2ccc3bf1060 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -303,6 +303,13 @@ export interface TableMetadata { * the workspace as source prints them. Written only by the reshape that applies them. */ sourceRows?: TableSourceRows + /** + * Each source column's key (`columns: { <key>: ... }` in workspace source) by **column id**, as + * the workspace source write that last wrote the table declared it, so reading the table as + * source prints a column under its key even after a rename. Written only by that write; a column + * without an entry is keyed by its name. + */ + sourceColumnKeys?: Record<string, string> } /** Rows workspace source manages: the whole table, each row matched by its key column. */ diff --git a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts index 7824cbb7272..6a3c2b3eb1c 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts @@ -49,6 +49,7 @@ import { deleteTableUseCase, listTableDefinitionsUseCase, } from '@/lib/table/application/tables' +import { getColumnId } from '@/lib/table/column-keys' import { getTableById } from '@/lib/table/service' import type { RowData } from '@/lib/table/types' import { validateRowSize } from '@/lib/table/validation' @@ -80,6 +81,7 @@ import { import { printStoredTable, type StoredTableState, + sourceColumns, storedFingerprint, tableAsDecl, } from '@/lib/workspace-source/application/kinds/table/stored' @@ -111,6 +113,11 @@ export interface TablePlan { /** The base's column bindings the plan's fingerprint was read under. */ bindings?: Record<string, BaseColumn> rows?: RowsPlan + /** + * The column keys the write records (`metadata.sourceColumnKeys`), by stored column name: each + * declared column whose key is not its name. Absent leaves the record alone. + */ + keys?: Record<string, string> } type TableChange = PlannedChange<'table'> @@ -162,6 +169,57 @@ function namesByKey(decl: TableDecl): Map<string, string> { ) } +/** The keys `decl` gives its columns where a key is not the column's name, by column name. */ +function declaredKeys(decl: TableDecl): Record<string, string> { + const keys: Record<string, string> = {} + for (const [key, column] of Object.entries(decl.columns)) { + const name = columnName(key, column) + if (name !== key) setOwn(keys, name, key) + } + return keys +} + +/** + * Whether a read of the stored table without a base (a fresh get, or a promotion's read of the + * target) would print a declared column under another key than `decl` gives it. + */ +function keysUnrecorded(state: StoredTableState, decl: TableDecl): boolean { + const { keyById } = tableAsDecl(state) + const idByName = new Map( + sourceColumns(state.table).map((column) => [column.name, getColumnId(column)]) + ) + return Object.entries(decl.columns).some(([key, column]) => { + const id = idByName.get(columnName(key, column)) + return id !== undefined && keyById.get(id) !== key + }) +} + +/** + * `change` recording `decl`'s column keys: every table write records them, and a no-op becomes a + * write of the record alone when a read without a base would print a column under another key. + */ +function withKeys( + change: TableChange, + state: StoredTableState, + decl: TableDecl, + entry: BaseResource | undefined +): TableChange { + const keys = declaredKeys(decl) + if (change.action === 'update' || change.action === 'adopt') { + return { ...change, detail: { ...change.detail, keys } } + } + if (change.action !== 'noop' || !keysUnrecorded(state, decl)) return change + const bindings = entry?.id === state.table.id ? entry.columns : undefined + return { + ...change, + action: 'update', + storedId: state.table.id, + fields: ['column keys'], + storedFingerprint: storedFingerprint(state, bindings), + detail: { ...(bindings ? { bindings } : {}), keys }, + } +} + /** The managed mark a declaration asks for, by its key column's stored name. */ function markOf(decl: TableDecl): { mode: 'managed'; keyColumn: string } | null { const key = decl.rows?.mode === 'managed' ? decl.rows.key : undefined @@ -456,7 +514,8 @@ async function planTable( // A table whose declaration is unchanged can still hold rows to write. const settled = planned.settled const resource = resolution.action === 'noop' ? resolution.stored : undefined - if (!resource || settled.action !== 'noop' || !decl.rows) return settled + if (!resource || settled.action !== 'noop') return settled + if (!decl.rows) return withKeys(settled, resource.state, decl, entry) const rows = await planRows(key, decl, resource.state, entry, ctx) const renamed = rows.plan?.write ? columnsRenamedOnServer(decl, resource.state, entry) : [] if (renamed.length > 0) { @@ -471,7 +530,7 @@ async function planTable( ) return settled } - return withRows(settled, rows, resource.state, entry) + return withKeys(withRows(settled, rows, resource.state, entry), resource.state, decl, entry) } switch (planned.action) { case 'create': { @@ -491,7 +550,7 @@ async function planTable( ...rows.fields, ], pending: pendingOf({ ...change, action: 'create' }, { columns: declaredColumnNames(decl) }), - detail: { rows: rows.plan }, + detail: { rows: rows.plan, keys: declaredKeys(decl) }, } } case 'adopt': @@ -533,45 +592,51 @@ async function planTable( } const action = planned.action === 'adopt' ? 'adopt' : changed ? 'update' : 'noop' const storedId = planned.stored.id - return { - ...change, - action, - storedId, - // A later workflow's dry run checks its row writes against the columns this declares. - ...(action !== 'noop' - ? { - pending: pendingOf( - { ...change, action, storedId }, - { columns: declaredColumnNames(decl) } - ), - } - : {}), - ...(changes.fields.length + rows.fields.length > 0 - ? { fields: [...changes.fields, ...rows.fields] } - : {}), - ...(dropped > 0 || changes.lostCells > 0 || rows.removed > 0 - ? { - destructive: { - what: 'cells', - counts: { - columns: dropped, - ...(changes.removedOptions > 0 ? { options: changes.removedOptions } : {}), - cells: changes.lostCells, - ...(rows.removed > 0 ? { rows: rows.removed } : {}), + const entryOfStored = planned.action === 'update' ? planned.base : undefined + return withKeys( + { + ...change, + action, + storedId, + // A later workflow's dry run checks its row writes against the columns this declares. + ...(action !== 'noop' + ? { + pending: pendingOf( + { ...change, action, storedId }, + { columns: declaredColumnNames(decl) } + ), + } + : {}), + ...(changes.fields.length + rows.fields.length > 0 + ? { fields: [...changes.fields, ...rows.fields] } + : {}), + ...(dropped > 0 || changes.lostCells > 0 || rows.removed > 0 + ? { + destructive: { + what: 'cells', + counts: { + columns: dropped, + ...(changes.removedOptions > 0 ? { options: changes.removedOptions } : {}), + cells: changes.lostCells, + ...(rows.removed > 0 ? { rows: rows.removed } : {}), + }, }, - }, - } - : {}), - storedFingerprint: planFingerprint(storedFingerprint(state, bindings), rows.plan), - ...(action === 'noop' && rows.base - ? { base: { ...printStoredTable(state, key, bindings).base, rows: rows.base } } - : {}), - detail: { - changes, - ...(bindings ? { bindings } : {}), - ...(rows.plan ? { rows: rows.plan } : {}), + } + : {}), + storedFingerprint: planFingerprint(storedFingerprint(state, bindings), rows.plan), + ...(action === 'noop' && rows.base + ? { base: { ...printStoredTable(state, key, bindings).base, rows: rows.base } } + : {}), + detail: { + changes, + ...(bindings ? { bindings } : {}), + ...(rows.plan ? { rows: rows.plan } : {}), + }, }, - } + state, + decl, + entryOfStored + ) } } } @@ -663,6 +728,7 @@ function reshapeInput( ...(rows?.write ? { rows: rows.write } : {}), ...(rows?.mark !== undefined ? { sourceRows: rows.mark } : {}), ...(rows?.version !== undefined ? { expectedRowsVersion: rows.version } : {}), + ...(plan?.keys !== undefined ? { sourceColumnKeys: plan.keys } : {}), } if (!plan?.changes) return { tableId, workspaceId, ...rowsInput } const { table, columns } = plan.changes @@ -804,17 +870,21 @@ export const tableAdapter: KindAdapter<'table', StoredTableState> = { }, }) const rows = change.detail?.rows - if (rows?.write || rows?.mark) { + const keys = change.detail?.keys + const recordsKeys = keys !== undefined && Object.keys(keys).length > 0 + if (rows?.write || rows?.mark || recordsKeys) { await reshapeTableUseCase.execute({ principal, - input: reshapeInput(ctx, table.id, { rows }), + input: reshapeInput(ctx, table.id, { rows, ...(recordsKeys ? { keys } : {}) }), }) } return written(ctx, change.key, table.id, decl) } const tableId = change.storedId const plan = change.detail - if (!tableId || !(plan?.changes || plan?.rows)) throw new Error('A table update needs its plan') + if (!tableId || !(plan?.changes || plan?.rows || plan?.keys)) { + throw new Error('A table update needs its plan') + } await reshapeTableUseCase.execute({ principal, input: reshapeInput(ctx, tableId, plan) }) return written(ctx, change.key, tableId, decl) }, diff --git a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts index d632e0425da..ba9e9e85f5e 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts @@ -55,23 +55,34 @@ function columnDecl(column: ColumnDefinition, key: string): TableColumnDecl { /** * A stored table as the declaration it prints as. A column keeps the key `bindings` (the base's - * column key → id) gives it; any other column is keyed by its name, so a fresh directory matches - * columns by name. + * column key → id) gives it, else the key the last source write recorded for it + * (`metadata.sourceColumnKeys`), so a fresh directory and another workspace's read print the key + * the declaration gave it; any other column is keyed by its name. */ export function tableAsDecl( state: StoredTableState, bindings?: Readonly<Record<string, BaseColumn>> ): TableAsDecl { const boundKey = new Map(Object.entries(bindings ?? {}).map(([key, column]) => [column.id, key])) + const recordedKey = state.table.metadata?.sourceColumnKeys ?? {} const stored = sourceColumns(state.table) const keyById = new Map<string, string>() const taken = new Set<string>() - // Bound keys first, so a column keyed by its name never takes a renamed column's key. - for (const column of stored) { - const key = boundKey.get(getColumnId(column)) - if (key === undefined || !isDeclarableColumnKey(key) || taken.has(key)) continue - keyById.set(getColumnId(column), key) - taken.add(key) + // Bound keys first, then recorded ones, so a column keyed by its name never takes a renamed + // column's key. + for (const keyOf of [ + (id: string) => boundKey.get(id), + (id: string) => (Object.hasOwn(recordedKey, id) ? recordedKey[id] : undefined), + ]) { + for (const column of stored) { + const id = getColumnId(column) + const key = keyOf(id) + if (keyById.has(id) || key === undefined || !isDeclarableColumnKey(key) || taken.has(key)) { + continue + } + keyById.set(id, key) + taken.add(key) + } } for (const column of stored) { const id = getColumnId(column) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index 444d1a986fc..71cf87543d5 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -19,6 +19,7 @@ import { generateId } from '@sim/utils/id' import { compareStrings } from '@sim/utils/string' import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, vi } from 'vitest' +import { updateTableColumnUseCase } from '@/lib/table/application/columns' import { createTableRows } from '@/lib/table/application/rows' import { readCurrentRowsVersion } from '@/lib/table/row-changes' import { getTableById } from '@/lib/table/service' @@ -347,6 +348,112 @@ describe('table rows as workspace source', () => { expect(firedEvents()).toEqual([]) }) + check( + "a column's key survives a fresh get and a rename in Sim, and a table written before keys were recorded records them", + async () => { + const workspaceId = await newWorkspace() + const keyed = `// @sim-sdk 1 +import { column, defineTable } from '@sim/workflow-sdk/workspace' + +export default defineTable({ + name: 'keyed', + columns: { + sku: column.string({ required: true, unique: true }), + note: column.string({ name: 'note_text' }), + }, +}) +` + const created = await apply(workspaceId, { 'tables/keyed.ts': keyed }) + expect(created.failed).toBeUndefined() + expect((await runner.get(workspaceId)).files['tables/keyed.ts']).toContain( + "note: column.string({ name: 'note_text' })" + ) + + const [table] = await db + .select({ id: userTableDefinitions.id, metadata: userTableDefinitions.metadata }) + .from(userTableDefinitions) + .where( + and( + eq(userTableDefinitions.workspaceId, workspaceId), + isNull(userTableDefinitions.archivedAt) + ) + ) + await updateTableColumnUseCase.execute({ + principal: owner, + input: { + tableId: table.id, + workspaceId, + columnName: 'note_text', + updates: { name: 'memo' }, + }, + }) + expect((await runner.get(workspaceId)).files['tables/keyed.ts']).toContain( + "note: column.string({ name: 'memo' })" + ) + + await db + .update(userTableDefinitions) + .set({ metadata: {} }) + .where(eq(userTableDefinitions.id, table.id)) + const legacy = await runner.get(workspaceId) + expect(legacy.files['tables/keyed.ts']).toContain('memo: column.string()') + const recorded = await apply( + workspaceId, + { 'tables/keyed.ts': keyed.replace("name: 'note_text'", "name: 'memo'") }, + { base: legacy.base } + ) + expect(recorded.applied).toMatchObject([ + { kind: 'table', key: 'keyed', action: 'update', fields: ['column keys'] }, + ]) + expect((await runner.get(workspaceId)).files['tables/keyed.ts']).toContain( + "note: column.string({ name: 'memo' })" + ) + } + ) + + check( + 'managed rows resent against a fresh read plan as no change, a column keyed apart from its name included, as a promotion resends them', + async () => { + const workspaceId = await newWorkspace() + const typed = `// @sim-sdk 1 +import { column, defineTable } from '@sim/workflow-sdk/workspace' + +export default defineTable({ + name: 'typed', + columns: { + sku: column.string({ required: true, unique: true }), + shipped: column.date(), + expires: column.ttl(), + price: column.currency({ currencyCode: 'EUR' }), + labels: column.select({ options: ['red', 'green', 'blue'], multiple: true }), + status: column.select({ options: ['new', 'won'] }), + meta: column.json(), + note_text: column.string({ name: 'memo' }), + }, + rows: { mode: 'managed', key: 'sku' }, +}) +` + const directory = { + 'tables/typed.ts': typed, + 'tables/typed.rows.jsonl': [ + '{"sku":"A-1","shipped":"2026-11-02T10:30:00Z","expires":"2027-01-01T00:00:00-00:00","price":19.99,"labels":["red","blue"],"status":"new","meta":{"a":[1,{"b":null}]}}', + '{"sku":"A-2","shipped":"2026-11-01","labels":[],"status":"won","meta":[1,"x",true]}', + '{"sku":"A-9","note_text":"line1\\nline2"}', + ].join('\n'), + } + const created = await apply(workspaceId, directory) + expect(created.failed).toBeUndefined() + const versionBefore = await version(workspaceId) + const read = await runner.get(workspaceId) + expect(read.files['tables/typed.rows.jsonl']).not.toBe(directory['tables/typed.rows.jsonl']) + + const resent = await apply(workspaceId, directory, { base: read.base }) + + expect(resent.applied).toEqual([]) + expect(await version(workspaceId)).toBe(versionBefore) + } + ) + check('a key rename is a delete and an insert, gated by allow-delete', async () => { const workspaceId = await newWorkspace() const read = await managedPlans(workspaceId) diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index ac558e68f5e..067ad309d2e 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -73,6 +73,13 @@ deleted. A workflow still names a workspace file by its resource key (`file('rep its path: a file an apply creates claims the key a get would first allocate it, so a workflow of the same apply resolves it. +A table's column keys persist with the table too: every table write records each declared column +whose key is not its name in `metadata.sourceColumnKeys` (column id → key), and a read prints a +column under the base's binding, else that record, else its name. So a fresh get, and the target's +read in a promotion, print the key the declaration gave the column (also after a rename in Sim), and +a promotion's managed rows match the stored ones by the same keys. An apply that finds a column a +read would print under another key records the keys alone (`fields: ['column keys']`). + ## Fingerprints and the three-way rule A resource's fingerprint is `fingerprintOf(canonical(decl))` = `rf1:` + SHA-256 of the canonical From 92c353913703361beb97f57d80c951550d4e7740 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:20:43 -0800 Subject: [PATCH 16/59] test(workspace-source): promoting managed rows converges Managed rows promoted to another workspace planned an update on every run: the target's read keyed a column whose key is not its name by that name, so the stored rows never fingerprinted like the directory's. Persisting column keys (previous commit) fixes it; this pins the promotion round trip. --- .../application/promote.integration.ts | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/apps/sim/lib/workspace-source/application/promote.integration.ts b/apps/sim/lib/workspace-source/application/promote.integration.ts index e8d4e4b3c73..a97775eb8aa 100644 --- a/apps/sim/lib/workspace-source/application/promote.integration.ts +++ b/apps/sim/lib/workspace-source/application/promote.integration.ts @@ -302,6 +302,36 @@ describe('workspace source promotion', () => { } ) + check('promoting managed rows converges: the second promotion changes nothing', async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + const directory = { + 'tables/stock.ts': `// @sim-sdk 1 +import { column, defineTable } from '@sim/workflow-sdk/workspace' + +export default defineTable({ + name: 'stock', + columns: { + sku: column.string({ required: true, unique: true }), + status: column.select({ options: ['new', 'won'] }), + note_text: column.string({ name: 'memo' }), + }, + rows: { mode: 'managed', key: 'sku' }, +}) +`, + 'tables/stock.rows.jsonl': '{"sku":"A-1","status":"new","note_text":"hi"}\n{"sku":"A-2"}\n', + } + const applied = await runner.apply(dev, { files: directory, kinds: ['table'] }) + const devDir = { ...(await runner.get(dev)), files: applied.files } + + const first = await promote(devDir, prod) + expect(first.applied.map((change) => `${change.action} ${change.kind}:${change.key}`)).toEqual([ + 'create table:stock', + ]) + const second = await promote(devDir, prod) + expect(second.applied).toEqual([]) + }) + check( "the target's own resources are kept, with a warning, unless the promotion allows deletes", async () => { From ebddf195701772a47bc2c19bd30f105859916827 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:29:36 -0800 Subject: [PATCH 17/59] fix(workspace-source): name unsent paths, exit 1 at the findings cap, keep seed rows - Each file under a kind folder the CLI does not send (a hidden name, or a character no workspace path holds) is a W_PATH_NOT_SENT warning, printed and in the JSON result and its warnings count, instead of one unstructured "Not sent" line. - The per-file findings cap ends a list with I_FINDINGS_TRUNCATED (info) instead of E_LIMIT_EXCEEDED, so plain source errors exit 1, not 7 (too large); the offline check keeps each rows finding's own severity. - A table seeded by source records { mode: 'seed' } in metadata.sourceRows, so a get prints rows: { mode: 'seed' }, and the CLI keeps the seed rows file beside it; a declaration that starts or stops seeding records or clears the mark. - I_KB_CHUNKING counts the stored documents that keep their chunks, not those the apply replaces or deletes. - A replaced knowledge document keeps its tag values: the SDK docs said they were lost. --- .../content/docs/workflows/workspace-code.mdx | 3 +- apps/docs/openapi-v2-workflows.json | 3 ++ apps/sim/lib/table/application/reshape.ts | 3 +- apps/sim/lib/table/types.ts | 24 +++++++---- .../application/kinds/knowledge/diff.ts | 15 ++++++- .../application/kinds/table/adapter.ts | 34 +++++++++++---- .../application/kinds/table/stored.ts | 13 ++++-- .../application/knowledge.integration.ts | 21 ++++++++++ .../application/rows.integration.ts | 41 ++++++++++++++++++- .../protocol/workspace-source-apply.test.ts | 23 +++++++++++ .../protocol/workspace-source-apply.ts | 34 +++++++++++---- .../protocol/workspace-source-check.test.ts | 24 +++++++++++ .../protocol/workspace-source-check.ts | 12 +++++- .../commands/protocol/workspace-source-dir.ts | 31 +++++++++++++- .../protocol/workspace-source-get.test.ts | 19 +++++++++ .../commands/protocol/workspace-source-get.ts | 3 ++ packages/sim-cli/src/generated/v2-api.ts | 3 ++ .../src/codegen/workspace-text.generated.ts | 2 +- .../workflow-sdk/src/workspace/knowledge.d.ts | 2 +- packages/workspace-code/DESIGN.md | 5 ++- .../workspace-code/src/declaration/fields.ts | 2 +- .../src/declaration/read.test.ts | 7 +++- .../workspace-code/src/diagnostics/codes.ts | 13 +++++- .../workspace-code/src/diagnostics/finding.ts | 4 +- .../src/kinds/knowledge.test.ts | 2 +- .../workspace-code/src/kinds/knowledge.ts | 5 ++- .../src/kinds/literal-secrets.test.ts | 2 +- 27 files changed, 297 insertions(+), 53 deletions(-) diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index 54cb4f379b9..84665419e93 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -79,6 +79,7 @@ To apply the rest with a workspace key, such as in CI, narrow the apply to the k - A resource the directory never managed is kept, and the apply names it in a warning. - Something deleted in Sim is not re-created by an unchanged directory, and an edit to it is refused as stale. - Table rows are part of the directory only when the table's `rows` opts in (`tables/<key>.rows.jsonl`). Connector-synced documents and run history never are. +- Seed rows (`rows: { mode: 'seed' }`) belong to the directory: `get` keeps the declaration and the rows file beside it, but Sim does not store them, so a get into a new directory has the declaration without the file. - A delete cannot be undone by applying an older copy of the directory, such as a reverted commit. The resource stays deleted, and its rows, documents, run history, webhook URLs and schedules are gone. The dry run marks each change that deletes data and counts them as `destructive`. ## Promote to another workspace @@ -110,7 +111,7 @@ Commit the whole directory, including `.sim/`: its base files are what an apply - Renaming a skill does not rewrite the workflows that use it: each agent keeps the name it stored until its skills list is next edited. - One apply sends at most 100 edited workflows and makes at most 1,000 changes; a directory holds at most 500 resources of a kind, and a `tables/<key>.rows.jsonl` at most 10,000 rows. An apply over a limit is refused before anything is written. - A document or file over 10 MB, or whose name a directory cannot hold, is left out of `get` and listed in `.sim/skipped.json`. An apply never writes or deletes it. -- Hidden files (a name starting with a dot, such as `files/.env`) are never sent. An apply names them in a warning. +- Hidden files (a name starting with a dot, such as `files/.env`) and names with a character no workspace path holds (such as `:` or `?`) are never sent. An apply names each one in a `W_PATH_NOT_SENT` warning. - A knowledge base's `embeddingModel` is fixed when it is created. An apply that changes it is refused. - An apply that stops part-way exits `8` and saves what it wrote into the directory. Run the same apply again until it exits `0`. - An apply exits `5` when a resource changed in Sim after you read it. Run `get` into a new directory and reapply your edit. diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 8bdfbfac91f..5ff4b95da7c 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -25117,6 +25117,7 @@ "E_DECL_SHAPE", "E_DECL_UNKNOWN_FIELD", "E_DECL_UNKNOWN_FILE", + "W_PATH_NOT_SENT", "E_BLOB_MANIFEST", "E_BLOB_MISSING", "E_LIMIT_EXCEEDED", @@ -25680,6 +25681,7 @@ "E_DECL_SHAPE", "E_DECL_UNKNOWN_FIELD", "E_DECL_UNKNOWN_FILE", + "W_PATH_NOT_SENT", "E_BLOB_MANIFEST", "E_BLOB_MISSING", "E_LIMIT_EXCEEDED", @@ -26086,6 +26088,7 @@ "E_DECL_SHAPE", "E_DECL_UNKNOWN_FIELD", "E_DECL_UNKNOWN_FILE", + "W_PATH_NOT_SENT", "E_BLOB_MANIFEST", "E_BLOB_MISSING", "E_LIMIT_EXCEEDED", diff --git a/apps/sim/lib/table/application/reshape.ts b/apps/sim/lib/table/application/reshape.ts index edc08005051..f61f12172e4 100644 --- a/apps/sim/lib/table/application/reshape.ts +++ b/apps/sim/lib/table/application/reshape.ts @@ -79,7 +79,7 @@ export interface ReshapeTableInput { * Marks the rows as managed by workspace source, matched by the column named `keyColumn` as the * reshape leaves the columns (`null` clears the mark); absent leaves it alone. */ - sourceRows?: { mode: 'managed'; keyColumn: string } | null + sourceRows?: { mode: 'managed'; keyColumn: string } | { mode: 'seed' } | null /** * Records each source column's key, by column name as the reshape leaves the columns * (`metadata.sourceColumnKeys`, replaced whole); absent leaves the record alone. @@ -132,6 +132,7 @@ function sourceRowsMark( idByName: ReadonlyMap<string, string> ): TableSourceRows | null { if (!input) return null + if (input.mode === 'seed') return input const keyColumnId = idByName.get(input.keyColumn) if (!keyColumnId) { throw new OrchestrationError('validation', `Column "${input.keyColumn}" does not exist`) diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 2ccc3bf1060..7ad7a797cf5 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -299,8 +299,10 @@ export interface TableMetadata { */ hiddenColumns?: string[] /** - * Set while workspace source manages the table's rows (`rows: { mode: 'managed' }`), so reading - * the workspace as source prints them. Written only by the reshape that applies them. + * Set while a workspace source declaration opts the table's rows in (`rows: { mode: 'managed' }` + * or `{ mode: 'seed' }`), so reading the workspace as source prints them, or for seed rows (which + * the directory keeps) the declaration that keeps its rows file. Written only by the reshape that + * applies them. */ sourceRows?: TableSourceRows /** @@ -312,12 +314,18 @@ export interface TableMetadata { sourceColumnKeys?: Record<string, string> } -/** Rows workspace source manages: the whole table, each row matched by its key column. */ -export interface TableSourceRows { - mode: 'managed' - /** The id (`getColumnId`) of the unique column each row is matched by. */ - keyColumnId: string -} +/** + * Rows a workspace source declaration opts in: `managed`, the whole table, each row matched by its + * key column; `seed`, rows written into the table when source created it, which the directory + * keeps and the server does not. + */ +export type TableSourceRows = + | { + mode: 'managed' + /** The id (`getColumnId`) of the unique column each row is matched by. */ + keyColumnId: string + } + | { mode: 'seed' } /** * The saved shape of a table view: everything `TableMetadata` covers plus the diff --git a/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts b/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts index 83a439bc37e..60bc716bf2f 100644 --- a/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts +++ b/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts @@ -166,7 +166,10 @@ export async function hashDocs( return [...left, ...failed] } -/** The info line a change gets for the documents it indexes, and for chunking it changes. */ +/** + * The info line a change gets for the documents it indexes, and for chunking it changes: + * `storedDocs` counts the stored documents that keep their chunks (neither replaced nor deleted). + */ export function indexingFindings( key: string, uploads: readonly DocUpload[], @@ -355,6 +358,14 @@ export async function diffKnowledgeBase(input: { ) } } - changes.diagnostics.push(...indexingFindings(key, changes.uploads, chunkingChanged, byName.size)) + const reindexed = new Set([ + ...changes.uploads.flatMap((upload) => (upload.replaces ? [upload.replaces.id] : [])), + ...changes.finishes.map((finish) => finish.replaces.id), + ...changes.deletes.map((doc) => doc.id), + ]) + const keepingChunks = [...byName.values()].filter((doc) => !reindexed.has(doc.id)).length + changes.diagnostics.push( + ...indexingFindings(key, changes.uploads, chunkingChanged, keepingChunks) + ) return changes } diff --git a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts index 6a3c2b3eb1c..0fd4e3a9ee2 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts @@ -100,12 +100,18 @@ import type { interface RowsPlan { /** The rows the reshape writes (only those that differ); absent leaves the rows alone. */ write?: ReshapeRows - /** The managed mark to set, by key column name, or `null` to clear it; absent leaves it. */ - mark?: { mode: 'managed'; keyColumn: string } | null + /** + * The rows mark to set (managed rows by key column name, or seed rows), or `null` to clear it; + * absent leaves it. + */ + mark?: RowsMark | null /** The row version the plan read the stored rows at; the apply refuses one that moved. */ version?: number } +/** How a declaration opts its rows in, as the table's mark records it. */ +type RowsMark = { mode: 'managed'; keyColumn: string } | { mode: 'seed' } + /** What the apply carries out for one table, decided at plan time. */ export interface TablePlan { /** The changes an update or adopt makes; absent for a create or delete. */ @@ -220,8 +226,9 @@ function withKeys( } } -/** The managed mark a declaration asks for, by its key column's stored name. */ -function markOf(decl: TableDecl): { mode: 'managed'; keyColumn: string } | null { +/** The rows mark a declaration asks for: managed rows by their key column's stored name, or seed. */ +function markOf(decl: TableDecl): RowsMark | null { + if (decl.rows?.mode === 'seed') return { mode: 'seed' } const key = decl.rows?.mode === 'managed' ? decl.rows.key : undefined const column = key !== undefined ? decl.columns[key] : undefined return key !== undefined && column @@ -235,7 +242,7 @@ function rowsWrite(decl: TableDecl, allowDelete: boolean): ReshapeRows | undefin if (!rows) return undefined const named = namedRows(rows, namesByKey(decl)) const mark = markOf(decl) - return rows.mode === 'managed' && mark + return rows.mode === 'managed' && mark?.mode === 'managed' ? { mode: 'managed', keyColumn: mark.keyColumn, @@ -343,9 +350,16 @@ async function planRows( const { decl: storedDecl, keyById } = tableAsDecl(state, entry?.columns) const storedKey = storedDecl.rows?.key if (rows?.mode !== 'managed') { - if (storedKey !== undefined) { - outcome.plan = { mark: null } - outcome.fields.push('rows (no longer managed)') + const storedMode = storedDecl.rows?.mode + if (storedMode !== rows?.mode) { + outcome.plan = { mark: markOf(decl) } + outcome.fields.push( + storedMode === 'managed' + ? 'rows (no longer managed)' + : rows + ? 'rows (seed)' + : 'rows (no longer seeded)' + ) } if (rows?.mode === 'seed' && rows.data.length > 0 && !entry) { if (state.table.rowCount === 0) { @@ -515,7 +529,9 @@ async function planTable( const settled = planned.settled const resource = resolution.action === 'noop' ? resolution.stored : undefined if (!resource || settled.action !== 'noop') return settled - if (!decl.rows) return withKeys(settled, resource.state, decl, entry) + if (!decl.rows && !resource.state.table.metadata?.sourceRows) { + return withKeys(settled, resource.state, decl, entry) + } const rows = await planRows(key, decl, resource.state, entry, ctx) const renamed = rows.plan?.write ? columnsRenamedOnServer(decl, resource.state, entry) : [] if (renamed.length > 0) { diff --git a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts index ba9e9e85f5e..81c109b6608 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts @@ -99,15 +99,20 @@ export function tableAsDecl( const key = keyById.get(getColumnId(column)) ?? column.name setOwn(columns, key, columnDecl(column, key)) } - // Rows source manages print as `rows`, keyed by the column the marker names while it exists. + // Rows source manages print as `rows`, keyed by the column the marker names while it exists; + // seed rows print as their `rows` alone, since the directory keeps their file. const marker = state.table.metadata?.sourceRows - const rowsKey = marker ? keyById.get(marker.keyColumnId) : undefined + const rowsKey = marker?.mode === 'managed' ? keyById.get(marker.keyColumnId) : undefined const decl: TableDecl = { name: state.table.name, ...(state.table.description ? { description: state.table.description } : {}), ...(state.folderPath !== '/' ? { folder: state.folderPath } : {}), columns, - ...(rowsKey !== undefined ? { rows: { mode: 'managed', key: rowsKey, data: [] } } : {}), + ...(rowsKey !== undefined + ? { rows: { mode: 'managed', key: rowsKey, data: [] } } + : marker?.mode === 'seed' + ? { rows: { mode: 'seed', data: [] } } + : {}), } return { decl, keyById, fingerprint: fingerprintOf(tableCodec.canonical(decl)) } } @@ -145,7 +150,7 @@ export function printStoredTable( const { decl, keyById, fingerprint } = tableAsDecl(state, bindings) let rowsEntry: BaseRows | undefined if (rows && 'seed' in rows) { - if (!decl.rows) decl.rows = rows.seed + if (decl.rows?.mode !== 'managed') decl.rows = rows.seed } else if (decl.rows && rows) { decl.rows = { ...decl.rows, data: rows.data } rowsEntry = baseRows(decl.rows, rows.version) diff --git a/apps/sim/lib/workspace-source/application/knowledge.integration.ts b/apps/sim/lib/workspace-source/application/knowledge.integration.ts index 56f8e4b5cc0..d857bda8383 100644 --- a/apps/sim/lib/workspace-source/application/knowledge.integration.ts +++ b/apps/sim/lib/workspace-source/application/knowledge.integration.ts @@ -517,6 +517,27 @@ describe('knowledge bases as workspace source', () => { } ) + check( + 'a chunking change counts only the stored documents that keep their chunks, not the ones it replaces', + async () => { + const { workspaceId, read } = await setUp() + + const planned = await apply( + workspaceId, + { + files: { + [POLICIES_FILE]: declaration('Policies', '\n chunking: { maxSize: 512 },'), + }, + docs: { ...POLICIES.docs, [REFUNDS]: Buffer.from('# Refunds\nWithin 60 days.\n') }, + }, + { base: read.base, dryRun: true, carry: 'none' } + ) + + const chunking = planned.diagnostics.find((finding) => finding.code === 'I_KB_CHUNKING') + expect(chunking?.message).toContain('1 stored document(s) keep their chunks') + } + ) + check('a get reproduces the apply, and re-applying it changes nothing', async () => { const workspaceId = await newWorkspace() const created = await apply(workspaceId, POLICIES) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index 71cf87543d5..9dce9850bac 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -745,8 +745,10 @@ export default defineTable({ expect(again.applied).toEqual([]) expect((await storedRows(workspaceId))[0]).toMatchObject({ seats: 5 }) - // A get leaves seed rows out: they were never the table's identity. + // A get keeps the declaration of seed rows but leaves the rows out: the directory holds them, + // and they were never the table's identity. const read = await runner.get(workspaceId) + expect(read.files['tables/plans.ts']).toContain("rows: { mode: 'seed' }") expect(read.files['tables/plans.rows.jsonl']).toBeUndefined() expect((await apply(workspaceId, read.files, { base: read.base })).applied).toEqual([]) @@ -756,6 +758,43 @@ export default defineTable({ expect(await storedRows(workspaceId)).toHaveLength(2) }) + check( + 'a declaration that stops seeding rows clears it, and one that starts records it', + async () => { + const workspaceId = await newWorkspace() + const seeded = await apply(workspaceId, { + 'tables/plans.ts': SEED, + 'tables/plans.rows.jsonl': ROWS, + }) + const unseeded = PLANS('undefined').replace(' rows: undefined,\n', '') + + const dropped = await apply( + workspaceId, + { 'tables/plans.ts': unseeded }, + { base: seeded.base } + ) + expect(dropped.applied).toMatchObject([ + { key: 'plans', action: 'update', fields: ['rows (no longer seeded)'] }, + ]) + expect((await runner.get(workspaceId)).files['tables/plans.ts']).not.toContain('rows:') + + const read = await runner.get(workspaceId) + const again = await apply( + workspaceId, + { 'tables/plans.ts': SEED, 'tables/plans.rows.jsonl': ROWS }, + { base: read.base } + ) + expect(again.applied).toMatchObject([ + { key: 'plans', action: 'update', fields: ['rows (seed)'] }, + ]) + expect(codes(again.diagnostics, 'info')).not.toContain('I_TABLE_SEED_SKIPPED') + expect((await runner.get(workspaceId)).files['tables/plans.ts']).toContain( + "rows: { mode: 'seed' }" + ) + expect(await storedRows(workspaceId)).toHaveLength(2) + } + ) + check('makes an existing column required in the apply whose managed rows fill it', async () => { const workspaceId = await newWorkspace() const gappy = '{"code":"solo","seats":1,"tier":"basic"}\n{"code":"team","tier":"pro"}\n' diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 110ae4ac83e..29dfd16785f 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -1080,6 +1080,29 @@ describe('workspaces source apply', () => { expect(warned).not.toContain('files/.gitkeep') }) + it('names each file it does not send in its own warning, in the JSON result and its count', async () => { + write('files/a:b.txt', 'x') + write('files/q?.txt', 'x') + write('workflows/.hidden-wf/workflow.ts', INGEST) + request.mockResolvedValue(answer()) + + await apply('--yes') + + const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) + const notSent = summary.diagnostics.filter( + (finding: { code: string }) => finding.code === 'W_PATH_NOT_SENT' + ) + expect(notSent.map((finding: { file: string }) => finding.file).sort()).toEqual([ + 'files/a:b.txt', + 'files/q?.txt', + 'workflows/.hidden-wf/workflow.ts', + ]) + expect(notSent.every((finding: { severity: string }) => finding.severity === 'warning')).toBe( + true + ) + expect(summary.warnings).toBe(3) + }) + it('sends secrets.ts and credentials.ts as their kinds, and no other root file', async () => { const secrets = "// @sim-sdk 1\nexport default defineSecrets({ OPENAI_API_KEY: { description: 'Models' } })\n" diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 4bed6286492..de5c84a7978 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -574,13 +574,15 @@ export function attachWorkspaceSourceApply(source: Command): void { ? await promoted(client, options.dir, read, options.to, options.allowDelete === true) : null const submission = promotion?.submission ?? read - if (submission.ignored.length > 0) { - printError( - `Not sent (a hidden name, or a character no workspace path holds): ${submission.ignored.join(', ')}` - ) - } // The apply repeats findings the preview already showed. const printed = new Set<string>() + const notSent = notSentFindings(submission.ignored) + printNewDiagnostics(notSent, printed) + /** `data` with the findings about files the CLI did not send, which it reports too. */ + const withNotSent = (data: ApplyData): ApplyData => ({ + ...data, + diagnostics: [...notSent, ...data.diagnostics], + }) let preview: ApplyData | null = null if (options.dryRun || !options.yes) { preview = await send(submission, true) @@ -590,8 +592,8 @@ export function attachWorkspaceSourceApply(source: Command): void { if (errors) setSoftExitCode(refusedExitCode(preview.diagnostics)) if (options.dryRun || refusesApply(preview)) { printProtocolResult(profile.output, { - ...summarize(preview), - ...resultDiagnostics(profile.output, preview.diagnostics), + ...summarize(withNotSent(preview)), + ...resultDiagnostics(profile.output, withNotSent(preview).diagnostics), }) return } @@ -632,8 +634,8 @@ export function attachWorkspaceSourceApply(source: Command): void { setSoftExitCode(failedExitCode(data.failed)) } printProtocolResult(profile.output, { - ...summarize(data), - ...resultDiagnostics(profile.output, data.diagnostics), + ...summarize(withNotSent(data)), + ...resultDiagnostics(profile.output, withNotSent(data).diagnostics), }) }, { @@ -647,6 +649,20 @@ export function attachWorkspaceSourceApply(source: Command): void { }) } +/** + * A `W_PATH_NOT_SENT` warning for each file under a kind folder the CLI does not send: a hidden + * name, or a character no workspace path holds. + */ +function notSentFindings(paths: readonly string[]): ApplyData['diagnostics'] { + return paths.map((path) => ({ + code: 'W_PATH_NOT_SENT' as const, + severity: 'warning' as const, + message: `${path} is not sent: a hidden name, or a character no workspace path holds. The apply writes and deletes nothing for it.`, + hint: 'Rename it to send it, or move it out of the kind folder.', + file: path, + })) +} + /** * The next step for an apply write whose answer never came back (a timeout, a dropped connection, * an interrupt): it may or may not have been written. diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts index 2db74283680..292094ef8a8 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts @@ -303,6 +303,30 @@ describe('sim workspaces source check', () => { expect(process.exitCode).toBe(1) }) + it('a rows file with more bad lines than one file lists exits 1, as source errors, not 7 as a size limit', async () => { + writeFile( + 'tables/plans.ts', + [ + '// @sim-sdk 1', + "import { column, defineTable } from '@sim/workflow-sdk/workspace'", + 'export default defineTable({', + " name: 'plans',", + ' columns: { code: column.string({ required: true, unique: true }) },', + " rows: { mode: 'managed', key: 'code' },", + '})', + '', + ].join('\n') + ) + writeFile('tables/plans.rows.jsonl', `${Array.from({ length: 150 }, () => '{').join('\n')}\n`) + + await check() + + const found = stdoutJson().diagnostics as Array<{ code: string; severity: string }> + expect(found.at(-1)).toMatchObject({ code: 'I_FINDINGS_TRUNCATED', severity: 'info' }) + expect(found.filter((finding) => finding.severity === 'error')).toHaveLength(100) + expect(process.exitCode).toBe(1) + }) + it('accepts seed rows the columns hold, coercing a numeric string as the write does', async () => { writeFile( 'tables/seeds.ts', diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts index cf7c90ffdfc..e4a39f2306f 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts @@ -135,10 +135,18 @@ export function attachWorkspaceSourceCheck(source: Command): void { ) findings.push(...checked.findings) } - for (const { code, message, hint, file, line, column } of declarations.rowFindings) { + for (const { + code, + severity, + message, + hint, + file, + line, + column, + } of declarations.rowFindings) { findings.push({ code, - severity: 'error', + severity, message, ...(hint !== undefined ? { hint } : {}), file, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index 728d6871713..f0a3d14f149 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -18,6 +18,7 @@ import { import { APPLY_ORDER, type BlobEntry, + type FlatDiagnostic, isFileKind, KIND_DIRS, WORKSPACE_KIND_SPECS, @@ -278,9 +279,13 @@ export interface WorkspaceDeclarations { workflowInputs?: ReadonlyMap<string, readonly WorkflowInput[] | null> } -/** A finding about a line of a table's rows file (`E_TABLE_ROWS`, an error). */ +/** + * A finding about a line of a table's rows file: `E_TABLE_ROWS`, an error, or the + * `I_FINDINGS_TRUNCATED` that ends a long list. + */ interface RowFinding { code: WorkspaceDiagnosticCode + severity: FlatDiagnostic['severity'] message: string hint?: string file: string @@ -348,8 +353,9 @@ function tableRowFindings(key: string, decl: TableDecl, text: string | undefined if (!rows || text === undefined) return [] const parsed = parseTableRows(key, text, rows, new Set(Object.keys(decl.columns))) const read = parsed.diagnostics.map( - ({ code, message, hint, span }): RowFinding => ({ + ({ code, severity, message, hint, span }): RowFinding => ({ code, + severity, message, ...(hint ? { hint } : {}), file, @@ -360,6 +366,7 @@ function tableRowFindings(key: string, decl: TableDecl, text: string | undefined const refused = refusedTableRows(decl, parsed.data).map( ({ row, errors }): RowFinding => ({ code: 'E_TABLE_ROWS', + severity: 'error', message: `Line ${parsed.lines[row] ?? 1} does not fit the declared columns: ${errors.join('; ')}.`, hint: `Write a value each column of ${KIND_DIRS.table}/${key}${DECLARATION_EXTENSION} holds (a select cell names one of its options), fill every required column, and keep unique values unique.`, file, @@ -407,6 +414,26 @@ function declaredTable( return codec.parse(key, text, []) } +/** + * The rows files of the tables `files` (a get's answer) declares seed rows for. A server read + * prints a seed table's `rows` but never its rows, which only the directory holds, so a get keeps + * the file it finds there. + */ +export function seedRowsFiles(files: Readonly<Record<string, string>>): Set<string> { + const kept = new Set<string>() + const paths = Object.keys(files).filter((path) => kindOfPath(path) === 'table') + for (const [key, tablePaths] of KIND_LAYOUTS.table.group(paths).groups) { + const rowsPath = tableRowsPath(key) + const { decl } = declaredTable( + key, + tablePaths.filter((path) => path !== rowsPath), + files + ) + if (decl?.rows?.mode === 'seed') kept.add(rowsPath) + } + return kept +} + /** A table declaration's columns by stored name, as a workflow's row writes name them. */ function declaredColumns(decl: TableDecl): ReadonlyMap<string, DeclaredColumn> { return new Map( diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts index fb633377108..383f6e24b6e 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts @@ -125,6 +125,25 @@ describe('workspaces source get', () => { expect(readFileSync(join(dir, 'tables/leads.ts'), 'utf8')).toBe(`${LEADS}// server\n`) }) + it('keeps the seed rows file of a table the answer declares seed rows for, which no read prints', async () => { + const seeded = (name: string) => + `// @sim-sdk 1\nimport { column, defineTable } from '@sim/workflow-sdk/workspace'\n\nexport default defineTable({ name: '${name}', columns: { code: column.string() }, rows: { mode: 'seed' } })\n` + const rows = '{"code":"a"}\n' + serve({ + 'tables/leads.ts': seeded('leads'), + 'tables/leads.rows.jsonl': rows, + 'tables/accounts.ts': seeded('accounts'), + 'tables/accounts.rows.jsonl': rows, + }) + await get() + + serve({ 'tables/leads.ts': seeded('leads'), 'tables/accounts.ts': ACCOUNTS }) + await get() + + expect(readFileSync(join(dir, 'tables/leads.rows.jsonl'), 'utf8')).toBe(rows) + expect(existsSync(join(dir, 'tables/accounts.rows.jsonl'))).toBe(false) + }) + it('refuses to restore a file deleted since the CLI wrote it, a pending delete', async () => { serve({ 'tables/leads.ts': LEADS, 'tables/accounts.ts': ACCOUNTS }) await get() diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index 2c25ff88f9e..96b0daba06e 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -59,6 +59,7 @@ import { keepFile, kindFolders, localDeclarations, + seedRowsFiles, skippedFile, WORKSPACE_TREE, } from './workspace-source-dir' @@ -167,11 +168,13 @@ async function writeWorkspaceSource( const blobs = new Map(source.blobs.map((blob) => [blob.path, blob])) const written = await readWritten(dir) const { paths: onDisk } = await kindFolders(dir, source.kinds) + const seedRows = seedRowsFiles(source.files) const gone = new Set( onDisk.filter( (path) => !Object.hasOwn(source.files, path) && !blobs.has(path) && + !seedRows.has(path) && Object.hasOwn(written.digests, path) ) ) diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 329cfe3139d..1f9f85b8ffc 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -1209,6 +1209,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_DECL_SHAPE' | 'E_DECL_UNKNOWN_FIELD' | 'E_DECL_UNKNOWN_FILE' + | 'W_PATH_NOT_SENT' | 'E_BLOB_MANIFEST' | 'E_BLOB_MISSING' | 'E_LIMIT_EXCEEDED' @@ -1487,6 +1488,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_DECL_SHAPE' | 'E_DECL_UNKNOWN_FIELD' | 'E_DECL_UNKNOWN_FILE' + | 'W_PATH_NOT_SENT' | 'E_BLOB_MANIFEST' | 'E_BLOB_MISSING' | 'E_LIMIT_EXCEEDED' @@ -10917,6 +10919,7 @@ export type GetWorkspaceSourceResponse = { | 'E_DECL_SHAPE' | 'E_DECL_UNKNOWN_FIELD' | 'E_DECL_UNKNOWN_FILE' + | 'W_PATH_NOT_SENT' | 'E_BLOB_MANIFEST' | 'E_BLOB_MISSING' | 'E_LIMIT_EXCEEDED' diff --git a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts index 0c3c7975a9d..724b21ef94e 100644 --- a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts +++ b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts @@ -11,7 +11,7 @@ export const WORKSPACE_SDK_DTS: Readonly<Record<string, string>> = { 'index.d.ts': '/**\n * The workspace declaration SDK: `import { … } from \u0027@sim/workflow-sdk/workspace\u0027` in the\n * declaration files of a workspace source directory (`tables/<key>.ts`, …). Like the workflow\n * SDK, nothing here runs: the server reads each file statically.\n */\nexport * from \u0027./credentials\u0027\nexport * from \u0027./deployment\u0027\nexport * from \u0027./file\u0027\nexport * from \u0027./knowledge\u0027\nexport * from \u0027./mcp\u0027\nexport * from \u0027./secrets\u0027\nexport * from \u0027./skill\u0027\nexport * from \u0027./table\u0027\nexport * from \u0027./tool\u0027\n', 'knowledge.d.ts': - '/**\n * Knowledge base declarations: `knowledge/<key>/knowledge.ts` in a workspace source directory,\n * beside `knowledge/<key>/docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload, so its tag values are not kept), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n', + '/**\n * Knowledge base declarations: `knowledge/<key>/knowledge.ts` in a workspace source directory,\n * beside `knowledge/<key>/docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload that takes the old one\u0027s tag values), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n', 'mcp.d.ts': '/**\n * MCP server declarations: `mcp/<key>.ts` declares one server the workspace\u0027s agents call tools\n * on.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineMcpServer, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineMcpServer({\n * name: \u0027GitHub\u0027,\n * url: \u0027https://api.githubcopilot.com/mcp/\u0027,\n * headers: { Authorization: \u0027Bearer {{GITHUB_TOKEN}}\u0027, \u0027X-Team\u0027: env.GITHUB_TEAM },\n * })\n * ```\n *\n * A workflow names the server by its key and one of its tools by name:\n * `mcp(\u0027github\u0027).tool(\u0027create_issue\u0027)`. Secrets never sit in source: a credential header\n * (`Authorization`, `Cookie`, or a name with `key`, `token`, `secret`, `auth`, …) holds env\n * references (`env.NAME`, or `{{NAME}}` in its text after a scheme such as `Bearer`) or\n * `stored()`, which keeps the value the server holds. The server\u0027s id derives from its URL, so a\n * `url` with another host or path replaces the server; a new query string updates it in place.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** A header value: text (`{{NAME}}` names an env variable), `env.NAME`, or `stored()`. */\nexport type McpHeaderValue = string | EnvRef\n\n/** How the server authenticates. */\nexport interface McpServerAuth {\n /** `headers` (the default) sends the declared headers; `oauth` signs in with an OAuth grant. */\n type: \u0027none\u0027 | \u0027headers\u0027 | \u0027oauth\u0027\n /** A pre-registered OAuth client, for a server without dynamic client registration. */\n clientId?: string\n}\n\n/** An MCP server declaration. */\nexport interface McpServerDeclaration {\n /** The display name; the key when left out. */\n name?: string\n description?: string\n /** An `http://` or `https://` URL. A new host or path replaces the server; a new query does not. */\n url: string\n transport?: \u0027streamable-http\u0027\n headers?: { [name: string]: McpHeaderValue }\n /** `{ type: \u0027headers\u0027 }` when left out. The OAuth client secret is set in the server settings. */\n auth?: McpServerAuth\n /** Milliseconds a request waits; 30000 when left out. */\n timeout?: number\n /** 3 when left out. */\n retries?: number\n /** `true` when left out. */\n enabled?: boolean\n}\n\n/** Declares an MCP server. Its default export is the whole file. */\nexport declare function defineMcpServer(server: McpServerDeclaration): McpServerDeclaration\n', 'secrets.d.ts': diff --git a/packages/workflow-sdk/src/workspace/knowledge.d.ts b/packages/workflow-sdk/src/workspace/knowledge.d.ts index c3345960a1f..b97a78ec431 100644 --- a/packages/workflow-sdk/src/workspace/knowledge.d.ts +++ b/packages/workflow-sdk/src/workspace/knowledge.d.ts @@ -14,7 +14,7 @@ * ``` * * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored - * document (a new upload, so its tag values are not kept), and removing one deletes it only with + * document (a new upload that takes the old one's tag values), and removing one deletes it only with * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a * connector syncs are never part of source. */ diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index 067ad309d2e..1714c2ac513 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -138,7 +138,10 @@ apart from the declarations'). - `{ mode: 'seed' }` writes the rows into a table the apply creates, or adopts while it holds no rows and the base has no entry for it (`I_TABLE_SEED_SKIPPED` otherwise). Seed rows are outside - every fingerprint, and a get never prints them. + every fingerprint, and a get never prints them: they are the directory's. The table's + `metadata.sourceRows` records `{ mode: 'seed' }`, so a get prints the declaration's `rows` and + the CLI keeps the rows file it finds beside it (`seedRowsFiles`), and a promotion from a + re-read directory still seeds the tables it creates. - `{ mode: 'managed', key }` makes the file the table's whole content, each row matched to its stored row by the unique `key` column. The table's `canonical()` carries only `{ mode, key }` (so tables without managed rows fingerprint as before); the rows have their own fingerprint diff --git a/packages/workspace-code/src/declaration/fields.ts b/packages/workspace-code/src/declaration/fields.ts index a33d2e75a9c..1db74cf767a 100644 --- a/packages/workspace-code/src/declaration/fields.ts +++ b/packages/workspace-code/src/declaration/fields.ts @@ -41,7 +41,7 @@ const SECRET_VALUE_CODES: ReadonlySet<WorkspaceDiagnosticCode> = new Set([ /** * A {@link FieldReport} that appends to `diagnostics`, located through the reader's spans. Once it * has appended `MAX_FINDINGS_PER_FILE` findings (findings already in `diagnostics`, such as ones - * about other files of the resource, do not count) it records one `E_LIMIT_EXCEEDED` (with a span + * about other files of the resource, do not count) it records one `I_FINDINGS_TRUNCATED` (with a span * but no frame, since the value it points at may be a secret) and drops the rest. */ export function fieldReporter( diff --git a/packages/workspace-code/src/declaration/read.test.ts b/packages/workspace-code/src/declaration/read.test.ts index 97f9a36b09a..f79eb3a091f 100644 --- a/packages/workspace-code/src/declaration/read.test.ts +++ b/packages/workspace-code/src/declaration/read.test.ts @@ -59,7 +59,10 @@ describe('readDeclaration on large files', () => { code: 'E_DECL_UNSUPPORTED', span: { line: MAX_FINDINGS_PER_FILE + 2, column: 1 }, }) - expect(read.diagnostics[MAX_FINDINGS_PER_FILE]).toMatchObject({ code: 'E_LIMIT_EXCEEDED' }) + expect(read.diagnostics[MAX_FINDINGS_PER_FILE]).toMatchObject({ + code: 'I_FINDINGS_TRUNCATED', + severity: 'info', + }) expect(fullTextScans()).toBeLessThanOrEqual(1) }) @@ -84,7 +87,7 @@ describe('readDeclaration on large files', () => { expect(parsed.diagnostics).toHaveLength(MAX_FINDINGS_PER_FILE + 1) const options = parsed.diagnostics.filter((finding) => finding.code === 'E_TABLE_OPTIONS') expect(options.at(-1)?.span).toMatchObject({ line: options.length + 3 }) - expect(parsed.diagnostics.at(-1)).toMatchObject({ code: 'E_LIMIT_EXCEEDED' }) + expect(parsed.diagnostics.at(-1)).toMatchObject({ code: 'I_FINDINGS_TRUNCATED' }) expect(fullTextScans()).toBeLessThanOrEqual(2) }) diff --git a/packages/workspace-code/src/diagnostics/codes.ts b/packages/workspace-code/src/diagnostics/codes.ts index dce974fd914..6690141853c 100644 --- a/packages/workspace-code/src/diagnostics/codes.ts +++ b/packages/workspace-code/src/diagnostics/codes.ts @@ -26,6 +26,12 @@ const DECLARATION_CODES = [ 'E_DECL_UNKNOWN_FIELD', /** A path inside a kind's folder that no resource owns. */ 'E_DECL_UNKNOWN_FILE', + /** + * A file under a kind's folder whose path no workspace holds (a hidden name such as `.env`, or a + * character such as `:` or `?`): the CLI does not send it, so an apply neither writes nor + * deletes anything for it. + */ + 'W_PATH_NOT_SENT', /** A blob manifest entry listed twice, or at a path that holds no binary file: the apply writes nothing. */ 'E_BLOB_MANIFEST', /** @@ -33,7 +39,7 @@ const DECLARATION_CODES = [ * apply writes nothing. Stage it, then apply again. */ 'E_BLOB_MISSING', - /** A size, count or nesting limit exceeded (a file, a kind, the staging area, findings). */ + /** A size, count or nesting limit exceeded (a file, a kind, the staging area). */ 'E_LIMIT_EXCEEDED', /** A resource key outside `^[a-z][a-z0-9_-]{0,63}$`. */ 'E_KEY_INVALID', @@ -262,7 +268,10 @@ const PLAN_CODES = [ 'I_KEY_REBOUND', /** A declaration under a new key for a resource the base read under another: keys never move. */ 'E_KEY_RENAMED', - /** The plan had more findings than one answer reports; this one counts those left out. */ + /** + * More findings than one answer (or one file's list) reports; this one says where the list + * stopped. Informational: the findings it follows decide whether the apply is refused. + */ 'I_FINDINGS_TRUNCATED', ] as const diff --git a/packages/workspace-code/src/diagnostics/finding.ts b/packages/workspace-code/src/diagnostics/finding.ts index a42e66c07ac..b6f866905c8 100644 --- a/packages/workspace-code/src/diagnostics/finding.ts +++ b/packages/workspace-code/src/diagnostics/finding.ts @@ -41,7 +41,7 @@ export type WorkspaceDiagnostic<C extends WorkspaceDiagnosticCode = WorkspaceDia type DiagnosticExtras = Omit<DiagnosticFields, 'message'> & ResourceFields /** - * Findings one declaration file lists; past it the file gets one `E_LIMIT_EXCEEDED` and the rest + * Findings one declaration file lists; past it the file gets one `I_FINDINGS_TRUNCATED` and the rest * are dropped, so a file of bad values costs no more than its first hundred. */ export const MAX_FINDINGS_PER_FILE = 100 @@ -51,7 +51,7 @@ const MAX_MESSAGE_LENGTH = 1_000 /** The finding that ends a file's list at {@link MAX_FINDINGS_PER_FILE}. */ export function findingLimitReached(extras: DiagnosticExtras = {}): WorkspaceDiagnostic { return diagnostic( - 'E_LIMIT_EXCEEDED', + 'I_FINDINGS_TRUNCATED', `Stopped after ${MAX_FINDINGS_PER_FILE} findings in this file.`, { ...extras, hint: 'Fix these and apply again to see the rest.' } ) diff --git a/packages/workspace-code/src/kinds/knowledge.test.ts b/packages/workspace-code/src/kinds/knowledge.test.ts index 34820e64766..1df52328d46 100644 --- a/packages/workspace-code/src/kinds/knowledge.test.ts +++ b/packages/workspace-code/src/kinds/knowledge.test.ts @@ -209,7 +209,7 @@ describe('knowledgeCodec', () => { } expect(Object.fromEntries(counts)).toEqual({ E_DECL_SHAPE: 100, - E_LIMIT_EXCEEDED: 1, + I_FINDINGS_TRUNCATED: 1, E_DECL_UNKNOWN_FIELD: 1, }) expect(parsed.diagnostics.at(-1)).toMatchObject({ diff --git a/packages/workspace-code/src/kinds/knowledge.ts b/packages/workspace-code/src/kinds/knowledge.ts index 962317364e0..8626244fbf2 100644 --- a/packages/workspace-code/src/kinds/knowledge.ts +++ b/packages/workspace-code/src/kinds/knowledge.ts @@ -238,14 +238,15 @@ function fileFinding( /** * `findings` about a knowledge base's document files, past {@link MAX_FINDINGS_PER_FILE} folded - * into one `E_LIMIT_EXCEEDED`: a folder of bad documents costs no more than its first hundred. + * into one `I_FINDINGS_TRUNCATED`: a folder of bad documents costs no more than its first + * hundred, and refuses for what those hundred are, not as a size limit. */ function cappedFileFindings(key: string, findings: WorkspaceDiagnostic[]): WorkspaceDiagnostic[] { if (findings.length <= MAX_FINDINGS_PER_FILE) return findings return [ ...findings.slice(0, MAX_FINDINGS_PER_FILE), diagnostic( - 'E_LIMIT_EXCEEDED', + 'I_FINDINGS_TRUNCATED', `Stopped after ${MAX_FINDINGS_PER_FILE} findings about the files in knowledge/${key}/.`, { kind: 'knowledge-base', key, hint: 'Fix these and apply again to see the rest.' } ), diff --git a/packages/workspace-code/src/kinds/literal-secrets.test.ts b/packages/workspace-code/src/kinds/literal-secrets.test.ts index 5b97d99159c..386b98b9b6a 100644 --- a/packages/workspace-code/src/kinds/literal-secrets.test.ts +++ b/packages/workspace-code/src/kinds/literal-secrets.test.ts @@ -24,7 +24,7 @@ describe('findings about a literal secret', () => { const source = `${HEADER}import { defineMcpServer } from '@sim/workflow-sdk/workspace'\n\nexport default defineMcpServer({\n name: 'Docs',\n url: 'https://mcp.example.com/mcp',\n headers: {\n Authorization: 'Bearer ${SECRET}',\n ${headers},\n },\n})\n` const parsed = mcpCodec.parse('docs', { 'mcp/docs.ts': source }, []) const codes = new Set(parsed.diagnostics.map((finding) => finding.code)) - expect(codes).toEqual(new Set(['E_MCP_LITERAL_SECRET', 'E_LIMIT_EXCEEDED'])) + expect(codes).toEqual(new Set(['E_MCP_LITERAL_SECRET', 'I_FINDINGS_TRUNCATED'])) expect(parsed.diagnostics[0].span).toMatchObject({ file: 'mcp/docs.ts', line: 8 }) expect(echoed(parsed.diagnostics)).not.toContain(SECRET) }) From 9a1c7787a9c65d600080216376444a2605d36221 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:10:04 -0800 Subject: [PATCH 18/59] fix(workflow-code): say a hidden field is dropped, and which value hides it --- ...ly-workflow-source.fidelity.integration.ts | 47 +++++++++++++++++++ packages/workflow-code/src/compile/SYNTAX.md | 6 ++- packages/workflow-code/src/compile/blocks.ts | 14 +++++- .../workflow-code/src/compile/checks.test.ts | 47 +++++++++++++++++++ .../src/compile/validate-catalog.ts | 43 +++++++++++++++++ 5 files changed, 153 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts index 95f68dbb334..71f8c0c3653 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts @@ -483,6 +483,53 @@ check( } ) +check( + 'a field the model hides, added in source, is reported as dropped and apply keeps the stored value', + async () => { + const workflowId = await insertWorkflow('Fidelity hidden field') + const start = generateId() + const agentId = generateId() + await putState(workflowId, { + blocks: { + [start]: block(start, 'start_trigger', 'Start'), + [agentId]: { + ...block(agentId, 'agent', 'Answer', { + model: { id: 'model', type: 'combobox', value: 'gpt-5.4' }, + }), + advancedMode: true, + }, + }, + edges: [ + { + id: generateId(), + source: start, + target: agentId, + sourceHandle: 'source', + targetHandle: 'target', + }, + ], + loops: {}, + parallels: {}, + variables: {}, + }) + const source = await getSource(workflowId) + expect(source.files['workflow.ts']).not.toContain('temperature') + const before = await snapshot(workflowId) + + const result = await applySource( + source, + editEntry(source.files, "model: 'gpt-5.4' }", "model: 'gpt-5.4', temperature: 0.5 }") + ) + const dropped = result.diagnostics.filter((d) => d.code === 'W_FIELD_NOT_APPLICABLE') + expect(dropped.map((d) => d.message)).toEqual([ + 'temperature is hidden (model gpt-5.4 does not use it), so it never runs: an apply does not save it over the stored value, and the source it writes back drops this line.', + ]) + expect(result.diff.noop).toBe(true) + expect(result.files['workflow.ts']).not.toContain('temperature') + expect(await snapshot(workflowId)).toEqual(before) + } +) + check( 'a workflow built by editing operations and one created from its source read the same through the JSON API', async () => { diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index 752f9cd7d1e..32d85376438 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -84,7 +84,9 @@ export default workflow('Name', { description: '…', variables: { … } }, ({ v `E_UNSUPPORTED_SYNTAX`. - A pinned param (`operation`, or a builder's `fixed` entry) is `E_FIXED_PARAM`; the compiler adds pinned values. -- A param not visible under the block's values is `W_FIELD_NOT_APPLICABLE`. A required field left +- A param not visible under the block's values is `W_FIELD_NOT_APPLICABLE`, naming the value that + hides it: it never runs, so apply keeps the stored value (preserved state) and the written-back + source drops the line. A required field left empty (or `todo()`) is `W_REQUIRED_MISSING`. A value whose JSON type differs from the field's is `W_VALUE_KIND`; it is never coerced. @@ -416,7 +418,7 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `E_UNKNOWN_BUILDER` | A builder path the catalog does not have, a bare builder-shaped call (`apii('Name', …)`) nothing binds, or a namespace the SDK lacks or the file did not import; `hint` names the builder a common other name stands for (`http.request` → `api`), then prefers a synonymous verb (`postMessage` → `sendMessage`), then the closest path, then the namespace's builders. | | `E_UNKNOWN_PARAM` | A param the block does not have; the hint has a did-you-mean. | | `E_FIXED_PARAM` | A param the builder already pins. | -| `W_FIELD_NOT_APPLICABLE` | A param not visible under the block's values. | +| `W_FIELD_NOT_APPLICABLE` | A param not visible under the block's values; apply keeps the stored value and the written-back source drops it. | | `W_REQUIRED_MISSING` | A required field left empty, including `todo()`; not repeated for a field an `E_UNKNOWN_PARAM` did-you-mean names, or one written with a value that failed to compile. | | `W_UNKNOWN_OPTION` | A dropdown value none of its options names; `hint` lists them. | | `E_UNKNOWN_OPERATION` | An `operation` the block does not offer; `hint` lists them. A retired one the stored block holds is `W_UNKNOWN_OPTION`. | diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 06d2973bcbd..cc090c49dd6 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -76,6 +76,7 @@ import type { Range } from './report' import { compileToolList } from './tools' import { fieldFor, + hiddenReason, isVisible, missingRequiredFields, paramLookup, @@ -1521,6 +1522,15 @@ function valueKindHint(sub: CatalogSubBlock): string { } } +/** + * `W_FIELD_NOT_APPLICABLE`: a hidden field never runs, and source holds only what runs, so an + * apply restores the stored value over it (preserved state) and the printed source drops it. + */ +function notApplicableMessage(param: string, reason: string | null): string { + const hidden = reason ? `hidden (${reason})` : "hidden under the block's current values" + return `${param} is ${hidden}, so it never runs: an apply does not save it over the stored value, and the source it writes back drops this line.` +} + /** Catalog warnings: visibility, required fields, value kinds, slider ranges. */ function checkFields(ctx: CompileContext, index: FieldIndex, state: BlockState): void { const { decl, params } = state @@ -1532,10 +1542,10 @@ function checkFields(ctx: CompileContext, index: FieldIndex, state: BlockState): if (!isVisible(sub, values)) { ctx.reporter.report( 'W_FIELD_NOT_APPLICABLE', - `${entry.param} is hidden under the block's current values; it is kept but not used.`, + notApplicableMessage(entry.param, hiddenReason(sub, values)), rangeOf(ctx, entry.property), { - hint: 'Remove it; apply keeps the stored value.', + hint: `Remove ${entry.param}, or change the values that hide it so it runs.`, blockKey: decl.key, param: entry.param, } diff --git a/packages/workflow-code/src/compile/checks.test.ts b/packages/workflow-code/src/compile/checks.test.ts index d342923b928..f21c682b051 100644 --- a/packages/workflow-code/src/compile/checks.test.ts +++ b/packages/workflow-code/src/compile/checks.test.ts @@ -146,6 +146,53 @@ describe("references into a start trigger's declared inputs", () => { }) }) +describe("a field hidden under the block's values", () => { + const DROPPED = + 'so it never runs: an apply does not save it over the stored value, and the source it writes back drops this line.' + + it('says the value is dropped, and names the condition it shows under', () => { + const [hidden] = findings( + run("const s = slack.sendMessage('S', { authMethod: 'oauth', text: 'x', limit: 3 })"), + 'W_FIELD_NOT_APPLICABLE' + ) + expect(hidden?.message).toBe( + `limit is hidden (it shows only when operation is list_channels), ${DROPPED}` + ) + expect(hidden?.hint).toBe('Remove limit, or change the values that hide it so it runs.') + }) + + it('names the model a model-gated field does not apply to', () => { + const agentBlock = TEST_CATALOG.blocks.agent as CatalogBlock + const gated: BlockCatalog = { + ...TEST_CATALOG, + blocks: { + ...TEST_CATALOG.blocks, + agent: { + ...agentBlock, + subBlocks: agentBlock.subBlocks.map((sub) => + sub.id === 'temperature' + ? { + ...sub, + condition: { opaque: true }, + visibleForModel: { 'gpt-5': false, 'claude-sonnet-4-5': true }, + } + : sub + ), + }, + }, + } + const written = (model: string) => + findings( + run(`const a = agent('A', { model: '${model}', temperature: 0.5 })`, undefined, gated), + 'W_FIELD_NOT_APPLICABLE' + ) + expect(written('gpt-5')[0]?.message).toBe( + `temperature is hidden (model gpt-5 does not use it), ${DROPPED}` + ) + expect(written('claude-sonnet-4-5')).toEqual([]) + }) +}) + describe('references into an agent responseFormat', () => { const FORMAT = "responseFormat: { name: 'classify', schema: { type: 'object', properties: { priority: { type: 'string' }, score: { type: 'number' } } } }" diff --git a/packages/workflow-code/src/compile/validate-catalog.ts b/packages/workflow-code/src/compile/validate-catalog.ts index 2b03684aef4..b129699fa8b 100644 --- a/packages/workflow-code/src/compile/validate-catalog.ts +++ b/packages/workflow-code/src/compile/validate-catalog.ts @@ -18,6 +18,49 @@ export function isVisible(sub: CatalogSubBlock, values: ReadonlyMap<string, Valu return evaluateCondition(sub, paramLookup(values)) !== false } +/** A clause list longer than this is described by the value it rejects, not by what it lists. */ +const LISTED_VALUES = 3 + +/** + * Why {@link isVisible} hides a field, as a clause: the value that hides it (`model gpt-5 does + * not use it`), or for a short data condition the values it shows under (`it shows only when + * operation is send`); `null` when the condition data does not say. + */ +export function hiddenReason( + sub: CatalogSubBlock, + values: ReadonlyMap<string, Value> +): string | null { + const { condition, visibleFor, visibleForModel } = sub + const rejected = (field: string) => { + const value = values.get(field) + return `${field} ${value === undefined || value === null ? 'unset' : String(value)} does not use it` + } + if (!condition) return null + if ('opaque' in condition) { + const operation = values.get('operation') + if (typeof operation === 'string' && visibleFor?.[operation] === false) { + return rejected('operation') + } + const model = values.get('model') + if (typeof model === 'string' && visibleForModel?.[model] === false) return rejected('model') + return null + } + const lookup = paramLookup(values) + const failing = [condition, condition.and].find( + (part): part is NonNullable<typeof part> => + part !== undefined && + evaluateClause({ field: part.field, value: part.value, not: part.not }, lookup) === false + ) + if (!failing) return null + if (Array.isArray(failing.value) && failing.value.length > LISTED_VALUES) { + return rejected(failing.field) + } + const wanted = Array.isArray(failing.value) + ? failing.value.map(String).join(' or ') + : String(failing.value) + return `it shows only when ${failing.field} is ${failing.not ? 'not ' : ''}${wanted}` +} + /** * The fields a block leaves empty although they are required under the values it runs with * (`W_REQUIRED_MISSING`). `values` are the block's compiled params, which already hold each From 1d48f5cfe574842efb29a3c4a9ca5023705131fa Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:25:46 -0800 Subject: [PATCH 19/59] fix(workflow-code): bind renames of name and const by the lock, and quiet $refs a rename resolves Three or more blocks renamed at once bind by params; renames nothing ties are refused naming the look-alike blocks instead of offering --allow-delete. A dangling $ref in a function file is one warning at the file line in check and dry run alike. --- .../apply-workflow-source.integration.ts | 161 ++++++++++++++++++ .../workflows/application/source-evaluate.ts | 138 +++++++++++++-- apps/sim/lib/workflows/code/compile-source.ts | 2 +- packages/workflow-code/DESIGN.md | 6 +- .../workflow-code/src/compile/grammar.test.ts | 42 +++++ packages/workflow-code/src/compile/index.ts | 18 +- packages/workflow-code/src/compile/values.ts | 14 +- .../workflow-code/src/materialize/identity.ts | 112 +++++++++--- 8 files changed, 447 insertions(+), 46 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index e2d516f70a7..87af5119f9a 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -2199,3 +2199,164 @@ export default workflow('', {}, () => { expect(await snapshot(seeded.workflowId)).toEqual(before) }) }) + +describe('renames in source', () => { + /** Start → Boom → Tally → Report, each a function; Report's code reads Boom and Tally by $ref. */ + async function seedChain() { + const workflowId = generateId() + const now = new Date() + await db.insert(workflow).values({ + id: workflowId, + userId: ownerId, + workspaceId, + name: `Source renames ${workflowId}`, + lastSynced: now, + createdAt: now, + updatedAt: now, + }) + const fn = (name: string, code: string): BlockState => ({ + id: generateId(), + type: 'function', + name, + position: { x: 0, y: 0 }, + subBlocks: { code: { id: 'code', type: 'code', value: code } }, + outputs: {}, + enabled: true, + }) + const start: BlockState = { ...fn('Start', ''), type: 'start_trigger', subBlocks: {} } + const boom = fn('Boom', 'return 1') + const tally = fn('Tally', 'return 2') + const report = fn('Report', 'return <boom.result> + <tally.result>') + const chain = [start, boom, tally, report] + const saved = await saveWorkflowToNormalizedTables(workflowId, { + blocks: Object.fromEntries(chain.map((block) => [block.id, block])), + edges: chain.slice(1).map((block, index) => ({ + id: generateId(), + source: chain[index].id, + target: block.id, + })), + loops: {}, + parallels: {}, + }) + if (!saved.success) throw new Error(`Seeding the workflow failed: ${saved.error}`) + return { workflowId, ids: { boom: boom.id, tally: tally.id, report: report.id } } + } + + const codes = (diagnostics: ReadonlyArray<{ code: string }>) => diagnostics.map((d) => d.code) + + /** Replaces every whole-word `from` in `workflow.ts`. */ + function renameIdent(files: Record<string, string>, from: string, to: string) { + return { + ...files, + 'workflow.ts': files['workflow.ts'].replace(new RegExp(`\\b${from}\\b(?!\\.js)`, 'g'), to), + } + } + + check('two display-name renames in one save keep both ids and resolve their $refs', async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = edit(edit(source.files, "fn('Boom'", "fn('Bang'"), "fn('Tally'", "fn('Total'") + + const dryRun = await apply(source, files, { dryRun: true }) + expect(codes(dryRun.diagnostics)).toEqual([]) + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed).toEqual([ + { id: seeded.ids.boom, from: 'boom', to: 'bang' }, + { id: seeded.ids.tally, from: 'tally', to: 'total' }, + ]) + const after = await snapshot(seeded.workflowId) + expect(after.state?.blocks[seeded.ids.report]?.subBlocks.code?.value).toBe( + 'return <bang.result> + <total.result>' + ) + }) + + check( + 'a block whose name and const both change while its wiring changes keeps its id', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = edit( + edit(renameIdent(source.files, 'boom', 'bang'), "fn('Boom'", "fn('Bang'"), + ' tally.to(report)', + ' tally.to(report)\n bang.to(report)' + ) + + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed).toEqual([ + { id: seeded.ids.boom, from: 'boom', to: 'bang' }, + ]) + expect(codes(result.diagnostics)).toContain('W_MAT_INFERRED_RENAME') + } + ) + + check('three blocks of one type renamed name and const in one save keep their ids', async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + let files = source.files + for (const [ident, name, newIdent, newName] of [ + ['boom', 'Boom', 'bang', 'Bang'], + ['tally', 'Tally', 'total', 'Total'], + ['report', 'Report', 'summary', 'Summary'], + ]) { + files = edit(renameIdent(files, ident, newIdent), `fn('${name}'`, `fn('${newName}'`) + } + + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed.map((r) => r.id).sort()).toEqual( + [seeded.ids.boom, seeded.ids.tally, seeded.ids.report].sort() + ) + expect(codes(result.diagnostics)).not.toContain('W_UNRESOLVED_REF') + const after = await snapshot(seeded.workflowId) + expect(after.state?.blocks[seeded.ids.report]?.subBlocks.code?.value).toBe( + 'return <bang.result> + <total.result>' + ) + }) + + check( + 'renames nothing ties to the stored blocks are refused with how to keep the ids, not --allow-delete', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + let files = source.files + for (const [ident, name, newIdent, newName] of [ + ['boom', 'Boom', 'bang', 'Bang'], + ['tally', 'Tally', 'total', 'Total'], + ]) { + files = edit(renameIdent(files, ident, newIdent), `fn('${name}'`, `fn('${newName}'`) + } + files = edit(edit(files, 'return 1', 'return 10'), 'return 2', 'return 20') + + const dryRun = await apply(source, files, { dryRun: true }) + + const refusal = dryRun.diagnostics.find((d) => d.code === 'E_APPLY_BLOCK_DELETE') + expect(refusal?.message).toBe( + 'This apply deletes 2 blocks the source no longer declares: Boom, Tally. It adds Bang, Total of the same type: if those are the same blocks renamed, both their name and const changed, so nothing ties them to the stored blocks and they would get new ids.' + ) + expect(refusal?.hint).not.toContain('allow-delete') + } + ) + + check( + 'a $ref to a missing block in a function file is the same warning in check and dry run, at the file', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = { ...source.files, 'functions/boom.js': 'return $ref.nosuchblock.value\n' } + + const dryRun = await apply(source, files, { dryRun: true }) + + expect( + dryRun.diagnostics.map((d) => ({ code: d.code, severity: d.severity, file: d.file })) + ).toEqual([{ code: 'W_UNRESOLVED_REF', severity: 'warning', file: 'functions/boom.js' }]) + } + ) +}) diff --git a/apps/sim/lib/workflows/application/source-evaluate.ts b/apps/sim/lib/workflows/application/source-evaluate.ts index 7892e1ecef7..bbe483f7e6a 100644 --- a/apps/sim/lib/workflows/application/source-evaluate.ts +++ b/apps/sim/lib/workflows/application/source-evaluate.ts @@ -4,10 +4,13 @@ import { workflowBlocks } from '@sim/db/schema' import { isRecordLike } from '@sim/utils/object' import { type BlockCatalog, + type CanonicalBlock, type CanonicalWorkflow, type CanonicalWorkflowMeta, + type CompileResult, canonicalize, canonicalizeUnderLock, + compileWorkflow, type Diagnostic, diffCanonical, encodeAnchor, @@ -20,6 +23,7 @@ import { renamesBetween, type SourceBase, type SourceHints, + type SourceSpan, type WorkflowLockBlock, type WorkflowLockV1, type WorkflowSourceMetaV1, @@ -229,6 +233,102 @@ function withoutPendingReferences( } } +/** + * `lint` without the block-output references compile already reported in a function's code + * file (`W_UNRESOLVED_REF` on that block, located at the file line): the same dangling `$ref` + * is one warning there, as an offline check reports it, not also a lint error on the + * `codeFile()` call. + */ +function withoutCodeFileReferences( + lint: WorkflowLintReport, + compiled: CompileResult['diagnostics'], + keyOf: (id: string) => string | undefined +): WorkflowLintReport { + const reported = new Set( + compiled.flatMap((diagnostic) => + diagnostic.code === 'W_UNRESOLVED_REF' && + diagnostic.blockKey !== undefined && + diagnostic.span.file !== SOURCE_ENTRY_FILE + ? [diagnostic.blockKey] + : [] + ) + ) + if (reported.size === 0) return lint + return { + ...lint, + unresolvedReferences: lint.unresolvedReferences.filter((reference) => { + const key = keyOf(reference.blockId) + return !( + reference.kind === 'block-output' && + reference.field === 'code' && + key !== undefined && + reported.has(key) + ) + }), + } +} + +/** + * `diagnostics` without the code-file `W_UNRESOLVED_REF` findings a block rename resolves: an + * apply rewrites every `<old key.path>` to the new key, so `$ref.<old key>` runs. The compile + * already knew the renames the lock names by ident; a rename materialize inferred (name and + * const both changed) is known only now, so the source is compiled again with every rename and + * a finding stays only where that compile still reports it. + */ +function withoutRenamedCodeFileRefs( + diagnostics: CompileResult['diagnostics'], + renames: ReadonlyMap<string, string>, + recompile: () => CompileResult +): CompileResult['diagnostics'] { + if (renames.size === 0 || !diagnostics.some((d) => d.code === 'W_UNRESOLVED_REF')) { + return diagnostics + } + const at = (d: CompileResult['diagnostics'][number]) => + `${d.span.file}:${d.span.line}:${d.span.column}` + const still = new Set( + recompile() + .diagnostics.filter((d) => d.code === 'W_UNRESOLVED_REF') + .map(at) + ) + return diagnostics.filter((d) => d.code !== 'W_UNRESOLVED_REF' || still.has(at(d))) +} + +/** Names in a refusal, the first ten and a count of the rest. */ +function listNames(names: readonly string[]): string { + const shown = names.slice(0, 10).join(', ') + return names.length > 10 ? `${shown}, and ${names.length - 10} more` : shown +} + +/** + * `E_APPLY_BLOCK_DELETE` for the stored blocks an apply removes. When the source adds blocks of + * a removed block's type, they may be renames whose name and const both changed, which no rule + * could tie to the stored blocks (several alike, and none reads as the one renamed): the finding + * names them and says how to keep the ids, instead of offering to delete. + */ +function blockDeleteRefusal( + removed: readonly CanonicalBlock[], + added: readonly CanonicalBlock[], + span: SourceSpan | undefined +): Diagnostic { + const removedTypes = new Set(removed.map((block) => block.type)) + const lookalikes = added.filter((block) => removedTypes.has(block.type)) + const deletes = `This apply deletes ${removed.length} block${removed.length === 1 ? '' : 's'} the source no longer declares: ${listNames(removed.map((block) => block.name))}.` + return { + code: 'E_APPLY_BLOCK_DELETE', + severity: 'error', + ...(lookalikes.length > 0 + ? { + message: `${deletes} It adds ${listNames(lookalikes.map((block) => block.name))} of the same type: if those are the same blocks renamed, both their name and const changed, so nothing ties them to the stored blocks and they would get new ids.`, + hint: 'To rename a block and keep its id, change its name or its const in one apply and the other in the next (the lock binds a block by either); or declare the removed blocks again.', + } + : { + message: deletes, + hint: 'Pass --allow-delete (allowDelete=true) to delete them, or declare them again to keep them.', + }), + ...(span ? { span } : {}), + } +} + /** Everything a commit (or a dry run) writes and reports, once every check passed. */ export interface SourcePlan { /** @@ -411,7 +511,7 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour input.storedCanonical.canonical, lock ) - const compiled = compileSource({ + const compileInput = { entry: { path: SOURCE_ENTRY_FILE, text: files[SOURCE_ENTRY_FILE] }, files: new Map(Object.entries(files).filter(([path]) => path !== SOURCE_ENTRY_FILE)), catalog, @@ -420,10 +520,9 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour ...Object.keys(lock.blocks), ...Object.keys(storedCanonical.identity.blocks), ]), - stored: storedCanonical.workflow, - lock, envNames: input.envNames, - }) + } + const compiled = compileSource({ ...compileInput, stored: storedCanonical.workflow, lock }) const compiledWorkflow = compiled.workflow if (!compiledWorkflow || hasErrors(compiled.diagnostics)) { const refused: Diagnostic[] = [...compiled.diagnostics] @@ -609,10 +708,15 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour * so a field source never prints (a forEach loop's empty while condition, a pair's stored * member) that materialize keeps from the stored block is no change. */ + const blockRenames = renamesBetween(storedCanonical.identity, materialized.identity) const canonicalDiff = diffCanonical( storedCanonical.workflow, { ...materialized.workflow, meta: storedCanonical.workflow.meta }, - renamesBetween(storedCanonical.identity, materialized.identity) + blockRenames + ) + const renamedKeys = new Map([...blockRenames].map(([key, storedKey]) => [storedKey, key])) + const compileDiagnostics = withoutRenamedCodeFileRefs(compiled.diagnostics, renamedKeys, () => + compileWorkflow({ ...compileInput, stored: compiled.stored, renamedKeys }) ) const sourceRefusals: Diagnostic[] = [] const headerSpan = compiled.spans[encodeAnchor({ kind: 'header' })] @@ -641,15 +745,13 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour ) const removedKeys = isNew ? [] : canonicalDiff.blocks.removed if (removedKeys.length > 0 && !input.allowDelete) { - const names = removedKeys.map((key) => storedCanonical.workflow.blocks[key]?.name ?? key) - const shown = names.slice(0, 10).join(', ') - sourceRefusals.push({ - code: 'E_APPLY_BLOCK_DELETE', - severity: 'error', - message: `This apply deletes ${removedKeys.length} block${removedKeys.length === 1 ? '' : 's'} the source no longer declares: ${shown}${names.length > 10 ? `, and ${names.length - 10} more` : ''}.`, - hint: 'Pass --allow-delete (allowDelete=true) to delete them, or declare them again to keep them.', - ...(headerSpan ? { span: headerSpan } : {}), - }) + sourceRefusals.push( + blockDeleteRefusal( + removedKeys.map((key) => storedCanonical.workflow.blocks[key]), + canonicalDiff.blocks.added.map((key) => materialized.workflow.blocks[key]), + headerSpan + ) + ) } const storedGraph: SourceGraph = { blocks: state.blocks, edges: state.edges } @@ -718,13 +820,17 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour ) const changedIds = new Set([...graphDiff.added, ...graphDiff.modified.map((m) => m.id)]) const diagnostics = [ - ...compiled.diagnostics, + ...compileDiagnostics, ...keptDiagnostics, ...materializeDiagnostics, ...refusals, ...normalized.diagnostics, ...lintDiagnostics( - withoutPendingReferences(lint, input.pendingResources ?? []), + withoutCodeFileReferences( + withoutPendingReferences(lint, input.pendingResources ?? []), + compileDiagnostics, + (id) => keyById.get(id) + ), changedIds, (id) => keyById.get(id) ), diff --git a/apps/sim/lib/workflows/code/compile-source.ts b/apps/sim/lib/workflows/code/compile-source.ts index d89f3f66042..4bba3732d24 100644 --- a/apps/sim/lib/workflows/code/compile-source.ts +++ b/apps/sim/lib/workflows/code/compile-source.ts @@ -56,5 +56,5 @@ export function compileSource( ) if (renames.size === 0) return { ...first, stored: input.stored } const stored = rekeyStored(input.stored, renames) - return { ...compileWorkflow({ ...compileInput, stored }), stored } + return { ...compileWorkflow({ ...compileInput, stored, renamedKeys: renames }), stored } } diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index f7aaa16a25f..6f26864e5b1 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -197,7 +197,11 @@ key-keyed lock; `mergeSourceMetaFromLock` stores a write's hints back by id. 3. `db-name`: the prev block with the same key (the lockless and stale-lock path); 4. `inferred`: exactly one unmatched prev block and one unmatched compiled block with the same type and parent and agreeing edge neighbourhoods, so renaming the variable and the display - name together keeps the id (`W_MAT_INFERRED_RENAME`; to get a new block instead, delete the old one in one apply and add the new one in the next); + name together keeps the id; failing that, a prev block the lock binds (the source dropped its + key and its ident) pairs with the one new block of its type and parent whatever its wiring, + or, when several were renamed at once, with the one new block whose params agree with it + (`W_MAT_INFERRED_RENAME`; to get a new block instead, delete the old one in one apply and add the new one in the next). Renames nothing ties + are refused as `E_APPLY_BLOCK_DELETE`, naming the look-alike new blocks and how to keep the ids; 5. otherwise created: the lock's id when unclaimed, else minted (`idSource`). A match across types is a delete plus a create (`W_MAT_RETYPE_RECREATED`), except a catalog diff --git a/packages/workflow-code/src/compile/grammar.test.ts b/packages/workflow-code/src/compile/grammar.test.ts index 63b32b6b411..cb7792dac36 100644 --- a/packages/workflow-code/src/compile/grammar.test.ts +++ b/packages/workflow-code/src/compile/grammar.test.ts @@ -1062,6 +1062,48 @@ describe('errors, each with its code, location and hint', () => { ]) }) + it('resolves a code file $ref to the stored key of a block the source renamed', () => { + const text = source( + [ + START, + "const lookup = api('Fetch Page', { url: 'x' })", + "const f = fn('F', { code: codeFile('./fns/f.js') })", + ].join('\n') + ) + const files = { 'workflows/fns/f.js': 'return $ref.lookup.data + $ref.lookup.nope' } + const lockBlock = (id: string, ident: string, order: number) => ({ + id, + parent: null, + ident, + order, + }) + const storedUnread = { + version: 1 as const, + blocks: { + start: lockBlock('b-start', 'start', 0), + lookup: lockBlock('b-lookup', 'lookup', 1), + f: lockBlock('b-f', 'f', 2), + }, + variables: {}, + } + const compile = (input: Partial<Parameters<typeof compileWorkflow>[0]>) => + compileWorkflow({ + entry: { path: ENTRY, text }, + files: new Map(Object.entries(files)), + catalog: TEST_CATALOG, + sdkMajor: 1, + ...input, + }) + const codes = (result: CompileResult) => + result.diagnostics + .filter((d) => d.span.file === 'workflows/fns/f.js') + .map((d) => `${d.code} ${d.span.column}`) + const expected = ['W_UNKNOWN_OUTPUT_PATH 27'] + expect(codes(compile({ storedUnread }))).toEqual(expected) + expect(codes(compile({ renamedKeys: new Map([['lookup', 'fetchpage']]) }))).toEqual(expected) + expect(codes(compile({}))).toEqual(['W_UNRESOLVED_REF 8', 'W_UNRESOLVED_REF 27']) + }) + it('reports a syntax error inside a code file at its own location', () => { const result = run(source("const f = fn('F', { code: codeFile('./fns/bad.js') })"), { 'workflows/fns/bad.js': 'return (', diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 6810aedac00..0785d92a1dc 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -82,6 +82,13 @@ export interface CompileInput { * be stored. */ storedUnreadEdges?: ReadonlyArray<StoredEdgeRef> + /** + * Stored block keys the source renamed, stored key → source key (`lockIdentRenames`, which the + * server computes against `stored`; with `storedUnread` the compile computes them itself). An + * apply rewrites every `<stored.path>` to the new key, so a code file's `$ref.<stored key>` + * still resolves and is checked against the renamed block instead of warning. + */ + renamedKeys?: ReadonlyMap<BlockKey, BlockKey> /** * The env variable names a run resolves (the workspace's plus the caller's personal ones). * `env.NAME` outside it compiles with `W_ENV_UNKNOWN`; absent, no name is checked. @@ -615,7 +622,11 @@ function compileEntry( }) } - checkCodeFileRefs(ctx) + const unread = input.stored === undefined ? input.storedUnread : undefined + const unreadRenames = unread + ? lockIdentRenames(hints, unread, (key) => Object.hasOwn(unread.blocks, key)) + : new Map<BlockKey, BlockKey>() + checkCodeFileRefs(ctx, new Map([...unreadRenames, ...(input.renamedKeys ?? [])])) const refTypes: RefTypeScope = { catalog: input.catalog, @@ -641,16 +652,13 @@ function compileEntry( checkDynamicOutputRefs(ctx, (key) => compiled.get(key)) - const unread = input.stored === undefined ? input.storedUnread : undefined const edges = new EdgeCollector( ctx, compiled, unread && { lockKeys: new Map([ ...Object.keys(unread.blocks).map((key) => [key, key] as const), - ...[...lockIdentRenames(hints, unread, (key) => Object.hasOwn(unread.blocks, key))].map( - ([lockKey, key]) => [key, lockKey] as const - ), + ...[...unreadRenames].map(([lockKey, key]) => [key, lockKey] as const), ]), ...(input.storedUnreadEdges ? { diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index 2f319a4da9f..8461ab6a913 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -857,9 +857,14 @@ export function checkDynamicOutputRefs( * that names no block (a block's ident, a renamed block's old key, a typo) or a variable that does * not exist is `W_UNRESOLVED_REF`, and a path no output has is `W_UNKNOWN_OUTPUT_PATH`. Warnings, * not errors, because stored code may already hold a dangling reference and must round-trip; at - * run time such a token is spliced into the code as text, which fails as JavaScript. + * run time such a token is spliced into the code as text, which fails as JavaScript. A head that + * is the stored key of a block the source renamed (`renamed`, stored key → source key) resolves: + * an apply rewrites it to the new key, so its path is checked against the renamed block. */ -export function checkCodeFileRefs(ctx: CompileContext): void { +export function checkCodeFileRefs( + ctx: CompileContext, + renamed: ReadonlyMap<BlockKey, BlockKey> +): void { for (const { path, blockKey, refs } of ctx.codeFiles.chains) { for (const ref of refs) { const at: Range = { file: path, start: ref.start, end: ref.end } @@ -887,7 +892,10 @@ export function checkCodeFileRefs(ctx: CompileContext): void { continue } if (isReservedWorkflowBlockName(head.name)) continue - const target = ctx.declsByKey.get(head.name) + const renamedTo = renamed.get(head.name) + const target = + ctx.declsByKey.get(head.name) ?? + (renamedTo === undefined ? undefined : ctx.declsByKey.get(renamedTo)) if (target) { checkOutputPath(ctx, at, target, rest, blockKey) continue diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index b23cba01984..4b1e36b9366 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -6,7 +6,7 @@ import type { MaterializeDiagnosticCode } from '../diagnostics/codes' import type { Diagnostic } from '../diagnostics/frame' import { pointer, type WorkflowLockBlock, type WorkflowLockV1 } from '../lock' import type { BlockKey, CanonicalWorkflow } from '../model' -import { getOwn } from '../records' +import { deepEqual, getOwn } from '../records' import type { MaterializeHooks } from './context' /** How identity rule matched a compiled block to a stored one (rules 1–4). */ @@ -248,6 +248,11 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { match, settled, diagnostics, + lockedIds: new Set( + Object.entries(lock?.blocks ?? {}).flatMap(([key, entry]) => + duplicates.has(key) ? [] : [entry.id] + ) + ), }) } @@ -319,6 +324,11 @@ interface InferenceState { match: (key: BlockKey, prevId: string, via: IdentityMatchRule) => boolean settled: Map<BlockKey, SettledKey> diagnostics: IdentityInputs['diagnostics'] + /** + * The ids the lock binds: blocks the author's source held. After rules 1–3 such a block is + * unmatched only when the source dropped both its key and its ident. + */ + lockedIds: ReadonlySet<string> } /** `in:<id>` / `out:<id>` for every already-bound neighbour of a block. */ @@ -345,14 +355,44 @@ function sameSet(left: Set<string>, right: Set<string>): boolean { return left.size === right.size && [...left].every((item) => right.has(item)) } +/** + * Whether two blocks' params agree: equal on every param both hold, and holding at least one in + * common. A param only one side holds is a default compile filled in or canvas state, which says + * nothing about identity. + */ +function paramsAgree( + left: CanonicalWorkflow['blocks'][string]['params'], + right: CanonicalWorkflow['blocks'][string]['params'] +): boolean { + const shared = Object.keys(left).filter((id) => Object.hasOwn(right, id)) + return shared.length > 0 && shared.every((id) => deepEqual(left[id], right[id])) +} + +/** + * How rule 4 pairs a group: `connections`, a single unmatched block on each side whose bound + * neighbours agree; `vacated`, a single unmatched block on each side where the stored one is a + * block the lock binds (the source dropped its key and its ident: a rename of both, with its + * wiring free to change); `params`, the one vacated stored block whose params agree with a + * new block's, when several of one type were renamed at once. + */ +type InferenceBasis = 'connections' | 'vacated' | 'params' + +const INFERENCE_REASON: Record<InferenceBasis, string> = { + connections: 'same type, parent and connections', + vacated: 'same type and parent, and the only block of them the source dropped', + params: 'same type, parent and params', +} + /** * Identity rule 4, repeated until it binds nothing new so a renamed container lets its - * renamed children pair next. A group is `(type, parent id)`; only a group with exactly one - * unmatched block on each side pairs, and only when both blocks have the same already-bound - * neighbours in the same directions. Each pairing is flagged with `W_MAT_INFERRED_RENAME`. + * renamed children pair next. A group is `(type, parent id)`. A group with exactly one unmatched + * block on each side pairs when both have the same already-bound neighbours in the same + * directions; failing that, and only once no group pairs that way, when the stored block is one + * the lock binds (its key and ident both left the source), or, for several such blocks, the one + * whose params agree with a new block's. Each pairing is flagged with `W_MAT_INFERRED_RENAME`. */ function inferRenames(state: InferenceState): void { - const { workflow, prev, resolved, claimed } = state + const { workflow, prev, resolved, claimed, lockedIds } = state const prevCanonical = prev.canonical.workflow const prevIdentity = prev.canonical.identity.blocks /** Prev blocks a settled key accounts for: a retype's own prev, else the prev under its key. */ @@ -368,9 +408,7 @@ function inferRenames(state: InferenceState): void { return id !== undefined && claimed.has(id) ? id : undefined } - let progressed = true - while (progressed) { - progressed = false + const groups = () => { const targetGroups = new Map<string, BlockKey[]>() for (const key of state.open()) { const block = workflow.blocks[key] @@ -392,25 +430,59 @@ function inferRenames(state: InferenceState): void { if (keys) keys.push(key) else prevGroups.set(group, [key]) } - for (const [group, targets] of targetGroups) { + return [...targetGroups].flatMap(([group, targets]) => { const prevKeys = prevGroups.get(group) - if (targets.length !== 1 || prevKeys?.length !== 1) continue + return prevKeys ? [{ targets, prevKeys }] : [] + }) + } + + const pair = (key: BlockKey, prevKey: BlockKey, basis: InferenceBasis): boolean => { + if (!state.match(key, prevIdentity[prevKey].id, 'inferred')) return false + state.diagnostics.push({ + code: 'W_MAT_INFERRED_RENAME', + severity: 'warning', + message: `Block "${key}" is taken as a rename of "${prevKey}" (${INFERENCE_REASON[basis]}), so it keeps that block's id and run history. To make it a new block instead, delete "${prevKey}" in one apply and add "${key}" in the next`, + blockKey: key, + }) + return true + } + + const byConnections = (): boolean => { + let progressed = false + for (const { targets, prevKeys } of groups()) { + if (targets.length !== 1 || prevKeys.length !== 1) continue const [key] = targets const [prevKey] = prevKeys const agree = sameSet( neighbourhood(workflow.edges, key, idOfTarget), neighbourhood(prevCanonical.edges, prevKey, idOfPrev) ) - if (!agree) continue - const prevId = prevIdentity[prevKey].id - if (!state.match(key, prevId, 'inferred')) continue - progressed = true - state.diagnostics.push({ - code: 'W_MAT_INFERRED_RENAME', - severity: 'warning', - message: `Block "${key}" is taken as a rename of "${prevKey}" (same type, parent and connections), so it keeps that block's id and run history. To make it a new block instead, delete "${prevKey}" in one apply and add "${key}" in the next`, - blockKey: key, - }) + if (agree && pair(key, prevKey, 'connections')) progressed = true } + return progressed } + + const byVacatedLock = (): boolean => { + let progressed = false + for (const { targets, prevKeys } of groups()) { + const vacated = prevKeys.filter((prevKey) => lockedIds.has(prevIdentity[prevKey].id)) + if (vacated.length === 0) continue + if (targets.length === 1 && prevKeys.length === 1 && vacated.length === 1) { + if (pair(targets[0], vacated[0], 'vacated')) progressed = true + continue + } + const equalParams = (key: BlockKey, prevKey: BlockKey) => + paramsAgree(workflow.blocks[key].params, prevCanonical.blocks[prevKey].params) + for (const key of targets) { + const [only, ...more] = vacated.filter((prevKey) => equalParams(key, prevKey)) + if (only === undefined || more.length > 0) continue + if (targets.filter((other) => equalParams(other, only)).length !== 1) continue + if (pair(key, only, 'params')) progressed = true + } + } + return progressed + } + + let progressed = true + while (progressed) progressed = byConnections() || byVacatedLock() } From 16330da2ebbfe46e30a5ea72a59c65571637ab0f Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:31:53 -0800 Subject: [PATCH 20/59] fix(workflow-code): point workflow-level findings at the name, name the stale workflow, and list changed variables E_NAME_TAKEN, E_WORKFLOW_META_INVALID and E_APPLY_BLOCK_DELETE point at the workflow name (or description) instead of the SDK header; an unknown MCP tool points at the agent's tools line; the stale finding names the workflow; the offline check stays quiet on a legacy block the lock binds, as the server does; the CLI apply result and preview name each changed variable. --- .../[workflowId]/source/route.test.ts | 2 +- .../apply-workflow-source.integration.ts | 32 +++++++---- .../application/apply-workflow-source.ts | 6 +-- .../application/source-analytics.test.ts | 4 +- .../workflows/application/source-commit.ts | 6 ++- .../lib/workflows/application/source-error.ts | 17 ++++-- .../workflows/application/source-evaluate.ts | 54 +++++++++++++------ .../application/kinds/mcp/adapter.ts | 53 ++++++++++++------ .../application/kinds/workflow/adapter.ts | 7 ++- .../application/mcp.integration.ts | 2 + .../lib/workspace-source/application/types.ts | 3 ++ .../protocol/workflow-source-apply.test.ts | 19 +++++++ .../protocol/workflow-source-apply.ts | 15 ++++-- packages/workflow-code/src/compile/blocks.ts | 12 ++++- .../workflow-code/src/compile/checks.test.ts | 18 +++++++ packages/workflow-code/src/compile/context.ts | 9 ++++ packages/workflow-code/src/compile/index.ts | 12 +++++ 17 files changed, 210 insertions(+), 61 deletions(-) diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts index f1b1b2ca24d..741d8ffe85e 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts @@ -215,7 +215,7 @@ describe('POST /api/v2/workflows/{workflowId}/source', () => { it('answers 409 with the current fingerprint when the draft moved', async () => { const current = `sf1:${'f'.repeat(64)}` - mocks.apply.mockRejectedValue(new WorkflowSourceStaleError(current)) + mocks.apply.mockRejectedValue(new WorkflowSourceStaleError(current, 'Lead Scoring')) const response = await POST(applyRequest(applyBody), context) const body = await response.json() diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index 87af5119f9a..9749b6babb9 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -174,6 +174,14 @@ function apply( ) } +/** The 1-based line and column where `needle` first starts in `text`. */ +function positionOf(text: string, needle: string): { line: number; column: number } { + const index = text.indexOf(needle) + expect(index, `the text holds "${needle}"`).toBeGreaterThanOrEqual(0) + const before = text.slice(0, index) + return { line: before.split('\n').length, column: index - before.lastIndexOf('\n') } +} + /** Replaces the one occurrence of `from` across the source files. */ function edit(files: Record<string, string>, from: string, to: string): Record<string, string> { const hits = Object.entries(files).filter(([, text]) => text.split(from).length === 2) @@ -1605,7 +1613,7 @@ describe('source apply edge cases', () => { ) check( - 'a workflow renamed in source to a name its folder holds is refused at the header', + 'a workflow renamed in source to a name its folder holds is refused at the name', async () => { const seeded = await seedWorkflow() const other = await seedWorkflow() @@ -1614,13 +1622,11 @@ describe('source apply edge cases', () => { const taken = await storedName(other.workflowId) const before = await snapshot(seeded.workflowId) - const error = await expectRejected( - apply(source, edit(source.files, `'${name}'`, `'${taken}'`)), - 'E_NAME_TAKEN' - ) + const files = edit(source.files, `'${name}'`, `'${taken}'`) + const error = await expectRejected(apply(source, files), 'E_NAME_TAKEN') expect(error.diagnostics.find((d) => d.code === 'E_NAME_TAKEN')).toMatchObject({ file: 'workflow.ts', - line: expect.any(Number), + ...positionOf(files['workflow.ts'], `'${taken}'`), }) expect(await storedName(seeded.workflowId)).toBe(name) expect(await snapshot(seeded.workflowId)).toEqual(before) @@ -1789,9 +1795,11 @@ export default workflow('', {}, () => { } const error = await expectRejected(apply(source, emptied), 'E_APPLY_BLOCK_DELETE') - expect(error.diagnostics.find((d) => d.code === 'E_APPLY_BLOCK_DELETE')?.message).toBe( - 'This apply deletes 2 blocks the source no longer declares: Start, Score.' - ) + expect(error.diagnostics.find((d) => d.code === 'E_APPLY_BLOCK_DELETE')).toMatchObject({ + message: 'This apply deletes 2 blocks the source no longer declares: Start, Score.', + file: 'workflow.ts', + ...positionOf(emptied['workflow.ts'], "'Source apply edge"), + }) expect(await snapshot(workflowId)).toEqual(before) const dryRun = await apply(source, emptied, { dryRun: true }) @@ -1971,7 +1979,11 @@ export default workflow('', {}, () => { expect(result.dryRun).toBe(true) expect(result.applied).toBe(false) expect(result.diagnostics).toContainEqual( - expect.objectContaining({ code: 'E_APPLY_STALE', severity: 'error' }) + expect.objectContaining({ + code: 'E_APPLY_STALE', + severity: 'error', + message: `Workflow "Source apply ${seeded.workflowId}" changed since this source was read. Get the source again, reapply your edit and retry.`, + }) ) expect(result.base.fingerprint).toBe(source.base.fingerprint) expect(await snapshot(seeded.workflowId)).toEqual(before) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.ts b/apps/sim/lib/workflows/application/apply-workflow-source.ts index a8f7f34280d..428edf4a5e5 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.ts @@ -49,10 +49,10 @@ import { } from '@/lib/workflows/application/source-commit' import { emptyDiffSummary, type SourceDiffSummary } from '@/lib/workflows/application/source-diff' import { - WORKFLOW_SOURCE_STALE_MESSAGE, type WorkflowSourceDiagnostic, WorkflowSourceRejectedError, WorkflowSourceStaleError, + workflowSourceStaleMessage, } from '@/lib/workflows/application/source-error' import { evaluateSource, @@ -559,12 +559,12 @@ async function runSourceApply( ]) } if (target.kind === 'existing' && storedFingerprint !== target.base.fingerprint) { - if (!dryRun) throw new WorkflowSourceStaleError(storedFingerprint) + if (!dryRun) throw new WorkflowSourceStaleError(storedFingerprint, stored.meta.name) return refuse([ { code: 'E_APPLY_STALE', severity: 'error', - message: WORKFLOW_SOURCE_STALE_MESSAGE, + message: workflowSourceStaleMessage(stored.meta.name), }, ]) } diff --git a/apps/sim/lib/workflows/application/source-analytics.test.ts b/apps/sim/lib/workflows/application/source-analytics.test.ts index 742fdd95586..b8b13648dec 100644 --- a/apps/sim/lib/workflows/application/source-analytics.test.ts +++ b/apps/sim/lib/workflows/application/source-analytics.test.ts @@ -35,7 +35,7 @@ describe('recordSourceRefused', () => { beforeEach(() => capture.mockClear()) it.each([ - ['stale', new WorkflowSourceStaleError('sf1:x')], + ['stale', new WorkflowSourceStaleError('sf1:x', 'Lead Scoring')], ['rejected', new WorkflowSourceRejectedError([])], ['locked', new OrchestrationError('locked', 'Workflow is locked')], ])('records a %s refusal as ok:false with its error_kind', (kind, error) => { @@ -74,7 +74,7 @@ describe('recordSourceRefused', () => { principal: createWorkspaceApiKeyPrincipal({ workspaceId: 'ws-1' }), workflowId: 'wf-1', }, - new WorkflowSourceStaleError('sf1:x') + new WorkflowSourceStaleError('sf1:x', 'Lead Scoring') ) expect(capture).not.toHaveBeenCalled() diff --git a/apps/sim/lib/workflows/application/source-commit.ts b/apps/sim/lib/workflows/application/source-commit.ts index 603d1cd6e24..5f86fc8a561 100644 --- a/apps/sim/lib/workflows/application/source-commit.ts +++ b/apps/sim/lib/workflows/application/source-commit.ts @@ -227,9 +227,11 @@ async function recheckAndWriteSourceMeta( canonicalize(current.state, { catalog, meta: current.meta, resources: plan.resources }), plan.resources ) - if (fingerprint !== args.storedFingerprint) throw new WorkflowSourceStaleError(fingerprint) + if (fingerprint !== args.storedFingerprint) { + throw new WorkflowSourceStaleError(fingerprint, current.meta.name) + } if (options.layoutOnly && !isEqual(withoutBlockLayout(current), withoutBlockLayout(stored))) { - throw new WorkflowSourceStaleError(fingerprint) + throw new WorkflowSourceStaleError(fingerprint, current.meta.name) } } await writeWorkflowSourceMeta( diff --git a/apps/sim/lib/workflows/application/source-error.ts b/apps/sim/lib/workflows/application/source-error.ts index b7b69406c4c..aa1ee7724e4 100644 --- a/apps/sim/lib/workflows/application/source-error.ts +++ b/apps/sim/lib/workflows/application/source-error.ts @@ -49,13 +49,20 @@ export class WorkflowSourceRejectedError extends OrchestrationError { * edited the workflow, toggled a block lock, or a migration changed it. Re-read the source, * rebase the edit onto it and apply again. */ -/** Why a source apply refuses a draft changed since its base, as both its 409 and dry run say. */ -export const WORKFLOW_SOURCE_STALE_MESSAGE = - 'The workflow changed since this source was read. Get the source again, reapply your edit and retry.' +/** + * Why a source apply refuses a draft changed since its base, as both its 409 and dry run say, + * naming the workflow so a workspace apply's line for it says which one. + */ +export function workflowSourceStaleMessage(workflowName: string): string { + return `Workflow ${JSON.stringify(workflowName)} changed since this source was read. Get the source again, reapply your edit and retry.` +} export class WorkflowSourceStaleError extends OrchestrationError { - constructor(readonly currentFingerprint: string) { - super('conflict', WORKFLOW_SOURCE_STALE_MESSAGE) + constructor( + readonly currentFingerprint: string, + workflowName: string + ) { + super('conflict', workflowSourceStaleMessage(workflowName)) this.name = 'WorkflowSourceStaleError' } } diff --git a/apps/sim/lib/workflows/application/source-evaluate.ts b/apps/sim/lib/workflows/application/source-evaluate.ts index bbe483f7e6a..8fedfb23158 100644 --- a/apps/sim/lib/workflows/application/source-evaluate.ts +++ b/apps/sim/lib/workflows/application/source-evaluate.ts @@ -120,6 +120,8 @@ export interface SourceEvaluationInput { */ export interface SourceReference extends ResourceKeyRef { member?: string + /** Where the entry file writes the reference, for an MCP tool: the agent's `tools`. */ + span?: SourceSpan } /** A resource an enclosing apply creates or adopts, by key, with the id that stands in for it. */ @@ -293,6 +295,11 @@ function withoutRenamedCodeFileRefs( return diagnostics.filter((d) => d.code !== 'W_UNRESOLVED_REF' || still.has(at(d))) } +/** `{ span }` when there is one, to spread into a diagnostic. */ +function withSpan(span: SourceSpan | undefined): { span?: SourceSpan } { + return span ? { span } : {} +} + /** Names in a refusal, the first ten and a count of the rest. */ function listNames(names: readonly string[]): string { const shown = names.slice(0, 10).join(', ') @@ -325,7 +332,7 @@ function blockDeleteRefusal( message: deletes, hint: 'Pass --allow-delete (allowDelete=true) to delete them, or declare them again to keep them.', }), - ...(span ? { span } : {}), + ...withSpan(span), } } @@ -392,9 +399,13 @@ function isResourceKind(kind: string): kind is ResourceKind { * Every `(kind, key)` resource reference in a compiled model, for ident lookup, and the tool each * MCP agent tool (`mcp(key).tool(name)`) names on its server. */ -function collectResourceKeys(value: unknown, refs: SourceReference[]): void { +function collectResourceKeys( + value: unknown, + refs: SourceReference[], + span: SourceSpan | undefined +): void { if (Array.isArray(value)) { - for (const item of value) collectResourceKeys(item, refs) + for (const item of value) collectResourceKeys(item, refs, span) return } if (value === null || typeof value !== 'object') return @@ -408,8 +419,8 @@ function collectResourceKeys(value: unknown, refs: SourceReference[]): void { refs.push({ kind: record.kind, key: record.key }) } const tool = mcpToolReference(record) - if (tool) refs.push(tool) - for (const nested of Object.values(record)) collectResourceKeys(nested, refs) + if (tool) refs.push(span ? { ...tool, span } : tool) + for (const nested of Object.values(record)) collectResourceKeys(nested, refs, span) } /** The server and tool an MCP agent tool entry names, or `null` for anything else. */ @@ -459,14 +470,18 @@ function boundLock(identity: LockIdentity, submitted: WorkflowLockV1): WorkflowL function metaLimitRefusals( meta: CanonicalWorkflowMeta, fields: readonly (keyof CanonicalWorkflowMeta)[] -): Diagnostic[] { - const refusals: Diagnostic[] = [] - const refuse = (message: string, hint: string) => - refusals.push({ code: 'E_WORKFLOW_META_INVALID', severity: 'error', message, hint }) +): Array<{ field: keyof CanonicalWorkflowMeta; refusal: Diagnostic }> { + const refusals: Array<{ field: keyof CanonicalWorkflowMeta; refusal: Diagnostic }> = [] + const refuse = (field: keyof CanonicalWorkflowMeta, message: string, hint: string) => + refusals.push({ + field, + refusal: { code: 'E_WORKFLOW_META_INVALID', severity: 'error', message, hint }, + }) if (fields.includes('name') && meta.name.trim() === '') { - refuse('The workflow name is empty.', "Name the workflow: workflow('Name', ...).") + refuse('name', 'The workflow name is empty.', "Name the workflow: workflow('Name', ...).") } else if (fields.includes('name') && meta.name.length > V2_WORKFLOW_NAME_MAX_LENGTH) { refuse( + 'name', `The workflow name is ${meta.name.length} characters, over the ${V2_WORKFLOW_NAME_MAX_LENGTH}-character limit.`, `Shorten the name to ${V2_WORKFLOW_NAME_MAX_LENGTH} characters or fewer.` ) @@ -474,6 +489,7 @@ function metaLimitRefusals( const description = meta.description ?? '' if (fields.includes('description') && description.length > V2_WORKFLOW_DESCRIPTION_MAX_LENGTH) { refuse( + 'description', `The workflow description is ${description.length} characters, over the ${V2_WORKFLOW_DESCRIPTION_MAX_LENGTH}-character limit.`, `Shorten the description to ${V2_WORKFLOW_DESCRIPTION_MAX_LENGTH} characters or fewer.` ) @@ -549,7 +565,12 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour } const compiledKeys: SourceReference[] = [] - collectResourceKeys(compiledWorkflow.blocks, compiledKeys) + for (const [key, block] of Object.entries(compiledWorkflow.blocks)) { + const span = + compiled.spans[encodeAnchor({ kind: 'param', block: key, param: 'tools' })] ?? + compiled.spans[encodeAnchor({ kind: 'block', block: key })] + collectResourceKeys(block, compiledKeys, span) + } const storedResources = resources const newKeys = compiledKeys.filter(({ kind, key }) => storedResources.toId(kind, key) === null) if (newKeys.length > 0) { @@ -720,6 +741,9 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour ) const sourceRefusals: Diagnostic[] = [] const headerSpan = compiled.spans[encodeAnchor({ kind: 'header' })] + /** Where a finding about the workflow itself points: the field it names, else the name. */ + const metaSpan = (field: keyof CanonicalWorkflowMeta = 'name') => + compiled.metaSpans[field] ?? compiled.metaSpans.name ?? headerSpan if ( metaChanged.includes('name') && (await workflowNameExistsInFolder({ @@ -734,13 +758,13 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour severity: 'error', message: `Another workflow in this workflow's folder is named ${JSON.stringify(meta.name)}.`, hint: `Choose another name, or keep workflow(${JSON.stringify(storedMeta.name)}, ...).`, - ...(headerSpan ? { span: headerSpan } : {}), + ...withSpan(metaSpan('name')), }) } sourceRefusals.push( - ...metaLimitRefusals(meta, isNew ? META_FIELDS : metaChanged).map((refusal) => ({ + ...metaLimitRefusals(meta, isNew ? META_FIELDS : metaChanged).map(({ field, refusal }) => ({ ...refusal, - ...(headerSpan ? { span: headerSpan } : {}), + ...withSpan(metaSpan(field)), })) ) const removedKeys = isNew ? [] : canonicalDiff.blocks.removed @@ -749,7 +773,7 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour blockDeleteRefusal( removedKeys.map((key) => storedCanonical.workflow.blocks[key]), canonicalDiff.blocks.added.map((key) => materialized.workflow.blocks[key]), - headerSpan + metaSpan() ) ) } diff --git a/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts index 761c9de0d78..2ae18d1be15 100644 --- a/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts @@ -23,9 +23,14 @@ import { createLogger } from '@sim/logger' import { mapWithConcurrency } from '@sim/utils/concurrency' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { didYouMean, type Json } from '@sim/workflow-code' +import { didYouMean, type Json, type SourceSpan } from '@sim/workflow-code' import { SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' -import { type BaseResource, fingerprintOf, resourceAddress } from '@sim/workspace-code' +import { + type BaseResource, + type FlatDiagnostic, + fingerprintOf, + resourceAddress, +} from '@sim/workspace-code' import { MCP_DEFAULTS, type McpServerDecl, @@ -50,6 +55,7 @@ import type { McpServerRow } from '@/lib/mcp/queries' import { generateMcpServerId } from '@/lib/mcp/utils' import { admitOnce } from '@/lib/workspace-source/application/context' import { + finding, kindFinding, planUndeclared, serverGoneFinding, @@ -427,8 +433,11 @@ async function listTools( /** Each change that calls tools of an MCP server, with the tools, by server key. */ function toolUsers( changes: readonly PlannedChange[] -): Map<string, Array<{ change: PlannedChange; tools: Set<string> }>> { - const byServer = new Map<string, Array<{ change: PlannedChange; tools: Set<string> }>>() +): Map<string, Array<{ change: PlannedChange; tools: Map<string, SourceSpan | undefined> }>> { + const byServer = new Map< + string, + Array<{ change: PlannedChange; tools: Map<string, SourceSpan | undefined> }> + >() for (const change of changes) { if (change.action === 'delete') continue for (const ref of change.refs ?? []) { @@ -436,21 +445,33 @@ function toolUsers( const users = byServer.get(ref.key) ?? [] let user = users.find((entry) => entry.change === change) if (!user) { - user = { change, tools: new Set() } + user = { change, tools: new Map() } users.push(user) } - user.tools.add(ref.member) + if (!user.tools.has(ref.member)) user.tools.set(ref.member, ref.span) byServer.set(ref.key, users) } } return byServer } -/** Where a finding about the tools a change calls points: its workflow file. */ -function fileOf(change: PlannedChange): { file?: string } { - return change.kind === 'workflow' - ? { file: `${workflowFolder(change.key)}/${SOURCE_ENTRY_FILE}` } - : {} +/** + * Where a finding about the tools a change calls points: its workflow file, at the line that + * writes the tool when the workflow's compile located it. + */ +function fileOf(change: PlannedChange, span?: SourceSpan): Partial<FlatDiagnostic> { + if (change.kind !== 'workflow') return {} + return { + file: `${workflowFolder(change.key)}/${SOURCE_ENTRY_FILE}`, + ...(span + ? { + line: span.line, + column: span.column, + endLine: span.endLine, + endColumn: span.endColumn, + } + : {}), + } } export const mcpAdapter: KindAdapter<'mcp-server', McpServerRow> = { @@ -631,18 +652,18 @@ export const mcpAdapter: KindAdapter<'mcp-server', McpServerRow> = { } const names = new Set(listed.names) for (const { change, tools } of callers) { - for (const tool of tools) { + for (const [tool, span] of tools) { if (names.has(tool)) continue const suggestion = didYouMean(tool, names, (name) => `"${name}"`) change.diagnostics.push( - kindFinding( - change.kind, - change.key, + finding( 'E_MCP_TOOL_UNKNOWN', `${where(change.kind, change.key)} calls tool "${tool}" on MCP server "${key}", which does not list it.${suggestion ? ` ${suggestion}` : ''}`, { hint: `The server lists: ${[...names].slice(0, 20).join(', ') || 'no tools'}.`, - ...fileOf(change), + ...fileOf(change, span), + kind: change.kind, + key: change.key, } ) ) diff --git a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts index e83e9c99daf..963ac7223ba 100644 --- a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts @@ -209,13 +209,16 @@ function sourceRefs( evaluated: EvaluatedSource | undefined ): NonNullable<WorkflowChange['refs']> { const refs = new Map<string, NonNullable<WorkflowChange['refs']>[number]>() - for (const { kind, key, member } of references) { + for (const { kind, key, member, span } of references) { if (!isWorkspaceKind(kind)) continue if (isResourceKeyed(kind)) { - refs.set(JSON.stringify([kind, key, member ?? null]), { + const id = JSON.stringify([kind, key, member ?? null]) + if (refs.has(id)) continue + refs.set(id, { kind, key, ...(member !== undefined ? { member } : {}), + ...(span !== undefined ? { span } : {}), }) continue } diff --git a/apps/sim/lib/workspace-source/application/mcp.integration.ts b/apps/sim/lib/workspace-source/application/mcp.integration.ts index 63e55d86bee..cdbd3876195 100644 --- a/apps/sim/lib/workspace-source/application/mcp.integration.ts +++ b/apps/sim/lib/workspace-source/application/mcp.integration.ts @@ -309,6 +309,8 @@ describe('MCP servers as workspace source', () => { kind: 'workflow', key: 'triage', file: 'workflows/triage/workflow.ts', + line: 9, + column: 5, }) expect(unknown?.message).toContain('Did you mean "create_issue"?') } diff --git a/apps/sim/lib/workspace-source/application/types.ts b/apps/sim/lib/workspace-source/application/types.ts index a033afdea00..4f8422e34a1 100644 --- a/apps/sim/lib/workspace-source/application/types.ts +++ b/apps/sim/lib/workspace-source/application/types.ts @@ -1,4 +1,5 @@ import type { Principal } from '@sim/auth/principal' +import type { SourceSpan } from '@sim/workflow-code' import type { BaseResource, BlobEntry, @@ -143,6 +144,8 @@ interface ChangeFields { key: string /** The part of the resource the reference names: an MCP server's tool. */ member?: string + /** Where the declaration's entry file writes the reference, when the pipeline located it. */ + span?: SourceSpan /** * The resource's id, for a kind keyed otherwise than a workflow names it (a workspace file, * keyed by its path): the plan finds that kind's change by id and names it by its key. diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index ddcf921d563..d1e706a1acd 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -325,6 +325,25 @@ describe('sim workflows source apply', () => { }) }) + it('names each changed variable in the result and the preview', async () => { + setTerminal(true) + const changed = outcome({ dryRun: true }) + changed.data.diff.variables = { added: ['retries'], removed: [], modified: ['threshold'] } + request.mockResolvedValue(changed) + prompt.question.mockResolvedValue('n') + + await apply(WORKFLOW_ID, '--dir', dir).catch(() => {}) + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect( + stderrLines().some((line) => line.includes('Variables: added retries; modified threshold')) + ).toBe(true) + const printed = vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '' + expect(JSON.parse(printed)).toMatchObject({ + variables: { added: ['retries'], removed: [], modified: ['threshold'] }, + }) + }) + it('prints the modified blocks readably in text output', async () => { contextMockState.output = 'text' const modified = outcome({ dryRun: true }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 26c2abae0e9..7f7d8afd7bb 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -129,12 +129,13 @@ function diffSummary(data: SourceApplyData): string { `${blocks.renamed.length} renamed`, `${blocks.modified.length} modified`, ] - const variableCount = - variables.added.length + variables.removed.length + variables.modified.length + const variableChanges = (['added', 'removed', 'modified'] as const).flatMap((change) => + variables[change].length > 0 ? [`${change} ${variables[change].join(', ')}`] : [] + ) const modified = blocks.modified.map(({ key, fields }) => `${key} (${fields.join(', ')})`) return `Blocks: ${parts.join(', ')}. Connections: +${edges.added} -${edges.removed}.${ - variableCount > 0 ? ` Variables changed: ${variableCount}.` : '' - }${modified.length > 0 ? `\nModified: ${modified.join('; ')}` : ''}` + modified.length > 0 ? `\nModified: ${modified.join('; ')}` : '' + }${variableChanges.length > 0 ? `\nVariables: ${variableChanges.join('; ')}` : ''}` } const GET_HINT = 'Write the directory with `sim workflows source get` first.' @@ -400,6 +401,11 @@ export function attachWorkflowSourceCreate(source: Command): void { ) } +/** Whether a diff adds, removes or changes any workflow variable. */ +function variablesChanged(variables: SourceApplyData['diff']['variables']): boolean { + return variables.added.length > 0 || variables.removed.length > 0 || variables.modified.length > 0 +} + /** A create that did not apply stored nothing, so its provisional workflow id names no workflow. */ function summarize(data: SourceApplyData, created: boolean) { return { @@ -416,6 +422,7 @@ function summarize(data: SourceApplyData, created: boolean) { }), edgesAdded: data.diff.edges.added, edgesRemoved: data.diff.edges.removed, + ...(variablesChanged(data.diff.variables) && { variables: data.diff.variables }), errors: data.diagnostics.filter((diagnostic) => diagnostic.severity === 'error').length, warnings: data.diagnostics.filter((diagnostic) => diagnostic.severity === 'warning').length, notes: data.notes.length, diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index cc090c49dd6..0a3dc59abf8 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -1816,7 +1816,9 @@ export function compileBlock(ctx: CompileContext, decl: BlockDecl): CompiledBloc /** * `W_LEGACY_BLOCK_TYPE` for a new block of a legacy or deprecated type, naming its replacement and - * that type's builders. A block the stored workflow already holds under that type stays quiet. + * that type's builders. A block the stored workflow already holds under that type stays quiet; an + * offline check, which has only the lock, stays quiet on a block the lock binds by key or ident, + * as the server does for the stored block it continues. */ function checkLegacyType(ctx: CompileContext, decl: BlockDecl): void { const block = catalogBlockOf(decl) @@ -1824,6 +1826,14 @@ function checkLegacyType(ctx: CompileContext, decl: BlockDecl): void { if (!block || !sunset) return const stored = ctx.stored ? getOwn(ctx.stored.blocks, decl.key) : undefined if (stored?.type === block.type) return + const lock = ctx.storedUnread + if ( + lock && + (Object.hasOwn(lock.blocks, decl.key) || + Object.values(lock.blocks).some((entry) => entry.ident === decl.ident)) + ) { + return + } const replacement = sunset.replacedBy const namespaces = replacement ? [ diff --git a/packages/workflow-code/src/compile/checks.test.ts b/packages/workflow-code/src/compile/checks.test.ts index f21c682b051..eb60ba48dd9 100644 --- a/packages/workflow-code/src/compile/checks.test.ts +++ b/packages/workflow-code/src/compile/checks.test.ts @@ -353,6 +353,24 @@ describe('legacy block types', () => { if (!stored) throw new Error('expected a workflow') expect(findings(run(body, stored, withLegacy), 'W_LEGACY_BLOCK_TYPE')).toEqual([]) }) + + it('stays quiet offline for a legacy block the lock binds, as the server does', () => { + const offline = (lockKey: string, ident: string) => + compileWorkflow({ + entry: { path: 'workflow.ts', text: source("const s = block('slack_legacy', 'S', {})") }, + files: new Map(), + catalog: withLegacy, + sdkMajor: 1, + storedUnread: { + version: 1, + blocks: { [lockKey]: { id: 'b-1', parent: null, ident, order: 0 } }, + variables: {}, + }, + }) + expect(findings(offline('s', 's'), 'W_LEGACY_BLOCK_TYPE')).toEqual([]) + expect(findings(offline('old', 's'), 'W_LEGACY_BLOCK_TYPE')).toEqual([]) + expect(findings(offline('other', 'other'), 'W_LEGACY_BLOCK_TYPE')).toHaveLength(1) + }) }) describe('unwired branches', () => { diff --git a/packages/workflow-code/src/compile/context.ts b/packages/workflow-code/src/compile/context.ts index 2e5079e2f9f..e898733b931 100644 --- a/packages/workflow-code/src/compile/context.ts +++ b/packages/workflow-code/src/compile/context.ts @@ -11,6 +11,7 @@ import type { CodeFileRef } from '../code-files' import type { RefSegment } from '../code-files/ref-path' import type { SourceSpan } from '../diagnostics/frame' import type { JsonTree } from '../json-fields' +import type { WorkflowLockV1 } from '../lock' import type { Anchor, AnchorKey, @@ -18,6 +19,7 @@ import type { CanonicalParam, CanonicalVariable, CanonicalWorkflow, + CanonicalWorkflowMeta, LoopType, ParallelType, SubBlockId, @@ -143,10 +145,17 @@ export interface CompileContext { wiring: WiringStatement[] implied: ImpliedEdge[] spans: Map<AnchorKey, SourceSpan> + /** Where `workflow()` writes the name and the description: `CompileResult.metaSpans`. */ + metaSpans: Partial<Record<keyof CanonicalWorkflowMeta, SourceSpan>> /** `CompileInput.knownKeys`: the only keys a `meta.key` override may name. */ knownKeys: ReadonlySet<BlockKey> /** `CompileInput.stored`: the stored workflow `stored()` reads its values from. */ stored: CanonicalWorkflow | undefined + /** + * `CompileInput.storedUnread` when `stored` is absent: the lock of a stored workflow an offline + * check continues without reading it. + */ + storedUnread: WorkflowLockV1 | undefined /** `CompileInput.envNames`: the env names a run resolves, when known. */ envNames: ReadonlySet<string> | undefined /** `CompileInput.envDeclaredIn`: the file the env names are declared in, when there is one. */ diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 0785d92a1dc..f0d4fce090c 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -21,6 +21,7 @@ import type { CanonicalBlock, CanonicalVariable, CanonicalWorkflow, + CanonicalWorkflowMeta, PortKey, VarKey, } from '../model' @@ -109,6 +110,11 @@ export interface CompileResult { diagnostics: LocatedDiagnostic<CompileDiagnosticCode>[] /** Where each anchor sits, to map later server findings back to lines. */ spans: Partial<Record<AnchorKey, SourceSpan>> + /** + * Where `workflow()` writes the name and the description, so a server finding about the + * workflow itself (a name another workflow holds, a block delete) points there. + */ + metaSpans: Partial<Record<keyof CanonicalWorkflowMeta, SourceSpan>> /** Idents as written, `order` as declaration index, and code files, for the lock. */ hints: SourceHints /** The header's major, or `null` when the header is missing. */ @@ -312,6 +318,7 @@ function compileWorkflowCall( } ) } else if (nameNode) { + ctx.metaSpans.name = ctx.reporter.span(rangeOf(ctx, nameNode)) checkSuspiciousName(ctx, name, nameNode, 'workflow') } const optionsNode: Node | undefined = third ? second : undefined @@ -329,6 +336,7 @@ function compileWorkflowCall( } else { for (const [key, property] of objectEntries(ctx, optionsNode)) { if (key === 'description') { + ctx.metaSpans.description = ctx.reporter.span(rangeOf(ctx, property.value)) description = literalText(property.value) if (description === null) unsupported( @@ -413,6 +421,7 @@ function emptyResult(reporter: Reporter, sdkMajor: number | null): CompileResult workflow: null, diagnostics: reporter.diagnostics, spans: {}, + metaSpans: {}, hints: { blocks: {} }, sdkMajor, envReferences: [], @@ -503,8 +512,10 @@ function compileEntry( wiring: [], implied: [], spans: new Map(), + metaSpans: {}, knownKeys: input.knownKeys ?? new Set(), stored: input.stored, + storedUnread: input.stored === undefined ? input.storedUnread : undefined, envNames: input.envNames, envDeclaredIn: input.envDeclaredIn, envReferences: new Set(), @@ -705,6 +716,7 @@ function compileEntry( workflow, diagnostics: reporter.diagnostics, spans, + metaSpans: ctx.metaSpans, hints, sdkMajor: parsed.sdkMajor, envReferences: [...ctx.envReferences].sort(compareStrings), From e56fbccbc8ac20412b33aa2f1d97d016d53bff31 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:42:17 -0800 Subject: [PATCH 21/59] fix(sim-cli): source commands act on the workspace a directory was read from check, status, diff and apply (workflow and workspace) now take the workspace recorded in .sim/base.json whatever the profile default, as get already did; only a --workspace naming another one is refused (SOURCE_OTHER_WORKSPACE). check refuses a directory without workflow.ts before any catalog lookup, and every --dir is made absolute before the command runs, so messages name the directory read wherever the CLI was started. --- apps/docs/content/docs/cli/reference.mdx | 2 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- .../content/docs/workflows/workspace-code.mdx | 2 +- .../src/commands/protocol/source-dir.ts | 41 ++++++++++++++++ .../protocol/workflow-source-apply.ts | 10 +++- .../protocol/workflow-source-check.test.ts | 31 +++++++++++- .../protocol/workflow-source-check.ts | 33 ++++++++++--- .../commands/protocol/workflow-source-get.ts | 2 + .../protocol/workflow-source-status.test.ts | 10 ++-- .../protocol/workflow-source-status.ts | 3 ++ .../protocol/workspace-source-apply.test.ts | 36 ++++++++++---- .../protocol/workspace-source-apply.ts | 49 +++++++------------ .../protocol/workspace-source-check.test.ts | 8 ++- .../protocol/workspace-source-check.ts | 17 +++++-- .../commands/protocol/workspace-source-dir.ts | 2 +- .../protocol/workspace-source-get.test.ts | 10 ++-- .../commands/protocol/workspace-source-get.ts | 42 ++++++---------- .../protocol/workspace-source-status.test.ts | 24 ++++++--- .../protocol/workspace-source-status.ts | 28 ++++++++--- packages/sim-cli/src/test/context-mock.ts | 29 ++++++----- 20 files changed, 263 insertions(+), 118 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index ea0a5625766..9a939e1c8b7 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -9950,7 +9950,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 89641bd048d..3bfd5bab460 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -760,7 +760,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index bdbbc7e1df5..1f68ff3371e 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -90,7 +90,7 @@ sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --dry sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --yes ``` -Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`. The directory is not changed and stays bound to the workspace it was read from. Without `--to`, an apply refuses a directory read from another workspace. +Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`. The directory is not changed and stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. ## Git and CI diff --git a/packages/sim-cli/src/commands/protocol/source-dir.ts b/packages/sim-cli/src/commands/protocol/source-dir.ts index 08169a37d5c..3d04b1c0dd9 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.ts @@ -37,6 +37,8 @@ import { SOURCE_LOCK_FILE, } from '@sim/workflow-code/limits' import { fingerprintOf, KIND_DIRS } from '@sim/workspace-code' +import type { Command } from 'commander' +import type { ResolvedProfile } from '../../config/index' import { embedStore } from '../../embed-context' import { SOURCE_BUSY_EXIT_CODE } from '../../exit-codes' import { SimApiError } from '../../http/client' @@ -181,6 +183,45 @@ export async function readSourceJson(dir: string, path: string): Promise<unknown } } +/** + * Makes `command`'s `--dir` (given or defaulted) an absolute path before its action runs, so every + * message names the directory it read wherever the CLI ran: a wrapper that changes directory + * first made `.` name a folder its caller never meant. An embedded run keeps its host's path. + */ +export function absoluteDirOption(command: Command): Command { + return command.hook('preAction', (_hooked, action) => { + const dir: unknown = action.getOptionValue('dir') + if (typeof dir === 'string' && !embedStore.getStore()) { + action.setOptionValue('dir', resolve(dir)) + } + }) +} + +/** + * The workspace a source command acts on for a directory read from `readFrom`: that workspace, + * whatever the profile's default, so a directory stays bound to where it came from. Only a + * `--workspace` naming another one is refused, with `next` saying what to run instead (given the + * workspace the directory was read from and the one named). A directory that records no workspace + * takes the profile's (`null` when none is selected). + */ +export function directoryWorkspace( + dir: string, + readFrom: string | null, + profile: Pick<ResolvedProfile, 'workspaceId' | 'sources'>, + next: (readFrom: string, named: string) => string +): string | null { + const named = profile.workspaceId + if (readFrom === null) return named + if (profile.sources.workspaceId !== 'flag' || named === null || named === readFrom) { + return readFrom + } + throw new SimApiError( + `${dir} was read from workspace ${readFrom}, but --workspace names ${named}. ${next(readFrom, named)}`, + 0, + 'SOURCE_OTHER_WORKSPACE' + ) +} + /** The workspace {@link SOURCE_STARTER_FILE} names, or `null` when the directory holds no starter. */ export async function readStarterWorkspace(dir: string): Promise<string | null> { const workspaceId = toRecord(await readSourceJson(dir, SOURCE_STARTER_FILE)).workspaceId diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index a2d68e9cc86..00329046d34 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -21,6 +21,7 @@ import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' import { printProtocolResult } from './result' import { + absoluteDirOption, FILE_CONCURRENCY, lockCodeFiles, readSourceText, @@ -197,6 +198,11 @@ export async function readLock(directory: string): Promise<WorkflowLockV1> { ) } +/** The directory's `workflow.ts`, or a refusal naming the command that writes one. */ +export function requireEntry(directory: string): Promise<string> { + return requireText(directory, ENTRY_FILE, GET_HINT) +} + /** * The files a write sends: the entry file and the code files it or the lock references. A * referenced file that is missing is left out, and the server reports it with its source line. @@ -205,7 +211,7 @@ export async function readSourceFiles( directory: string, lock: WorkflowLockV1 | null ): Promise<Record<string, string>> { - const entry = await requireText(directory, ENTRY_FILE, GET_HINT) + const entry = await requireEntry(directory) const files: Record<string, string> = { [ENTRY_FILE]: entry } const paths = codeFilePaths(entry, lock) const texts = await mapWithConcurrency(paths, FILE_CONCURRENCY, (path) => @@ -315,6 +321,7 @@ const DIR_DESCRIPTION = */ export function attachWorkflowSourceApply(source: Command): void { const [command, connectApply] = apiCommand(source, 'apply', ['applyWorkflowSource']) + absoluteDirOption(command) command .argument( '[workflowId]', @@ -376,6 +383,7 @@ export function attachWorkflowSourceCreate(source: Command): void { 'getSourceCatalog', 'getSourceResources', ]) + absoluteDirOption(command) command .requiredOption( '--dir <dir>', diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index 261d7f85fc6..4f5a42e3a31 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -5,7 +5,7 @@ import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { SimApiError } from '../../http/client' import { cacheCatalogText } from '../../test/catalog-cache' -import { contextMockFns } from '../../test/context-mock' +import { contextMockFns, mockedProfile } from '../../test/context-mock' import { attachWorkflowSourceCheck } from './workflow-source-check' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -205,7 +205,11 @@ beforeEach(() => { JSON.stringify({ workflowId: 'wf', workspaceId: WORKSPACE_ID, fingerprint: 'sf1:a' }) ) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) - contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_other', output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_other', + output: 'json', + }) contextMockFns.mockRequest.mockReset() vi.spyOn(console, 'log').mockImplementation(() => {}) vi.spyOn(console, 'error').mockImplementation(() => {}) @@ -296,6 +300,29 @@ describe('sim workflows source check', () => { expect((error as SimApiError).message).toContain('sim workflows source get') }) + it('refuses a directory without workflow.ts before looking for a catalog, by its absolute path', async () => { + const error = await check().catch((caught: unknown) => caught) + + expect((error as SimApiError).code).not.toBe('CATALOG_NOT_CACHED') + expect((error as SimApiError).message).toContain(`${dir} has no workflow.ts`) + }) + + it('refuses a --workspace naming another workspace than .sim/base.json, and only that', async () => { + cacheCatalog() + writeFileSync(join(dir, 'workflow.ts'), CLEAN) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_other', + output: 'json', + sources: { workspaceId: 'flag' }, + }) + + const error = await check().catch((caught: unknown) => caught) + + expect((error as SimApiError).code).toBe('SOURCE_OTHER_WORKSPACE') + expect((error as SimApiError).message).toContain(`was read from workspace ${WORKSPACE_ID}`) + }) + it('points a workspace root at the workspace check', async () => { cacheCatalog() writeWorkspace() diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index e02b9c08fdb..057ddcc9121 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -23,7 +23,14 @@ import { profileFrom } from '../../context' import { setSoftExitCode } from '../../embed-context' import { SimApiError } from '../../http/client' import { printProtocolResult } from './result' -import { readSourceJson, readSourceText, workspaceFolderOf } from './source-dir' +import { + absoluteDirOption, + directoryWorkspace, + readSourceJson, + readSourceText, + readStarterWorkspace, + workspaceFolderOf, +} from './source-dir' import { cachedCatalog, cachedSourceCatalog, @@ -33,7 +40,7 @@ import { } from './source-tree' import { printNewDiagnostics, resultDiagnostics } from './source-ui' import { readStoredEdges } from './stored-edges' -import { readSourceFiles } from './workflow-source-apply' +import { readSourceFiles, requireEntry } from './workflow-source-apply' import { WORKFLOW_TREE } from './workflow-source-get' import { kindFolders, @@ -528,9 +535,11 @@ export function requireCachedCatalog(workspaceId: string, get: string): BlockCat */ export function attachWorkflowSourceCheck(source: Command): void { strictFlags( - source - .command('check') - .option('--dir <dir>', 'Source directory: workflow.ts, functions/* and .sim/', '.') + absoluteDirOption(source.command('check')).option( + '--dir <dir>', + 'Source directory: workflow.ts, functions/* and .sim/', + '.' + ) ) .allowExcessArguments(false) .description('Compile a workflow source directory locally, without the server') @@ -548,11 +557,21 @@ export function attachWorkflowSourceCheck(source: Command): void { 0 ) } + await requireEntry(options.dir) const workspace = await workspaceFolderOf(options.dir) - const workspaceId = + const recorded = typeof base.workspaceId === 'string' ? base.workspaceId - : (workspace?.workspaceId ?? profile.workspaceId) + : (workspace?.workspaceId ?? + (profile.sources.workspaceId === 'flag' + ? null + : await readStarterWorkspace(options.dir))) + const workspaceId = directoryWorkspace( + options.dir, + recorded, + profile, + (from) => `Run it without --workspace to check it against the catalog of ${from}.` + ) if (!workspaceId) { throw new SimApiError( `${options.dir} has no ${SOURCE_BASE_FILE} and no workspace is selected: pass --workspace <workspaceId>.`, diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index fd5c5a2d915..e175db48dad 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -13,6 +13,7 @@ import { callsOperations, type OperationClient } from '../../runtime/called-oper import { buildRequest } from '../../runtime/request' import { printProtocolResult } from './result' import { + absoluteDirOption, assertPrintedSourcePaths, locallyEditedFiles, readPreviousLock, @@ -325,6 +326,7 @@ export function attachWorkflowSourceGet(source: Command): void { `--${FORCE_FLAG}`, "With --dir: replace another workflow's source or edits no apply has saved" ) + absoluteDirOption(get) get.addHelpText('after', `${GET_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}`) get.action(async (workflowId: string, _options: unknown, command: Command): Promise<void> => { const flags = command.optsWithGlobals() as Record<string, unknown> diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts index acaeec0bbcf..72cec73a101 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts @@ -4,7 +4,7 @@ import { basename, dirname, join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { REQUEST_CANCELLED, SimApiError } from '../../http/client' -import { contextMockFns, contextMockState } from '../../test/context-mock' +import { contextMockFns, contextMockState, mockedProfile } from '../../test/context-mock' import { digest } from './source-dir' import { attachWorkflowSourceDiff, attachWorkflowSourceStatus } from './workflow-source-status' @@ -102,7 +102,7 @@ const STALE = { code: 'E_APPLY_STALE', severity: 'error', message: 'The workflow beforeEach(() => { contextMockState.output = 'json' mockRequest.mockReset() - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'json' }) + mockProfileFrom.mockReturnValue({ ...mockedProfile(), workspaceId: WORKSPACE_ID, output: 'json' }) root = mkdtempSync(join(tmpdir(), 'sim-workflow-status-')) writeWorkflowDir(root, null) vi.spyOn(console, 'log').mockImplementation(() => {}) @@ -409,7 +409,11 @@ describe('workflows source diff', () => { it('prints a colorless unified diff for a person when stdout is not a terminal', async () => { write(root, 'functions/score.js', 'return 2\n') mockRequest.mockResolvedValue({ data: { ...dryRun().data, files: GOT } }) - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'text' }) + mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: WORKSPACE_ID, + output: 'text', + }) await run(root, 'diff') diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts index 33b84981d49..4bb903d4e7f 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts @@ -22,6 +22,7 @@ import type { ApplyWorkflowSourceResponse, GetWorkflowSourceResponse } from '../ import { SimApiError } from '../../http/client' import { apiCommand } from '../../runtime/called-operations' import { + absoluteDirOption, digest, listSourceFiles, readFolderWritten, @@ -134,6 +135,7 @@ interface StatusOptions { */ export function attachWorkflowSourceStatus(source: Command): void { const [command, connect] = apiCommand(source, 'status', ['applyWorkflowSource']) + absoluteDirOption(command) command .option('--dir <dir>', 'Source directory: workflow.ts, functions/* and .sim/', '.') .option('--offline', 'Report local changes only, without asking the server') @@ -231,6 +233,7 @@ interface DiffOptions { */ export function attachWorkflowSourceDiff(source: Command): void { const [command, connect] = apiCommand(source, 'diff', ['getWorkflowSource']) + absoluteDirOption(command) command .argument('[paths...]', 'Files or folders to compare (default: every file)') .option('--dir <dir>', 'Source directory: workflow.ts, functions/* and .sim/', '.') diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 6fb1071a98d..5d0108670c5 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -6,6 +6,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + realpathSync, rmSync, truncateSync, unlinkSync, @@ -13,7 +14,7 @@ import { writeFileSync, } from 'node:fs' import { hostname, tmpdir } from 'node:os' -import { dirname, join } from 'node:path' +import { basename, dirname, join } from 'node:path' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' @@ -196,9 +197,8 @@ describe('workspaces source apply', () => { expect(Object.keys(bodies()[0].files)).toContain('workflows/ingest/workflow.ts') }) - it('sends an unchanged workflow folder whole when applying a copy to another workspace', async () => { + it('sends an unchanged workflow folder copied from another workspace whole', async () => { const copied = folderBase('ws_other') - write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) write('workflows/ingest/.sim/base.json', copied) write( '.sim/written.json', @@ -209,7 +209,6 @@ describe('workspaces source apply', () => { }, }) ) - contextMockState.workspaceSource = 'flag' request.mockResolvedValue(answer()) await apply('--yes') @@ -767,17 +766,36 @@ describe('workspaces source apply', () => { await expect(apply('--yes')).rejects.toThrow(/unsafe file path/) }) - it('refuses a directory read from another workspace unless --workspace names the target', async () => { + it('applies to the workspace the directory was read from, refusing only a --workspace naming another', async () => { write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) request.mockResolvedValue(answer()) - await expect(apply('--yes')).rejects.toMatchObject({ code: 'SOURCE_OTHER_WORKSPACE' }) - expect(request).not.toHaveBeenCalled() - - contextMockState.workspaceSource = 'flag' await apply('--yes') expect(request).toHaveBeenCalledTimes(1) + expect(request.mock.calls[0][0]).toBe('/api/v2/workspaces/ws_other/source') + + write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) + contextMockState.workspaceSource = 'flag' + await expect(apply('--yes')).rejects.toMatchObject({ + code: 'SOURCE_OTHER_WORKSPACE', + message: expect.stringContaining('--to ws_local'), + }) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('names the directory by its absolute path when --dir is relative', async () => { + const cwd = process.cwd() + process.chdir(dirname(dir)) + try { + contextMockState.workspaceSource = 'flag' + write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) + await expect(runWorkspaceSource('apply', basename(dir), '--yes')).rejects.toMatchObject({ + message: expect.stringContaining(`${realpathSync(dir)} was read from workspace ws_other`), + }) + } finally { + process.chdir(cwd) + } }) it('refuses a base that records blobs no get downloaded, but sends one the user removed', async () => { diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 1315235984d..87da672b88c 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -27,6 +27,8 @@ import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' import { printProtocolResult } from './result' import { + absoluteDirOption, + directoryWorkspace, isMissing, orphanedCodeFiles, readSourceJson, @@ -82,8 +84,9 @@ Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printe file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from -another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to -to promote it), its +another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or +promote it with --to; without --workspace an apply targets the workspace the directory was read +from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a @@ -336,26 +339,6 @@ async function saveApplied( }) } -/** - * Refuses a directory read from another workspace than the one targeted, unless `--workspace` - * names the target: applying it would create every resource anew there, so the choice is the - * caller's to make explicitly. - */ -function assertBaseWorkspace( - dir: string, - base: unknown, - workspaceId: string, - explicit: boolean -): void { - const readFrom = baseWorkspaceOf(base) - if (readFrom === null || readFrom === workspaceId || explicit) return - throw new SimApiError( - `${dir} was read from workspace ${readFrom}, but this profile targets ${workspaceId}. Pass --workspace ${readFrom} to apply it there, or --to ${workspaceId} to promote it: its resources bind by key to that workspace's own, and the directory stays bound to ${readFrom}.`, - 0, - 'SOURCE_OTHER_WORKSPACE' - ) -} - /** The workflow a directory's base belongs to (a workflow source directory), or `null`. */ function baseWorkflowOf(base: unknown): string | null { return isRecordLike(base) && typeof base.workflowId === 'string' ? base.workflowId : null @@ -419,6 +402,7 @@ export function attachWorkspaceSourceApply(source: Command): void { 'getWorkspaceSource', 'getSourceCatalog', ]) + absoluteDirOption(command) command .requiredOption('--dir <dir>', 'Workspace source directory written by `workspaces source get`') .option('--dry-run', 'Plan and check without writing anything') @@ -436,7 +420,18 @@ export function attachWorkspaceSourceApply(source: Command): void { requireConsent(options, 'writes the workspace', 'apply') const only = parseKinds(options.only) const { client, profile } = connect() - const workspaceId = options.to ?? client.requireWorkspace() + const readFrom = baseWorkspaceOf(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) + const workspaceId = + options.to ?? + client.requireWorkspace( + directoryWorkspace( + options.dir, + readFrom, + profile, + (from, named) => + `Run it without --workspace to apply it to ${from}, or promote it with --to ${named}: its resources bind by key to that workspace's own, and the directory stays bound to ${from}.` + ) ?? undefined + ) const post = async (submission: WorkspaceSubmission, dryRun: boolean) => { const body: ApplyWorkspaceSourceBody = { files: submission.files, @@ -509,14 +504,6 @@ export function attachWorkspaceSourceApply(source: Command): void { if (read.kinds.length === 0 || baseWorkflowOf(read.base) !== null) { await assertWorkspaceDir(options.dir) } - if (!options.to) { - assertBaseWorkspace( - options.dir, - read.base, - workspaceId, - profile.sources.workspaceId === 'flag' - ) - } assertBlobsFetched(options.dir, read, written.digests) const submission = options.to ? await promoted(client, options.dir, read, options.to, options.allowDelete === true) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts index 2db74283680..d023b2ad853 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts @@ -5,7 +5,7 @@ import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' import { cacheCatalogText } from '../../test/catalog-cache' -import { contextMockFns } from '../../test/context-mock' +import { contextMockFns, mockedProfile } from '../../test/context-mock' import { attachWorkspaceSourceCheck } from './workspace-source-check' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -77,7 +77,11 @@ beforeEach(() => { ) cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, readFileSync(FIXTURE_CATALOG, 'utf8')) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) - contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_other', output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_other', + output: 'json', + }) contextMockFns.mockRequest.mockReset() vi.spyOn(console, 'log').mockImplementation(() => {}) vi.spyOn(console, 'error').mockImplementation(() => {}) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts index cf7c90ffdfc..5e63e0dc73a 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts @@ -7,7 +7,13 @@ import { profileFrom } from '../../context' import { AUTH_REQUIRED_CODE } from '../../exit-codes' import { SimApiError } from '../../http/client' import { apiCommand, type Connection } from '../../runtime/called-operations' -import { readSourceJson, SOURCE_DIR_BUSY_HELP, withSourceDirLock } from './source-dir' +import { + absoluteDirOption, + directoryWorkspace, + readSourceJson, + SOURCE_DIR_BUSY_HELP, + withSourceDirLock, +} from './source-dir' import { unreachableReason } from './source-status' import { cachedCatalog, loadCatalog, refreshWorkspaceTypes } from './source-tree' import { @@ -89,6 +95,7 @@ async function fetchedCatalog( */ export function attachWorkspaceSourceCheck(source: Command): void { const [command, connect] = apiCommand(source, 'check', ['getSourceCatalog']) + absoluteDirOption(command) strictFlags( command.option( '--dir <dir>', @@ -105,8 +112,12 @@ export function attachWorkspaceSourceCheck(source: Command): void { const profile = profileFrom(command) const base = toRecord(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) await assertWorkspaceDir(options.dir) - const workspaceId = - typeof base.workspaceId === 'string' ? base.workspaceId : profile.workspaceId + const workspaceId = directoryWorkspace( + options.dir, + typeof base.workspaceId === 'string' ? base.workspaceId : null, + profile, + (from) => `Run it without --workspace to check it against the catalog of ${from}.` + ) if (!workspaceId) { throw new SimApiError( `${options.dir} has no ${WORKSPACE_BASE_FILE} and no workspace is selected: pass --workspace <workspaceId>.`, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index f07abccffb4..857fa086a78 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -147,7 +147,7 @@ Secrets never appear in source: a workflow reads an environment variable as \`en ## Promote to another workspace -\`sim workspaces source apply --dir . --to <workspaceId> --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name), and a change made in the target since is refused as stale (exit 5). Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w <workspaceId>\`) and connect its credentials, or the workflows using them are held back. This directory is not changed and stays bound to the workspace it was read from. Without \`--to\`, an apply refuses a directory read from another workspace (\`SOURCE_OTHER_WORKSPACE\`). +\`sim workspaces source apply --dir . --to <workspaceId> --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name), and a change made in the target since is refused as stale (exit 5). Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w <workspaceId>\`) and connect its credentials, or the workflows using them are held back. This directory is not changed and stays bound to the workspace it was read from. Without \`--to\`, \`check\`, \`status\`, \`diff\` and \`apply\` act on the workspace this directory was read from, whatever the profile's default; a \`--workspace\` naming another is refused (\`SOURCE_OTHER_WORKSPACE\`). ## Good to know diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts index fb633377108..bce483607fd 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts @@ -18,7 +18,7 @@ import { fingerprintOf } from '@sim/workspace-code' import { knowledgeCodec } from '@sim/workspace-code/kinds/knowledge' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' -import { contextMockFns, contextMockState } from '../../test/context-mock' +import { contextMockFns, contextMockState, mockedProfile } from '../../test/context-mock' import { runWorkflowSourceCheck, runWorkspaceSource } from '../../test/workspace-source-command' import { blobBytes, blobDigests } from './workspace-source-blobs' import { kindFolders } from './workspace-source-dir' @@ -90,7 +90,11 @@ beforeEach(() => { contextMockState.workspaceSource = 'config' request.mockReset() requestRaw.mockReset() - contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_local', output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_local', + output: 'json', + }) rootDir = mkdtempSync(join(tmpdir(), 'sim-workspace-get-')) dir = join(rootDir, 'out') vi.stubEnv('SIM_CONFIG_DIR', join(rootDir, 'config')) @@ -650,7 +654,7 @@ describe('workspaces source get', () => { '---\nname: fresh\ndescription: New\n---\nAdded after the get.\n' ) expect(typecheck(uses('fresh'))).toContain('"fresh"') - contextMockFns.mockProfileFrom.mockReturnValue({ output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ ...mockedProfile(), output: 'json' }) request.mockClear() await runWorkspaceSource('check', dir) expect(request).not.toHaveBeenCalled() diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index 2c25ff88f9e..83d7656eef0 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -22,7 +22,9 @@ import { apiCommand } from '../../runtime/called-operations' import { removeStagingOnSignal } from './files-get' import { printProtocolResult } from './result' import { + absoluteDirOption, checkedPath, + directoryWorkspace, editedFiles, FILE_CONCURRENCY, readSourceJson, @@ -341,29 +343,6 @@ async function keepDownloads( ) } -/** - * The workspace a get writes `dir` from. A directory whose base was read from a workspace stays - * that workspace's: a get refreshes it, whatever the profile's default, so a refresh never - * rewrites it into another workspace. A `--workspace` naming another one is refused (get into - * another directory, or promote with `apply --to`). A directory without a base takes `profile`. - */ -async function directoryWorkspace( - dir: string, - profile: string, - explicit: boolean -): Promise<string> { - // Read as leniently as an apply reads it: a base the parser refuses still names its workspace. - const recorded = toRecord(await readSourceJson(dir, WORKSPACE_BASE_FILE)).workspaceId - const readFrom = typeof recorded === 'string' ? recorded : null - if (readFrom === null || readFrom === profile) return profile - if (!explicit) return readFrom - throw new SimApiError( - `${dir} was read from workspace ${readFrom}, but --workspace names ${profile}. Get ${profile} into another directory, or run the get without --workspace to refresh ${readFrom}; to apply this directory to ${profile}, use \`sim workspaces source apply --to ${profile}\`.`, - 0, - 'SOURCE_OTHER_WORKSPACE' - ) -} - /** * `sim workspaces source get --dir <dir> [--only kinds]`: writes the workspace as a source * directory: `tables/`, `tools/`, `skills/`, `knowledge/`, `files/` and `workflows/<key>/` (each a @@ -378,6 +357,7 @@ export function attachWorkspaceSourceGet(source: Command): void { 'downloadFile', 'downloadKnowledgeDocument', ]) + absoluteDirOption(command) command .requiredOption('--dir <dir>', 'Directory to write the workspace source into') .option('--only <kinds>', `Read only these kinds (${ONLY_KINDS_HELP})`) @@ -389,10 +369,18 @@ export function attachWorkspaceSourceGet(source: Command): void { const kinds = parseKinds(options.only) const { client, profile } = connect() await withSourceDirLock(options.dir, async () => { - const workspaceId = await directoryWorkspace( - options.dir, - client.requireWorkspace(), - profile.sources.workspaceId === 'flag' + // Read as leniently as an apply reads it: a base the parser refuses still names its workspace. + const recorded = toRecord( + await readSourceJson(options.dir, WORKSPACE_BASE_FILE) + ).workspaceId + const workspaceId = client.requireWorkspace( + directoryWorkspace( + options.dir, + typeof recorded === 'string' ? recorded : null, + profile, + (readFrom, named) => + `Get ${named} into another directory, or run the get without --workspace to refresh ${readFrom}; to apply this directory to ${named}, use \`sim workspaces source apply --to ${named}\`.` + ) ?? undefined ) const { data } = await client .request<GetWorkspaceSourceResponse>('getWorkspaceSource', { diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts index ace1cee4fea..68858293f3a 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts @@ -5,7 +5,7 @@ import { dirname, join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' -import { contextMockFns, contextMockState } from '../../test/context-mock' +import { contextMockFns, contextMockState, mockedProfile } from '../../test/context-mock' import { digest } from './source-dir' import { attachWorkspaceSourceDiff, attachWorkspaceSourceStatus } from './workspace-source-status' @@ -94,7 +94,7 @@ beforeEach(() => { contextMockState.workspaceSource = 'config' contextMockState.workspaceId = WORKSPACE_ID mockRequest.mockReset() - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'json' }) + mockProfileFrom.mockReturnValue({ ...mockedProfile(), workspaceId: WORKSPACE_ID, output: 'json' }) dir = mkdtempSync(join(tmpdir(), 'sim-workspace-status-')) write('.sim/base.json', JSON.stringify(BASE)) for (const [path, text] of Object.entries(GOT)) write(path, text) @@ -277,19 +277,27 @@ describe('workspaces source status', () => { }) }) - it('recommends the workspace a status was pointed at with --workspace', async () => { - contextMockState.workspaceSource = 'flag' + it('compares with the workspace the directory was read from, whatever the profile default', async () => { contextMockState.workspaceId = 'ws_other' write('tables/leads.ts', LEADS.replace('columns: {}', 'columns: { email: column.string() }')) mockRequest.mockResolvedValue(dryRun([{ kind: 'table', key: 'leads', action: 'update' }])) await run('status') + expect(mockRequest.mock.calls[0][0]).toBe(`/api/v2/workspaces/${WORKSPACE_ID}/source`) expect(printed().next).toMatchObject({ - command: `sim workspaces source apply --dir ${dir} --yes --workspace ws_other`, + command: `sim workspaces source apply --dir ${dir} --yes`, }) }) + it('refuses a --workspace naming another workspace than the directory was read from', async () => { + contextMockState.workspaceSource = 'flag' + contextMockState.workspaceId = 'ws_other' + + await expect(run('status')).rejects.toMatchObject({ code: 'SOURCE_OTHER_WORKSPACE' }) + expect(mockRequest).not.toHaveBeenCalled() + }) + it('recommends --allow-delete for an update that drops data, as applying it needs', async () => { write('tables/leads.ts', LEADS.replace("name: 'leads'", "name: 'leads', description: 'x'")) mockRequest.mockResolvedValue( @@ -561,7 +569,11 @@ describe('workspaces source status', () => { it('prints one screen ending with the next command for a person', async () => { write('tables/leads.ts', `${LEADS}// edited\n`) mockRequest.mockResolvedValue(dryRun([{ kind: 'table', key: 'leads', action: 'update' }])) - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'text' }) + mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: WORKSPACE_ID, + output: 'text', + }) await run('status') diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts index 21bf1d09faf..2bd45bf9cf5 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts @@ -33,7 +33,9 @@ import type { import { SimApiError } from '../../http/client' import { apiCommand, type Connection } from '../../runtime/called-operations' import { + absoluteDirOption, digest, + directoryWorkspace, readSourceJson, readSourceText, readWritten, @@ -146,17 +148,24 @@ function localPaths(local: LocalWorkspace): string[] { } /** - * The workspace a status or diff asks about: the one `--workspace` names, else the one the - * directory was read from, else the profile's. + * The workspace a status or diff asks about: the one the directory was read from (a `--workspace` + * naming another is refused), else the profile's. */ function targetWorkspace( connection: Connection<'applyWorkspaceSource' | 'getWorkspaceSource'>, + dir: string, readFrom: string | null ): string { const { client, profile } = connection - return profile.sources.workspaceId === 'flag' || readFrom === null - ? client.requireWorkspace() - : readFrom + return client.requireWorkspace( + directoryWorkspace( + dir, + readFrom, + profile, + (from, named) => + `Run it without --workspace to compare with ${from}, or get ${named} into another directory.` + ) ?? undefined + ) } /** The resources `.sim/skipped.json` lists: never written by a get, so never drift. */ @@ -285,6 +294,7 @@ interface StatusOptions { */ export function attachWorkspaceSourceStatus(source: Command): void { const [command, connect] = apiCommand(source, 'status', ['applyWorkspaceSource']) + absoluteDirOption(command) command .option('--dir <dir>', 'Workspace source directory written by `workspaces source get`', '.') .option('--only <kinds>', `Report only these kinds (${ONLY_KINDS_HELP})`) @@ -314,10 +324,11 @@ export function attachWorkspaceSourceStatus(source: Command): void { let login: string | null = null if (!options.offline) { const connection = connect() - const target = targetWorkspace(connection, local.readFrom) + const target = targetWorkspace(connection, options.dir, local.readFrom) workspaceId = target const { profile } = connection - if (profile.sources.workspaceId === 'flag' || profile.workspaceId !== target) { + // A directory with a base is bound to its workspace; only one without names it. + if (local.readFrom === null && profile.sources.workspaceId === 'flag') { workspaceScope = ` --workspace ${target}` } let submission = local.submission @@ -523,6 +534,7 @@ function serverDiffs( */ export function attachWorkspaceSourceDiff(source: Command): void { const [command, connect] = apiCommand(source, 'diff', ['getWorkspaceSource']) + absoluteDirOption(command) command .argument('[paths...]', 'Files, folders or kind:key resources to compare (default: every file)') .option('--dir <dir>', 'Workspace source directory written by `workspaces source get`', '.') @@ -543,7 +555,7 @@ export function attachWorkspaceSourceDiff(source: Command): void { const selects = filter.selects const read = async (kinds: readonly WorkspaceKind[]) => { const connection = connect() - const workspaceId = targetWorkspace(connection, local.readFrom) + const workspaceId = targetWorkspace(connection, options.dir, local.readFrom) try { const response = await connection.client.request<GetWorkspaceSourceResponse>( 'getWorkspaceSource', diff --git a/packages/sim-cli/src/test/context-mock.ts b/packages/sim-cli/src/test/context-mock.ts index 21cac9375e4..712a194220d 100644 --- a/packages/sim-cli/src/test/context-mock.ts +++ b/packages/sim-cli/src/test/context-mock.ts @@ -16,10 +16,23 @@ export const contextMockState: { workspaceSource: 'config', } +/** The resolved profile `clientFrom` and `profileFrom` return, built from {@link contextMockState}. */ +export function mockedProfile() { + return { + workspaceId: contextMockState.workspaceId, + output: contextMockState.output, + name: 'default', + authProfile: 'default', + apiKey: 'k', + endpoint: 'https://sim.example', + sources: { workspaceId: contextMockState.workspaceSource }, + } +} + /** * The `SimClient` methods protocol commands call, plus the context entry * points. Non-bare defaults: `mockRequireWorkspace` returns - * `contextMockState.workspaceId`; `mockClientFrom` returns + * the workspace it is given, else `contextMockState.workspaceId`; `mockClientFrom` returns * `{ client: { request, requestRaw, requireWorkspace }, profile }` with the * profile built from {@link contextMockState} (name `default`, api key `k`, * endpoint `https://sim.example`). @@ -27,24 +40,16 @@ export const contextMockState: { export const contextMockFns = { mockRequest: vi.fn(), mockRequestRaw: vi.fn(), - mockRequireWorkspace: vi.fn(() => contextMockState.workspaceId), + mockRequireWorkspace: vi.fn((explicit?: string) => explicit ?? contextMockState.workspaceId), mockGlobalsOf: vi.fn(), - mockProfileFrom: vi.fn(), + mockProfileFrom: vi.fn(mockedProfile), mockClientFrom: vi.fn(() => ({ client: { request: contextMockFns.mockRequest, requestRaw: contextMockFns.mockRequestRaw, requireWorkspace: contextMockFns.mockRequireWorkspace, }, - profile: { - workspaceId: contextMockState.workspaceId, - output: contextMockState.output, - name: 'default', - authProfile: 'default', - apiKey: 'k', - endpoint: 'https://sim.example', - sources: { workspaceId: contextMockState.workspaceSource }, - }, + profile: mockedProfile(), })), } From 1079302683158982549485e1ad21c90bd88b21a1 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:47:44 -0800 Subject: [PATCH 22/59] feat(sim-cli): each source finding shown once, and the ids a write created With --output json (or yaml) a source check, apply or create prints its findings only in the document on stdout, no longer also as frames on stderr, and leaves info findings out unless --verbose. A display format prints the first warning of each code whole and the rest of that code on one line. Workflow create and apply, and workspace apply, list what they created as created: [{ kind, key, id }] (named on one line in a display format), and a workflow write lists its notes in a machine result. --- apps/docs/content/docs/cli/reference.mdx | 5 + apps/docs/content/docs/cli/workflows.mdx | 3 + apps/docs/content/docs/cli/workspaces.mdx | 2 + .../sim-cli/src/commands/protocol/result.ts | 13 +- .../src/commands/protocol/source-ui.ts | 137 ++++++++++++--- .../protocol/workflow-source-apply.test.ts | 133 ++++++++++++++- .../protocol/workflow-source-apply.ts | 79 ++++++--- .../protocol/workflow-source-check.test.ts | 69 +++++--- .../protocol/workflow-source-check.ts | 159 +++++++++--------- .../workflow-source-roundtrip.test.ts | 34 ++-- .../protocol/workspace-source-apply.test.ts | 8 +- .../protocol/workspace-source-apply.ts | 42 +++-- .../protocol/workspace-source-check.test.ts | 26 ++- .../protocol/workspace-source-check.ts | 128 +++++++------- 14 files changed, 586 insertions(+), 252 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 9a939e1c8b7..b5c009d8970 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8573,6 +8573,7 @@ sim workflows source apply [workflowId] [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir <dir>` | Yes | Source directory: workflow.ts, functions/*, .sim/lock.json and .sim/base.json. | | `--dry-run` | No | Check the source and show the outcome without saving or writing files. | | `-y, --yes` | No | Apply without asking for confirmation. | @@ -8598,6 +8599,7 @@ sim workflows source create [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir <dir>` | Yes | Source directory: workflow.ts and functions/*. Without workflow.ts, writes a starter and the types instead (an error with --dry-run or --yes). | | `--dry-run` | No | Check the source and show the outcome without creating anything. | | `--name <name>` | No | The workflow name (default: the source's). | @@ -8627,6 +8629,7 @@ sim workflows source check [options] | `--dir <dir>` | No | Source directory: workflow.ts, functions/* and .sim/. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | </CommandTable> @@ -9941,6 +9944,7 @@ sim workspaces source apply [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir <dir>` | Yes | Workspace source directory written by `workspaces source get`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | @@ -9971,6 +9975,7 @@ sim workspaces source check [options] | `--dir <dir>` | No | Workspace source directory: the kind folders and .sim/base.json. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | </CommandTable> diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index eafc3758d94..8f698cbb9cd 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1353,6 +1353,7 @@ sim workflows source create [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir <dir>` | Yes | Source directory: workflow.ts and functions/*. Without workflow.ts, writes a starter and the types instead (an error with --dry-run or --yes). | | `--dry-run` | No | Check the source and show the outcome without creating anything. | | `--name <name>` | No | The workflow name (default: the source's). | @@ -1387,6 +1388,7 @@ sim workflows source apply [workflowId] [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir <dir>` | Yes | Source directory: workflow.ts, functions/*, .sim/lock.json and .sim/base.json. | | `--dry-run` | No | Check the source and show the outcome without saving or writing files. | | `-y, --yes` | No | Apply without asking for confirmation. | @@ -1431,6 +1433,7 @@ sim workflows source check [options] | `--dir <dir>` | No | Source directory: workflow.ts, functions/* and .sim/. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | </CommandTable> diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 3bfd5bab460..565d6632de8 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -751,6 +751,7 @@ sim workspaces source apply [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir <dir>` | Yes | Workspace source directory written by `workspaces source get`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | @@ -779,6 +780,7 @@ sim workspaces source check [options] | `--dir <dir>` | No | Workspace source directory: the kind folders and .sim/base.json. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | </CommandTable> diff --git a/packages/sim-cli/src/commands/protocol/result.ts b/packages/sim-cli/src/commands/protocol/result.ts index 8034aa06e22..5f0de86ea88 100644 --- a/packages/sim-cli/src/commands/protocol/result.ts +++ b/packages/sim-cli/src/commands/protocol/result.ts @@ -8,10 +8,19 @@ function fieldText(value: unknown): string { return text(nested ? JSON.stringify(value) : value) } -export function printProtocolResult(format: OutputFormat, result: Record<string, unknown>): void { +/** + * Prints a protocol command's result: the record itself in a machine format, else one line per + * field, with `display` replacing the text of the fields it names (a list a person reads as a + * sentence rather than as JSON). + */ +export function printProtocolResult( + format: OutputFormat, + result: Record<string, unknown>, + display: Readonly<Record<string, string>> = {} +): void { const fields = Object.entries(result).map<[string, string]>(([key, value]) => [ key, - fieldText(value), + display[key] ?? fieldText(value), ]) printRecord(format, fields, result) } diff --git a/packages/sim-cli/src/commands/protocol/source-ui.ts b/packages/sim-cli/src/commands/protocol/source-ui.ts index 6c7a896a845..a15215c3529 100644 --- a/packages/sim-cli/src/commands/protocol/source-ui.ts +++ b/packages/sim-cli/src/commands/protocol/source-ui.ts @@ -6,6 +6,7 @@ import { isRecordLike, toRecord } from '@sim/utils/object' import { SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' import { type FlatDiagnostic, type RefusalClass, refusalClass } from '@sim/workspace-code' +import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' import { askLine } from '#sim-cli/terminal/line-input' import type { OutputFormat } from '../../config/index' @@ -26,7 +27,7 @@ import { noAnswer, REQUEST_CANCELLED, SimApiError } from '../../http/client' type WorkflowDiagnostic = ApplyWorkflowSourceResponse['data']['diagnostics'][number] /** A workflow finding, or a workspace one: the same fields, over both code vocabularies. */ -type Diagnostic = Omit<WorkflowDiagnostic, 'code'> & { +export type Diagnostic = Omit<WorkflowDiagnostic, 'code'> & { code: | WorkflowDiagnostic['code'] | ApplyWorkspaceSourceResponse['data']['diagnostics'][number]['code'] @@ -74,28 +75,49 @@ function formatDiagnostic(diagnostic: Diagnostic): string { return lines.join('\n') } +/** How many other locations a collapsed line lists before counting the rest. */ +const COLLAPSED_LOCATIONS = 8 + +/** Where a finding sits, as its printed line starts with: `file:line:column`, or empty. */ +function locationOf(diagnostic: Diagnostic): string { + return formatDiagnostic({ ...diagnostic, frame: undefined, hint: undefined }) + .split(` ${diagnostic.severity} `)[0] + .trim() +} + /** - * Prints a write's findings: every error and warning with its location, frame and hint, and each - * info code folded into one line (a new workflow places every block by autolayout, which is no - * reason to print a frame per block and bury the warnings). + * Prints a write's findings on stderr for a person: every error with its location, frame and + * hint; the first warning of each code the same way, and the rest of that code on one line + * listing where they are (one fix usually serves them all, and twenty frames of the same warning + * buried the errors); each info code folded into one line. */ -function printDiagnostics(diagnostics: Diagnostic[]): void { - const infos = new Map<string, Diagnostic[]>() +function printDiagnostics(diagnostics: readonly Diagnostic[]): void { + const groups = new Map<string, Diagnostic[]>() for (const diagnostic of diagnostics) { - if (diagnostic.severity !== 'info') { + if (diagnostic.severity === 'error') { printError(formatDiagnostic(diagnostic)) continue } - const group = infos.get(diagnostic.code) + const key = `${diagnostic.severity}:${diagnostic.code}` + const group = groups.get(key) if (group) group.push(diagnostic) - else infos.set(diagnostic.code, [diagnostic]) + else groups.set(key, [diagnostic]) } - for (const [code, group] of infos) { - const [first] = group + for (const [first, ...rest] of groups.values()) { + if (first.severity === 'info') { + printError( + rest.length === 0 + ? formatDiagnostic({ ...first, frame: undefined }) + : `info ${first.code} on ${rest.length + 1} blocks: ${first.message} (and ${rest.length} more like it)` + ) + continue + } + printError(formatDiagnostic(first)) + if (rest.length === 0) continue + const listed = rest.slice(0, COLLAPSED_LOCATIONS).map(locationOf).filter(Boolean) + const unlisted = rest.length - listed.length printError( - group.length === 1 - ? formatDiagnostic({ ...first, frame: undefined }) - : `info ${code} on ${group.length} blocks: ${first.message} (and ${group.length - 1} more like it)` + ` ${rest.length} more ${first.code} like it${listed.length > 0 ? `: ${listed.join(', ')}` : ''}${unlisted > 0 && listed.length > 0 ? `, and ${unlisted} more` : ''}` ) } } @@ -103,16 +125,28 @@ function printDiagnostics(diagnostics: Diagnostic[]): void { /** A finding in a source command's `--output json` (or `yaml`) result: the frame stays human-only. */ type ResultDiagnostic = Omit<FlatDiagnostic, 'frame' | 'code'> & { code: Diagnostic['code'] } +/** Whether `format` prints one machine document on stdout rather than lines for a person. */ +function isMachine(format: OutputFormat): boolean { + return format === 'json' || format === 'yaml' +} + /** * The `diagnostics` a source command's result carries in a machine format, so the one document on - * stdout holds every finding; a display format prints them on stderr only. + * stdout holds every finding: errors and warnings, and info only with `verbose` (a new workflow + * places every block by autolayout, which is no finding to act on). A display format prints them + * on stderr only. */ -export function resultDiagnostics( +function resultDiagnostics( format: OutputFormat, - diagnostics: readonly Diagnostic[] + diagnostics: readonly Diagnostic[], + verbose = false ): { diagnostics?: ResultDiagnostic[] } { - if (format !== 'json' && format !== 'yaml') return {} - return { diagnostics: diagnostics.map(({ frame: _frame, ...finding }) => finding) } + if (!isMachine(format)) return {} + return { + diagnostics: diagnostics + .filter((finding) => verbose || finding.severity !== 'info') + .map(({ frame: _frame, ...finding }) => finding), + } } function findingKey(diagnostic: Diagnostic): string { @@ -128,11 +162,64 @@ export function fresh(printed: Set<string>, key: string): boolean { } /** - * Prints the findings `printed` does not hold yet and records them, so a finding the preview - * showed is not shown again by the write that follows it. + * Where a source command's findings go, chosen once by its output format, so each is shown once: + * a display format prints them on stderr as frames (and never in the result), a machine format + * only in the one document on stdout (and never as frames, which an agent reading both streams + * read twice). */ -export function printNewDiagnostics(diagnostics: Diagnostic[], printed: Set<string>): void { - printDiagnostics(diagnostics.filter((diagnostic) => fresh(printed, findingKey(diagnostic)))) +export interface FindingsOutput { + /** Whether the result is one machine document on stdout. */ + machine: boolean + /** + * Prints the findings not printed yet on stderr in a display format, so a finding the preview + * showed is not shown again by the write that follows it; nothing in a machine format. + */ + print(diagnostics: readonly Diagnostic[]): void + /** The result's `diagnostics` field: see {@link resultDiagnostics}. */ + result(diagnostics: readonly Diagnostic[]): { diagnostics?: ResultDiagnostic[] } +} + +/** The {@link FindingsOutput} of `format`; `verbose` keeps info findings in a machine result. */ +export function findingsOutput(format: OutputFormat, verbose = false): FindingsOutput { + const printed = new Set<string>() + const machine = isMachine(format) + return { + machine, + print(diagnostics) { + const unseen = diagnostics.filter((diagnostic) => fresh(printed, findingKey(diagnostic))) + if (!machine) printDiagnostics(unseen) + }, + result: (diagnostics) => resultDiagnostics(format, diagnostics, verbose), + } +} + +/** A resource a source write created, as its result lists it; `id` once it is written. */ +export interface CreatedResource { + kind: string + key?: string + id?: string +} + +/** How many created resources the display line names before counting the rest. */ +const NAMED_CREATED = 5 + +/** The created resources as one line for a person: `workflow (abc), block start (b-1), …`. */ +export function createdText(created: readonly CreatedResource[]): string { + if (created.length === 0) return 'none' + const named = created.slice(0, NAMED_CREATED).map(({ kind, key, id }) => { + const label = [kind, key].filter((part) => part !== undefined).join(' ') + return id === undefined ? label : `${label} (${id})` + }) + const more = created.length - named.length + return `${named.join(', ')}${more > 0 ? `, and ${more} more` : ''}` +} + +/** Adds `--verbose` to a source command whose result can carry info findings. */ +export function verboseFlag(command: Command): Command { + return command.option( + '--verbose', + 'With --output json, also list info findings (such as blocks placed by autolayout)' + ) } function isDiagnosticList(value: unknown): value is Diagnostic[] { @@ -198,8 +285,8 @@ export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost? const details = toRecord(error.details) const diagnostics = details.diagnostics const printed = isDiagnosticList(diagnostics) - if (printed) printDiagnostics(diagnostics) - const machine = format === 'json' || format === 'yaml' + const machine = isMachine(format) + if (printed && !machine) printDiagnostics(diagnostics) const message = machine ? error.message : error.message.replace(DETAILS_REFERENCE, printed ? ' Each finding is printed above.' : '') diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index 5bd13c68794..a6c5eb1ed3c 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -435,12 +435,17 @@ describe('sim workflows source apply', () => { ]) const printed = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') - expect(printed).toMatchObject({ created: false, dryRun: true }) + expect(printed).toMatchObject({ + dryRun: true, + created: [{ kind: 'workflow' }, { kind: 'block', key: 'start' }], + }) expect(printed).not.toHaveProperty('workflowId') + expect(JSON.stringify(printed.created)).not.toContain('"id"') }) it('folds info findings into one line per code and keeps frames on warnings', async () => { setTerminal(false) + contextMockState.output = 'text' const layout = (key: string, line: number) => ({ code: 'I_MAT_NEEDS_LAYOUT', severity: 'info', @@ -480,6 +485,7 @@ describe('sim workflows source apply', () => { it('prints every note on a dry run, so a kept name or a dropped tool is never silent', async () => { setTerminal(false) + contextMockState.output = 'text' const note = "The workflow name 'New' differs from the stored one and was not changed." request.mockResolvedValue(outcome({ dryRun: true, notes: [note] })) @@ -488,18 +494,19 @@ describe('sim workflows source apply', () => { expect(stderrLines()).toContain(`note: ${note}`) }) - it('counts notes apart from warnings in the result', async () => { + it('lists notes apart from warnings in the JSON result, and prints none on stderr', async () => { setTerminal(false) request.mockResolvedValue(outcome({ dryRun: true, notes: ['a kept name'] })) await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') - const printed = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') - expect(printed).toMatchObject({ warnings: 0, notes: 1 }) + expect(stdoutJson()).toMatchObject({ warnings: 0, notes: ['a kept name'] }) + expect(stderrLines()).toEqual([]) }) it('at a terminal, prints the notes of the preview and of the applied write', async () => { setTerminal(true) + contextMockState.output = 'text' request.mockImplementation(async (_path: string, options: { query?: { dryRun?: boolean } }) => outcome({ dryRun: options.query?.dryRun === true, @@ -517,6 +524,7 @@ describe('sim workflows source apply', () => { it('at a terminal, prints a warning and a note the preview showed once, not again on the write', async () => { setTerminal(true) + contextMockState.output = 'text' const warning = { code: 'W_REQUIRED_MISSING', severity: 'warning', @@ -559,6 +567,123 @@ describe('sim workflows source apply', () => { }) }) + describe('findings and results an agent reads', () => { + const untyped = (line: number) => ({ + code: 'W_JSON_REF_UNTYPED', + severity: 'warning', + message: `Nothing declares the type of <score.result.n${line}>.`, + file: 'workflow.ts', + line, + column: 7, + frame: `> ${line} | n: score.result.n${line},`, + hint: 'Leave it bare when it holds a number.', + }) + const layout = { + code: 'I_MAT_NEEDS_LAYOUT', + severity: 'info', + message: 'Block "start" is placed by autolayout', + file: 'workflow.ts', + line: 3, + column: 3, + } + + it('with --output json, prints findings only in the JSON on stdout, never as frames on stderr', async () => { + setTerminal(false) + request.mockResolvedValue(outcome({ dryRun: true, diagnostics: [untyped(4)] })) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stdoutJson()).toMatchObject({ diagnostics: [{ code: 'W_JSON_REF_UNTYPED' }] }) + expect(stderrLines().join('\n')).not.toContain('W_JSON_REF_UNTYPED') + }) + + it('leaves info findings out of the JSON unless --verbose', async () => { + setTerminal(false) + request.mockResolvedValue(outcome({ dryRun: true, diagnostics: [layout, untyped(4)] })) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + expect(stdoutJson().diagnostics).toEqual([expect.objectContaining({ severity: 'warning' })]) + + vi.mocked(console.log).mockClear() + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run', '--verbose') + expect(stdoutJson().diagnostics).toEqual([ + expect.objectContaining({ code: 'I_MAT_NEEDS_LAYOUT' }), + expect.objectContaining({ code: 'W_JSON_REF_UNTYPED' }), + ]) + }) + + it('in a display format, prints the first warning of a code whole and the rest on one line', async () => { + setTerminal(false) + contextMockState.output = 'text' + request.mockResolvedValue( + outcome({ dryRun: true, diagnostics: [4, 5, 6, 7].map((line) => untyped(line)) }) + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + const lines = stderrLines() + expect(lines.filter((line) => line.includes('> '))).toHaveLength(1) + expect(lines).toContain( + ' 3 more W_JSON_REF_UNTYPED like it: workflow.ts:5:7, workflow.ts:6:7, workflow.ts:7:7' + ) + }) + + it('lists the workflow and blocks a create made, with their ids', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + rmSync(join(dir, '.sim/base.json')) + const catalog = JSON.parse(readFileSync(FIXTURE_CATALOG, 'utf8')) + request.mockImplementation(async (path: string) => { + if (path.endsWith('/workflows/source/catalog')) return { data: catalog } + if (path.endsWith('/workflows/source/resources')) { + return { + data: { + workspaceId: WORKSPACE_ID, + tables: [], + knowledgeBases: [], + credentials: [], + workflows: [], + env: [], + truncated: false, + }, + } + } + return outcome() + }) + + await program().parseAsync([ + 'node', + 'sim', + 'workflows', + 'source', + 'create', + '--dir', + dir, + '--yes', + ]) + + expect(stdoutJson()).toMatchObject({ + workflowId: WORKFLOW_ID, + created: [ + { kind: 'workflow', id: WORKFLOW_ID }, + { kind: 'block', key: 'start', id: 'b-1' }, + ], + }) + }) + + it('names what an apply created on one line in a display format', async () => { + setTerminal(false) + contextMockState.output = 'text' + request.mockResolvedValue(outcome()) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes') + + expect(vi.mocked(console.log).mock.calls.map((call) => call.join(' '))).toContain( + 'created\tblock start (b-1)' + ) + }) + }) + it('exits 5 when a dry run finds the directory stale', async () => { setTerminal(false) request.mockResolvedValue(outcome({ dryRun: true, diagnostics: [STALE] })) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 00329046d34..bc7dc403e09 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -37,13 +37,16 @@ import { import { SOURCE_WRITE_EXIT_CODES_HELP } from './source-exit-codes' import { cachedCatalogByHash } from './source-tree' import { + type CreatedResource, confirm, + createdText, + type FindingsOutput, + findingsOutput, fresh, - printNewDiagnostics, printRefusal, refusedExitCode, requireConsent, - resultDiagnostics, + verboseFlag, } from './source-ui' import { storedEdgesWrite } from './stored-edges' import { @@ -80,6 +83,7 @@ interface SourceWriteOptions { dryRun?: boolean yes?: boolean allowDelete?: boolean + verbose?: boolean } /** @@ -127,14 +131,15 @@ function codeFilePaths(entry: string, lock: WorkflowLockV1 | null): string[] { } /** - * Prints a write's diagnostics and notes (such as a custom tool the save could not store), - * skipping any already in `printed` and adding the rest: after a terminal preview, the + * Prints a write's diagnostics and, in a display format, its notes (such as a custom tool the + * save could not store; a machine result lists them), each once: after a terminal preview, the * write it confirmed prints only what the preview did not. */ -function printFindings(data: SourceApplyData, printed: Set<string>): void { - printNewDiagnostics(data.diagnostics, printed) +function printFindings(data: SourceApplyData, findings: FindingsOutput, notes: Set<string>): void { + findings.print(data.diagnostics) + if (findings.machine) return for (const note of data.notes) { - if (fresh(printed, `note:${note}`)) printError(`note: ${note}`) + if (fresh(notes, note)) printError(`note: ${note}`) } } @@ -266,16 +271,22 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { return printRefusal(error, args.output, dryRun ? undefined : args.ifAnswerLost) } } - const printed = new Set<string>() + const findings = findingsOutput(args.output, options.verbose) + const notes = new Set<string>() + const report = (data: SourceApplyData) => { + const result = summarize(data, creating, findings.machine) + printProtocolResult( + args.output, + { ...result, ...findings.result(data.diagnostics) }, + result.created ? { created: createdText(result.created) } : {} + ) + } if (!options.dryRun && !options.yes) { const preview = await send(true) - printFindings(preview, printed) + printFindings(preview, findings, notes) if (preview.diagnostics.some((diagnostic) => diagnostic.severity === 'error')) { setSoftExitCode(refusedExitCode(preview.diagnostics)) - printProtocolResult(args.output, { - ...summarize(preview, creating), - ...resultDiagnostics(args.output, preview.diagnostics), - }) + report(preview) return } printError(diffSummary(preview)) @@ -287,7 +298,7 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { const verb = creating ? 'create' : 'apply' const sendAndSave = async (): Promise<SourceApplyData> => { const data = await send(options.dryRun === true) - printFindings(data, printed) + printFindings(data, findings, notes) const refused = data.diagnostics.some((diagnostic) => diagnostic.severity === 'error') if (refused) setSoftExitCode(refusedExitCode(data.diagnostics)) if (!data.dryRun && !refused && (data.applied || !creating)) await args.save(data) @@ -300,10 +311,7 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { abandoned: `Stopped before the ${verb} answered. ${args.ifAnswerLost}`, }) - printProtocolResult(args.output, { - ...summarize(data, creating), - ...resultDiagnostics(args.output, data.diagnostics), - }) + report(data) } const DIR_DESCRIPTION = @@ -322,6 +330,7 @@ const DIR_DESCRIPTION = export function attachWorkflowSourceApply(source: Command): void { const [command, connectApply] = apiCommand(source, 'apply', ['applyWorkflowSource']) absoluteDirOption(command) + verboseFlag(command) command .argument( '[workflowId]', @@ -384,6 +393,7 @@ export function attachWorkflowSourceCreate(source: Command): void { 'getSourceResources', ]) absoluteDirOption(command) + verboseFlag(command) command .requiredOption( '--dir <dir>', @@ -474,14 +484,35 @@ async function assertStarterWorkspace( ) } -/** A create that did not apply stored nothing, so its provisional workflow id names no workflow. */ -function summarize(data: SourceApplyData, created: boolean) { +/** + * What a write created (on a dry run, would create; ids are only known once it is written): the + * workflow of a create, then each block added. + */ +function createdResources(data: SourceApplyData, creating: boolean): CreatedResource[] { + const written = !data.dryRun && data.applied + return [ + ...(creating ? [{ kind: 'workflow', ...(written ? { id: data.workflowId } : {}) }] : []), + ...data.diff.blocks.added.map(({ key, id }) => ({ + kind: 'block', + key, + ...(written ? { id } : {}), + })), + ] +} + +/** + * The result of a write. A create that did not apply stored nothing, so its provisional workflow + * id names no workflow. Notes are listed in a `machine` result, and counted in a display one, + * which prints them on stderr. + */ +function summarize(data: SourceApplyData, creating: boolean, machine: boolean) { + const created = data.dryRun || data.applied ? createdResources(data, creating) : [] return { - ...(created && !data.applied ? {} : { workflowId: data.workflowId }), - ...(created ? { created: data.applied } : {}), + ...(creating && !data.applied ? {} : { workflowId: data.workflowId }), applied: data.applied, dryRun: data.dryRun, noop: data.diff.noop, + ...(created.length > 0 ? { created } : {}), blocksAdded: data.diff.blocks.added.length, blocksRemoved: data.diff.blocks.removed.length, blocksModified: data.diff.blocks.modified.length, @@ -492,9 +523,9 @@ function summarize(data: SourceApplyData, created: boolean) { edgesRemoved: data.diff.edges.removed, errors: data.diagnostics.filter((diagnostic) => diagnostic.severity === 'error').length, warnings: data.diagnostics.filter((diagnostic) => diagnostic.severity === 'warning').length, - notes: data.notes.length, + notes: machine ? data.notes : data.notes.length, fingerprint: data.base.fingerprint, needsRedeployment: data.needsRedeployment, - ...(created && data.applied && !data.dryRun ? { guide: SOURCE_AGENTS_FILE } : {}), + ...(creating && data.applied && !data.dryRun ? { guide: SOURCE_AGENTS_FILE } : {}), } } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index 4f5a42e3a31..df0569d90d8 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -48,10 +48,18 @@ function stdoutJson(): Record<string, unknown> { return JSON.parse(calls[0].join(' ')) } -function stderr(): string { - return vi - .mocked(console.error) - .mock.calls.map((call) => call.join(' ')) +/** + * The findings of the JSON result on stdout as a person reads them on stderr in a display format + * (`file:line:column severity CODE: message`, then its hint): with `--output json` they are + * printed only there. + */ +function findingLines(): string { + const diagnostics = (result().diagnostics ?? []) as Array<Record<string, unknown>> + return diagnostics + .map( + ({ file, line, column, severity, code, message, hint }) => + `${file === undefined ? '' : `${file}:${line}:${column} `}${severity} ${code}: ${message}${hint === undefined ? '' : `\n hint: ${hint}`}` + ) .join('\n') } @@ -243,7 +251,7 @@ describe('sim workflows source check', () => { await check() expect(result()).toMatchObject({ errors: 1 }) - expect(stderr()).toMatch(/workflow\.ts:6:\d+ error E_UNKNOWN_PARAM/) + expect(findingLines()).toMatch(/workflow\.ts:6:\d+ error E_UNKNOWN_PARAM/) expect(process.exitCode).toBe(1) }) @@ -270,6 +278,19 @@ describe('sim workflows source check', () => { }) }) + it('with --output json, prints each finding in the JSON only, not again as a frame on stderr', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace('{ body: { received: start.message } }', '{ bdy: { received: start.message } }') + ) + + await check() + + expect(stdoutJson()).toMatchObject({ diagnostics: [{ code: 'E_UNKNOWN_PARAM' }] }) + expect(vi.mocked(console.error)).not.toHaveBeenCalled() + }) + it('leaves stored() to the server, which holds the stored values', async () => { cacheCatalog() writeFileSync( @@ -288,7 +309,7 @@ describe('sim workflows source check', () => { await check() - expect(stderr()).not.toContain('E_STORED_VALUE_MISSING') + expect(findingLines()).not.toContain('E_STORED_VALUE_MISSING') }) it('refuses without a cached catalog, naming the get that caches one', async () => { @@ -359,8 +380,10 @@ describe('sim workflows source check', () => { await checkIn(folder) expect(result()).toMatchObject({ errors: 2 }) - expect(stderr()).toMatch(/workflow\.ts:6:\d+ error E_MAT_UNKNOWN_RESOURCE: No table "contact"/) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( + /workflow\.ts:6:\d+ error E_MAT_UNKNOWN_RESOURCE: No table "contact"/ + ) + expect(findingLines()).toMatch( /error E_MAT_UNKNOWN_RESOURCE: No credential "ops-slak"[\s\S]*Keys: ops-slack/ ) expect(process.exitCode).toBe(1) @@ -377,7 +400,7 @@ describe('sim workflows source check', () => { await checkIn(folder) expect(result()).toMatchObject({ errors: 0, warnings: 1 }) - expect(stderr()).toContain('W_ENV_UNKNOWN') + expect(findingLines()).toContain('W_ENV_UNKNOWN') }) it('fails a row write naming a column or select option the table does not declare', async () => { @@ -394,10 +417,10 @@ describe('sim workflows source check', () => { await checkIn(folder) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( /error W_TABLE_UNKNOWN_COLUMN: data writes emial[\s\S]*Columns: email, status/ ) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( /error W_UNKNOWN_OPTION: status of table "contacts" has no option done/ ) expect(process.exitCode).toBe(1) @@ -408,7 +431,7 @@ describe('sim workflows source check', () => { writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) await check() - expect(stderr()).toMatch(/error W_UNKNOWN_OPTION/) + expect(findingLines()).toMatch(/error W_UNKNOWN_OPTION/) expect(process.exitCode).toBe(1) process.exitCode = undefined @@ -440,10 +463,12 @@ describe('sim workflows source check', () => { writeLock(['start', 'l', 'inner', 'fallback', 'a', 'route']) await check() - expect(stderr()).toMatch(/warning W_ERROR_PORT_STORED/) - expect(stderr()).toMatch(/warning W_EDGE_CYCLE/) - expect(stderr()).toMatch(/warning W_BRANCH_UNWIRED/) - expect(stderr()).toMatch(/keeps this error edge only if the stored workflow already holds it/) + expect(findingLines()).toMatch(/warning W_ERROR_PORT_STORED/) + expect(findingLines()).toMatch(/warning W_EDGE_CYCLE/) + expect(findingLines()).toMatch(/warning W_BRANCH_UNWIRED/) + expect(findingLines()).toMatch( + /keeps this error edge only if the stored workflow already holds it/ + ) expect(result()).toMatchObject({ errors: 0 }) expect(process.exitCode).toBeUndefined() @@ -458,8 +483,8 @@ describe('sim workflows source check', () => { writeLock(['start', 'l', 'inner', 'fallback', 'a', 'route']) await check() - expect(stderr()).toMatch(/warning W_ERROR_PORT_STORED/) - expect(stderr()).toMatch(/warning W_EDGE_CYCLE/) + expect(findingLines()).toMatch(/warning W_ERROR_PORT_STORED/) + expect(findingLines()).toMatch(/warning W_EDGE_CYCLE/) expect(result()).toMatchObject({ errors: 0 }) expect(process.exitCode).toBeUndefined() }) @@ -473,8 +498,8 @@ describe('sim workflows source check', () => { writeLock(['start', 'l', 'inner', 'fallback', 'a', 'route']) await check() - expect(stderr()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) - expect(stderr()).toMatch(/error E_EDGE_CYCLE/) + expect(findingLines()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) + expect(findingLines()).toMatch(/error E_EDGE_CYCLE/) expect(process.exitCode).toBe(1) }) @@ -484,8 +509,8 @@ describe('sim workflows source check', () => { writeLock(['start', 'inner', 'fallback', 'route']) await check() - expect(stderr()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) - expect(stderr()).toMatch(/error E_EDGE_CYCLE/) + expect(findingLines()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) + expect(findingLines()).toMatch(/error E_EDGE_CYCLE/) expect(process.exitCode).toBe(1) }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index 057ddcc9121..a6f72c07bec 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -38,7 +38,7 @@ import { refreshTsconfigs, refreshWorkspaceTypes, } from './source-tree' -import { printNewDiagnostics, resultDiagnostics } from './source-ui' +import { type Diagnostic, findingsOutput, verboseFlag } from './source-ui' import { readStoredEdges } from './stored-edges' import { readSourceFiles, requireEntry } from './workflow-source-apply' import { WORKFLOW_TREE } from './workflow-source-get' @@ -94,7 +94,7 @@ type StrictMode = 'new' | 'all' | 'off' const KEPT_AS_WRITTEN = /[;,] (?:it|the value|the text) is kept as written\.$/ /** One finding of a check, as the write commands print theirs. */ -export type CheckFinding = Parameters<typeof printNewDiagnostics>[0][number] +export type CheckFinding = Diagnostic /** The `--strict` / `--no-strict` pair: absent selects the default, {@link StrictMode} `new`. */ export interface StrictOptions { @@ -491,13 +491,18 @@ export async function checkWorkflowDir( return { findings, entry: entryText } } -/** Prints a check's findings and result, and exits 1 on any error. */ +/** + * Prints a check's findings and result, and exits 1 on any error. `verbose` keeps info findings in + * a machine result. + */ export function reportCheck( output: Parameters<typeof printProtocolResult>[0], findings: CheckFinding[], - result: Record<string, unknown> + result: Record<string, unknown>, + verbose = false ): void { - printNewDiagnostics(findings, new Set()) + const shown = findingsOutput(output, verbose) + shown.print(findings) const errors = findings.filter((finding) => finding.severity === 'error').length if (errors > 0) setSoftExitCode(1) printProtocolResult(output, { @@ -505,7 +510,7 @@ export function reportCheck( errors, warnings: findings.filter((finding) => finding.severity === 'warning').length, ...result, - ...resultDiagnostics(output, findings), + ...shown.result(findings), }) } @@ -534,82 +539,86 @@ export function requireCachedCatalog(workspaceId: string, get: string): BlockCat * milliseconds and without the network. The dry run stays the authority. */ export function attachWorkflowSourceCheck(source: Command): void { - strictFlags( - absoluteDirOption(source.command('check')).option( - '--dir <dir>', - 'Source directory: workflow.ts, functions/* and .sim/', - '.' + verboseFlag( + strictFlags( + absoluteDirOption(source.command('check')).option( + '--dir <dir>', + 'Source directory: workflow.ts, functions/* and .sim/', + '.' + ) ) ) .allowExcessArguments(false) .description('Compile a workflow source directory locally, without the server') .addHelpText('after', CHECK_HELP) - .action(async (options: { dir: string } & StrictOptions, command: Command) => { - const profile = profileFrom(command) - const base = toRecord(await readSourceJson(options.dir, SOURCE_BASE_FILE)) - if ( - typeof base.workspaceId === 'string' && - base.workflowId === undefined && - (await readSourceText(options.dir, SOURCE_ENTRY_FILE)) === null - ) { - throw new SimApiError( - `${options.dir} is a workspace source directory: check it with \`sim workspaces source check --dir ${options.dir}\`.`, - 0 - ) - } - await requireEntry(options.dir) - const workspace = await workspaceFolderOf(options.dir) - const recorded = - typeof base.workspaceId === 'string' - ? base.workspaceId - : (workspace?.workspaceId ?? - (profile.sources.workspaceId === 'flag' - ? null - : await readStarterWorkspace(options.dir))) - const workspaceId = directoryWorkspace( - options.dir, - recorded, - profile, - (from) => `Run it without --workspace to check it against the catalog of ${from}.` - ) - if (!workspaceId) { - throw new SimApiError( - `${options.dir} has no ${SOURCE_BASE_FILE} and no workspace is selected: pass --workspace <workspaceId>.`, - 0 - ) - } - const catalog = cachedOrRefuse( - (await cachedSourceCatalog(options.dir)) ?? cachedCatalog(workspaceId), - workspaceId, - workspace - ? `\`sim workspaces source get --dir ${workspace.root}\`` - : `\`sim workflows source get <workflowId> --dir ${options.dir}\` (or \`create\`)` - ) - const declarations = workspace - ? { - ...(await readWorkspaceDeclarations(workspace.root)), - workflowInputs: await workspaceWorkflowInputs(workspace.root, catalog), - } - : undefined - const { findings, entry } = await checkWorkflowDir(options.dir, catalog, { - strict: strictMode(options), - ...(declarations ? { declarations } : {}), - }) - if (workspace && declarations) { - await refreshWorkspaceTypes( - workspace.root, - withWorkflowInputs(declarations.local, declarations.workflowInputs) - ) - await refreshWorkspaceTsconfigs(workspace.root, workspaceId) - } else { - await refreshTsconfigs( + .action( + async (options: { dir: string; verbose?: boolean } & StrictOptions, command: Command) => { + const profile = profileFrom(command) + const base = toRecord(await readSourceJson(options.dir, SOURCE_BASE_FILE)) + if ( + typeof base.workspaceId === 'string' && + base.workflowId === undefined && + (await readSourceText(options.dir, SOURCE_ENTRY_FILE)) === null + ) { + throw new SimApiError( + `${options.dir} is a workspace source directory: check it with \`sim workspaces source check --dir ${options.dir}\`.`, + 0 + ) + } + await requireEntry(options.dir) + const workspace = await workspaceFolderOf(options.dir) + const recorded = + typeof base.workspaceId === 'string' + ? base.workspaceId + : (workspace?.workspaceId ?? + (profile.sources.workspaceId === 'flag' + ? null + : await readStarterWorkspace(options.dir))) + const workspaceId = directoryWorkspace( options.dir, + recorded, + profile, + (from) => `Run it without --workspace to check it against the catalog of ${from}.` + ) + if (!workspaceId) { + throw new SimApiError( + `${options.dir} has no ${SOURCE_BASE_FILE} and no workspace is selected: pass --workspace <workspaceId>.`, + 0 + ) + } + const catalog = cachedOrRefuse( + (await cachedSourceCatalog(options.dir)) ?? cachedCatalog(workspaceId), workspaceId, - WORKFLOW_TREE, - { sources: [entry] }, - { incremental: true } + workspace + ? `\`sim workspaces source get --dir ${workspace.root}\`` + : `\`sim workflows source get <workflowId> --dir ${options.dir}\` (or \`create\`)` ) + const declarations = workspace + ? { + ...(await readWorkspaceDeclarations(workspace.root)), + workflowInputs: await workspaceWorkflowInputs(workspace.root, catalog), + } + : undefined + const { findings, entry } = await checkWorkflowDir(options.dir, catalog, { + strict: strictMode(options), + ...(declarations ? { declarations } : {}), + }) + if (workspace && declarations) { + await refreshWorkspaceTypes( + workspace.root, + withWorkflowInputs(declarations.local, declarations.workflowInputs) + ) + await refreshWorkspaceTsconfigs(workspace.root, workspaceId) + } else { + await refreshTsconfigs( + options.dir, + workspaceId, + WORKFLOW_TREE, + { sources: [entry] }, + { incremental: true } + ) + } + reportCheck(profile.output, findings, { catalogHash: catalog.hash }, options.verbose) } - reportCheck(profile.output, findings, { catalogHash: catalog.hash }) - }) + ) } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 870b79dbfc6..61d89cb9857 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -18,23 +18,25 @@ import { SimApiError } from '../../http/client' import { buildGeneratedCommands } from '../../runtime/build' import { attachProtocolCommands } from './index' +/** The profile every command resolves; a test that reads display output sets `output` to text. */ +const profile = vi.hoisted(() => ({ + name: 'default', + endpoint: 'https://sim.example', + authProfile: 'default', + apiKey: 'key', + oauth: null, + workspaceId: '00000000-0000-4000-8000-00000000000b', + output: 'json' as 'json' | 'text', + sources: { + endpoint: 'default' as const, + credential: 'env' as const, + workspaceId: 'env' as const, + output: 'default' as const, + }, +})) + vi.mock('../../context', async () => { const { SimClient } = await import('../../http/client') - const profile = { - name: 'default', - endpoint: 'https://sim.example', - authProfile: 'default', - apiKey: 'key', - oauth: null, - workspaceId: '00000000-0000-4000-8000-00000000000b', - output: 'json' as const, - sources: { - endpoint: 'default' as const, - credential: 'env' as const, - workspaceId: 'env' as const, - output: 'default' as const, - }, - } return { globalsOf: () => ({}), profileFrom: () => profile, @@ -201,6 +203,7 @@ let catalogRequests: Headers[] let stderr: string[] beforeEach(() => { + profile.output = 'json' root = mkdtempSync(join(tmpdir(), 'sim-source-roundtrip-')) dir = join(root, 'source') vi.stubEnv('SIM_CONFIG_DIR', join(root, 'config')) @@ -786,6 +789,7 @@ describe('apply refusals', () => { it('prints each compile error as file:line:column with its frame and hint, then exits 1', async () => { await get() + profile.output = 'text' server.apply = () => refusal(400, 'SOURCE_INVALID', { diagnostics: [ diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 5d0108670c5..308b8adf0b6 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -358,7 +358,7 @@ describe('workspaces source apply', () => { expect(summary).toMatchObject({ workspaceId: 'ws_prod', dryRun: true, - created: 1, + created: [{ kind: 'table', key: 'leads' }], diagnostics: [expect.objectContaining({ code: 'W_UNDECLARED', key: 'audit' })], }) }) @@ -525,7 +525,11 @@ describe('workspaces source apply', () => { .mocked(console.log) .mock.calls.map(([line]) => String(line)) .join('\n') - expect(JSON.parse(printed)).toMatchObject({ applied: 1, created: 1, failed: 'workflow:ingest' }) + expect(JSON.parse(printed)).toMatchObject({ + applied: 1, + created: [{ kind: 'table', key: 'leads', id: 'tbl_1' }], + failed: 'workflow:ingest', + }) expect(readFileSync(join(dir, 'tables/leads.ts'), 'utf8')).toBe(`${LEADS}// stored\n`) expect(JSON.parse(readFileSync(join(dir, '.sim/base.json'), 'utf8')).resources).toEqual({ 'table:leads': { id: 'tbl_1' }, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 87da672b88c..4981c9eb313 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -45,11 +45,12 @@ import { SOURCE_WRITE_EXIT_CODES_HELP } from './source-exit-codes' import { cachedCatalog } from './source-tree' import { confirm, - printNewDiagnostics, + createdText, + findingsOutput, printRefusal, refusedExitCode, requireConsent, - resultDiagnostics, + verboseFlag, } from './source-ui' import { removedWorkflowRecords, workflowFolderRecords } from './stored-edges' import { @@ -72,6 +73,7 @@ type Change = ApplyData['plan'][number] interface ApplyOptions { dir: string + verbose?: boolean dryRun?: boolean yes?: boolean allowDelete?: boolean @@ -403,6 +405,7 @@ export function attachWorkspaceSourceApply(source: Command): void { 'getSourceCatalog', ]) absoluteDirOption(command) + verboseFlag(command) command .requiredOption('--dir <dir>', 'Workspace source directory written by `workspaces source get`') .option('--dry-run', 'Plan and check without writing anything') @@ -513,20 +516,25 @@ export function attachWorkspaceSourceApply(source: Command): void { `Not sent (a hidden name, or a character no workspace path holds): ${submission.ignored.join(', ')}` ) } - // The apply repeats findings the preview already showed. - const printed = new Set<string>() + // The apply repeats findings the preview already showed: each is printed once. + const findings = findingsOutput(profile.output, options.verbose) + const report = (data: ApplyData) => { + const result = summarize(data) + printProtocolResult( + profile.output, + { ...result, ...findings.result(data.diagnostics) }, + { created: createdText(result.created) } + ) + } let preview: ApplyData | null = null if (options.dryRun || !options.yes) { preview = await send(submission, true) printPlan(preview.plan) - printNewDiagnostics(preview.diagnostics, printed) + findings.print(preview.diagnostics) const errors = preview.diagnostics.some((finding) => finding.severity === 'error') if (errors) setSoftExitCode(refusedExitCode(preview.diagnostics)) if (options.dryRun || refusesApply(preview)) { - printProtocolResult(profile.output, { - ...summarize(preview), - ...resultDiagnostics(profile.output, preview.diagnostics), - }) + report(preview) return } if ( @@ -542,7 +550,7 @@ export function attachWorkspaceSourceApply(source: Command): void { const data = await deferInterrupts( async () => { const answer = await send(submission, false) - printNewDiagnostics(answer.diagnostics, printed) + findings.print(answer.diagnostics) // A promotion leaves the directory bound to the workspace it was read from. if (!options.to) { const workspaceCatalog = cachedCatalog(workspaceId) @@ -569,10 +577,7 @@ export function attachWorkspaceSourceApply(source: Command): void { printError(`Stopped at ${data.failed.kind} ${data.failed.key}: ${data.failed.message}`) setSoftExitCode(failedExitCode(data.failed)) } - printProtocolResult(profile.output, { - ...summarize(data), - ...resultDiagnostics(profile.output, data.diagnostics), - }) + report(data) }, { readOnly: options.dryRun === true } ) @@ -587,7 +592,10 @@ function answerLostHint(dir: string): string { return `It may have been written. Run \`sim workspaces source apply --dir ${dir} --dry-run\`: no changes, or only the workflows you edited refused as stale (exit 5), means it may have been, or someone else changed those resources. Run \`sim workspaces source get --dir <fresh dir>\` and compare it with ${dir}: if it holds your edits, \`sim workspaces source get --dir ${dir} --force\` brings this directory up to date; if not, reapply your edits in the fresh directory. Any other outcome means it was not written: apply again.` } -/** The result line: what a dry run would write, or what an apply wrote (not what it planned). */ +/** + * The result line: what a dry run would write, or what an apply wrote (not what it planned). + * `created` lists each resource created, with its id once written. + */ function summarize(data: ApplyData) { const changes = data.dryRun ? data.plan : data.applied const count = (action: Change['action']) => @@ -596,7 +604,9 @@ function summarize(data: ApplyData) { workspaceId: data.workspaceId, dryRun: data.dryRun, applied: data.applied.length, - created: count('create'), + created: changes + .filter((change) => change.action === 'create' && !change.held) + .map(({ kind, key, id }) => ({ kind, key, ...(id !== undefined ? { id } : {}) })), updated: count('update') + count('adopt'), deleted: count('delete'), destructive: destructiveChanges(changes), diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts index d023b2ad853..8eae19d47fd 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts @@ -53,10 +53,18 @@ function stdoutJson(): Record<string, unknown> { return JSON.parse(calls[0].join(' ')) } -function stderr(): string { - return vi - .mocked(console.error) - .mock.calls.map((call) => call.join(' ')) +/** + * The findings of the JSON result on stdout as a person reads them on stderr in a display format + * (`file:line:column severity CODE: message`, then its hint): with `--output json` they are + * printed only there. + */ +function findingLines(): string { + const diagnostics = (result().diagnostics ?? []) as Array<Record<string, unknown>> + return diagnostics + .map( + ({ file, line, column, severity, code, message, hint }) => + `${file === undefined ? '' : `${file}:${line}:${column} `}${severity} ${code}: ${message}${hint === undefined ? '' : `\n hint: ${hint}`}` + ) .join('\n') } @@ -113,10 +121,12 @@ describe('sim workspaces source check', () => { await check() expect(result()).toMatchObject({ errors: 2, workflows: 3 }) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( /workflows\/second\/workflow\.ts:5:\d+ error E_MAT_UNKNOWN_RESOURCE: No table "contact"/ ) - expect(stderr()).toMatch(/workflows\/third\/workflow\.ts:5:\d+ error W_TABLE_UNKNOWN_COLUMN/) + expect(findingLines()).toMatch( + /workflows\/third\/workflow\.ts:5:\d+ error W_TABLE_UNKNOWN_COLUMN/ + ) expect(process.exitCode).toBe(1) }) @@ -134,8 +144,8 @@ describe('sim workspaces source check', () => { await check() - expect(stderr()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes emial/) - expect(stderr()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes seen/) + expect(findingLines()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes emial/) + expect(findingLines()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes seen/) expect(process.exitCode).toBe(1) }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts index 5e63e0dc73a..221b67a3b5c 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts @@ -16,6 +16,7 @@ import { } from './source-dir' import { unreachableReason } from './source-status' import { cachedCatalog, loadCatalog, refreshWorkspaceTypes } from './source-tree' +import { verboseFlag } from './source-ui' import { type CheckFinding, checkWorkflowDir, @@ -96,11 +97,13 @@ async function fetchedCatalog( export function attachWorkspaceSourceCheck(source: Command): void { const [command, connect] = apiCommand(source, 'check', ['getSourceCatalog']) absoluteDirOption(command) - strictFlags( - command.option( - '--dir <dir>', - `Workspace source directory: the kind folders and ${WORKSPACE_BASE_FILE}`, - '.' + verboseFlag( + strictFlags( + command.option( + '--dir <dir>', + `Workspace source directory: the kind folders and ${WORKSPACE_BASE_FILE}`, + '.' + ) ) ) .allowExcessArguments(false) @@ -108,61 +111,68 @@ export function attachWorkspaceSourceCheck(source: Command): void { 'Compile every workflow of a workspace source directory locally, without the server' ) .addHelpText('after', `${CHECK_HELP}\n${SOURCE_DIR_BUSY_HELP}`) - .action(async (options: { dir: string } & StrictOptions, command: Command) => { - const profile = profileFrom(command) - const base = toRecord(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) - await assertWorkspaceDir(options.dir) - const workspaceId = directoryWorkspace( - options.dir, - typeof base.workspaceId === 'string' ? base.workspaceId : null, - profile, - (from) => `Run it without --workspace to check it against the catalog of ${from}.` - ) - if (!workspaceId) { - throw new SimApiError( - `${options.dir} has no ${WORKSPACE_BASE_FILE} and no workspace is selected: pass --workspace <workspaceId>.`, - 0 + .action( + async (options: { dir: string; verbose?: boolean } & StrictOptions, command: Command) => { + const profile = profileFrom(command) + const base = toRecord(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) + await assertWorkspaceDir(options.dir) + const workspaceId = directoryWorkspace( + options.dir, + typeof base.workspaceId === 'string' ? base.workspaceId : null, + profile, + (from) => `Run it without --workspace to check it against the catalog of ${from}.` ) - } - await withSourceDirLock(options.dir, async () => { - const catalog = - cachedCatalog(workspaceId) ?? (await fetchedCatalog(connect, workspaceId, options.dir)) - const workflowInputs = await workspaceWorkflowInputs(options.dir, catalog) - const declarations = { - ...(await readWorkspaceDeclarations(options.dir)), - workflowInputs, - } - const keys = await workspaceWorkflowKeys(options.dir) - const findings: CheckFinding[] = [] - for (const key of keys) { - const checked = await checkWorkflowDir( - join(options.dir, KIND_DIRS.workflow, key), - catalog, - { - declarations, - strict: strictMode(options), - prefix: `${KIND_DIRS.workflow}/${key}/`, - } + if (!workspaceId) { + throw new SimApiError( + `${options.dir} has no ${WORKSPACE_BASE_FILE} and no workspace is selected: pass --workspace <workspaceId>.`, + 0 ) - findings.push(...checked.findings) - } - for (const { code, message, hint, file, line, column } of declarations.rowFindings) { - findings.push({ - code, - severity: 'error', - message, - ...(hint !== undefined ? { hint } : {}), - file, - line, - column, - }) } - await refreshWorkspaceTypes( - options.dir, - withWorkflowInputs(declarations.local, workflowInputs) - ) - await refreshWorkspaceTsconfigs(options.dir, workspaceId) - reportCheck(profile.output, findings, { workflows: keys.length, catalogHash: catalog.hash }) - }) - }) + await withSourceDirLock(options.dir, async () => { + const catalog = + cachedCatalog(workspaceId) ?? (await fetchedCatalog(connect, workspaceId, options.dir)) + const workflowInputs = await workspaceWorkflowInputs(options.dir, catalog) + const declarations = { + ...(await readWorkspaceDeclarations(options.dir)), + workflowInputs, + } + const keys = await workspaceWorkflowKeys(options.dir) + const findings: CheckFinding[] = [] + for (const key of keys) { + const checked = await checkWorkflowDir( + join(options.dir, KIND_DIRS.workflow, key), + catalog, + { + declarations, + strict: strictMode(options), + prefix: `${KIND_DIRS.workflow}/${key}/`, + } + ) + findings.push(...checked.findings) + } + for (const { code, message, hint, file, line, column } of declarations.rowFindings) { + findings.push({ + code, + severity: 'error', + message, + ...(hint !== undefined ? { hint } : {}), + file, + line, + column, + }) + } + await refreshWorkspaceTypes( + options.dir, + withWorkflowInputs(declarations.local, workflowInputs) + ) + await refreshWorkspaceTsconfigs(options.dir, workspaceId) + reportCheck( + profile.output, + findings, + { workflows: keys.length, catalogHash: catalog.hash }, + options.verbose + ) + }) + } + ) } From 3cd992c417c94bf20420130b401f00ca22f1f180 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:52:23 -0800 Subject: [PATCH 23/59] fix(workflow-code): W_JSON_REF_UNTYPED only where the edit wrote it, and not on typed Function returns The warning fired on every bare Function-output reference in a JSON field, untouched blocks included, so one apply of a large workflow printed thousands of tokens of it. Against a stored workflow (the server's dry run and apply) it is now raised only for a JSON field the source changed, and an offline check reports it only on blocks the lock does not bind (--strict: on every block). A Function result field the JavaScript returns as a number, boolean, array or object literal (directly, or through a local never reassigned to another kind) no longer warns; a text field and anything uncertain keep the warning. --- apps/docs/content/docs/cli/reference.mdx | 4 +- apps/docs/content/docs/cli/workflows.mdx | 2 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- .../protocol/workflow-source-check.test.ts | 25 ++ .../protocol/workflow-source-check.ts | 16 +- packages/workflow-code/src/compile/SYNTAX.md | 4 +- .../src/compile/function-returns.ts | 425 ++++++++++++++++++ packages/workflow-code/src/compile/index.ts | 24 +- .../workflow-code/src/json-fields.test.ts | 72 ++- 9 files changed, 564 insertions(+), 10 deletions(-) create mode 100644 packages/workflow-code/src/compile/function-returns.ts diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index b5c009d8970..2989f1273a0 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8637,7 +8637,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -9983,7 +9983,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 8f698cbb9cd..554bb053e62 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1441,7 +1441,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 565d6632de8..f0cf2a7677c 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -788,7 +788,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index df0569d90d8..17df1322368 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -441,6 +441,31 @@ describe('sim workflows source check', () => { expect(process.exitCode).toBeUndefined() }) + it('reports an untyped JSON reference only on a block the lock does not bind, and on all with --strict', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace( + "const done = response('Done', { body: { received: start.message } })", + "const call = api('Call', { url: 'https://x', method: 'POST', body: { n: start.missing.deep } })\n const done = response('Done', { body: { received: start.message } })" + ).replace('import { response,', 'import { api, response,') + ) + const untyped = () => + ((result().diagnostics ?? []) as Array<{ code: string }>).filter( + (finding) => finding.code === 'W_JSON_REF_UNTYPED' + ) + + await check() + expect(untyped()).toHaveLength(1) + + writeLock(['start', 'done', 'call']) + await check() + expect(untyped()).toEqual([]) + + await checkIn(dir, '--strict') + expect(untyped()).toHaveLength(1) + }) + it('fails on a bound block with --strict and on nothing with --no-strict', async () => { cacheCatalog() writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index a6f72c07bec..10012138909 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -79,6 +79,14 @@ const STRICT_CODES: ReadonlySet<string> = new Set([ */ const STORED_EXEMPT_CODES: ReadonlySet<string> = new Set(['W_BRANCH_UNWIRED']) +/** + * Warnings a check reports only on a block the lock does not bind (one added since the get), and + * on every block with `--strict`: printed source holds them on blocks nobody edited, and the + * server, which compares with the stored workflow, already reports them only for an edited field. + * `W_JSON_REF_UNTYPED`: a bare reference nothing types, in a JSON field. + */ +const EDIT_SCOPED_CODES: ReadonlySet<string> = new Set(['W_JSON_REF_UNTYPED']) + /** * Which {@link STRICT_CODES} warnings fail a check: `new` those on a block the lock does not bind * (one added since the get, or every block of a workflow that has no lock yet), `all` every one, @@ -117,7 +125,8 @@ does not bind (a block added since the get, or every block of a new workflow): ${[...STRICT_CODES].map((code) => ` ${code}`).join(',\n')}. --strict fails on them in every block, except ${[...STORED_EXEMPT_CODES].join(', ')} on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never -fails on a warning.` +fails on a warning. ${[...EDIT_SCOPED_CODES].join(', ')} is reported only on blocks the lock does not +bind, and with --strict on every block.` /** Adds the `--strict` / `--no-strict` pair to a check command. */ export function strictFlags(command: Command): Command { @@ -474,8 +483,11 @@ export async function checkWorkflowDir( !(STORED_EXEMPT_CODES.has(code) && bound(blockKey)) && (options.strict === 'all' || (options.strict === 'new' && !bound(blockKey))) const prefix = options.prefix ?? '' + const reported = (code: string, blockKey: string | undefined) => + !SERVER_ONLY_CODES.has(code) && + !(EDIT_SCOPED_CODES.has(code) && options.strict !== 'all' && bound(blockKey)) const findings = [ - ...compiled.diagnostics.filter((diagnostic) => !SERVER_ONLY_CODES.has(diagnostic.code)), + ...compiled.diagnostics.filter((diagnostic) => reported(diagnostic.code, diagnostic.blockKey)), ...declared, ].map(({ code, severity, message, hint, frame, span, blockKey }): CheckFinding => { const raised = severity === 'warning' && fails(code, blockKey) diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index cb7c733d100..c6d3463f292 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -215,7 +215,9 @@ A JSON code field (subBlock type `code`: `responseFormat`, table row `data`, a r written as an object or array literal compiles to `JSON.stringify(value, null, 2)` text. Each bare reference is stored by its output type: quoted (`"<a.b>"`) when the catalog or the workflow types it `string`, bare (`<a.b>`) otherwise, because the executor splices a string raw and any other value as -JSON. A reference nothing types is stored bare with `W_JSON_REF_UNTYPED`. A template is a JSON +JSON. A reference nothing types is stored bare with `W_JSON_REF_UNTYPED`, which is not raised for a +Function `result` field its JavaScript returns as a non-string literal (`return { total: a + b }`), +nor, against a stored workflow, for a field the source left as stored. A template is a JSON string, which is how a value becomes text on purpose. A resource (`table('x')`, `credential()`, `file()`) or `raw(node)` is not data, so inside the literal it is `E_UNSUPPORTED_SYNTAX`. The printer writes a field's text back as the literal only when serializing that literal gives the same bytes; otherwise the text prints as a diff --git a/packages/workflow-code/src/compile/function-returns.ts b/packages/workflow-code/src/compile/function-returns.ts new file mode 100644 index 00000000000..4307e2080a9 --- /dev/null +++ b/packages/workflow-code/src/compile/function-returns.ts @@ -0,0 +1,425 @@ +/** + * What a Function block's `result` fields hold, read from the object literal its JavaScript code + * returns (`return { total: subtotal + tax, tier: 'gold' }`), so a bare reference to one in a JSON + * field is not reported as untyped when the code says it is a number, boolean, array or object. + * + * The reading is deliberately narrow: every `return` outside a nested function must return an + * object literal, and a field counts only when each of them sets it to an expression whose kind + * is certain (a literal, arithmetic, a comparison, `String(…)`, `.length`, a local variable never + * reassigned to another kind, …). Anything else stays unknown, and the reference keeps its + * warning: a wrong "typed" would store a string bare and break the JSON at run time. + */ + +import { parse } from '@babel/parser' +import type { Expression, Node, ObjectExpression, PatternLike } from '@babel/types' +import { childNodes } from '../babel' +import { parseStoredPath } from '../code-files/ref-path' +import { toCodeFile } from '../code-files/to-file' +import type { BlockKey, CanonicalBlock } from '../model' +import { getOwn } from '../records' + +/** A value's kind as far as JSON splicing goes: text, or any other JSON value. */ +export type ReturnedKind = 'string' | 'typed' + +/** The JavaScript kinds an expression can be inferred to have; `typed` is a mix of non-strings. */ +type JsKind = 'string' | 'number' | 'boolean' | 'typed' + +const FUNCTION_TYPE = 'function' + +const NUMBER_FUNCTIONS: ReadonlySet<string> = new Set(['Number', 'parseInt', 'parseFloat']) +const BOOLEAN_FUNCTIONS: ReadonlySet<string> = new Set(['Boolean', 'isNaN', 'isFinite']) +const STRING_METHODS: ReadonlySet<string> = new Set([ + 'toFixed', + 'toPrecision', + 'toString', + 'toUpperCase', + 'toLowerCase', + 'trim', + 'trimStart', + 'trimEnd', + 'padStart', + 'padEnd', + 'repeat', + 'toISOString', + 'toLocaleString', + 'join', + 'charAt', + 'replace', + 'replaceAll', + 'normalize', + 'substring', +]) +const BOOLEAN_METHODS: ReadonlySet<string> = new Set([ + 'includes', + 'startsWith', + 'endsWith', + 'some', + 'every', + 'test', + 'hasOwnProperty', +]) +const NUMBER_METHODS: ReadonlySet<string> = new Set([ + 'indexOf', + 'lastIndexOf', + 'findIndex', + 'getTime', + 'charCodeAt', +]) +const ARRAY_METHODS: ReadonlySet<string> = new Set(['map', 'filter', 'split', 'flat', 'flatMap']) +const STATIC_KINDS: Readonly<Record<string, JsKind>> = { + 'Number.isFinite': 'boolean', + 'Number.isInteger': 'boolean', + 'Number.isNaN': 'boolean', + 'Number.isSafeInteger': 'boolean', + 'Number.parseFloat': 'number', + 'Number.parseInt': 'number', + 'Array.isArray': 'boolean', + 'Object.keys': 'typed', + 'Object.values': 'typed', + 'Object.entries': 'typed', + 'JSON.stringify': 'string', + 'Date.now': 'number', +} +const ARITHMETIC: ReadonlySet<string> = new Set([ + '-', + '*', + '/', + '%', + '**', + '|', + '&', + '^', + '<<', + '>>', + '>>>', +]) +const COMPARISON: ReadonlySet<string> = new Set([ + '==', + '===', + '!=', + '!==', + '<', + '<=', + '>', + '>=', + 'in', + 'instanceof', +]) + +/** Each function block's returned field kinds, read once per compiled block object. */ +const cache = new WeakMap<CanonicalBlock, ReadonlyMap<string, ReturnedKind> | null>() + +function isFunctionNode(node: Node): boolean { + return ( + node.type === 'FunctionDeclaration' || + node.type === 'FunctionExpression' || + node.type === 'ArrowFunctionExpression' || + node.type === 'ObjectMethod' || + node.type === 'ClassMethod' || + node.type === 'ClassPrivateMethod' + ) +} + +/** Every node under `root`, calling `visit` on each; `enterFunctions` false skips nested bodies. */ +function walk(root: Node, enterFunctions: boolean, visit: (node: Node) => void): void { + const stack: Node[] = [root] + while (stack.length > 0) { + const node = stack.pop() + if (!node) break + visit(node) + if (!enterFunctions && node !== root && isFunctionNode(node)) continue + stack.push(...childNodes(node)) + } +} + +/** The names a declaration pattern binds; `null` for a destructuring, whose kinds are unknown. */ +function boundName(pattern: PatternLike | Node): string | null { + return pattern.type === 'Identifier' ? pattern.name : null +} + +/** The local variables of the code and what each is set to, for {@link Inference.identifier}. */ +interface Bindings { + /** The initializer of each name declared once with one; `null` for any other declaration. */ + initial: Map<string, Expression | null> + /** Every assignment to each name, anywhere in the code (callbacks included). */ + writes: Map<string, Array<{ operator: string; value: Expression | null }>> +} + +function collectBindings(program: Node): Bindings { + const initial = new Map<string, Expression | null>() + const writes = new Map<string, Array<{ operator: string; value: Expression | null }>>() + const declare = (name: string, value: Expression | null) => { + initial.set(name, initial.has(name) ? null : value) + } + walk(program, true, (node) => { + if (node.type === 'VariableDeclarator') { + const name = boundName(node.id) + if (name !== null) declare(name, node.init ?? null) + else for (const child of childNodes(node.id)) walkPatternNames(child, (n) => declare(n, null)) + } else if (node.type === 'FunctionDeclaration' && node.id) { + declare(node.id.name, null) + } else if (isFunctionNode(node) && 'params' in node) { + for (const param of node.params) walkPatternNames(param, (n) => declare(n, null)) + } else if (node.type === 'CatchClause' && node.param) { + walkPatternNames(node.param, (n) => declare(n, null)) + } else if (node.type === 'AssignmentExpression' && node.left.type === 'Identifier') { + const list = writes.get(node.left.name) ?? [] + list.push({ operator: node.operator, value: node.right }) + writes.set(node.left.name, list) + } else if (node.type === 'AssignmentExpression') { + walkPatternNames(node.left, (n) => { + const list = writes.get(n) ?? [] + list.push({ operator: '=', value: null }) + writes.set(n, list) + }) + } else if (node.type === 'UpdateExpression' && node.argument.type === 'Identifier') { + const list = writes.get(node.argument.name) ?? [] + list.push({ operator: '++', value: null }) + writes.set(node.argument.name, list) + } + }) + return { initial, writes } +} + +/** Calls `found` with every identifier a pattern binds. */ +function walkPatternNames(pattern: Node, found: (name: string) => void): void { + walk(pattern, false, (node) => { + if (node.type === 'Identifier') found(node.name) + }) +} + +/** Two kinds of the same value (both branches of `a ? b : c`), as one kind, or unknown. */ +function merge(a: JsKind | undefined, b: JsKind | undefined): JsKind | undefined { + if (a === undefined || b === undefined) return undefined + if (a === b) return a + return a !== 'string' && b !== 'string' ? 'typed' : undefined +} + +/** The member name a non-computed (or string-keyed) member expression or call reads. */ +function memberName(node: Node): string | null { + if (node.type !== 'MemberExpression' && node.type !== 'OptionalMemberExpression') return null + if (!node.computed && node.property.type === 'Identifier') return node.property.name + return node.property.type === 'StringLiteral' ? node.property.value : null +} + +/** Infers expression kinds over one program's bindings, guarding against self-reference. */ +class Inference { + private readonly resolving = new Set<string>() + + constructor(private readonly bindings: Bindings) {} + + kind(node: Node | null | undefined): JsKind | undefined { + if (!node) return undefined + switch (node.type) { + case 'NumericLiteral': + case 'BigIntLiteral': + return 'number' + case 'StringLiteral': + case 'TemplateLiteral': + return 'string' + case 'BooleanLiteral': + return 'boolean' + case 'NullLiteral': + case 'ArrayExpression': + case 'ObjectExpression': + return 'typed' + case 'UnaryExpression': + if (node.operator === '!' || node.operator === 'delete') return 'boolean' + if (node.operator === 'typeof') return 'string' + if (node.operator === '-' || node.operator === '+' || node.operator === '~') return 'number' + return undefined + case 'BinaryExpression': + return this.binary(node.operator, node.left, node.right) + case 'LogicalExpression': + return merge(this.kind(node.left), this.kind(node.right)) + case 'ConditionalExpression': + return merge(this.kind(node.consequent), this.kind(node.alternate)) + case 'SequenceExpression': + return this.kind(node.expressions.at(-1)) + case 'AssignmentExpression': + return node.operator === '=' ? this.kind(node.right) : undefined + case 'UpdateExpression': + return 'number' + case 'MemberExpression': + case 'OptionalMemberExpression': + return memberName(node) === 'length' ? 'number' : undefined + case 'CallExpression': + case 'OptionalCallExpression': + return this.call(node.callee) + case 'Identifier': + return this.identifier(node.name) + default: + return undefined + } + } + + private binary(operator: string, left: Node, right: Node): JsKind | undefined { + if (COMPARISON.has(operator)) return 'boolean' + if (ARITHMETIC.has(operator)) return 'number' + if (operator !== '+') return undefined + const [a, b] = [this.kind(left), this.kind(right)] + if (a === 'string' || b === 'string') return 'string' + return a === 'number' && b === 'number' ? 'number' : undefined + } + + private call(callee: Node): JsKind | undefined { + if (callee.type === 'Identifier') { + if (callee.name === 'String') return 'string' + if (NUMBER_FUNCTIONS.has(callee.name)) return 'number' + if (BOOLEAN_FUNCTIONS.has(callee.name)) return 'boolean' + return undefined + } + const method = memberName(callee) + if (method === null || !('object' in callee)) return undefined + if (callee.object.type === 'Identifier') { + if (callee.object.name === 'Math') return 'number' + const known = STATIC_KINDS[`${callee.object.name}.${method}`] + if (known) return known + } + if (STRING_METHODS.has(method)) return 'string' + if (BOOLEAN_METHODS.has(method)) return 'boolean' + if (NUMBER_METHODS.has(method)) return 'number' + if (ARRAY_METHODS.has(method)) return 'typed' + return undefined + } + + /** + * A local variable's kind: its initializer's, kept only when every later assignment keeps it + * (`+=` of a number to a number, of anything to a string; `++`; `=` of the same kind). + */ + private identifier(name: string): JsKind | undefined { + const init = this.bindings.initial.get(name) + if (!init || this.resolving.has(name)) return undefined + this.resolving.add(name) + try { + const declared = this.kind(init) + if (declared === undefined) return undefined + for (const { operator, value } of this.bindings.writes.get(name) ?? []) { + if (operator === '++') { + if (declared !== 'number') return undefined + } else if (operator === '=') { + if (merge(declared, this.kind(value)) !== declared) return undefined + } else if (operator === '+=') { + const added = this.kind(value) + if (!(declared === 'string' || (declared === 'number' && added === 'number'))) { + return undefined + } + } else if (ARITHMETIC.has(operator.slice(0, -1))) { + if (declared !== 'number') return undefined + } else { + return undefined + } + } + return declared + } finally { + this.resolving.delete(name) + } + } +} + +/** The fields one returned object literal sets, by name; a spread makes earlier ones unknown. */ +function returnedFields( + object: ObjectExpression, + inference: Inference +): Map<string, JsKind | undefined> { + const fields = new Map<string, JsKind | undefined>() + for (const property of object.properties) { + if (property.type === 'SpreadElement') { + for (const key of fields.keys()) fields.set(key, undefined) + continue + } + if (property.type !== 'ObjectProperty' || property.computed) continue + const key = + property.key.type === 'Identifier' + ? property.key.name + : property.key.type === 'StringLiteral' + ? property.key.value + : null + if (key === null) continue + fields.set(key, inference.kind(property.value)) + } + return fields +} + +/** The field kinds a function body's returns agree on, or `null` when any return is no literal. */ +function readReturns(code: string): ReadonlyMap<string, ReturnedKind> | null { + let program: Node + try { + const file = parse(toCodeFile(code, 'javascript'), { + sourceType: 'unambiguous', + allowReturnOutsideFunction: true, + allowAwaitOutsideFunction: true, + errorRecovery: true, + }) + if (file.errors && file.errors.length > 0) return null + program = file.program + } catch { + return null + } + const returns: Array<Node | null | undefined> = [] + walk(program, false, (node) => { + if (node.type === 'ReturnStatement') returns.push(node.argument) + }) + if (returns.length === 0) return null + const inference = new Inference(collectBindings(program)) + let agreed: Map<string, JsKind | undefined> | null = null + for (const argument of returns) { + if (argument?.type !== 'ObjectExpression') return null + const fields = returnedFields(argument, inference) + if (agreed === null) { + agreed = fields + continue + } + for (const [key, kind] of agreed) { + agreed.set(key, fields.has(key) ? merge(kind, fields.get(key)) : undefined) + } + } + const kinds = new Map<string, ReturnedKind>() + for (const [key, kind] of agreed ?? []) { + if (kind !== undefined) kinds.set(key, kind === 'string' ? 'string' : 'typed') + } + return kinds +} + +/** + * The kind of `result.<field>` of a JavaScript Function block, from the object literal its code + * returns; `undefined` for any other block, path or language, or when the code does not say. + */ +function functionResultKind( + block: CanonicalBlock, + path: readonly string[] +): ReturnedKind | undefined { + if (block.type !== FUNCTION_TYPE || path.length !== 2 || path[0] !== 'result') return undefined + let fields = cache.get(block) + if (fields === undefined) { + const code = getOwn(block.params, 'code') + const language = getOwn(block.params, 'language') + const javascript = + language === undefined || + (language.kind === 'value' && (language.value === 'javascript' || language.value === '')) + fields = + javascript && code?.kind === 'value' && typeof code.value === 'string' + ? readReturns(code.value) + : null + cache.set(block, fields) + } + return fields?.get(path[1]) +} + +/** + * What a stored reference (`<score.result.total>`) holds when it names a field a Function block's + * code returns as a literal: see {@link functionResultKind}. `block` finds a compiled block by key. + */ +export function referencedResultKind( + token: string, + block: (key: BlockKey) => CanonicalBlock | undefined +): ReturnedKind | undefined { + if (!token.startsWith('<') || !token.endsWith('>')) return undefined + const path = parseStoredPath(token.slice(1, -1)) + if (!path) return undefined + const names = path.map((segment) => (segment.kind === 'name' ? segment.name : null)) + const [head, ...rest] = names + if (head === null || head === undefined || rest.some((name) => name === null)) return undefined + const target = block(head) + return target ? functionResultKind(target, rest as string[]) : undefined +} diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 3ea902e54c4..61a2225fbb4 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -24,13 +24,14 @@ import type { PortKey, VarKey, } from '../model' -import { setOwn } from '../records' +import { getOwn, setOwn } from '../records' import { OLDEST_SDK_MAJOR, SDK_MODULE } from '../sdk' import { compileBlock } from './blocks' import { checkStorableLiterals, checkStorableText } from './characters' import { CommentCollector } from './comments' import { type CompileContext, rangeOf, recordSpan, type Scope, unsupported } from './context' import { EdgeCollector } from './edges' +import { referencedResultKind } from './function-returns' import { didYouMean, HINTS, hintForNode, nodeLabel } from './hints' import { type CompileLimits, DEFAULT_LIMITS, parseWorkflowSource, sdkHeader } from './parse' import { ErrorLimitReached, Reporter } from './report' @@ -415,6 +416,22 @@ function emptyResult(reporter: Reporter, sdkMajor: number | null): CompileResult const BYTE_ORDER_MARK = '\uFEFF' +/** + * Whether the stored workflow holds `value` in `param` of block `key` already: a JSON field the + * source left as it was printed, whose untyped references the edit did not write, so + * `W_JSON_REF_UNTYPED` is not repeated on every apply of a workflow it was saved with. + */ +function storedHolds( + stored: CanonicalWorkflow | undefined, + key: BlockKey, + param: string, + value: string +): boolean { + const held = stored && getOwn(stored.blocks, key) + const storedParam = held ? getOwn(held.params, param) : undefined + return storedParam?.kind === 'value' && storedParam.value === value +} + /** * Statically interprets a workflow source file into the canonical model. Pass 1 collects * imports and every block declaration, nested bodies included; pass 2 evaluates values, @@ -624,9 +641,12 @@ function compileEntry( } const isString = (token: string) => refValueKind(token, refTypes) === 'string' for (const { params, id, tree, refs } of ctx.jsonFields) { - setOwn(params, id, { kind: 'value', value: jsonFieldText(tree, isString) }) + const value = jsonFieldText(tree, isString) + setOwn(params, id, { kind: 'value', value }) for (const ref of refs) { if (refValueKind(ref.token, refTypes) !== 'untyped') continue + if (referencedResultKind(ref.token, refTypes.block) === 'typed') continue + if (storedHolds(input.stored, ref.blockKey, id, value)) continue reporter.report( 'W_JSON_REF_UNTYPED', `Nothing declares the type of ${ref.token}; it is stored bare, so a string value would break the JSON.`, diff --git a/packages/workflow-code/src/json-fields.test.ts b/packages/workflow-code/src/json-fields.test.ts index 96ea924d69a..ac042748fc1 100644 --- a/packages/workflow-code/src/json-fields.test.ts +++ b/packages/workflow-code/src/json-fields.test.ts @@ -3,7 +3,7 @@ import type { BlockCatalog, CatalogBlock } from './catalog/types' import { compileWorkflow } from './compile' import { TEST_CATALOG } from './compile/catalog.test-helpers' import { refValueKind, valueTree } from './json-fields' -import type { CanonicalBlock } from './model' +import type { CanonicalBlock, CanonicalWorkflow } from './model' const IMPORTS = "import { agent, api, raw, table, trigger, workflow } from '@sim/workflow-sdk'" @@ -142,3 +142,73 @@ describe('refValueKind for a block acting as a trigger', () => { expect(kind('<svc.status>', block)).toBe('untyped') }) }) + +describe('W_JSON_REF_UNTYPED on a Function block output', () => { + const FN_IMPORTS = "import { api, fn, trigger, workflow } from '@sim/workflow-sdk'" + + /** Compiles a Function block with `code` whose result `field` a JSON body uses bare. */ + function compile(code: string, field: string, stored?: CanonicalWorkflow) { + const text = `// @sim-sdk 1\n${FN_IMPORTS}\n\nexport default workflow('Test', {}, () => { + const start = trigger.start('Start') + const score = fn('Score', { code: ${JSON.stringify(code)} }) + const next = api('Next', { url: 'https://x', body: { v: score.result.${field} } }) + start.to(score) + score.to(next) +})\n` + return compileWorkflow({ + entry: { path: 'workflow.ts', text }, + files: new Map(), + catalog: TEST_CATALOG, + sdkMajor: 1, + ...(stored ? { stored } : {}), + }) + } + + function untyped(code: string, field: string, stored?: CanonicalWorkflow) { + return compile(code, field, stored).diagnostics.filter((d) => d.code === 'W_JSON_REF_UNTYPED') + } + + it('stays quiet on a field the code returns as a number, boolean, array or object literal', () => { + const code = [ + 'const items = $ref.start.items || []', + 'const tax = Math.round(items.length * 10) / 100', + 'let score = $ref.start.total > 2000 ? 90 : 20', + "if ($ref.start.tier === 'new') score += 10", + 'return { tax, score, ok: score >= 70, count: items.length, list: items.map((i) => i.sku), meta: {} }', + ].join('\n') + for (const field of ['tax', 'score', 'ok', 'count', 'list', 'meta']) { + expect(untyped(code, field), field).toEqual([]) + } + }) + + it('keeps the warning on a field returned as text, which stored bare breaks the JSON', () => { + const code = "return { tier: 'gold', label: String(1) + 'x' }" + expect(untyped(code, 'tier')).toHaveLength(1) + expect(untyped(code, 'label')).toHaveLength(1) + }) + + it('keeps the warning when nothing certain says what the field holds', () => { + // A reference, a variable reassigned to text, a field only one return sets, a non-literal + // return, and a spread that may replace the field. + expect(untyped('return { v: $ref.start.v }', 'v')).toHaveLength(1) + expect(untyped("let v = 1\nif ($ref.start.x) v = 'one'\nreturn { v }", 'v')).toHaveLength(1) + expect(untyped('if ($ref.start.x) return { v: 1 }\nreturn { w: 2 }', 'v')).toHaveLength(1) + expect(untyped('if ($ref.start.x) return { v: 1 }\nreturn $ref.start.o', 'v')).toHaveLength(1) + expect(untyped('return { v: 1, ...$ref.start.o }', 'v')).toHaveLength(1) + expect(untyped('return { v: 1 ', 'v')).toHaveLength(1) + }) + + it('reads only the returns of the body, not those of a callback inside it', () => { + const code = "const f = (x) => { return { v: 'text' } }\nreturn { v: f(1).length }" + expect(untyped(code, 'v')).toEqual([]) + }) + + it('on an apply, warns only on a JSON field the source changed from the stored workflow', () => { + const code = 'return { v: $ref.start.v, w: $ref.start.w }' + const printed = compile(code, 'v').workflow + if (!printed) throw new Error('the source did not compile') + + expect(untyped(code, 'v', printed)).toEqual([]) + expect(untyped(code, 'w', printed)).toHaveLength(1) + }) +}) From af0886a6f61980cf1ff0397874cc58bb9aa2538d Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:53:50 -0800 Subject: [PATCH 24/59] feat(sim-cli): workflows source apply and create --deploy --deploy deploys the workflow once the write is saved, through the same deployWorkflow operation (and so the same server-side permission check) as `sim workflows deploy`; a failed deploy says the write itself was saved and names the command to retry. Without it, a write to a deployed workflow whose draft now differs names that command as next. --deploy with --dry-run is refused. --- apps/docs/content/docs/cli/reference.mdx | 6 +- apps/docs/content/docs/cli/workflows.mdx | 6 +- .../protocol/workflow-source-apply.test.ts | 77 +++++++++++ .../protocol/workflow-source-apply.ts | 122 +++++++++++++++++- 4 files changed, 200 insertions(+), 11 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 2989f1273a0..0edfa376cf6 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8577,13 +8577,14 @@ sim workflows source apply [workflowId] [options] | `--dir <dir>` | Yes | Source directory: workflow.ts, functions/*, .sim/lock.json and .sim/base.json. | | `--dry-run` | No | Check the source and show the outcome without saving or writing files. | | `-y, --yes` | No | Apply without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy <workflowId>` does). | | `--allow-delete` | No | Delete blocks the source no longer declares; without it, an apply that deletes a block is refused. | </CommandTable> Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### sim workflows source create @@ -8605,12 +8606,13 @@ sim workflows source create [options] | `--name <name>` | No | The workflow name (default: the source's). | | `--folder <path>` | No | The folder to create it in (default: the root). | | `-y, --yes` | No | Create without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy <workflowId>` does). | </CommandTable> Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### sim workflows source check diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 554bb053e62..2905e725b56 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1359,12 +1359,13 @@ sim workflows source create [options] | `--name <name>` | No | The workflow name (default: the source's). | | `--folder <path>` | No | The folder to create it in (default: the root). | | `-y, --yes` | No | Create without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy <workflowId>` does). | </CommandTable> Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### Apply a workflow source directory to a workflow draft @@ -1392,13 +1393,14 @@ sim workflows source apply [workflowId] [options] | `--dir <dir>` | Yes | Source directory: workflow.ts, functions/*, .sim/lock.json and .sim/base.json. | | `--dry-run` | No | Check the source and show the outcome without saving or writing files. | | `-y, --yes` | No | Apply without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy <workflowId>` does). | | `--allow-delete` | No | Delete blocks the source no longer declares; without it, an apply that deletes a block is refused. | </CommandTable> Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### Show the block catalog workflow source is checked against diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index a6c5eb1ed3c..d619bd226fe 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -684,6 +684,83 @@ describe('sim workflows source apply', () => { }) }) + describe('--deploy', () => { + const DEPLOY_PATH = `/api/v2/workflows/${WORKFLOW_ID}/deploy` + const deployed = { + data: { + id: WORKFLOW_ID, + isDeployed: true, + deployedAt: null, + warnings: [], + activeDeployment: null, + latestDeploymentAttempt: null, + version: 4, + lint: null, + }, + } + + it('deploys through the deploy operation once the apply is saved', async () => { + setTerminal(false) + request.mockImplementation(async (path: string) => + path === DEPLOY_PATH ? deployed : outcome() + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH, DEPLOY_PATH]) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(`${ENTRY}// stored\n`) + expect(stdoutJson()).toMatchObject({ + applied: true, + deployed: { version: 4, isDeployed: true }, + needsRedeployment: false, + }) + }) + + it('without --deploy, names the deploy command when the deployed workflow now differs', async () => { + setTerminal(false) + request.mockResolvedValue(outcome()) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes') + + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH]) + expect(stdoutJson()).toMatchObject({ + needsRedeployment: true, + next: { command: `sim workflows deploy ${WORKFLOW_ID}` }, + }) + }) + + it('deploys nothing after a refused apply, and refuses beside --dry-run', async () => { + setTerminal(false) + request.mockResolvedValue(outcome({ diagnostics: [STALE] })) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH]) + + request.mockClear() + await expect(apply(WORKFLOW_ID, '--dir', dir, '--dry-run', '--deploy')).rejects.toThrow( + /--dry-run saves nothing/ + ) + expect(request).not.toHaveBeenCalled() + }) + + it('says the apply was saved when the deploy after it fails', async () => { + setTerminal(false) + request.mockImplementation(async (path: string) => { + if (path === DEPLOY_PATH) throw new SimApiError('Forbidden', 403, 'FORBIDDEN') + return outcome() + }) + + const error = await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy').catch( + (caught: unknown) => caught + ) + + expect(error).toMatchObject({ status: 403, code: 'FORBIDDEN' }) + expect((error as Error).message).toContain(`The apply was saved into ${dir}`) + expect((error as Error).message).toContain(`sim workflows deploy ${WORKFLOW_ID}`) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(`${ENTRY}// stored\n`) + }) + }) + it('exits 5 when a dry run finds the directory stale', async () => { setTerminal(false) request.mockResolvedValue(outcome({ dryRun: true, diagnostics: [STALE] })) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index bc7dc403e09..455acf6d045 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -16,6 +16,7 @@ import type { ApplyWorkflowSourceBody, ApplyWorkflowSourceResponse, CreateWorkflowFromSourceBody, + DeployWorkflowResponse, } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' @@ -84,6 +85,7 @@ interface SourceWriteOptions { yes?: boolean allowDelete?: boolean verbose?: boolean + deploy?: boolean } /** @@ -242,7 +244,7 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { options: SourceWriteOptions creating: boolean question: string - client: OperationClient<Operation> + client: OperationClient<Operation | 'deployWorkflow'> operation: Operation params: Record<string, string> body: ApplyWorkflowSourceBody | CreateWorkflowFromSourceBody @@ -273,12 +275,18 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { } const findings = findingsOutput(args.output, options.verbose) const notes = new Set<string>() - const report = (data: SourceApplyData) => { - const result = summarize(data, creating, findings.machine) + const report = (data: SourceApplyData, deployment: DeployOutcome = {}) => { + const result = { ...summarize(data, creating, findings.machine), ...deployment } printProtocolResult( args.output, { ...result, ...findings.result(data.diagnostics) }, - result.created ? { created: createdText(result.created) } : {} + { + ...(result.created ? { created: createdText(result.created) } : {}), + ...(deployment.next ? { next: deployment.next.command } : {}), + ...(deployment.deployed + ? { deployed: `version ${deployment.deployed.version ?? '?'}` } + : {}), + } ) } if (!options.dryRun && !options.yes) { @@ -296,10 +304,11 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { } const verb = creating ? 'create' : 'apply' + let refused = false const sendAndSave = async (): Promise<SourceApplyData> => { const data = await send(options.dryRun === true) printFindings(data, findings, notes) - const refused = data.diagnostics.some((diagnostic) => diagnostic.severity === 'error') + refused = data.diagnostics.some((diagnostic) => diagnostic.severity === 'error') if (refused) setSoftExitCode(refusedExitCode(data.diagnostics)) if (!data.dryRun && !refused && (data.applied || !creating)) await args.save(data) return data @@ -311,7 +320,92 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { abandoned: `Stopped before the ${verb} answered. ${args.ifAnswerLost}`, }) - report(data) + const stored = !data.dryRun && !refused && (data.applied || !creating) + report( + data, + stored + ? await deployStep(args.client, data, { ...options, verb, machine: findings.machine }) + : {} + ) +} + +/** What a write adds to its result about deploying: the deploy it ran, or the one to run. */ +interface DeployOutcome { + deployed?: { version?: number; isDeployed: boolean; warnings?: string[] } + needsRedeployment?: boolean + next?: { command: string; reason: string } +} + +/** + * After a stored write: with `--deploy`, deploys the workflow through the same operation as + * `sim workflows deploy` (so the same permission check, on the server); without it, names that + * command as the next step when the workflow is deployed and its draft now differs. A failed + * deploy says the write itself was saved. + */ +async function deployStep( + client: OperationClient<'deployWorkflow'>, + data: SourceApplyData, + options: { deploy?: boolean; dir: string; verb: string; machine: boolean } +): Promise<DeployOutcome> { + const deployCommand = `sim workflows deploy ${data.workflowId}` + if (!options.deploy) { + if (!data.needsRedeployment) return {} + return { + next: { + command: deployCommand, + reason: `The workflow is deployed and its draft now differs from the live version: deploy to make the change live (or ${options.verb} with --deploy).`, + }, + } + } + const deployed = await client + .request<DeployWorkflowResponse>('deployWorkflow', { + params: { workflowId: data.workflowId }, + body: {}, + }) + .then( + (response) => response.data, + (error: unknown) => { + throw deployFailure(error, options, deployCommand) + } + ) + if (!options.machine) { + for (const warning of deployed.warnings) printError(`deploy warning: ${warning}`) + } + return { + deployed: { + ...(deployed.version !== undefined ? { version: deployed.version } : {}), + isDeployed: deployed.isDeployed, + ...(options.machine && deployed.warnings.length > 0 ? { warnings: deployed.warnings } : {}), + }, + needsRedeployment: false, + } +} + +/** A failed deploy after a saved write, saying the write itself stands and how to deploy it. */ +function deployFailure( + error: unknown, + options: { dir: string; verb: string }, + deployCommand: string +): unknown { + if (!(error instanceof SimApiError)) return error + return new SimApiError( + `The ${options.verb} was saved into ${options.dir}, but the deploy failed: ${error.message} Fix that, then run \`${deployCommand}\`.`, + error.status, + error.code, + error.details, + error.exitCode, + error.requestId + ) +} + +/** Refuses `--deploy` beside `--dry-run`: a dry run stores nothing to deploy. */ +function assertDeployable(options: SourceWriteOptions): void { + if (options.deploy && options.dryRun) { + throw new SimApiError( + '--deploy deploys what the write saves, and --dry-run saves nothing: drop one of them.', + 0 + ) + } } const DIR_DESCRIPTION = @@ -328,7 +422,10 @@ const DIR_DESCRIPTION = * needs `--yes`. `--dry-run` writes nothing, so it never asks. */ export function attachWorkflowSourceApply(source: Command): void { - const [command, connectApply] = apiCommand(source, 'apply', ['applyWorkflowSource']) + const [command, connectApply] = apiCommand(source, 'apply', [ + 'applyWorkflowSource', + 'deployWorkflow', + ]) absoluteDirOption(command) verboseFlag(command) command @@ -339,6 +436,10 @@ export function attachWorkflowSourceApply(source: Command): void { .requiredOption('--dir <dir>', DIR_DESCRIPTION) .option('--dry-run', 'Check the source and show the outcome without saving or writing files') .option('-y, --yes', 'Apply without asking for confirmation') + .option( + '--deploy', + 'After the write is saved, deploy the workflow (as `sim workflows deploy <workflowId>` does)' + ) .option( '--allow-delete', 'Delete blocks the source no longer declares; without it, an apply that deletes a block is refused' @@ -351,6 +452,7 @@ export function attachWorkflowSourceApply(source: Command): void { ) .action(async (named: string | undefined, options: SourceWriteOptions) => withWorkspaceFolderLock(options.dir, async () => { + assertDeployable(options) requireConsent(options, 'writes the workflow draft', 'apply') const { client, profile } = connectApply() const lock = await readLock(options.dir) @@ -391,6 +493,7 @@ export function attachWorkflowSourceCreate(source: Command): void { 'createWorkflowFromSource', 'getSourceCatalog', 'getSourceResources', + 'deployWorkflow', ]) absoluteDirOption(command) verboseFlag(command) @@ -403,6 +506,10 @@ export function attachWorkflowSourceCreate(source: Command): void { .option('--name <name>', "The workflow name (default: the source's)") .option('--folder <path>', 'The folder to create it in (default: the root)') .option('-y, --yes', 'Create without asking for confirmation') + .option( + '--deploy', + 'After the write is saved, deploy the workflow (as `sim workflows deploy <workflowId>` does)' + ) .allowExcessArguments(false) .description('Create a workflow from a workflow source directory') .addHelpText( @@ -427,6 +534,7 @@ export function attachWorkflowSourceCreate(source: Command): void { printProtocolResult(profile.output, { created: false, scaffolded: true, ...tree }) return } + assertDeployable(options) requireConsent(options, 'creates a workflow', 'create') const { client, profile } = connectCreate() const workspaceId = client.requireWorkspace() From 63f760607f7aa1f0582c0bbc276c014ef823dbbc Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:57:01 -0800 Subject: [PATCH 25/59] feat(sim-cli): source apply without --yes prints the plan instead of refusing Where no terminal can ask, workflows source apply/create and workspaces source apply without --yes now run the dry run, print the plan and its findings, write nothing, and exit 14 naming the same command with --yes (or exit with the refusal when the plan has errors), instead of refusing with CONFIRMATION_REQUIRED and nothing to act on. apply --yes prints the summary of what it applied, so an agent's loop is edit, then apply --yes. --- apps/docs/content/docs/cli/reference.mdx | 6 +- apps/docs/content/docs/cli/scripting.mdx | 1 + apps/docs/content/docs/cli/workflows.mdx | 4 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- apps/docs/content/docs/workflows/code.mdx | 2 +- .../commands/protocol/source-exit-codes.ts | 6 ++ .../src/commands/protocol/source-ui.ts | 44 ++++++++----- .../protocol/workflow-source-apply.test.ts | 62 ++++++++++++++----- .../protocol/workflow-source-apply.ts | 27 ++++++-- .../commands/protocol/workflow-source-get.ts | 2 +- .../protocol/workspace-source-apply.test.ts | 32 ++++++++++ .../protocol/workspace-source-apply.ts | 44 +++++++++---- .../commands/protocol/workspace-source-dir.ts | 2 +- packages/sim-cli/src/exit-codes.ts | 7 +++ packages/workflow-sdk/README.md | 1 + 15 files changed, 186 insertions(+), 56 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 0edfa376cf6..047d3933101 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8582,7 +8582,7 @@ sim workflows source apply [workflowId] [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). @@ -8610,7 +8610,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). @@ -9956,7 +9956,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). diff --git a/apps/docs/content/docs/cli/scripting.mdx b/apps/docs/content/docs/cli/scripting.mdx index 7edbc6ed28b..13ecf0f7e05 100644 --- a/apps/docs/content/docs/cli/scripting.mdx +++ b/apps/docs/content/docs/cli/scripting.mdx @@ -175,6 +175,7 @@ The CLI resends a request the API answers `429` only when sending it twice is sa | `9` | Still rate limited (`429`) after the CLI's retries; nothing was written | | `10` | `--dir` is a workflow folder of a workspace directory another `sim` command is using (`SOURCE_DIR_BUSY`); nothing was written: run it again once the other finished | | `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; nothing was sent | +| `14` | Run without `--yes` and with no terminal to ask: the plan was printed with no errors and nothing was written; run the same command with `--yes` | | `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check | | Code | `sim workflows source status` or `sim workspaces source status` | diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 2905e725b56..01d6614e7b1 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1363,7 +1363,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). @@ -1398,7 +1398,7 @@ sim workflows source apply [workflowId] [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index f0cf2a7677c..49a6d0e7089 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -761,7 +761,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index eca8b571710..608ec3a9b06 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -84,7 +84,7 @@ Nothing in the file runs on your machine. Sim reads it statically, checks it aga sim workflows source apply --dir ./support-triage --yes ``` - The workflow builder updates for anyone who has the workflow open. Deploy when you are ready. The save rewrites `workflow.ts` in canonical form, so read it again before the next edit. + The workflow builder updates for anyone who has the workflow open. Deploy when you are ready, or add `--deploy` to deploy right after the save; without it, a deployed workflow whose draft now differs prints the `sim workflows deploy` command to run. `--yes` refuses everything a dry run would, so step 4 is optional: without `--yes`, and with no terminal to ask, `apply` prints the plan, writes nothing and exits `14`. The save rewrites `workflow.ts` in canonical form, so read it again before the next edit. Editing the name or description in `workflow('…')` renames or redescribes the workflow; a name another workflow in its folder already has is refused with `E_NAME_TAKEN`. Lines starting with `note:` report something the save did beyond what the file says; read them. diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts index 9d210e3f79a..04260a4727d 100644 --- a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts @@ -6,6 +6,7 @@ import { SOURCE_LOCAL_CHANGES_EXIT_CODE, SOURCE_LOCKED_EXIT_CODE, SOURCE_PARTIAL_EXIT_CODE, + SOURCE_PLANNED_EXIT_CODE, SOURCE_STALE_EXIT_CODE, SOURCE_TOO_LARGE_EXIT_CODE, } from '../../exit-codes' @@ -73,6 +74,11 @@ export function sourceExitCodesTable( 'Not logged in, the login expired, or the server refused the credential (`AUTH_REQUIRED`); nothing was sent', '`sim login`, or fix or unset `SIM_API_KEY` when the key comes from it; then the same command again', ], + [ + SOURCE_PLANNED_EXIT_CODE, + '`apply` or `create` without `--yes` and no terminal to ask: it printed the plan, found no errors, and wrote nothing', + 'Run the same command with `--yes`', + ], [ INTERRUPTED_EXIT_CODE, 'Interrupted (Ctrl-C). An apply or create already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written', diff --git a/packages/sim-cli/src/commands/protocol/source-ui.ts b/packages/sim-cli/src/commands/protocol/source-ui.ts index a15215c3529..a5d5ef9adbc 100644 --- a/packages/sim-cli/src/commands/protocol/source-ui.ts +++ b/packages/sim-cli/src/commands/protocol/source-ui.ts @@ -10,10 +10,11 @@ import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' import { askLine } from '#sim-cli/terminal/line-input' import type { OutputFormat } from '../../config/index' -import { embedStore } from '../../embed-context' +import { embedStore, setSoftExitCode } from '../../embed-context' import { SOURCE_BUSY_EXIT_CODE, SOURCE_LOCKED_EXIT_CODE, + SOURCE_PLANNED_EXIT_CODE, SOURCE_STALE_EXIT_CODE, SOURCE_TOO_LARGE_EXIT_CODE, } from '../../exit-codes' @@ -317,19 +318,34 @@ function refusalDetails( } /** - * A write without `--yes` or `--dry-run` where no terminal can ask: the generic refusal, naming - * what the write does (`writes the workflow draft`) and its verb (`apply`). + * How a source write proceeds: `dry-run` checks only; `write` writes at once (`--yes`); `ask` + * previews and asks on a terminal; `plan` (no `--yes` and no terminal to ask) previews, prints the + * plan and stops with {@link SOURCE_PLANNED_EXIT_CODE}, so an agent's edit loop sees the plan + * without a separate dry run and applies it with `--yes`. */ -export function requireConsent( - options: { dryRun?: boolean; yes?: boolean }, - action: string, - verb: string -): void { - if (!options.dryRun && !options.yes && !canPrompt()) { - throw new SimApiError( - `This ${action}. Re-run with --yes to ${verb} it, or with --dry-run to check it without writing.`, - 0, - 'CONFIRMATION_REQUIRED' - ) +export type WriteMode = 'dry-run' | 'write' | 'ask' | 'plan' + +/** The {@link WriteMode} of a write's `--dry-run` and `--yes` flags, here. */ +export function writeMode(options: { dryRun?: boolean; yes?: boolean }): WriteMode { + if (options.dryRun) return 'dry-run' + if (options.yes) return 'write' + return canPrompt() ? 'ask' : 'plan' +} + +/** `value` as one shell word: as it is when it needs no quoting, else single-quoted. */ +export function shellWord(value: string): string { + return /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'` +} + +/** The next step a `plan` run names: the same write with `--yes`. */ +export function planNext(command: string): { command: string; reason: string } { + return { + command: `${command} --yes`, + reason: 'Nothing was written: this is the plan. Run it with --yes to apply it.', } } + +/** Marks a `plan` run that found no errors: nothing was written, applying needs `--yes`. */ +export function stopAtPlan(): void { + setSoftExitCode(SOURCE_PLANNED_EXIT_CODE) +} diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index d619bd226fe..f9ebc60ec5c 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -158,28 +158,62 @@ function dryRunFlags() { } describe('sim workflows source apply', () => { - it('refuses without --yes when nobody is at a terminal, sending nothing', async () => { + it('without --yes and nobody at a terminal, prints the plan, writes nothing and exits 14', async () => { setTerminal(false) + request.mockResolvedValue(outcome({ dryRun: true })) - const error = await apply(WORKFLOW_ID, '--dir', dir).catch((caught: unknown) => caught) + await apply(WORKFLOW_ID, '--dir', dir, '--allow-delete') - expect(error).toBeInstanceOf(SimApiError) - expect((error as SimApiError).code).toBe('CONFIRMATION_REQUIRED') - expect((error as SimApiError).message).toContain('--yes') - expect(request).not.toHaveBeenCalled() + expect(dryRunFlags()).toEqual([true]) + expect(process.exitCode).toBe(14) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(ENTRY) + expect(stdoutJson()).toMatchObject({ + applied: false, + dryRun: true, + next: { command: `sim workflows source apply --dir ${dir} --allow-delete --yes` }, + }) + expect(stderrLines().join('\n')).toContain('Blocks: 1 added') }) - it('refuses a create without --yes and sends nothing', async () => { + it('without --yes, exits with the refusal and names no next step when the plan has errors', async () => { setTerminal(false) + request.mockResolvedValue( + outcome({ + dryRun: true, + diagnostics: [{ code: 'E_UNKNOWN_PARAM', severity: 'error', message: 'No param x.' }], + }) + ) + + await apply(WORKFLOW_ID, '--dir', dir) - const error = await program() - .parseAsync(['node', 'sim', 'workflows', 'source', 'create', '--dir', dir]) - .catch((caught: unknown) => caught) + expect(process.exitCode).toBe(1) + expect(stdoutJson()).not.toHaveProperty('next') + }) - expect(error).toBeInstanceOf(SimApiError) - expect((error as SimApiError).code).toBe('CONFIRMATION_REQUIRED') - expect((error as SimApiError).message).toContain('--yes to create') - expect(request).not.toHaveBeenCalled() + it('plans a create without --yes and creates nothing', async () => { + setTerminal(false) + rmSync(join(dir, '.sim/base.json')) + request.mockResolvedValue(outcome({ dryRun: true })) + + await program().parseAsync(['node', 'sim', 'workflows', 'source', 'create', '--dir', dir]) + + expect(dryRunFlags()).toEqual([true]) + expect(process.exitCode).toBe(14) + expect(stdoutJson()).toMatchObject({ + next: { + command: `sim workflows source create --dir ${dir} --workspace ${WORKSPACE_ID} --yes`, + }, + }) + }) + + it('with --yes, prints the summary of what it applied', async () => { + setTerminal(false) + request.mockResolvedValue(outcome()) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes') + + expect(dryRunFlags()).toEqual([false]) + expect(stderrLines().join('\n')).toContain('Blocks: 1 added, 0 removed') }) it('applies at once with --yes and saves what the server stored', async () => { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 455acf6d045..2b350537408 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -44,10 +44,13 @@ import { type FindingsOutput, findingsOutput, fresh, + planNext, printRefusal, refusedExitCode, - requireConsent, + shellWord, + stopAtPlan, verboseFlag, + writeMode, } from './source-ui' import { storedEdgesWrite } from './stored-edges' import { @@ -65,7 +68,9 @@ file:line:column with its frame and hint) or the command failed; 5 the workflow directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's -retries, so nothing was written: wait a minute and run it again.` +retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no +terminal to ask: the plan was printed with no errors and nothing was written (run the same +command with --yes to apply it).` const CREATE_EXIT_CODES_HELP = ` Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each @@ -73,7 +78,8 @@ printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: -wait a minute and run it again.` +wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was +printed with no errors and nothing was created (run the same command with --yes).` type SourceApplyData = ApplyWorkflowSourceResponse['data'] @@ -253,6 +259,8 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { save: (data: SourceApplyData) => Promise<void> /** The next step when the write's answer never came back, so it may have been stored. */ ifAnswerLost: string + /** This write as a command, without `--yes`: what a `plan` run names next with it. */ + command: string }): Promise<void> { const { options, creating } = args /** Sends the write (or its dry run); a refusal prints its diagnostics and rethrows. */ @@ -289,7 +297,8 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { } ) } - if (!options.dryRun && !options.yes) { + const mode = writeMode(options) + if (mode === 'ask' || mode === 'plan') { const preview = await send(true) printFindings(preview, findings, notes) if (preview.diagnostics.some((diagnostic) => diagnostic.severity === 'error')) { @@ -298,6 +307,11 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { return } printError(diffSummary(preview)) + if (mode === 'plan') { + stopAtPlan() + report(preview, { next: planNext(args.command) }) + return + } if (!(await confirm(args.question))) { throw new SimApiError('Nothing was applied.', 0, 'CANCELLED') } @@ -321,6 +335,7 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { }) const stored = !data.dryRun && !refused && (data.applied || !creating) + if (mode === 'write' && stored) printError(diffSummary(data)) report( data, stored @@ -453,7 +468,6 @@ export function attachWorkflowSourceApply(source: Command): void { .action(async (named: string | undefined, options: SourceWriteOptions) => withWorkspaceFolderLock(options.dir, async () => { assertDeployable(options) - requireConsent(options, 'writes the workflow draft', 'apply') const { client, profile } = connectApply() const lock = await readLock(options.dir) const base = await readBase(options.dir, named) @@ -463,6 +477,7 @@ export function attachWorkflowSourceApply(source: Command): void { options, creating: false, question: `Apply these changes to workflow ${workflowId}?`, + command: `sim workflows source apply --dir ${shellWord(options.dir)}${options.allowDelete ? ' --allow-delete' : ''}${options.deploy ? ' --deploy' : ''}`, output: profile.output, client, operation: 'applyWorkflowSource', @@ -535,7 +550,6 @@ export function attachWorkflowSourceCreate(source: Command): void { return } assertDeployable(options) - requireConsent(options, 'creates a workflow', 'create') const { client, profile } = connectCreate() const workspaceId = client.requireWorkspace() await assertStarterWorkspace( @@ -548,6 +562,7 @@ export function attachWorkflowSourceCreate(source: Command): void { options, creating: true, question: 'Create this workflow?', + command: `${createCommand(shellWord(options.dir), workspaceId)}${options.name !== undefined ? ` --name ${shellWord(options.name)}` : ''}${options.folder !== undefined ? ` --folder ${shellWord(options.folder)}` : ''}${options.deploy ? ' --deploy' : ''}`, output: profile.output, client, operation: 'createWorkflowFromSource', diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index e175db48dad..0e59c25db85 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -87,7 +87,7 @@ This directory is a Sim workflow written as TypeScript. The server compiles \`${ 2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. Fix and rerun until it exits 0. 3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks with the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. 4. \`sim workflows source status --dir .\` before applying: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed it since (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\`: fix every error it prints, rerun until it exits 0. -5. \`sim workflows source apply --dir . --yes\`: saves the draft (never deploys). It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out), so read the file again before the next edit rather than editing from memory. +5. \`sim workflows source apply --dir . --yes\`: saves the draft, refusing errors, block deletions without \`--allow-delete\` and a stale directory, so it needs no separate dry run (without \`--yes\` and a terminal it prints the plan and exits 14). \`--deploy\` also deploys it; without it, a deployed workflow's next \`sim workflows deploy <id>\` is printed. It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out), so read the file again before the next edit rather than editing from memory. 6. Exit 5 (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get <workflowId> --dir <fresh dir>\`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare it with this one. Only when it holds your edit, run \`sim workflows source get <workflowId> --dir . --force\`; otherwise reapply your edit in the fresh directory. An apply that deletes blocks (a declaration removed from \`${SOURCE_ENTRY_FILE}\`) is refused with \`E_APPLY_BLOCK_DELETE\` naming them; pass \`--allow-delete\` when the deletion is intended. Editing the \`workflow('…')\` name or description renames or redescribes the workflow; a name another workflow in its folder holds is refused with \`E_NAME_TAKEN\`. diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 308b8adf0b6..5e14ee0be67 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -130,6 +130,38 @@ afterEach(() => { }) describe('workspaces source apply', () => { + it('without --yes and nobody at a terminal, prints the plan, writes nothing and exits 14', async () => { + request.mockResolvedValue( + answer({ dryRun: true, plan: [{ kind: 'table', key: 'leads', action: 'create' }] }) + ) + + await apply('--allow-delete') + + expect(request.mock.calls.map((call) => call[1].query?.dryRun === true)).toEqual([true]) + expect(process.exitCode).toBe(14) + const printed = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) + expect(printed).toMatchObject({ + dryRun: true, + created: [{ kind: 'table', key: 'leads' }], + next: { command: `sim workspaces source apply --dir ${dir} --allow-delete --yes` }, + }) + expect(vi.mocked(console.error).mock.calls.flat().join('\n')).toContain('+ leads') + }) + + it('with --yes, prints the plan it applied', async () => { + request.mockResolvedValue( + answer({ + plan: [{ kind: 'table', key: 'leads', action: 'create' }], + applied: [{ kind: 'table', key: 'leads', action: 'create', id: 'tbl_1' }], + }) + ) + + await apply('--yes') + + expect(request).toHaveBeenCalledTimes(1) + expect(vi.mocked(console.error).mock.calls.flat().join('\n')).toContain('+ leads') + }) + it('puts each finding of a dry run in the JSON on stdout', async () => { const finding = { code: 'E_DECL_SHAPE', diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 4981c9eb313..5e97d872580 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -47,10 +47,13 @@ import { confirm, createdText, findingsOutput, + planNext, printRefusal, refusedExitCode, - requireConsent, + shellWord, + stopAtPlan, verboseFlag, + writeMode, } from './source-ui' import { removedWorkflowRecords, workflowFolderRecords } from './stored-edges' import { @@ -98,7 +101,9 @@ part-way (stale, locked, forbidden) saves what was written too, and exits as tha or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory -(SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished.` +(SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 +without --yes and with no terminal to ask: the plan was printed with no errors and nothing was +written (run the same command with --yes to apply it).` const SYMBOLS: Record<Change['action'], string> = { create: '+', @@ -420,7 +425,7 @@ export function attachWorkspaceSourceApply(source: Command): void { .description('Apply a workspace source directory') .addHelpText('after', `${EXIT_CODES_HELP}${SOURCE_WRITE_EXIT_CODES_HELP}`) .action(async (options: ApplyOptions) => { - requireConsent(options, 'writes the workspace', 'apply') + const mode = writeMode(options) const only = parseKinds(options.only) const { client, profile } = connect() const readFrom = baseWorkspaceOf(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) @@ -518,29 +523,31 @@ export function attachWorkspaceSourceApply(source: Command): void { } // The apply repeats findings the preview already showed: each is printed once. const findings = findingsOutput(profile.output, options.verbose) - const report = (data: ApplyData) => { - const result = summarize(data) + const report = (data: ApplyData, next?: { command: string; reason: string }) => { + const result = { ...summarize(data), ...(next ? { next } : {}) } printProtocolResult( profile.output, { ...result, ...findings.result(data.diagnostics) }, - { created: createdText(result.created) } + { created: createdText(result.created), ...(next ? { next: next.command } : {}) } ) } let preview: ApplyData | null = null - if (options.dryRun || !options.yes) { + if (mode !== 'write') { preview = await send(submission, true) printPlan(preview.plan) findings.print(preview.diagnostics) const errors = preview.diagnostics.some((finding) => finding.severity === 'error') if (errors) setSoftExitCode(refusedExitCode(preview.diagnostics)) - if (options.dryRun || refusesApply(preview)) { + if (mode === 'dry-run' || refusesApply(preview)) { report(preview) return } - if ( - !options.yes && - !(await confirm(`Apply these changes to workspace ${workspaceId}?`)) - ) { + if (mode === 'plan') { + stopAtPlan() + report(preview, planNext(applyCommand(options))) + return + } + if (!(await confirm(`Apply these changes to workspace ${workspaceId}?`))) { throw new SimApiError('Nothing was applied.', 0, 'CANCELLED') } } @@ -566,6 +573,7 @@ export function attachWorkspaceSourceApply(source: Command): void { abandoned: `Stopped before the apply answered. ${answerLostHint(options.dir)}`, } ) + if (mode === 'write') printPlan(data.plan) const held = data.plan.filter((change) => change.held) if (held.length > 0) { printError( @@ -579,11 +587,21 @@ export function attachWorkspaceSourceApply(source: Command): void { } report(data) }, - { readOnly: options.dryRun === true } + { readOnly: mode === 'dry-run' || mode === 'plan' } ) }) } +/** This apply as a command, without `--yes` or `--dry-run`: what a plan run names next. */ +function applyCommand(options: ApplyOptions): string { + return [ + `sim workspaces source apply --dir ${shellWord(options.dir)}`, + ...(options.only ? [`--only ${shellWord(options.only)}`] : []), + ...(options.to ? [`--to ${shellWord(options.to)}`] : []), + ...(options.allowDelete ? ['--allow-delete'] : []), + ].join(' ') +} + /** * The next step for an apply write whose answer never came back (a timeout, a dropped connection, * an interrupt): it may or may not have been written. diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index 857fa086a78..37acb9714f7 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -124,7 +124,7 @@ Secrets never appear in source: a workflow reads an environment variable as \`en 2. \`sim workspaces source check --dir .\`: compiles every workflow offline with the server's compiler, against what this directory declares: an unknown resource key (skills included) or table column, a child workflow's inputs, and an env name \`${KIND_DIRS.secret}\` lacks (a warning); and checks each \`${KIND_DIRS.table}/<key>.rows.jsonl\` line against its table as the apply would. It fails (exit 1) on errors and on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block), regenerates \`.sim/sdk/workspace.d.ts\` and adds the blocks the workflows now use to \`tsconfig.json\`. In a workflow folder, \`sim workflows source check --dir .\` checks that one workflow the same way. Fix and rerun until it exits 0. 3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks the files with the TypeScript release the declarations are built for. \`tsconfig.json\` includes only the blocks the workflows used at the last check or get, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). \`${SOURCE_SDK_DIR}/${SDK_GUIDE_FILE}\` lists every builder with its file and params type; \`grep\` it rather than reading the declarations. The dry run is the authority: a printed file you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. 4. \`sim workspaces source status --dir .\` before applying: your changes by resource (\`sim workspaces source diff --dir .\` shows the edits), what changed on the server since the get (a workflow edited in the editor included), the conflicts, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed since (\`--offline\`: skip the server). Then \`sim workspaces source apply --dir . --dry-run\`: shows the plan (\`+\` create, \`~\` update, \`=\` adopt an existing resource of the same name, \`-\` delete) and every error; fix and rerun until it exits 0. -5. \`sim workspaces source apply --dir . --yes\`: writes it and rewrites this directory from what was stored. +5. \`sim workspaces source apply --dir . --yes\`: writes it and rewrites this directory from what was stored; it prints the plan it applied (without \`--yes\` and a terminal it prints the plan and exits 14, writing nothing). ## Deletes diff --git a/packages/sim-cli/src/exit-codes.ts b/packages/sim-cli/src/exit-codes.ts index 4b4e9f73ba9..ff1a53caad9 100644 --- a/packages/sim-cli/src/exit-codes.ts +++ b/packages/sim-cli/src/exit-codes.ts @@ -72,5 +72,12 @@ export const SOURCE_DRIFT_EXIT_CODE = 12 */ export const AUTH_REQUIRED_EXIT_CODE = 13 +/** + * A `workflows source apply`/`create` or `workspaces source apply` run without `--yes` where no + * terminal can ask: it printed the plan a dry run found, with no errors, and wrote nothing. Run + * the same command with `--yes` to apply that plan. + */ +export const SOURCE_PLANNED_EXIT_CODE = 14 + /** The error code of an {@link AUTH_REQUIRED_EXIT_CODE} refusal the CLI raises itself. */ export const AUTH_REQUIRED_CODE = 'AUTH_REQUIRED' diff --git a/packages/workflow-sdk/README.md b/packages/workflow-sdk/README.md index 20a1ade672b..cb34282ba90 100644 --- a/packages/workflow-sdk/README.md +++ b/packages/workflow-sdk/README.md @@ -118,6 +118,7 @@ Every `sim workflows source` and `sim workspaces source` command exits with one | 11 | `status`: only your local edits | Dry-run, then apply | | 12 | `status`: the server changed since the get (drift, or a conflict with your edits) | Follow the `Next:` line it prints | | 13 | Not logged in, or the server refused the credential (`AUTH_REQUIRED`); nothing was sent | `sim login`, or fix or unset `SIM_API_KEY` | +| 14 | `apply` or `create` without `--yes` and no terminal to ask: it printed the plan, found no errors, and wrote nothing | Run the same command with `--yes` | | 130 | Interrupted (Ctrl-C). An apply or create already sent is finished and saved first; a second Ctrl-C stops at once and says it may have been written | `sim workflows source status --dir ./wf` | ## Write declarations, not programs From 38797fe430c6b8c118ec423424648ab578d3c3cd Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:33:02 -0800 Subject: [PATCH 26/59] fix(sim-cli): fail check on a dangling $ref in a new function file, and on any with --strict A rename's apply already rewrites $ref.<old key> in function files to the new key and writes the files back; the integration proof now asserts the written-back function file too. --- apps/docs/content/docs/cli/reference.mdx | 4 +-- apps/docs/content/docs/cli/workflows.mdx | 2 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- .../apply-workflow-source.integration.ts | 1 + .../protocol/workflow-source-check.test.ts | 32 +++++++++++++++++++ .../protocol/workflow-source-check.ts | 1 + 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 153ec878e47..5122451e7dd 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8634,7 +8634,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. ### sim workflows source status @@ -9978,7 +9978,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 318297758b1..ee12260462d 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1438,7 +1438,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 12b9dcf1f8e..1b6f4629ec4 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -786,7 +786,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index 9749b6babb9..bf8175319a3 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -2283,6 +2283,7 @@ describe('renames in source', () => { expect(after.state?.blocks[seeded.ids.report]?.subBlocks.code?.value).toBe( 'return <bang.result> + <total.result>' ) + expect(result.files['functions/report.js']).toBe('return $ref.bang.result + $ref.total.result') }) check( diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index 261d7f85fc6..b0277bcbe9e 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -391,6 +391,38 @@ describe('sim workflows source check', () => { expect(process.exitCode).toBeUndefined() }) + it('fails on a dangling $ref in a new function file by default, and on any with --strict', async () => { + cacheCatalog() + mkdirSync(join(dir, 'functions'), { recursive: true }) + writeFileSync(join(dir, 'functions/total.js'), 'return $ref.ordertotals.sum\n') + writeFileSync( + join(dir, 'workflow.ts'), + `// @sim-sdk 1 +import { codeFile, fn, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('W', {}, () => { + const start = trigger.start('Start', {}) + const total = fn('Total', { code: codeFile('./functions/total.js') }) + start.to(total) +}) +` + ) + + await check() + expect(stderr()).toMatch(/functions\/total\.js:1:8 error W_UNRESOLVED_REF/) + expect(process.exitCode).toBe(1) + + process.exitCode = undefined + writeLock(['start', 'total']) + await check() + expect(result()).toMatchObject({ errors: 0, warnings: 1 }) + expect(process.exitCode).toBeUndefined() + + await checkIn(dir, '--strict') + expect(result()).toMatchObject({ errors: 1 }) + expect(process.exitCode).toBe(1) + }) + it('fails on a bound block with --strict and on nothing with --no-strict', async () => { cacheCatalog() writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index e02b9c08fdb..c8b50add769 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -62,6 +62,7 @@ const STRICT_CODES: ReadonlySet<string> = new Set([ 'W_TABLE_UNKNOWN_COLUMN', 'W_CHILD_INPUT_UNKNOWN', 'W_BRANCH_UNWIRED', + 'W_UNRESOLVED_REF', ]) /** From b1f74ae9f41640f19e9433d95c8b0e08cfc87e2d Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 17:04:40 -0800 Subject: [PATCH 27/59] fix(sim-cli): a source create goes to the workspace its starter was typed for Without --workspace, a create now binds to the workspace .sim/starter.json records, as check, status, diff and apply bind to the one .sim/base.json records, instead of refusing with SOURCE_OTHER_WORKSPACE when the profile default differs; --workspace still chooses another. --- apps/docs/content/docs/cli/reference.mdx | 2 +- apps/docs/content/docs/cli/workflows.mdx | 2 +- apps/docs/content/docs/workflows/code.mdx | 2 +- .../protocol/workflow-source-apply.test.ts | 9 ++--- .../protocol/workflow-source-apply.ts | 34 ++++--------------- .../commands/protocol/workflow-source-get.ts | 2 +- 6 files changed, 14 insertions(+), 37 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 047d3933101..5253a6169e6 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8610,7 +8610,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 01d6614e7b1..5b4f4982155 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1363,7 +1363,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace (SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index 608ec3a9b06..10e22c68b09 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -90,7 +90,7 @@ Nothing in the file runs on your machine. Sim reads it statically, checks it aga A save that removes a block (its declaration is gone from `workflow.ts`) is refused with `E_APPLY_BLOCK_DELETE`, naming the blocks. Pass `--allow-delete` when the deletion is intended. -To create a new workflow, run `sim workflows source create --dir <dir> --workspace <workspaceId>` on an empty directory: without `--dry-run` or `--yes` it creates nothing and writes a starter `workflow.ts` with that workspace's type declarations, `tsconfig.json` and `.sim/AGENTS.md`, and records the workspace in `.sim/starter.json`. Edit the starter, check it, then run `sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run` and `--yes` (the starter prints both). A create into another workspace than the starter's is refused (`SOURCE_OTHER_WORKSPACE`) unless `--workspace` names it. Until the workflow is created the directory has no `.sim/base.json`, so `status`, `diff` and `apply` refuse it. The file starts with the `// @sim-sdk 1` line and default-exports one `workflow('Name', {}, () => { ... })`. The command prints the new workflow ID and writes the same type declarations, `tsconfig.json`, and `.sim/` files that `get` writes, so you can continue editing with `apply`. If another workflow already has that name, the command exits `1`; pass `--name` to choose another. +To create a new workflow, run `sim workflows source create --dir <dir> --workspace <workspaceId>` on an empty directory: without `--dry-run` or `--yes` it creates nothing and writes a starter `workflow.ts` with that workspace's type declarations, `tsconfig.json` and `.sim/AGENTS.md`, and records the workspace in `.sim/starter.json`. Edit the starter, check it, then run `sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run` and `--yes` (the starter prints both). A create goes to the starter's workspace whatever your default workspace, unless `--workspace` names another. Until the workflow is created the directory has no `.sim/base.json`, so `status`, `diff` and `apply` refuse it. The file starts with the `// @sim-sdk 1` line and default-exports one `workflow('Name', {}, () => { ... })`. The command prints the new workflow ID and writes the same type declarations, `tsconfig.json`, and `.sim/` files that `get` writes, so you can continue editing with `apply`. If another workflow already has that name, the command exits `1`; pass `--name` to choose another. Keep the `.sim/` folder the commands write. `.sim/lock.json` ties each block in the file to the stored block, so renaming a block keeps the same block, with its webhook URL, schedules and run history. Changing its name string keeps it through the lock; renaming the variable too keeps it when the block's type, parent and connections are unchanged, and the dry run reports that as `W_MAT_INFERRED_RENAME`. Renaming two blocks of the same type in the same place in one save is ambiguous, so do those one save at a time. `.sim/base.json` records the version you read. diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index f9ebc60ec5c..43fabf08808 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -1099,7 +1099,7 @@ describe('sim workflows source create on a directory with no workflow.ts', () => ) }) - it('refuses to create a starter in a profile workspace other than its own, creating nothing', async () => { + it('creates a starter in the workspace it was typed for, whatever the profile default', async () => { setTerminal(false) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) const starter = join(dir, 'new') @@ -1109,11 +1109,8 @@ describe('sim workflows source create on a directory with no workflow.ts', () => contextMockState.workspaceSource = 'config' try { - await expect(create('--dir', starter, '--yes')).rejects.toMatchObject({ - code: 'SOURCE_OTHER_WORKSPACE', - message: expect.stringContaining(`--workspace ${WORKSPACE_ID}`), - }) - expect(createCalls()).toEqual([]) + await create('--dir', starter, '--yes') + expect(createCalls()).toEqual([`/api/v2/workspaces/${WORKSPACE_ID}/workflows/source`]) } finally { contextMockState.workspaceId = WORKSPACE_ID } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 2b350537408..945bff41386 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -75,8 +75,8 @@ command with --yes to apply it).` const CREATE_EXIT_CODES_HELP = ` Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken -(SOURCE_NAME_TAKEN: pass --name), a starter written for another workspace -(SOURCE_OTHER_WORKSPACE: pass --workspace), or the command failed; 6 the folder is locked; 7 the +(SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the +workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes).` @@ -551,12 +551,11 @@ export function attachWorkflowSourceCreate(source: Command): void { } assertDeployable(options) const { client, profile } = connectCreate() - const workspaceId = client.requireWorkspace() - await assertStarterWorkspace( - options.dir, - workspaceId, - profile.sources.workspaceId === 'flag' - ) + // A starter is bound to the workspace its types were written for, as a get directory + // is to the one it was read from; --workspace still chooses another. + const starterWorkspace = + profile.sources.workspaceId === 'flag' ? null : await readStarterWorkspace(options.dir) + const workspaceId = client.requireWorkspace(starterWorkspace ?? undefined) const files = await readSourceFiles(options.dir, null) await runSourceWrite<'createWorkflowFromSource'>({ options, @@ -588,25 +587,6 @@ function createCommand(dir: string, workspaceId: string): string { return `sim workflows source create --dir ${dir} --workspace ${workspaceId}` } -/** - * Refuses a create into another workspace than the one a starter's types were written for - * ({@link SOURCE_STARTER_FILE}), unless `--workspace` names it: the profile's default workspace - * silently differing from the starter's sent the workflow somewhere its author never looked. - */ -async function assertStarterWorkspace( - dir: string, - workspaceId: string, - explicit: boolean -): Promise<void> { - const starterWorkspace = await readStarterWorkspace(dir) - if (starterWorkspace === null || starterWorkspace === workspaceId || explicit) return - throw new SimApiError( - `${dir} holds a starter written for workspace ${starterWorkspace}, but this profile targets ${workspaceId}; nothing was created. Run \`${createCommand(dir, starterWorkspace)}\` to create it where its types came from, or pass --workspace ${workspaceId} to create it there.`, - 0, - 'SOURCE_OTHER_WORKSPACE' - ) -} - /** * What a write created (on a dry run, would create; ids are only known once it is written): the * workflow of a create, then each block added. diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index 0e59c25db85..7e598bac251 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -94,7 +94,7 @@ An apply that deletes blocks (a declaration removed from \`${SOURCE_ENTRY_FILE}\ Lines starting \`note:\` explain something the write did differently than the source says; read them. -A new workflow: \`sim workflows source create --dir <new dir> --workspace <workspaceId>\` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, the types of that workspace, \`tsconfig.json\` and this guide, records the workspace in \`${SOURCE_STARTER_FILE}\`, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, run step 2 and 3, then \`sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run\` and the same with \`--yes\` (the starter prints both with its workspace filled in): it prints the new workflow id and writes the lock files. A create into another workspace than the starter's is refused (\`SOURCE_OTHER_WORKSPACE\`) unless \`--workspace\` names it. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir <copy> --workspace <workspaceId> --name <new name> --yes\`. +A new workflow: \`sim workflows source create --dir <new dir> --workspace <workspaceId>\` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, the types of that workspace, \`tsconfig.json\` and this guide, records the workspace in \`${SOURCE_STARTER_FILE}\`, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, run step 2 and 3, then \`sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run\` and the same with \`--yes\` (the starter prints both with its workspace filled in): it prints the new workflow id and writes the lock files. A create goes to the starter's workspace whatever the profile's default, unless \`--workspace\` names another. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir <copy> --workspace <workspaceId> --name <new name> --yes\`. ## File shape From efcbf80002ada34de0316f9386e10e0701870c1a Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:35:23 -0800 Subject: [PATCH 28/59] fix(workflow-code): name added and removed blocks with their types, and warn on a probable rename The diff lists each added and removed block's type; an apply passed --allow-delete that deletes a block and adds one of the same type and parent warns W_APPLY_PROBABLE_RENAME, since that block gets a new id. The CLI result and preview name the added and removed blocks. --- apps/docs/openapi-v2-workflows.json | 44 +++++++++++++----- .../workflows/source/route.test.ts | 7 ++- .../contracts/v2/openapi/workflow-source.ts | 2 +- .../lib/api/contracts/v2/workflow-source.ts | 8 +++- .../apply-workflow-source.integration.ts | 18 ++++++++ .../application/source-analytics.test.ts | 4 +- .../lib/workflows/application/source-diff.ts | 31 +++++++++---- .../workflows/application/source-evaluate.ts | 45 +++++++++++++++---- .../protocol/workflow-source-apply.test.ts | 30 +++++++++++-- .../protocol/workflow-source-apply.ts | 10 +++++ packages/sim-cli/src/generated/v2-api.ts | 10 +++++ packages/workflow-code/src/compile/SYNTAX.md | 3 +- .../workflow-code/src/diagnostics/codes.ts | 5 +++ 13 files changed, 179 insertions(+), 38 deletions(-) diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 847540cbd2a..4c021fbcd72 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -6343,7 +6343,8 @@ "added": [ { "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "key": "start" + "key": "start", + "type": "start_trigger" } ], "removed": [], @@ -18915,6 +18916,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -19060,11 +19062,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "New blocks." }, @@ -19080,11 +19086,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "Deleted blocks." }, @@ -19350,6 +19360,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -20012,11 +20023,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "New blocks." }, @@ -20032,11 +20047,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "Deleted blocks." }, @@ -20302,6 +20321,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -20623,7 +20643,8 @@ "added": [ { "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "key": "start" + "key": "start", + "type": "start_trigger" } ], "removed": [], @@ -25340,6 +25361,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -25903,6 +25925,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -26309,6 +26332,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts index 229dfc3afc3..f82f20c20d1 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts @@ -49,7 +49,12 @@ const created = { applied: true, diff: { noop: false, - blocks: { added: [{ id: 'b-1', key: 'start' }], removed: [], renamed: [], modified: [] }, + blocks: { + added: [{ id: 'b-1', key: 'start', type: 'start_trigger' }], + removed: [], + renamed: [], + modified: [], + }, edges: { added: 0, removed: 0 }, variables: { added: [], removed: [], modified: [] }, }, diff --git a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts index 0c5da604194..6a25d2449a0 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts @@ -181,7 +181,7 @@ export const workflowSourceOpenApiRoutes = [ diff: { noop: false, blocks: { - added: [{ id: BLOCK_ID, key: 'start' }], + added: [{ id: BLOCK_ID, key: 'start', type: 'start_trigger' }], removed: [], renamed: [], modified: [], diff --git a/apps/sim/lib/api/contracts/v2/workflow-source.ts b/apps/sim/lib/api/contracts/v2/workflow-source.ts index 003f57b94a7..42b34189693 100644 --- a/apps/sim/lib/api/contracts/v2/workflow-source.ts +++ b/apps/sim/lib/api/contracts/v2/workflow-source.ts @@ -123,6 +123,10 @@ const blockRefSchema = z }) .describe('A block by id and source key.') +const typedBlockRefSchema = blockRefSchema + .extend({ type: z.string().describe('Block type.') }) + .describe('An added or removed block by id, source key and type.') + const sourceDiffSchema = z .object({ noop: z.boolean().describe('True when the source matches the saved draft.'), @@ -132,8 +136,8 @@ const sourceDiffSchema = z .describe('The workflow name and description fields the source changes.'), blocks: z .object({ - added: z.array(blockRefSchema).describe('New blocks.'), - removed: z.array(blockRefSchema).describe('Deleted blocks.'), + added: z.array(typedBlockRefSchema).describe('New blocks.'), + removed: z.array(typedBlockRefSchema).describe('Deleted blocks.'), renamed: z .array( z diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index bf8175319a3..cf58125805c 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -2355,6 +2355,24 @@ describe('renames in source', () => { 'This apply deletes 2 blocks the source no longer declares: Boom, Tally. It adds Bang, Total of the same type: if those are the same blocks renamed, both their name and const changed, so nothing ties them to the stored blocks and they would get new ids.' ) expect(refusal?.hint).not.toContain('allow-delete') + + const allowed = await apply(source, files, { dryRun: true, allowDelete: true }) + expect(allowed.diff.blocks.removed.map(({ key, type }) => `${key} ${type}`)).toEqual([ + 'boom function', + 'tally function', + ]) + expect(allowed.diff.blocks.added.map(({ key, type }) => `${key} ${type}`)).toEqual([ + 'bang function', + 'total function', + ]) + expect( + allowed.diagnostics + .filter((d) => d.code === 'W_APPLY_PROBABLE_RENAME') + .map((d) => d.message) + ).toEqual([ + 'Boom (function) is deleted and Bang (function) added: probably a rename, but its name and const both changed, so it gets a new id and loses its run history and webhook URL.', + 'Tally (function) is deleted and Total (function) added: probably a rename, but its name and const both changed, so it gets a new id and loses its run history and webhook URL.', + ]) } ) diff --git a/apps/sim/lib/workflows/application/source-analytics.test.ts b/apps/sim/lib/workflows/application/source-analytics.test.ts index b8b13648dec..29edc1a30ea 100644 --- a/apps/sim/lib/workflows/application/source-analytics.test.ts +++ b/apps/sim/lib/workflows/application/source-analytics.test.ts @@ -86,7 +86,7 @@ describe('recordSourceWritten', () => { it('counts a create from source as a workflow creation, attributed to the Copilot subject', () => { const diff = emptyDiffSummary() - diff.blocks.added.push({ id: 'b-1', key: 'start' }) + diff.blocks.added.push({ id: 'b-1', key: 'start', type: 'start_trigger' }) recordSourceWritten( { @@ -125,7 +125,7 @@ describe('recordSourceDryRun', () => { it('records a clean dry run as ok with the changes a commit would make', () => { const diff = emptyDiffSummary() - diff.blocks.added.push({ id: 'b-1', key: 'start' }) + diff.blocks.added.push({ id: 'b-1', key: 'start', type: 'start_trigger' }) diff.edges.added = 2 recordSourceDryRun( diff --git a/apps/sim/lib/workflows/application/source-diff.ts b/apps/sim/lib/workflows/application/source-diff.ts index 305ad10d919..7ba815b8cb2 100644 --- a/apps/sim/lib/workflows/application/source-diff.ts +++ b/apps/sim/lib/workflows/application/source-diff.ts @@ -1,16 +1,21 @@ -import type { CanonicalDiff, LockIdentity } from '@sim/workflow-code' +import type { CanonicalDiff, CanonicalWorkflow, LockIdentity } from '@sim/workflow-code' interface SourceBlockRef { id: string key: string } +/** An added or removed block, with its type so a reader can spot a probable rename. */ +interface TypedSourceBlockRef extends SourceBlockRef { + type: string +} + /** What the apply changed (or, on a dry run, would change), by stored id and source key. */ export interface SourceDiffSummary { noop: boolean blocks: { - added: SourceBlockRef[] - removed: SourceBlockRef[] + added: TypedSourceBlockRef[] + removed: TypedSourceBlockRef[] renamed: Array<{ id: string; from: string; to: string }> modified: Array<SourceBlockRef & { fields: string[]; reparented?: boolean }> } @@ -26,19 +31,27 @@ export interface SourceDiffSummary { */ export function summarizeDiff( diff: CanonicalDiff, - baseIdentity: LockIdentity, - targetIdentity: LockIdentity, + base: { workflow: CanonicalWorkflow; identity: LockIdentity }, + target: { workflow: CanonicalWorkflow; identity: LockIdentity }, noop: boolean, meta: ReadonlyArray<'name' | 'description'> = [] ): SourceDiffSummary { - const targetId = (key: string) => targetIdentity.blocks[key]?.id ?? '' - const baseId = (key: string) => baseIdentity.blocks[key]?.id ?? '' + const targetId = (key: string) => target.identity.blocks[key]?.id ?? '' + const baseId = (key: string) => base.identity.blocks[key]?.id ?? '' return { noop: noop && meta.length === 0, ...(meta.length > 0 ? { meta: [...meta] } : {}), blocks: { - added: diff.blocks.added.map((key) => ({ id: targetId(key), key })), - removed: diff.blocks.removed.map((key) => ({ id: baseId(key), key })), + added: diff.blocks.added.map((key) => ({ + id: targetId(key), + key, + type: target.workflow.blocks[key]?.type ?? '', + })), + removed: diff.blocks.removed.map((key) => ({ + id: baseId(key), + key, + type: base.workflow.blocks[key]?.type ?? '', + })), renamed: diff.blocks.renamed.map(({ from, to }) => ({ id: targetId(to), from, to })), modified: diff.blocks.modified.map((change) => { const fields = change.changes.map((item) => diff --git a/apps/sim/lib/workflows/application/source-evaluate.ts b/apps/sim/lib/workflows/application/source-evaluate.ts index 8fedfb23158..76330f6e9e0 100644 --- a/apps/sim/lib/workflows/application/source-evaluate.ts +++ b/apps/sim/lib/workflows/application/source-evaluate.ts @@ -312,6 +312,25 @@ function listNames(names: readonly string[]): string { * could tie to the stored blocks (several alike, and none reads as the one renamed): the finding * names them and says how to keep the ids, instead of offering to delete. */ +/** + * Each removed block with the added block of the same type and parent it most probably became: + * one pair per removed block, in order, each added block used once. + */ +function probableRenames( + removed: readonly CanonicalBlock[], + added: readonly CanonicalBlock[] +): Array<{ removed: CanonicalBlock; added: CanonicalBlock }> { + const free = [...added] + return removed.flatMap((block) => { + const index = free.findIndex( + (candidate) => candidate.type === block.type && candidate.parent === block.parent + ) + if (index === -1) return [] + const [match] = free.splice(index, 1) + return [{ removed: block, added: match }] + }) +} + function blockDeleteRefusal( removed: readonly CanonicalBlock[], added: readonly CanonicalBlock[], @@ -768,15 +787,22 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour })) ) const removedKeys = isNew ? [] : canonicalDiff.blocks.removed + const removedBlocks = removedKeys.map((key) => storedCanonical.workflow.blocks[key]) + const addedBlocks = canonicalDiff.blocks.added.map((key) => materialized.workflow.blocks[key]) if (removedKeys.length > 0 && !input.allowDelete) { - sourceRefusals.push( - blockDeleteRefusal( - removedKeys.map((key) => storedCanonical.workflow.blocks[key]), - canonicalDiff.blocks.added.map((key) => materialized.workflow.blocks[key]), - metaSpan() - ) - ) + sourceRefusals.push(blockDeleteRefusal(removedBlocks, addedBlocks, metaSpan())) } + const renameWarnings = input.allowDelete + ? probableRenames(removedBlocks, addedBlocks).map( + ({ removed, added }): Diagnostic => ({ + code: 'W_APPLY_PROBABLE_RENAME', + severity: 'warning', + message: `${removed.name} (${removed.type}) is deleted and ${added.name} (${added.type}) added: probably a rename, but its name and const both changed, so it gets a new id and loses its run history and webhook URL.`, + hint: `To keep the id, apply ${added.name} under the const or name ${removed.name} had first, then change the other.`, + blockKey: added.key, + }) + ) + : [] const storedGraph: SourceGraph = { blocks: state.blocks, edges: state.edges } let blocks = materialized.state.blocks @@ -804,6 +830,7 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour ]) const refusals: Diagnostic[] = [ ...sourceRefusals, + ...renameWarnings, ...findLockedBlockRefusals(storedGraph, { blocks, edges }, graphDiff, keyOf), ...findInvalidSchedules({ blocks, edges }, graphDiff, keyOf), ...gateDiagnostics, @@ -891,8 +918,8 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise<Sour diagnostics, diff: summarizeDiff( canonicalDiff, - storedCanonical.identity, - materialized.identity, + storedCanonical, + { workflow: materialized.workflow, identity: materialized.identity }, noop, metaChanged ), diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index d1e706a1acd..c86441a7903 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -48,13 +48,17 @@ function outcome(overrides: { dryRun?: boolean; diagnostics?: unknown[]; notes?: diff: { noop: false, blocks: { - added: [{ id: 'b-1', key: 'start' }], - removed: [], + added: [{ id: 'b-1', key: 'start', type: 'start_trigger' }] as Array<{ + id: string + key: string + type: string + }>, + removed: [] as Array<{ id: string; key: string; type: string }>, renamed: [], modified: [] as Array<{ id: string; key: string; fields: string[] }>, }, edges: { added: 1, removed: 0 }, - variables: { added: [], removed: [], modified: [] }, + variables: { added: [] as string[], removed: [] as string[], modified: [] as string[] }, }, diagnostics: overrides.diagnostics ?? [], files: { 'workflow.ts': `${ENTRY}// stored\n` }, @@ -325,6 +329,26 @@ describe('sim workflows source apply', () => { }) }) + it('names each added and removed block with its type in the result and the preview', async () => { + setTerminal(true) + const changed = outcome({ dryRun: true }) + changed.data.diff.blocks.added = [{ id: 'b-3', key: 'bang', type: 'function' }] + changed.data.diff.blocks.removed = [{ id: 'b-2', key: 'boom', type: 'function' }] + request.mockResolvedValue(changed) + prompt.question.mockResolvedValue('n') + + await apply(WORKFLOW_ID, '--dir', dir).catch(() => {}) + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stderrLines().some((line) => line.includes('Added: bang (function)'))).toBe(true) + expect(stderrLines().some((line) => line.includes('Removed: boom (function)'))).toBe(true) + const printed = vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '' + expect(JSON.parse(printed)).toMatchObject({ + added: [{ block: 'bang', type: 'function' }], + removed: [{ block: 'boom', type: 'function' }], + }) + }) + it('names each changed variable in the result and the preview', async () => { setTerminal(true) const changed = outcome({ dryRun: true }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 7f7d8afd7bb..e9606fe4408 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -133,7 +133,11 @@ function diffSummary(data: SourceApplyData): string { variables[change].length > 0 ? [`${change} ${variables[change].join(', ')}`] : [] ) const modified = blocks.modified.map(({ key, fields }) => `${key} (${fields.join(', ')})`) + const typed = (list: ReadonlyArray<{ key: string; type: string }>) => + list.map(({ key, type }) => `${key} (${type})`).join('; ') return `Blocks: ${parts.join(', ')}. Connections: +${edges.added} -${edges.removed}.${ + blocks.added.length > 0 ? `\nAdded: ${typed(blocks.added)}` : '' + }${blocks.removed.length > 0 ? `\nRemoved: ${typed(blocks.removed)}` : ''}${ modified.length > 0 ? `\nModified: ${modified.join('; ')}` : '' }${variableChanges.length > 0 ? `\nVariables: ${variableChanges.join('; ')}` : ''}` } @@ -415,7 +419,13 @@ function summarize(data: SourceApplyData, created: boolean) { dryRun: data.dryRun, noop: data.diff.noop, blocksAdded: data.diff.blocks.added.length, + ...(data.diff.blocks.added.length > 0 && { + added: data.diff.blocks.added.map(({ key, type }) => ({ block: key, type })), + }), blocksRemoved: data.diff.blocks.removed.length, + ...(data.diff.blocks.removed.length > 0 && { + removed: data.diff.blocks.removed.map(({ key, type }) => ({ block: key, type })), + }), blocksModified: data.diff.blocks.modified.length, ...(data.diff.blocks.modified.length > 0 && { modified: data.diff.blocks.modified.map(({ key, fields }) => ({ block: key, fields })), diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 329cfe3139d..444997d17be 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -780,10 +780,12 @@ export type ApplyWorkflowSourceResponse = { added: Array<{ id: string key: string + type: string }> removed: Array<{ id: string key: string + type: string }> renamed: Array<{ id: string @@ -949,6 +951,7 @@ export type ApplyWorkflowSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -1432,6 +1435,7 @@ export type ApplyWorkspaceSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -1710,6 +1714,7 @@ export type ApplyWorkspaceSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -5199,10 +5204,12 @@ export type CreateWorkflowFromSourceResponse = { added: Array<{ id: string key: string + type: string }> removed: Array<{ id: string key: string + type: string }> renamed: Array<{ id: string @@ -5368,6 +5375,7 @@ export type CreateWorkflowFromSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -10462,6 +10470,7 @@ export type GetWorkflowSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -11140,6 +11149,7 @@ export type GetWorkspaceSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index 32d85376438..9ada4de1e5f 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -514,7 +514,8 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `W_APPLY_TRIGGER_REMOVED` | A deleted or retyped block still serves a live webhook, schedule, chat output or MCP tool until the next deploy. | | `W_APPLY_CATALOG_MOVED` | Compile failed and the block catalog changed since the read: get the source again. | | `E_APPLY_LINT`, `W_APPLY_LINT` | A workflow lint finding, mapped to a source line. | -| `E_APPLY_BLOCK_DELETE` | The apply deletes blocks the source no longer declares; pass `--allow-delete`. | +| `E_APPLY_BLOCK_DELETE` | The apply deletes blocks the source no longer declares; pass `--allow-delete`. When it also adds blocks of a deleted one's type, it names them as possible renames and says how to keep the ids instead. | +| `W_APPLY_PROBABLE_RENAME` | With `--allow-delete`, a deleted block and an added one share a type and parent: a probable rename that gets a new id. | | `E_NAME_TAKEN` | The `workflow('…')` name renames the workflow to one another workflow in its folder holds. | | `E_WORKFLOW_META_INVALID` | The `workflow('…')` name is empty or over 255 characters, or its description is over 50,000 characters: limits the workflow API holds every workflow to. | | `E_APPLY_INVALID_SCHEDULE` | An added or changed schedule whose cron, timezone or interval deploy would refuse. | diff --git a/packages/workflow-code/src/diagnostics/codes.ts b/packages/workflow-code/src/diagnostics/codes.ts index 9b40b36e773..6dd3ae5eb1d 100644 --- a/packages/workflow-code/src/diagnostics/codes.ts +++ b/packages/workflow-code/src/diagnostics/codes.ts @@ -448,6 +448,11 @@ const APPLY_DIAGNOSTIC_CODES = [ 'W_APPLY_TRIGGER_REMOVED', /** Compile failed and the catalog changed since the source was read: get the source again. */ 'W_APPLY_CATALOG_MOVED', + /** + * An apply passed `allowDelete` deletes a block and adds one of the same type and parent: a + * probable rename whose name and const both changed, which gets a new id. + */ + 'W_APPLY_PROBABLE_RENAME', /** A blocking workflow lint finding, mapped to a source line. */ 'E_APPLY_LINT', /** A non-blocking workflow lint finding, mapped to a source line. */ From 3e73d329b5930bcdc6e77964ee22cb0b34e7ef3c Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:40:04 -0800 Subject: [PATCH 29/59] fix(workspace-source): warn when a table column rename leaves a workflow writing the old name A plan that renames a column scans the stored workflows that use the table and warns W_TABLE_COLUMN_RENAMED_IN_USE on each Table block that still writes, filters or sorts by the old name, naming the workflow and block, at the line of the declared workflow's file. --- apps/docs/openapi-v2-workflows.json | 3 + .../application/cross-kind.integration.ts | 35 +++++ .../application/kinds/table/adapter.ts | 2 + .../application/kinds/table/diff.ts | 9 ++ .../application/kinds/table/renames.ts | 139 ++++++++++++++++++ packages/sim-cli/src/generated/v2-api.ts | 3 + packages/workspace-code/DESIGN.md | 5 +- .../workspace-code/src/diagnostics/codes.ts | 5 + 8 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 apps/sim/lib/workspace-source/application/kinds/table/renames.ts diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 4c021fbcd72..38c20a2b199 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -25212,6 +25212,7 @@ "E_PLAN_REF_DELETED", "E_PLAN_REF_HELD", "W_RESOURCE_IN_USE", + "W_TABLE_COLUMN_RENAMED_IN_USE", "E_NAME_TAKEN", "E_WORKFLOW_FILES_MISSING", "E_WORKFLOW_FOLDER_COPIED", @@ -25776,6 +25777,7 @@ "E_PLAN_REF_DELETED", "E_PLAN_REF_HELD", "W_RESOURCE_IN_USE", + "W_TABLE_COLUMN_RENAMED_IN_USE", "E_NAME_TAKEN", "E_WORKFLOW_FILES_MISSING", "E_WORKFLOW_FOLDER_COPIED", @@ -26183,6 +26185,7 @@ "E_PLAN_REF_DELETED", "E_PLAN_REF_HELD", "W_RESOURCE_IN_USE", + "W_TABLE_COLUMN_RENAMED_IN_USE", "E_NAME_TAKEN", "E_WORKFLOW_FILES_MISSING", "E_WORKFLOW_FOLDER_COPIED", diff --git a/apps/sim/lib/workspace-source/application/cross-kind.integration.ts b/apps/sim/lib/workspace-source/application/cross-kind.integration.ts index 78388601d77..b3a3d13647f 100644 --- a/apps/sim/lib/workspace-source/application/cross-kind.integration.ts +++ b/apps/sim/lib/workspace-source/application/cross-kind.integration.ts @@ -630,6 +630,41 @@ export default workflow('Answer', {}, () => { } ) + check( + 'renaming a table column an unchanged declared workflow writes warns at the line that writes it', + async () => { + const { workspaceId, dir, base } = await setUp() + const printed = (await get(workspaceId)).files['workflows/ingest/workflow.ts'] + const lines = printed.split('\n') + const dataLine = lines.findIndex((line) => line.includes('email: start.email')) + 1 + const files = { + ...dir.files, + 'tables/leads.ts': dir.files['tables/leads.ts'].replace( + 'column.string({ required: true })', + "column.string({ required: true, name: 'email_address' })" + ), + } + expect(files['tables/leads.ts']).toContain('email_address') + + const planned = await apply(workspaceId, { ...dir, files }, { base, dryRun: true }) + + const renamed = planned.diagnostics.filter( + (finding) => finding.code === 'W_TABLE_COLUMN_RENAMED_IN_USE' + ) + expect(renamed).toEqual([ + expect.objectContaining({ + severity: 'warning', + kind: 'table', + key: 'leads', + message: + 'tables/leads renames column "email" to "email_address", but block "Save Lead" of workflow "Ingest" still writes "email"; the run drops it.', + file: 'workflows/ingest/workflow.ts', + line: dataLine, + }), + ]) + } + ) + check( 'deleting a table a workflow outside the directory uses warns, naming that workflow', async () => { diff --git a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts index 7824cbb7272..430f5bd43d9 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts @@ -70,6 +70,7 @@ import { isColumnOp, type TableChanges, } from '@/lib/workspace-source/application/kinds/table/diff' +import { checkColumnRenamesInUse } from '@/lib/workspace-source/application/kinds/table/renames' import { baseRows, namedRows, @@ -700,6 +701,7 @@ async function managedRowsMovedSinceBase( } export const tableAdapter: KindAdapter<'table', StoredTableState> = { + checkRefs: checkColumnRenamesInUse, codec: tableCodec, async load(ctx, { allocate }) { diff --git a/apps/sim/lib/workspace-source/application/kinds/table/diff.ts b/apps/sim/lib/workspace-source/application/kinds/table/diff.ts index 88e09ec06c1..6f53119171f 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/diff.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/diff.ts @@ -66,6 +66,10 @@ export interface TableChanges { columns: ColumnChange[] /** Fields that change, for the plan display. */ fields: string[] + /** Stored columns this update renames, old name → new name. */ + renamedColumns: Array<{ from: string; to: string }> + /** Every column name the table holds after the update, in declaration order. */ + columnNames: string[] /** Cells a drop empties, and cells a removed option takes a selection from. */ lostCells: number /** @@ -249,6 +253,10 @@ export async function diffTable( table: {}, columns: [], fields: [], + renamedColumns: [], + columnNames: Object.entries(decl.columns).map(([columnKey, column]) => + columnName(columnKey, column) + ), lostCells: 0, removedOptions: 0, diagnostics, @@ -320,6 +328,7 @@ export async function diffTable( if (target.name !== current.name) { updates.name = target.name fields.push('name') + changes.renamedColumns.push({ from: current.name, to: target.name }) } const retyped = target.type !== current.type const becomesUnique = Boolean(target.unique) && !current.unique diff --git a/apps/sim/lib/workspace-source/application/kinds/table/renames.ts b/apps/sim/lib/workspace-source/application/kinds/table/renames.ts new file mode 100644 index 00000000000..b2a10fbfa0f --- /dev/null +++ b/apps/sim/lib/workspace-source/application/kinds/table/renames.ts @@ -0,0 +1,139 @@ +/** + * Workflows that still write a table column by the name a plan renames. A row write, filter or + * sort names a column by its name, so after the rename the run drops the cell (or matches + * nothing); the plan warns on each block, at the line its workflow file writes the old name when + * the directory declares that workflow. A workflow folder the apply evaluates is checked by its own + * lint against the declared columns, so only workflows the apply leaves as stored are scanned here. + */ +import { escapeRegExp } from '@sim/utils/string' +import { splitLines } from '@sim/workflow-code' +import { SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' +import { workflowFolder } from '@sim/workspace-code/kinds/workflow' +import { readWorkflowSource } from '@/lib/workflows/application/read-workflow-source' +import { collectTableBlockFieldIssues } from '@/lib/workflows/editing/lint' +import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' +import { finding, where } from '@/lib/workspace-source/application/findings' +import { workflowsUsing } from '@/lib/workspace-source/application/in-use' +import type { TablePlan } from '@/lib/workspace-source/application/kinds/table/adapter' +import type { PlanContext, PlannedChange } from '@/lib/workspace-source/application/types' + +/** A table this plan renames columns of: its id, name, and each old name → new name. */ +interface RenamingTable { + change: PlannedChange + tableId: string + tableName: string + columnNames: string[] + renamed: Map<string, string> +} + +function renamingTables(changes: readonly PlannedChange[]): RenamingTable[] { + return changes.flatMap((change) => { + if (change.kind !== 'table' || change.held || change.storedId === undefined) return [] + const plan = change.detail as TablePlan | undefined + const renamed = new Map( + (plan?.changes?.renamedColumns ?? []).map(({ from, to }) => [from, to] as const) + ) + if (renamed.size === 0 || !plan?.changes) return [] + return [ + { + change, + tableId: change.storedId, + tableName: plan.changes.table.name ?? change.key, + columnNames: plan.changes.columnNames, + renamed, + }, + ] + }) +} + +/** + * The 1-based line and column of the first `name` key written after the declaration of block + * `blockName` in `text`, else of the declaration itself; `undefined` when neither is found. + */ +function locateWrite( + text: string, + blockName: string, + name: string +): { line: number; column: number } | undefined { + const lines = splitLines(text) + const quoted = [`'${blockName}'`, `"${blockName}"`] + const declaration = lines.findIndex((line) => quoted.some((name) => line.includes(name))) + if (declaration === -1) return undefined + const keyPattern = new RegExp(`(^|[\\s{,])(['"]?)${escapeRegExp(name)}\\2\\s*:`) + for (let index = declaration; index < lines.length; index++) { + const match = keyPattern.exec(lines[index]) + if (match) return { line: index + 1, column: match.index + match[1].length + 1 } + } + return { line: declaration + 1, column: 1 } +} + +/** + * Warns `W_TABLE_COLUMN_RENAMED_IN_USE` on each table change that renames a column some stored + * workflow's Table block still writes, filters or sorts by the old name: one finding per block, + * naming the workflow and the block, pinned to the declared workflow's file and line. + */ +export async function checkColumnRenamesInUse( + changes: readonly PlannedChange[], + ctx: PlanContext +): Promise<void> { + const tables = renamingTables(changes) + if (tables.length === 0) return + const using = await workflowsUsing( + ctx.workspace.workspaceId, + tables.map((table) => table.tableId) + ) + if (using === null) return + /** Declared workflows by stored id: their folder key, and whether the apply evaluates them. */ + const declared = new Map<string, { key: string; evaluated: boolean }>() + for (const change of changes) { + if (change.kind !== 'workflow' || change.storedId === undefined) continue + declared.set(change.storedId, { key: change.key, evaluated: change.refs !== undefined }) + } + for (const table of tables) { + for (const workflow of using.get(table.tableId) ?? []) { + if (!workflow.draft) continue + const folder = declared.get(workflow.id) + if ( + folder?.evaluated || + changes.some( + (c) => c.kind === 'workflow' && c.storedId === workflow.id && c.action === 'delete' + ) + ) + continue + const loaded = await loadWorkflowFromNormalizedTables(workflow.id) + if (!loaded) continue + const issues = collectTableBlockFieldIssues( + loaded.blocks, + new Map([[table.tableId, { name: table.tableName, columnNames: table.columnNames }]]) + ).filter((issue) => table.renamed.has(issue.field)) + if (issues.length === 0) continue + const text = folder + ? ( + await readWorkflowSource.execute({ + principal: ctx.principal, + input: { workflowId: workflow.id }, + }) + ).files[SOURCE_ENTRY_FILE] + : undefined + for (const issue of issues) { + const renamedTo = table.renamed.get(issue.field) + const blockName = issue.blockName ?? issue.blockId + const at = text !== undefined ? locateWrite(text, blockName, issue.field) : undefined + table.change.diagnostics.push( + finding( + 'W_TABLE_COLUMN_RENAMED_IN_USE', + `${where('table', table.change.key)} renames column "${issue.field}" to "${renamedTo}", but block "${blockName}" of workflow "${workflow.name}" still ${issue.rowWrite ? 'writes' : 'reads'} "${issue.field}"; the run drops it.`, + { + hint: `Write "${renamedTo}" in that block${folder ? ` (${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE})` : ''}, in this apply or right after it.`, + ...(folder + ? { file: `${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE}`, ...(at ?? {}) } + : {}), + kind: 'table', + key: table.change.key, + } + ) + ) + } + } + } +} diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 444997d17be..316679b2c37 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -1286,6 +1286,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_PLAN_REF_DELETED' | 'E_PLAN_REF_HELD' | 'W_RESOURCE_IN_USE' + | 'W_TABLE_COLUMN_RENAMED_IN_USE' | 'E_NAME_TAKEN' | 'E_WORKFLOW_FILES_MISSING' | 'E_WORKFLOW_FOLDER_COPIED' @@ -1565,6 +1566,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_PLAN_REF_DELETED' | 'E_PLAN_REF_HELD' | 'W_RESOURCE_IN_USE' + | 'W_TABLE_COLUMN_RENAMED_IN_USE' | 'E_NAME_TAKEN' | 'E_WORKFLOW_FILES_MISSING' | 'E_WORKFLOW_FOLDER_COPIED' @@ -11000,6 +11002,7 @@ export type GetWorkspaceSourceResponse = { | 'E_PLAN_REF_DELETED' | 'E_PLAN_REF_HELD' | 'W_RESOURCE_IN_USE' + | 'W_TABLE_COLUMN_RENAMED_IN_USE' | 'E_NAME_TAKEN' | 'E_WORKFLOW_FILES_MISSING' | 'E_WORKFLOW_FOLDER_COPIED' diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index 3ce623354cf..eca28036fa3 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -238,7 +238,10 @@ resolves keys an earlier kind creates), check references across kinds, authorize cap the plan. Any error refuses the whole apply before the first write, so the plan refuses what a write would refuse: a delete of a locked resource (`E_APPLY_LOCKED`), of one a stored workflow still uses (`E_PLAN_REF_DELETED` for a declared caller, `W_RESOURCE_IN_USE` for one outside the -directory, workflows included), and a table change whose data breaks a new column rule. A plan +directory, workflows included), and a table change whose data breaks a new column rule. A column +rename warns `W_TABLE_COLUMN_RENAMED_IN_USE` on each Table block of a stored workflow that still writes, +filters or sorts by the old name (the run drops it), at the line of the declared workflow's file; an +edited workflow folder is checked by its own lint against the declared columns instead. A plan reports at most 1000 findings, errors first, then warnings, then infos, and an info counts the rest, so cutting findings never refuses a plan. diff --git a/packages/workspace-code/src/diagnostics/codes.ts b/packages/workspace-code/src/diagnostics/codes.ts index dce974fd914..23290515df2 100644 --- a/packages/workspace-code/src/diagnostics/codes.ts +++ b/packages/workspace-code/src/diagnostics/codes.ts @@ -239,6 +239,11 @@ const PLAN_CODES = [ 'E_PLAN_REF_HELD', /** A resource this apply deletes is still used by stored workflows. */ 'W_RESOURCE_IN_USE', + /** + * A table column this apply renames is still written, filtered or sorted by its old name in a + * stored workflow's Table block, which the run then drops. + */ + 'W_TABLE_COLUMN_RENAMED_IN_USE', /** Another resource of the kind holds this name or key. */ 'E_NAME_TAKEN', /** A workflow folder sent without its files whose base names no workflow of this workspace. */ From 3ea9d3ad09875dbd79aa4b80294e6bfc5b540c46 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:45:55 -0800 Subject: [PATCH 30/59] fix(workflow-code): leave out a container's subBlocks that block.data supersedes Editing operations store a loop's iterations/loopType inputs beside data.count; the serializer reads only data, so source no longer prints them under extra. They are preserved state: apply restores them from the stored block, so a no-op apply stays byte-identical. --- ...ly-workflow-source.fidelity.integration.ts | 58 ++++++++++++++++++- packages/workflow-code/DESIGN.md | 7 ++- .../src/canonicalize/subflows.ts | 12 ++++ .../src/equivalence/executable.ts | 11 +++- .../workflow-code/src/materialize/preserve.ts | 10 +++- packages/workflow-code/src/print/printer.ts | 4 +- 6 files changed, 92 insertions(+), 10 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts index 71f8c0c3653..83472ee5cbf 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts @@ -577,7 +577,63 @@ check( const canvas = byName(await readGraph(canvasId)) const fromSource = byName(await readGraph(created.workflowId)) - expect(findDifferences(canvas, fromSource, 20)).toEqual([]) + // The loop's `iterations`/`loopType` subBlocks the editing operations leave beside + // `data.count` never run, so source does not carry them to the created workflow. + const differences = findDifferences(canvas, fromSource, 20).map((d) => d.path) + expect(differences.sort()).toEqual([ + '/blocks/Retry/subBlocks/iterations', + '/blocks/Retry/subBlocks/loopType', + ]) + } +) + +check( + 'a loop the editing operations built prints no superseded subBlocks, and a no-op apply keeps them stored', + async () => { + const workflowId = await insertWorkflow('Fidelity operations loop') + const start = generateId() + const loop = generateId() + await applyOperations(workflowId, [ + { operation_type: 'add', block_id: start, params: { type: 'start_trigger', name: 'Start' } }, + { + operation_type: 'add', + block_id: loop, + params: { type: 'loop', name: 'Retry', inputs: { loopType: 'for', iterations: 3 } }, + }, + { + operation_type: 'insert_into_subflow', + block_id: generateId(), + params: { + subflowId: loop, + type: 'function', + name: 'Attempt', + inputs: { code: 'return 1' }, + }, + }, + { operation_type: 'edit', block_id: start, params: { connections: { source: loop } } }, + ]) + const stored = (await readGraph(workflowId)).blocks[loop] + expect(Object.keys(stored.subBlocks ?? {}).sort()).toEqual(['iterations', 'loopType']) + expect(stored.data?.count).toBe(3) + + const source = await getSource(workflowId) + expect(source.files['workflow.ts']).toContain("loop.for('Retry', { count: 3 }") + expect(source.files['workflow.ts']).not.toContain('iterations') + expect(source.files['workflow.ts']).not.toContain('extra') + + const before = await snapshot(workflowId) + const unchanged = await applySource(source, source.files) + expect(unchanged.diff.noop).toBe(true) + expect(await snapshot(workflowId)).toEqual(before) + + const counted = await applySource( + source, + editEntry(source.files, '{ count: 3 }', '{ count: 5 }') + ) + expect(counted.applied).toBe(true) + const after = (await readGraph(workflowId)).blocks[loop] + expect(after.data?.count).toBe(5) + expect(after.subBlocks).toEqual(stored.subBlocks) } ) diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index 6f26864e5b1..07f5ca29f84 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -38,7 +38,9 @@ stored workflow (see "Preserved state"). There is no verbose mode. Two escape hatches carry state that runs but has no SDK syntax: - `meta.extra`: stored params of a schema-agnostic custom block, the agent's legacy prompt fields, - a container's stored params. + a container's stored params other than those `block.data` supersedes (the config inputs the + editing engine copies into `data`, `SUPERSEDED_CONTAINER_PARAMS`), which never run and are + preserved state. - `meta.subflowData`: a loop or parallel's `block.data` subflow keys canonicalize could not read as config, which the serializer passes on. Kept apart from `extra` because canvas containers often store params under the same names. @@ -372,7 +374,8 @@ then writes the stored graph so that a no-op is byte-identical (O1): filled into fields that never run are dropped); - the fields of the loop or parallel types not in effect, and prev's whole config when it runs the same; -- unknown `block.data` source does not carry. +- unknown `block.data` source does not carry; +- a container's subBlocks `block.data` supersedes (`iterations`, `loopType`, …). Canonicalize's output already holds all of this, so restoring onto it changes nothing. diff --git a/packages/workflow-code/src/canonicalize/subflows.ts b/packages/workflow-code/src/canonicalize/subflows.ts index d85ff1f58f0..77f88100cb4 100644 --- a/packages/workflow-code/src/canonicalize/subflows.ts +++ b/packages/workflow-code/src/canonicalize/subflows.ts @@ -32,6 +32,18 @@ export const SUBFLOW_DATA_KEYS: ReadonlySet<string> = new Set([ 'batchSize', ]) +/** + * Container subBlocks `block.data` supersedes: the config inputs the editing engine copies into + * `data` (`iterations` → `count`, `condition` → `whileCondition`) and may also leave stored as + * subBlocks. The serializer reads a container's config from `data` only, so these never run: + * preserved state, which print leaves out and apply restores from the stored block. + */ +export const SUPERSEDED_CONTAINER_PARAMS: ReadonlySet<string> = new Set([ + ...SUBFLOW_DATA_KEYS, + 'iterations', + 'condition', +]) + /** A subflow config with the `block.data` keys whose stored value it could not represent. */ interface SubflowResult<S> { spec: S diff --git a/packages/workflow-code/src/equivalence/executable.ts b/packages/workflow-code/src/equivalence/executable.ts index e7d3398e015..a50b2e1e5d5 100644 --- a/packages/workflow-code/src/equivalence/executable.ts +++ b/packages/workflow-code/src/equivalence/executable.ts @@ -16,7 +16,7 @@ */ import { compareEdges } from '../canonicalize/edges' -import { SUBFLOW_DATA_KEYS } from '../canonicalize/subflows' +import { SUBFLOW_DATA_KEYS, SUPERSEDED_CONTAINER_PARAMS } from '../canonicalize/subflows' import { executableInactiveParams } from '../catalog/activity' import { isCanonicalPair } from '../catalog/canonical-index' import { serializedCanonicalIndex } from '../catalog/surface' @@ -89,7 +89,14 @@ function executableParams( declared: CatalogBlock | undefined, catalog: BlockCatalog ): Record<string, CanonicalParam> { - if (!declared) return { ...block.params } + if (!declared) { + if (!block.subflow) return { ...block.params } + const params: Record<string, CanonicalParam> = {} + for (const [id, param] of Object.entries(block.params)) { + if (!SUPERSEDED_CONTAINER_PARAMS.has(id)) setOwn(params, id, param) + } + return params + } const inactive = new Set(executableInactiveParams(block, catalog)) const params: Record<string, CanonicalParam> = {} for (const [id, param] of Object.entries(block.params)) { diff --git a/packages/workflow-code/src/materialize/preserve.ts b/packages/workflow-code/src/materialize/preserve.ts index 4146f83a46b..7b07c7c3eb8 100644 --- a/packages/workflow-code/src/materialize/preserve.ts +++ b/packages/workflow-code/src/materialize/preserve.ts @@ -9,13 +9,14 @@ * editor defaults compile filled into fields that never run; * - the config fields of the loop or parallel types not in effect, and prev's spelling of a config * that runs the same (`count: 0` and `count: 5`); - * - unknown `block.data` extra data source does not carry. + * - unknown `block.data` extra data source does not carry; + * - a container's subBlocks `block.data` supersedes (`SUPERSEDED_CONTAINER_PARAMS`). * * Canonicalize's output already holds all of this, so restoring onto it changes nothing and the * no-op round trip stays byte-identical. */ -import { SUBFLOW_DATA_KEYS } from '../canonicalize/subflows' +import { SUBFLOW_DATA_KEYS, SUPERSEDED_CONTAINER_PARAMS } from '../canonicalize/subflows' import { canonicalActivityInputs, isInactive, withStoredValue } from '../catalog/activity' import { type CanonicalGroup, isCanonicalPair } from '../catalog/canonical-index' import { serializedCanonicalIndex } from '../catalog/surface' @@ -245,7 +246,10 @@ export function restorePreserved( restorePairs(next, prev, declared, catalog) restoreInactive(next, prev, declared, catalog) } - if (target.subflow) next.subflow = restoreSubflow(target.subflow, prev.subflow) + if (target.subflow) { + next.subflow = restoreSubflow(target.subflow, prev.subflow) + for (const id of SUPERSEDED_CONTAINER_PARAMS) setParam(next.params, id, getOwn(prev.params, id)) + } setOrDelete(next, 'extra', restoreExtra(target, prev)) return next } diff --git a/packages/workflow-code/src/print/printer.ts b/packages/workflow-code/src/print/printer.ts index 8a4f3460f3e..b4d08a207d0 100644 --- a/packages/workflow-code/src/print/printer.ts +++ b/packages/workflow-code/src/print/printer.ts @@ -11,7 +11,7 @@ import { isReservedWorkflowBlockName, normalizeWorkflowBlockName, } from '@sim/workflow-types/workflow' -import { SUBFLOW_DATA_KEYS } from '../canonicalize/subflows' +import { SUBFLOW_DATA_KEYS, SUPERSEDED_CONTAINER_PARAMS } from '../canonicalize/subflows' import { builderPins } from '../catalog/surface' import { catalogBlock } from '../catalog/view' import type { PrintDiagnosticCode } from '../diagnostics/codes' @@ -235,7 +235,7 @@ function containerCall(ctx: PrintContext, block: CanonicalBlock): Expr { ctx, block, catalogBlock(ctx.catalog, block.type), - new Set() + new Set(SUPERSEDED_CONTAINER_PARAMS) ) const subflowData: Record<string, Json> = {} for (const [key, value] of Object.entries(block.extra ?? {})) { From fd511af0a7b918354a414482ec8683979ba32a3d Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:40:53 -0800 Subject: [PATCH 31/59] feat(sim-cli): sim workflows source params <builder> prints a builder's params and outputs offline Reads the catalog the directory was printed with (else the workspace's cached one): required params first with enums and defaults, the gate field and its variants, outputs, and the agent tool form. A namespace lists its builders, a core.d.ts construct prints its declaration, and a miss names the closest builder. The facts come from the generator's own builder types, so they match the .d.ts tsc checks against. --- apps/docs/content/docs/cli/reference.mdx | 32 ++ apps/docs/content/docs/cli/workflows.mdx | 32 ++ .../sim-cli/src/commands/protocol/index.ts | 2 + .../protocol/workflow-source-params.test.ts | 123 ++++++++ .../protocol/workflow-source-params.ts | 90 ++++++ packages/workflow-code/src/index.ts | 2 +- packages/workflow-sdk/src/codegen/generate.ts | 89 +++++- packages/workflow-sdk/src/codegen/index.ts | 8 + packages/workflow-sdk/src/codegen/names.ts | 12 + .../workflow-sdk/src/codegen/reference.ts | 287 ++++++++++++++++++ 10 files changed, 672 insertions(+), 5 deletions(-) create mode 100644 packages/sim-cli/src/commands/protocol/workflow-source-params.test.ts create mode 100644 packages/sim-cli/src/commands/protocol/workflow-source-params.ts create mode 100644 packages/workflow-sdk/src/codegen/reference.ts diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index ea0a5625766..e00bd476bd5 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8694,6 +8694,38 @@ Without --server, each file created, edited or deleted since the directory was r API request and response schemas: [getWorkflowSource](/api-reference/workflows/getWorkflowSource). +### sim workflows source params + +Print a builder's params and outputs from the cached block catalog (offline) + +```bash +sim workflows source params <builder> [options] +``` + +**Arguments** + +<CommandTable> + +| Argument | Required | Description | +| --- | --- | --- | +| `builder` | Yes | Builder path (table.upsertRow, agent, trigger.schedule) or namespace | + +</CommandTable> + +**Options** + +<CommandTable> + +| Option | Required | Description | +| --- | --- | --- | +| `--dir <dir>` | No | Source directory whose catalog to read. Defaults to `.`. | + +</CommandTable> + +Prints a builder's params (required first, with enums and defaults), the gate field that picks which others apply, and its outputs, from the block catalog the last get, create or check cached. No request is made. <builder> is the dotted builder path as source writes it: table.upsertRow, agent, trigger.schedule, slack.sendMessage (slack.tool.sendMessage names the same builder). A namespace (gmail, trigger.gmail) lists its builders; a construct core.d.ts declares (loop, parallel, t, env) prints its declaration. Run it from a workflow or workspace source directory, or pass --dir. + +Exit codes: 0 printed; 1 no builder or namespace of that name (the closest one is named), or no cached catalog (run a get once). + ### sim workflows get Get workflow diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index eafc3758d94..00598895dfa 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1472,6 +1472,38 @@ Without --server, each file created, edited or deleted since the directory was r API request and response schemas: [getWorkflowSource](/api-reference/workflows/getWorkflowSource). +### Print a builder's params and outputs from the cached block catalog + +Offline. + +```bash +sim workflows source params <builder> [options] +``` + +**Arguments** + +<CommandTable> + +| Argument | Required | Description | +| --- | --- | --- | +| `builder` | Yes | Builder path (table.upsertRow, agent, trigger.schedule) or namespace | + +</CommandTable> + +**Options** + +<CommandTable> + +| Option | Required | Description | +| --- | --- | --- | +| `--dir <dir>` | No | Source directory whose catalog to read. Defaults to `.`. | + +</CommandTable> + +Prints a builder's params (required first, with enums and defaults), the gate field that picks which others apply, and its outputs, from the block catalog the last get, create or check cached. No request is made. <builder> is the dotted builder path as source writes it: table.upsertRow, agent, trigger.schedule, slack.sendMessage (slack.tool.sendMessage names the same builder). A namespace (gmail, trigger.gmail) lists its builders; a construct core.d.ts declares (loop, parallel, t, env) prints its declaration. Run it from a workflow or workspace source directory, or pass --dir. + +Exit codes: 0 printed; 1 no builder or namespace of that name (the closest one is named), or no cached catalog (run a get once). + ### List the tables, knowledge bases, credentials, workflows, skills, custom tools, MCP servers and env names source can name ```bash diff --git a/packages/sim-cli/src/commands/protocol/index.ts b/packages/sim-cli/src/commands/protocol/index.ts index 31782f51591..d9b37e51f24 100644 --- a/packages/sim-cli/src/commands/protocol/index.ts +++ b/packages/sim-cli/src/commands/protocol/index.ts @@ -21,6 +21,7 @@ import { attachWorkflowRunWait } from './workflow-run-wait' import { attachWorkflowSourceApply, attachWorkflowSourceCreate } from './workflow-source-apply' import { attachWorkflowSourceCheck } from './workflow-source-check' import { attachWorkflowSourceGet } from './workflow-source-get' +import { attachWorkflowSourceParams } from './workflow-source-params' import { attachWorkflowSourceDiff, attachWorkflowSourceStatus } from './workflow-source-status' import { attachWorkspaceOperationWait } from './workspace-operation-wait' import { attachWorkspaceSourceApply } from './workspace-source-apply' @@ -98,6 +99,7 @@ export function attachProtocolCommands(program: Command): void { attachWorkflowSourceCheck(source) attachWorkflowSourceStatus(source) attachWorkflowSourceDiff(source) + attachWorkflowSourceParams(source) const workspaces = group(program, 'workspaces') attachWorkspaceOperationWait(group(workspaces, 'operations')) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-params.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-params.test.ts new file mode 100644 index 00000000000..c5d00d1467b --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/workflow-source-params.test.ts @@ -0,0 +1,123 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Command } from 'commander' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { SimApiError } from '../../http/client' +import { cacheCatalogText } from '../../test/catalog-cache' +import { contextMockFns } from '../../test/context-mock' +import { attachWorkflowSourceParams } from './workflow-source-params' + +vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) + +const WORKSPACE_ID = 'ws_params' +/** A real catalog projected from registry blocks, shared with the SDK's own type checks. */ +const FIXTURE_TEXT = readFileSync( + join(__dirname, '../../../../workflow-sdk/test/fixtures/catalog.json'), + 'utf8' +) + +let dir: string + +async function params(...argv: string[]): Promise<void> { + const source = new Command('source') + attachWorkflowSourceParams(source) + source.commands.forEach((command) => command.exitOverride()) + await source.parseAsync(['node', 'source', 'params', ...argv, '--dir', dir]) +} + +function stdoutJson(): Record<string, unknown> { + const calls = vi.mocked(console.log).mock.calls + expect(calls).toHaveLength(1) + return JSON.parse(calls[0].join(' ')) +} + +async function refusal(run: Promise<void>): Promise<SimApiError> { + return (await run.then( + () => { + throw new Error('expected the command to refuse') + }, + (error: unknown) => error + )) as SimApiError +} + +/** The fixture catalog under another hash, without the `table.upsertRow` builder. */ +function catalogWithoutUpsert(): string { + const catalog = JSON.parse(FIXTURE_TEXT) + const { 'table.upsertRow': _removed, ...builders } = catalog.builders + return JSON.stringify({ ...catalog, builders, hash: 'f'.repeat(64) }) +} + +function writeBase(catalogHash?: string): void { + writeFileSync( + join(dir, '.sim/base.json'), + JSON.stringify({ + workflowId: 'wf', + workspaceId: WORKSPACE_ID, + fingerprint: 'sf1:a', + catalogHash, + }) + ) +} + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'sim-source-params-')) + mkdirSync(join(dir, '.sim')) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_other', output: 'json' }) + contextMockFns.mockRequest.mockReset() + vi.spyOn(console, 'log').mockImplementation(() => {}) +}) + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }) +}) + +describe('sim workflows source params', () => { + it("reads the catalog the directory was printed with, not the workspace's newer one", async () => { + const printedWith = JSON.parse(FIXTURE_TEXT).hash as string + cacheCatalogText(join(dir, 'config'), null, FIXTURE_TEXT) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, catalogWithoutUpsert()) + writeBase(printedWith) + + await params('table.upsertRow') + + const { builder } = stdoutJson() as { + builder: { key: string; params: { common: Array<{ key: string; required: boolean }> } } + } + expect(builder.key).toBe('table.upsertRow') + expect(builder.params.common.filter((param) => param.required).map((p) => p.key)).toEqual([ + 'tableId', + 'data', + ]) + expect(contextMockFns.mockRequest).not.toHaveBeenCalled() + }) + + it('reads an agent tool form as the builder it names', async () => { + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, FIXTURE_TEXT) + writeBase() + + await params('slack.tool.sendMessage') + + expect(stdoutJson()).toMatchObject({ builder: { key: 'slack.sendMessage' } }) + }) + + it('refuses a builder the catalog lacks, naming the closest one', async () => { + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, FIXTURE_TEXT) + writeBase() + + const error = await refusal(params('slack.postMessage')) + + expect(error.code).toBe('BUILDER_UNKNOWN') + expect(error.message).toContain('Did you mean slack.sendMessage?') + }) + + it('refuses without a cached catalog, making no request', async () => { + writeBase() + + const error = await refusal(params('agent')) + + expect(error.code).toBe('CATALOG_NOT_CACHED') + expect(contextMockFns.mockRequest).not.toHaveBeenCalled() + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-params.ts b/packages/sim-cli/src/commands/protocol/workflow-source-params.ts new file mode 100644 index 00000000000..632867ed1c2 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/workflow-source-params.ts @@ -0,0 +1,90 @@ +import { toRecord } from '@sim/utils/object' +import type { BlockCatalog } from '@sim/workflow-code' +import { SOURCE_BASE_FILE } from '@sim/workflow-code/limits' +import { lookupBuilder, renderBuilderReference } from '@sim/workflow-sdk/codegen' +import type { Command } from 'commander' +import { printLine } from '#sim-cli/output/io' +import { profileFrom } from '../../context' +import { SimApiError } from '../../http/client' +import { printProtocolResult } from './result' +import { readSourceJson, workspaceFolderOf } from './source-dir' +import { cachedCatalog, cachedSourceCatalog } from './source-tree' + +const PARAMS_HELP = ` +Prints a builder's params (required first, with enums and defaults), the gate field that picks +which others apply, and its outputs, from the block catalog the last get, create or check cached. +No request is made. <builder> is the dotted builder path as source writes it: table.upsertRow, +agent, trigger.schedule, slack.sendMessage (slack.tool.sendMessage names the same builder). A +namespace (gmail, trigger.gmail) lists its builders; a construct core.d.ts declares (loop, +parallel, t, env) prints its declaration. Run it from a workflow or workspace source directory, +or pass --dir. + +Exit codes: 0 printed; 1 no builder or namespace of that name (the closest one is named), or no +cached catalog (run a get once).` + +/** + * The catalog the directory was printed with (its `.sim/base.json` hash), else the one cached for + * its workspace, the workspace a workflow folder belongs to, or the profile's. + */ +async function lookupCatalog(dir: string, profileWorkspace: string | null): Promise<BlockCatalog> { + const printedWith = await cachedSourceCatalog(dir) + if (printedWith) return printedWith + const base = toRecord(await readSourceJson(dir, SOURCE_BASE_FILE)) + const workspaceId = + typeof base.workspaceId === 'string' + ? base.workspaceId + : ((await workspaceFolderOf(dir))?.workspaceId ?? profileWorkspace) + const cached = workspaceId ? cachedCatalog(workspaceId) : null + if (cached) return cached + throw new SimApiError( + workspaceId + ? `No block catalog is cached for workspace ${workspaceId}. Run \`sim workflows source get <workflowId> --dir <dir>\` (or \`sim workspaces source get --dir <dir>\`) once, then look it up again.` + : `${dir} is not a source directory and no workspace is selected: pass --dir <source dir> or --workspace <workspaceId>.`, + 0, + 'CATALOG_NOT_CACHED' + ) +} + +/** + * `sim workflows source params <builder> [--dir <dir>]`: one builder's params and outputs from the + * cached catalog, so an agent reads the shape of a block it is about to write in one command + * instead of searching the declarations. + */ +export function attachWorkflowSourceParams(source: Command): void { + source + .command('params') + .argument('<builder>', 'Builder path (table.upsertRow, agent, trigger.schedule) or namespace') + .option('--dir <dir>', 'Source directory whose catalog to read', '.') + .allowExcessArguments(false) + .description("Print a builder's params and outputs from the cached block catalog (offline)") + .addHelpText('after', PARAMS_HELP) + .action(async (query: string, options: { dir: string }, command: Command) => { + const profile = profileFrom(command) + const catalog = await lookupCatalog(options.dir, profile.workspaceId) + const found = lookupBuilder(catalog, query) + const machine = profile.output === 'json' || profile.output === 'yaml' + switch (found.kind) { + case 'builder': + if (machine) printProtocolResult(profile.output, { builder: found.reference }) + else printLine(renderBuilderReference(found.reference)) + return + case 'namespace': + if (machine) printProtocolResult(profile.output, found) + else { + printLine(`${found.namespace} has ${found.builders.length} builders:`) + for (const builder of found.builders) printLine(` ${builder.key} — ${builder.summary}`) + } + return + case 'core': + if (machine) printProtocolResult(profile.output, found) + else printLine(`${found.name} is declared in .sim/sdk/core.d.ts:\n\n${found.declaration}`) + return + case 'missing': + throw new SimApiError( + `No builder or namespace is named ${query}.${found.hint ? ` ${found.hint}` : ''}`, + 0, + 'BUILDER_UNKNOWN' + ) + } + }) +} diff --git a/packages/workflow-code/src/index.ts b/packages/workflow-code/src/index.ts index e56652be09b..1cc4ce34d88 100644 --- a/packages/workflow-code/src/index.ts +++ b/packages/workflow-code/src/index.ts @@ -38,7 +38,7 @@ export { } from './code-files' export { type CompileResult, compileWorkflow, type StoredEdgeRef } from './compile' export { codePointLabel, scriptName } from './compile/characters' -export { didYouMean } from './compile/hints' +export { didYouMean, suggestBuilder } from './compile/hints' export { babelMessage, bracketDepthOverflow, diff --git a/packages/workflow-sdk/src/codegen/generate.ts b/packages/workflow-sdk/src/codegen/generate.ts index 5ae0c5a82ac..a7d11d3dafd 100644 --- a/packages/workflow-sdk/src/codegen/generate.ts +++ b/packages/workflow-sdk/src/codegen/generate.ts @@ -17,7 +17,7 @@ import { compareStrings } from '@sim/utils/string' import type { BlockCatalog, CatalogBlock, CatalogOutput, CatalogSubBlock } from '@sim/workflow-code' import { builderOutputSchemas, printKey, quoteString } from '@sim/workflow-code' -import { subBlockEditorDefault } from '@sim/workflow-code/catalog/defaults' +import { type EditorDefault, subBlockEditorDefault } from '@sim/workflow-code/catalog/defaults' import { builderPins, hasDefaultValue, @@ -184,7 +184,34 @@ interface BuilderOutput { body: string } -interface BuilderTypes { +/** One param of a builder, as `describeBuilder` reports it. */ +export interface ParamFact { + /** What source writes. */ + key: string + /** The TypeScript type, `| null` included where a stored empty value prints as `null`. */ + type: string + required: boolean + /** The param's doc lines: title, bounds, selector, dependencies. */ + doc: string[] + /** The value an omitted param means (the editor default), when it has one. */ + default?: EditorDefault +} + +/** + * The params of a builder as facts: the ones every call takes, and when one field (the gate) + * decides which others apply, its values and the params of each. + */ +export interface ParamFacts { + common: ParamFact[] + gate: { + key: string + /** The gate's value when it is omitted, if any. */ + default?: Scalar + variants: Array<{ values: Scalar[]; params: ParamFact[] }> + } | null +} + +export interface BuilderTypes { /** The params type declaration, exported. */ params: string output: BuilderOutput @@ -198,6 +225,10 @@ interface BuilderTypes { summary: string /** The params type's name. */ paramsName: string + /** The params as facts. */ + facts: ParamFacts + /** What a call returns: `Block<…>` of the output type. */ + returns: string } /** @@ -225,7 +256,8 @@ function operationOutputs( return { outputs: picked, nullable: new Set(returned.nullable ?? []) } } -function operationLabel(block: CatalogBlock, operation: string): string | undefined { +/** The label of `operation` in the block's operation dropdown, if it has one. */ +export function operationLabel(block: CatalogBlock, operation: string): string | undefined { const dropdown = block.subBlocks.find((subBlock) => subBlock.id === 'operation') if (!dropdown || !Array.isArray(dropdown.options)) return undefined return dropdown.options.find((option) => option.id === operation)?.label @@ -265,6 +297,52 @@ function requiredInVariant(param: Param, requiredByVisibility: boolean): boolean return requiredByVisibility && !hasDefaultValue(param.members) } +/** The params of a builder as {@link ParamFacts}, required as the params type renders them. */ +function paramFacts( + params: readonly Param[], + gate: Gate | null, + candidates: readonly CatalogSubBlock[], + pinned: VisibilityContext['pinned'] +): ParamFacts { + const fact = (param: Param, required: boolean): ParamFact => { + const editorDefault = param.members + .map((member) => subBlockEditorDefault(member)) + .find((value) => value !== undefined) + const nullable = param.members.some((member) => subBlockEditorDefault(member) !== undefined) + return { + key: param.key, + type: nullable ? `${param.type} | null` : param.type, + required, + doc: param.doc.filter((line) => !line.startsWith('@when')), + ...(editorDefault !== undefined ? { default: editorDefault } : {}), + } + } + const gated = new Set<string>() + if (gate) { + gated.add(gate.field) + for (const variant of gate.variants) for (const p of variant.params) gated.add(p.name) + } + const common = params + .filter((param) => !gated.has(param.name)) + .map((param) => fact(param, requiredForBuilder(param.members, pinned))) + if (!gate) return { common, gate: null } + const fallback = defaultGateValue(candidates, gate.field) + return { + common, + gate: { + key: params.find((param) => param.name === gate.field)?.key ?? gate.field, + ...(fallback !== undefined ? { default: fallback } : {}), + variants: gate.variants.map((variant) => ({ + values: [...variant.values], + params: variant.params.flatMap(({ name, required }) => { + const param = params.find((candidate) => candidate.name === name) + return param ? [fact(param, requiredInVariant(param, required))] : [] + }), + })), + }, + } +} + function renderParamsType( params: readonly Param[], gate: Gate | null, @@ -366,7 +444,8 @@ function buildParams( return { params, gate, candidates, context } } -function builderTypes(catalog: BlockCatalog, entry: BuilderEntry): BuilderTypes { +/** A builder's generated types: its params and output declarations, signatures and facts. */ +export function builderTypes(catalog: BlockCatalog, entry: BuilderEntry): BuilderTypes { const block = catalogBlock(catalog, entry.builder.type) if (!block) { throw new Error( @@ -433,6 +512,8 @@ function builderTypes(catalog: BlockCatalog, entry: BuilderEntry): BuilderTypes special, summary: [block.name, label].filter(Boolean).join(': '), paramsName, + facts: paramFacts(params, gate, candidates, context.pinned), + returns: `Block<${outputType}>`, } } diff --git a/packages/workflow-sdk/src/codegen/index.ts b/packages/workflow-sdk/src/codegen/index.ts index 54672b4ded8..910658290ee 100644 --- a/packages/workflow-sdk/src/codegen/index.ts +++ b/packages/workflow-sdk/src/codegen/index.ts @@ -16,6 +16,14 @@ export { } from './layout' export { SDK_MAJOR } from './major' export { SPECIAL_BUILDER_KEYS } from './names' +export { + type BuilderLookup, + type BuilderReference, + builderSnippet, + describeBuilder, + lookupBuilder, + renderBuilderReference, +} from './reference' export { SOURCE_COMPILER_OPTIONS, typecheckCommand } from './tsconfig' export { type GenerateWorkspaceTypesFn, diff --git a/packages/workflow-sdk/src/codegen/names.ts b/packages/workflow-sdk/src/codegen/names.ts index 74280ab02d7..ce148d1a525 100644 --- a/packages/workflow-sdk/src/codegen/names.ts +++ b/packages/workflow-sdk/src/codegen/names.ts @@ -84,3 +84,15 @@ export function builderEntries(catalog: BlockCatalog): BuilderEntry[] { } return entries } + +/** + * The entry of the catalog builder `key`, or `null` when the catalog has none, named as + * {@link builderEntries} names it. It does not validate the rest of the map. + */ +export function builderEntry(catalog: BlockCatalog, key: string): BuilderEntry | null { + const builder = catalogBuilder(catalog, key) + if (!builder || !catalogBlock(catalog, builder.type)) return null + const path = key.split('.') + const special = path.length === 1 ? SDK_SPECIAL_BUILDERS[path[0]] : undefined + return { key, path, builder, special, typeName: pascal(path) } +} diff --git a/packages/workflow-sdk/src/codegen/reference.ts b/packages/workflow-sdk/src/codegen/reference.ts new file mode 100644 index 00000000000..15cabede343 --- /dev/null +++ b/packages/workflow-sdk/src/codegen/reference.ts @@ -0,0 +1,287 @@ +/** + * One builder described at the terminal, from the catalog the declarations are generated from: + * what `sim workflows source params <builder>` prints, and the snippet the source primer carries + * for each builder a directory already uses. The params, their required-ness and the outputs are + * the generated declarations' own (`builderTypes`), so the two never disagree. + */ + +import { compareStrings } from '@sim/utils/string' +import type { BlockCatalog } from '@sim/workflow-code' +import { suggestBuilder } from '@sim/workflow-code' +import { catalogBlock, catalogBuilder, catalogBuilderPaths } from '@sim/workflow-code/catalog/view' +import { SDK_CORE_EXPORTS } from '@sim/workflow-code/sdk' +import { CORE_DTS } from './core-text.generated' +import { builderTypes, operationLabel, type ParamFact, type ParamFacts } from './generate' +import { isLegacyBuilder } from './layout' +import { builderEntry, TOOL_NAMESPACE } from './names' + +/** A builder as `sim workflows source params` reports it. */ +export interface BuilderReference { + /** The dotted builder path (`slack.sendMessage`). */ + key: string + blockType: string + /** `Block: Operation`. */ + summary: string + operation?: string + /** An older block version's builder, kept so stored workflows print; write the current one. */ + legacy: boolean + /** The call, as the declarations type it. */ + signature: string + /** The agent tool form (`slack.tool.sendMessage(…)`), when the builder has one. */ + tool: string | null + paramsName: string + params: ParamFacts + /** Params `core.d.ts` types for the builder itself (`fn`'s `code`), not listed in `params`. */ + coreParams: readonly string[] + outputName: string + /** The output type's members as the declaration holds them, one line each. */ + outputs: string[] +} + +/** A builder of a namespace, in a lookup's list. */ +interface BuilderSummary { + key: string + summary: string +} + +/** What a builder lookup found. */ +export type BuilderLookup = + | { kind: 'builder'; reference: BuilderReference } + | { kind: 'namespace'; namespace: string; builders: BuilderSummary[] } + | { kind: 'core'; name: string; declaration: string } + | { kind: 'missing'; hint?: string } + +/** `Block: Operation` for a catalog builder. */ +function summaryOf(catalog: BlockCatalog, key: string): string { + const builder = catalogBuilder(catalog, key) + const block = builder ? catalogBlock(catalog, builder.type) : undefined + if (!builder || !block) return key + const label = builder.operation ? operationLabel(block, builder.operation) : undefined + return [block.name, label].filter(Boolean).join(': ') +} + +/** `slack.tool.sendMessage` → `slack.sendMessage`: the tool form names the same builder. */ +function withoutToolSegment(query: string): string { + const path = query.split('.') + const at = path.indexOf(TOOL_NAMESPACE) + if (at > 0 && at === path.length - 2) path.splice(at, 1) + return path.join('.') +} + +/** The reference of the catalog builder `key`, or `null` when the catalog has none. */ +export function describeBuilder(catalog: BlockCatalog, key: string): BuilderReference | null { + const entry = builderEntry(catalog, key) + if (!entry) return null + const types = builderTypes(catalog, entry) + const call = entry.special ? key : (types.signature.at(-1) ?? '') + const signature = entry.special + ? `${call}(name: string, params: ${types.paramsName}, meta?: BlockMeta): ${types.returns}` + : `${entry.path.slice(0, -1).join('.')}.${call.replace(/^export function /, '')}`.replace( + /^\./, + '' + ) + const tool = types.tool + ? `${entry.path.slice(0, -1).join('.')}.${TOOL_NAMESPACE}.${(types.tool.at(-1) ?? '').replace(/^export function /, '')}` + : null + return { + key, + blockType: entry.builder.type, + summary: types.summary, + ...(entry.builder.operation ? { operation: entry.builder.operation } : {}), + legacy: isLegacyBuilder(key), + signature, + tool, + paramsName: types.paramsName, + params: types.facts, + coreParams: entry.special?.coreParams ?? [], + outputName: types.output.name, + outputs: types.output.body === '' ? [] : types.output.body.split('\n'), + } +} + +/** + * Looks `query` up as a builder (`table.upsertRow`, `slack.tool.sendMessage`), a namespace + * (`gmail`, `trigger`: its builders listed), or a construct `core.d.ts` declares (`loop`, `t`); + * otherwise reports the closest builder or namespace as a hint. + */ +export function lookupBuilder(catalog: BlockCatalog, query: string): BuilderLookup { + const key = withoutToolSegment(query.trim()) + const reference = describeBuilder(catalog, key) + if (reference) return { kind: 'builder', reference } + const paths = catalogBuilderPaths(catalog).sort(compareStrings) + const builders = paths + .filter((path) => path.startsWith(`${key}.`)) + .map((path) => ({ key: path, summary: summaryOf(catalog, path) })) + if (builders.length > 0) return { kind: 'namespace', namespace: key, builders } + const root = key.split('.')[0] ?? key + if ((SDK_CORE_EXPORTS as readonly string[]).includes(root)) { + return { kind: 'core', name: key, declaration: coreDeclaration(root) } + } + const hint = suggestBuilder( + key, + paths.filter((path) => !isLegacyBuilder(path)) + ) + return hint ? { kind: 'missing', hint } : { kind: 'missing' } +} + +const MAX_CORE_LINES = 80 + +/** + * Every top-level declaration of `name` in `core.d.ts` (overloads included) with the doc comment + * above it, as the file holds it: what a construct the catalog does not generate takes. + */ +function coreDeclaration(name: string): string { + const lines = CORE_DTS.split('\n') + const start = new RegExp( + `^export (?:declare )?(?:const|function|namespace|interface|type) ${name}\\b` + ) + const out: string[] = [] + for (let index = 0; index < lines.length; index++) { + if (!start.test(lines[index])) continue + let first = index + if (lines[index - 1]?.trim().endsWith('*/')) { + while (first > 0 && !lines[first].trim().startsWith('/**')) first-- + } + let depth = 0 + let end = index + for (; end < lines.length; end++) { + for (const char of lines[end]) { + if ('({['.includes(char)) depth++ + else if (')}]'.includes(char)) depth-- + } + const continues = /[=:,|&]\s*$/.test(lines[end]) || /^\s+[?:|&]/.test(lines[end + 1] ?? '') + if (depth <= 0 && !continues) break + } + if (out.length > 0) out.push('') + out.push(...lines.slice(first, end + 1)) + index = end + } + return out.slice(0, MAX_CORE_LINES).join('\n') +} + +const MAX_SNIPPET_TYPE = 60 +const MAX_SNIPPET_OPTIONAL = 12 + +/** A type short enough for one snippet line. */ +function shortType(type: string): string { + const flat = type.replace(/\s+/g, ' ') + return flat.length > MAX_SNIPPET_TYPE ? `${flat.slice(0, MAX_SNIPPET_TYPE - 1)}…` : flat +} + +/** A value of `type` for an example call: a literal it allows, else a placeholder. */ +function exampleValue(type: string): string { + const literal = /^\s*('(?:[^'\\]|\\.)*'|-?\d+(?:\.\d+)?|true|false)\s*(?:\||$)/.exec(type) + if (literal) return literal[1] + if (/^CredentialRef\b/.test(type)) return "credential('<key>')" + if (/^TableRef\b/.test(type)) return "table('<key>')" + if (/^TableData\b/.test(type)) return "{ column: '…' }" + if (/^KnowledgeBaseRef\b/.test(type)) return "knowledgeBase('<key>')" + if (/^WorkflowRef\b/.test(type)) return "childWorkflow('<key>')" + if (/^(?:Template|string|Value<string>)\b/.test(type)) return "'…'" + if (/^(?:number|Value<number>)\b/.test(type)) return '1' + if (/^(?:boolean|Value<boolean>)\b/.test(type)) return 'true' + if (/^(?:Array<|[\w<>]+\[\])/.test(type)) return '[]' + return `<${shortType(type.split(' | ')[0] ?? type)}>` +} + +/** The names of an output type's top-level members. */ +function outputNames(outputs: readonly string[]): string[] { + return outputs.flatMap((line) => { + const member = /^ {2}(?:readonly )?'?([\w$-]+)'?\??:/.exec(line) + return member ? [member[1]] : [] + }) +} + +function paramLine(param: ParamFact): string { + const fallback = param.default !== undefined ? ` = ${JSON.stringify(param.default)}` : '' + return `${param.key}: ${shortType(param.type)}${fallback}` +} + +/** + * A few lines for the source primer: the call with its required params filled in, the required + * params with their types, the optional ones by name, and the outputs. + */ +export function builderSnippet(reference: BuilderReference): string { + const required = reference.params.common.filter((param) => param.required) + const optional = reference.params.common.filter((param) => !param.required) + const gate = reference.params.gate + const variant = gate + ? (gate.variants.find((v) => gate.default !== undefined && v.values.includes(gate.default)) ?? + gate.variants[0]) + : undefined + const args = [ + ...required.map((param) => `${param.key}: ${exampleValue(param.type)}`), + ...(gate && variant && gate.default === undefined + ? [`${gate.key}: ${JSON.stringify(variant.values[0])}`] + : []), + ...(variant?.params ?? []) + .filter((param) => param.required) + .map((param) => `${param.key}: ${exampleValue(param.type)}`), + ] + const call = `${reference.key}('Name', ${args.length > 0 ? `{ ${args.join(', ')} }` : '{}'})` + const lines = [`- \`${reference.key}\` (${reference.summary}): \`const x = ${call}\``] + if (required.length > 0) lines.push(` required: ${required.map(paramLine).join('; ')}`) + if (gate) { + const variants = gate.variants.map((variant) => { + const needed = variant.params.filter((param) => param.required).map((param) => param.key) + return `${variant.values.map((value) => JSON.stringify(value)).join('|')}${needed.length > 0 ? ` → ${needed.join(', ')}` : ''}` + }) + const fallback = gate.default !== undefined ? ` (default ${JSON.stringify(gate.default)})` : '' + lines.push(` ${gate.key}${fallback} picks the rest: ${variants.join('; ')}`) + } + if (optional.length > 0) { + const shown = optional.slice(0, MAX_SNIPPET_OPTIONAL).map((param) => param.key) + const more = optional.length > shown.length ? `, +${optional.length - shown.length} more` : '' + lines.push(` optional: ${shown.join(', ')}${more}`) + } + if (reference.coreParams.length > 0) { + lines.push(` core.d.ts types: ${reference.coreParams.join(', ')}`) + } + const outputs = outputNames(reference.outputs) + if (outputs.length > 0) lines.push(` outputs: ${outputs.join(', ')}`) + return lines.join('\n') +} + +function describeParam(param: ParamFact, indent: string): string[] { + const fallback = param.default !== undefined ? ` (default ${JSON.stringify(param.default)})` : '' + const head = `${indent}${param.key}${param.required ? '' : '?'}: ${param.type}${fallback}` + return [head, ...param.doc.map((line) => `${indent} ${line}`)] +} + +/** The full text `sim workflows source params <builder>` prints for one builder. */ +export function renderBuilderReference(reference: BuilderReference): string { + const where = [ + `block ${reference.blockType}`, + reference.operation && `operation ${reference.operation}`, + ] + .filter(Boolean) + .join(', ') + const lines = [`${reference.key} — ${reference.summary} (${where})`] + if (reference.legacy) { + lines.push('Legacy: an older block version. Write the current builder instead.') + } + lines.push('', reference.signature) + if (reference.tool) lines.push(`${reference.tool} (as an agent tool)`) + const required = reference.params.common.filter((param) => param.required) + const optional = reference.params.common.filter((param) => !param.required) + lines.push('', `Params (${reference.paramsName}):`) + if (reference.params.common.length === 0 && !reference.params.gate) lines.push(' (none)') + for (const param of [...required, ...optional]) lines.push(...describeParam(param, ' ')) + const gate = reference.params.gate + if (gate) { + const fallback = gate.default !== undefined ? ` (default ${JSON.stringify(gate.default)})` : '' + lines.push(` ${gate.key}${fallback} picks which of these apply:`) + for (const variant of gate.variants) { + lines.push( + ` ${gate.key}: ${variant.values.map((value) => JSON.stringify(value)).join(' | ')}` + ) + for (const param of variant.params) lines.push(...describeParam(param, ' ')) + } + } + if (reference.coreParams.length > 0) { + lines.push(` Also typed in core.d.ts: ${reference.coreParams.join(', ')}`) + } + lines.push('', `Outputs (${reference.outputName}):`) + lines.push(...(reference.outputs.length > 0 ? reference.outputs : [' (none)'])) + return lines.join('\n') +} From b7301b60c2570882eb98f2bd87c73905f2e818ef Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:41:45 -0800 Subject: [PATCH 32/59] docs(workflow-sdk): document $ref.loop and $ref.parallel iteration refs in the cookbook A function code file inside a loop or parallel body reads the item and position as $ref.loop.currentItem / $ref.loop.index ($ref.parallel.* in a parallel); the examples and the code page now say so beside p.currentItem and p.index. --- apps/docs/content/docs/workflows/code.mdx | 2 +- packages/workflow-sdk/cookbook/examples/agents.ts | 6 ++++-- packages/workflow-sdk/cookbook/examples/cookbook.ts | 9 ++++++--- .../workflow-sdk/src/codegen/cookbook-text.generated.ts | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index eca8b571710..749a8d24a84 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -195,7 +195,7 @@ Code that does not fit a block goes in a Function block, kept in its own file: const score = fn('Score lead', { code: codeFile('./functions/score-lead.js') }) ``` -In the file, `$ref.block.path` reads another block's output and `$env.NAME` reads an environment variable. `block` is the block's key: its name lowercased with all whitespace and every `.` removed (`$ref.triageagent.content` for "Triage Agent", `$ref.v2result.text` for "V2. Result"). A `$ref` that names no block is a warning, because it would not resolve when the workflow runs. Python and shell files are saved as written. +In the file, `$ref.block.path` reads another block's output and `$env.NAME` reads an environment variable. `block` is the block's key: its name lowercased with all whitespace and every `.` removed (`$ref.triageagent.content` for "Triage Agent", `$ref.v2result.text` for "V2. Result"). Inside a loop body, `$ref.loop.currentItem` and `$ref.loop.index` read the iteration's item and position; inside a parallel body, `$ref.parallel.currentItem` and `$ref.parallel.index`. A `$ref` that names no block is a warning, because it would not resolve when the workflow runs. Python and shell files are saved as written. ## Use it in CI diff --git a/packages/workflow-sdk/cookbook/examples/agents.ts b/packages/workflow-sdk/cookbook/examples/agents.ts index 3d708abd73e..32e8859d7ea 100644 --- a/packages/workflow-sdk/cookbook/examples/agents.ts +++ b/packages/workflow-sdk/cookbook/examples/agents.ts @@ -64,8 +64,10 @@ export default workflow( inputMapping: { email: start.email }, }) - // Parallel: one branch per item, run at once. p.currentItem is the item; each.results lists - // every branch's output once all finish. (loop.forEach is the sequential form.) + // Parallel: one branch per item, run at once. p.currentItem is the item and p.index its + // position; each.results lists every branch's output once all finish. (loop.forEach is the + // sequential form.) In a function code file inside the body, the same values are + // $ref.parallel.currentItem and $ref.parallel.index. const each = parallel.collection('Each Domain', { collection: start.domains }, (p) => { const brief = agent('Company Brief', { model: 'claude-sonnet-4-6', diff --git a/packages/workflow-sdk/cookbook/examples/cookbook.ts b/packages/workflow-sdk/cookbook/examples/cookbook.ts index bd4acf69498..14ed8d9ed9f 100644 --- a/packages/workflow-sdk/cookbook/examples/cookbook.ts +++ b/packages/workflow-sdk/cookbook/examples/cookbook.ts @@ -103,9 +103,12 @@ export default workflow('Cookbook', { description: 'One example of each common b else: true, }) - // Loop over a list: p.currentItem is one element; the loop's results is the list of - // iteration outputs. In js`` code a reference is the value itself, not text spliced in: write - // it where an expression goes, never inside quotes ('${x}' would give the JSON, quotes included). + // Loop over a list: p.currentItem is one element and p.index its position; the loop's results + // is the list of iteration outputs. In js`` code a reference is the value itself, not text + // spliced in: write it where an expression goes, never inside quotes ('${x}' would give the + // JSON, quotes included). A function code file (code: codeFile('./functions/line.js')) reads + // other blocks as $ref.<block key>.<path> ($ref.recentorders.rows) and, inside the loop body, + // the iteration as $ref.loop.currentItem and $ref.loop.index. const each = loop.forEach('Each Order', { collection: recent.rows }, (p) => { const line = fn('Line', { code: js`return ${p.currentItem.data.orderId} + ': ' + ${p.currentItem.data.status}`, diff --git a/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts b/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts index feaf003c963..dfec30cef72 100644 --- a/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts +++ b/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts @@ -1,4 +1,4 @@ // Generated by scripts/generate-embedded-text.ts from cookbook/examples; do not edit. /** The example workflows, file name → text, written as `.sim/examples/<file>`: together one correct example of each common block. */ -export const COOKBOOK_EXAMPLES: Readonly<Record<string, string>> = {"agents.ts":"// @sim-sdk 1\nimport {\n agent,\n childWorkflow,\n credential,\n customTool,\n env,\n mcp,\n parallel,\n response,\n router,\n runWorkflow,\n slack,\n t,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `lookup-order`, `linear`, `ops-slack`,\n// `enrich-lead` and ANTHROPIC_API_KEY stand in for this workspace\u0027s resources\n// (.sim/sdk/workspace.d.ts; `sim workflows source resources` lists the real keys).\nexport default workflow(\n \u0027Cookbook: agents and subflows\u0027,\n { description: \u0027Router, agent tools, a child workflow and a parallel fan-out.\u0027 },\n () => {\n const start = trigger.start(\u0027Start\u0027, {\n inputFormat: { message: \u0027string\u0027, email: \u0027string\u0027, domains: \u0027array\u0027 },\n })\n\n // Router: the model picks one route by its description; wire with triage.<route>.to(...).\n const triage = router(\u0027Triage\u0027, {\n context: start.message,\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n routes: {\n support: \u0027A customer asking about an existing order\u0027,\n sales: \u0027A prospect asking about pricing or a demo\u0027,\n },\n })\n\n // Agent tools: a workspace custom tool, one tool of an MCP server, an integration operation\n // (<service>.tool.<operation>; the params given here are fixed, the model fills the rest) and\n // another workflow.\n const support = agent(\u0027Support Agent\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Answer the customer. Use your tools when they help.\u0027 },\n { role: \u0027user\u0027, content: start.message },\n ],\n tools: [\n customTool(\u0027lookup-order\u0027),\n mcp(\u0027linear\u0027).tool(\u0027create_issue\u0027),\n slack.tool.sendMessage({ credential: credential(\u0027ops-slack\u0027), channel: \u0027support-urgent\u0027 }),\n childWorkflow(\u0027enrich-lead\u0027).asTool(),\n ],\n })\n\n // Child workflow: inputMapping is checked against the child\u0027s start inputs. Its output is\n // enrich.result (untyped JSON): the child\u0027s response block output, so a field of the child\u0027s\n // response body is enrich.result.data.<field>.\n const enrich = runWorkflow(\u0027Enrich Lead\u0027, {\n workflowId: childWorkflow(\u0027enrich-lead\u0027),\n inputMapping: { email: start.email },\n })\n\n // Parallel: one branch per item, run at once. p.currentItem is the item; each.results lists\n // every branch\u0027s output once all finish. (loop.forEach is the sequential form.)\n const each = parallel.collection(\u0027Each Domain\u0027, { collection: start.domains }, (p) => {\n const brief = agent(\u0027Company Brief\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Describe ${p.currentItem} in two sentences.` }],\n })\n p.to(brief)\n })\n\n const answer = response(\u0027Answer\u0027, { body: { answer: support.content } })\n const briefs = response(\u0027Briefs\u0027, {\n body: { lead: t`${enrich.result}`, briefs: t`${each.results}` },\n })\n\n start.to(triage)\n triage.support.to(support)\n support.to(answer)\n triage.sales.to(enrich)\n enrich.to(each)\n each.to(briefs)\n }\n)\n","cookbook.ts":"// @sim-sdk 1\nimport {\n agent,\n condition,\n credential,\n env,\n fn,\n human,\n js,\n knowledge,\n knowledgeBase,\n loop,\n response,\n slack,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `orders`, `policies`, `ops-slack` and\n// ANTHROPIC_API_KEY stand in for this workspace\u0027s resources. `sim workflows source resources`\n// lists the real keys, and .sim/sdk/workspace.d.ts types them.\nexport default workflow(\u0027Cookbook\u0027, { description: \u0027One example of each common block.\u0027 }, () => {\n // Triggers: trigger.start (API/manual input), trigger.schedule, trigger.webhook.\n const start = trigger.start(\u0027Start\u0027, { inputFormat: { orderId: \u0027string\u0027, amount: \u0027number\u0027 } })\n const nightly = trigger.schedule(\u0027Nightly\u0027, {\n scheduleType: \u0027daily\u0027,\n dailyTime: \u002702:00\u0027,\n timezone: \u0027UTC\u0027,\n })\n\n // Agent: structured output via responseFormat; its fields become outputs (check.risk).\n // apiKey: needed on self-hosted Sim; hosted Sim supplies one.\n const check = agent(\u0027Check Order\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Rate the fraud risk of an order as low, med or high.\u0027 },\n { role: \u0027user\u0027, content: t`Order ${start.orderId} for ${start.amount}` },\n ],\n responseFormat: {\n name: \u0027risk\u0027,\n strict: true,\n schema: {\n type: \u0027object\u0027,\n properties: {\n risk: { type: \u0027string\u0027, enum: [\u0027low\u0027, \u0027med\u0027, \u0027high\u0027] },\n reason: { type: \u0027string\u0027 },\n },\n required: [\u0027risk\u0027, \u0027reason\u0027],\n additionalProperties: false,\n },\n },\n })\n\n // Condition: one branch per key, `else: true` last; wire with route.<key>.to(...).\n const route = condition(\u0027Route Risk\u0027, {\n high: js`${check.risk} === \u0027high\u0027`,\n med: js`${check.risk} === \u0027med\u0027`,\n else: true,\n })\n\n // Function: inline JavaScript; reference outputs with ${...}. Result is fn.result.\n const shape = fn(\u0027Shape\u0027, {\n code: js`return { id: ${start.orderId}, cents: Math.round(${start.amount} * 100) }`,\n })\n\n // Tables: query rows (each row is { id, data: { column } }) and insert a row.\n const recent = table.queryRows(\u0027Recent Orders\u0027, {\n tableId: table(\u0027orders\u0027),\n filterInput: { all: [{ field: \u0027status\u0027, op: \u0027eq\u0027, value: \u0027open\u0027 }] },\n limit: 50,\n })\n const save = table.insertRow(\u0027Save Order\u0027, {\n tableId: table(\u0027orders\u0027),\n data: { orderId: start.orderId, status: \u0027open\u0027, risk: check.risk },\n })\n\n // Knowledge base search: results is the list of matching chunks.\n const docs = knowledge.search(\u0027Policy Search\u0027, {\n knowledgeBaseId: knowledgeBase(\u0027policies\u0027),\n query: check.reason,\n topK: 3,\n })\n\n // Slack: a saved credential by key (see .sim/sdk/workspace.d.ts).\n const alert = slack.sendMessage(\u0027Alert Ops\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-alerts\u0027,\n messageFormat: \u0027text\u0027,\n text: t`High-risk order ${start.orderId}: ${check.reason}`,\n })\n\n // Human in the loop: inputFormat is the form; answers are approval.submission.<field>.\n const approval = human(\u0027Manager Review\u0027, {\n builderData: [{ name: \u0027orderId\u0027, type: \u0027string\u0027, value: start.orderId }],\n inputFormat: { approved: { type: \u0027boolean\u0027, required: true }, note: \u0027string\u0027 },\n })\n const decided = condition(\u0027Approved\u0027, {\n yes: js`${approval.submission.approved} === true`,\n else: true,\n })\n\n // Loop over a list: p.currentItem is one element; the loop\u0027s results is the list of\n // iteration outputs. In js`` code a reference is the value itself, not text spliced in: write\n // it where an expression goes, never inside quotes (\u0027${x}\u0027 would give the JSON, quotes included).\n const each = loop.forEach(\u0027Each Order\u0027, { collection: recent.rows }, (p) => {\n const line = fn(\u0027Line\u0027, {\n code: js`return ${p.currentItem.data.orderId} + \u0027: \u0027 + ${p.currentItem.data.status}`,\n })\n p.to(line)\n })\n const report = agent(\u0027Report\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Summarize these orders in 3 lines:\\n${each.results}` }],\n })\n const post = slack.sendMessage(\u0027Post Report\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-daily\u0027,\n messageFormat: \u0027text\u0027,\n text: report.content,\n })\n\n // Response: the API reply body. Wrap an untyped value (fn.result.x) in t`` when it is text.\n const reply = response(\u0027Respond\u0027, { body: { risk: check.risk, id: t`${shape.result.id}` } })\n const rejected = response(\u0027Respond Rejected\u0027, { body: { status: \u0027rejected\u0027 } })\n\n // Wiring: a.to(b) chains; a block can fan out to several targets.\n start.to(check)\n check.to(save)\n save.to(route)\n route.high.to(alert)\n alert.to(approval)\n approval.to(decided)\n decided.yes.to(shape)\n decided.else.to(rejected)\n route.med.to(docs)\n docs.to(shape)\n route.else.to(shape)\n shape.to(reply)\n nightly.to(recent)\n recent.to(each)\n each.to(report)\n report.to(post)\n})\n","integrations.ts":"// @sim-sdk 1\nimport {\n api,\n credential,\n env,\n gmail,\n response,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `leads`, `ops-gmail`, CLEARBIT_KEY and\n// LEADS_WEBHOOK_TOKEN stand in for this workspace\u0027s resources (.sim/sdk/workspace.d.ts).\n// Integration builders are named by service, not block type: `gmail_v2` is `gmail.*`, and a\n// hidden older version lives under `legacy.*`. .sim/sdk/INDEX.md lists every builder.\nexport default workflow(\n \u0027Cookbook: integrations\u0027,\n { description: \u0027Webhook, weekday schedule, HTTP with headers, table update, Gmail, error path.\u0027 },\n () => {\n // Webhook trigger: inputFormat types the posted JSON fields (hook.leadId). With requireAuth\n // the caller must send the token; keep it in an env var.\n const hook = trigger.webhook(\u0027Lead Webhook\u0027, {\n requireAuth: true,\n token: env.LEADS_WEBHOOK_TOKEN,\n inputFormat: { leadId: \u0027string\u0027, email: \u0027string\u0027, domain: \u0027string\u0027 },\n })\n\n // Schedule: no weekday preset exists, so scheduleType \u0027custom\u0027 takes a 5-field cron.\n const weekdays = trigger.schedule(\u0027Weekdays 8am\u0027, {\n scheduleType: \u0027custom\u0027,\n cronExpression: \u00270 8 * * 1-5\u0027,\n timezone: \u0027America/New_York\u0027,\n })\n\n // HTTP request: headers (and query params) are table rows, { cells: { Key, Value } }.\n const lookup = api(\u0027Company Lookup\u0027, {\n url: t`https://company.clearbit.com/v2/companies/find?domain=${hook.domain}`,\n method: \u0027GET\u0027,\n headers: [{ cells: { Key: \u0027Authorization\u0027, Value: t`Bearer ${env.CLEARBIT_KEY}` } }],\n })\n\n // Update one row by id: data holds only the columns to change. lookup.data is untyped JSON,\n // so a text field from it is wrapped in t``.\n const save = table.updateRowById(\u0027Save Company\u0027, {\n tableId: table(\u0027leads\u0027),\n rowId: hook.leadId,\n data: { company: t`${lookup.data.name}`, enriched: true },\n })\n\n // Gmail (gmail_v2): send one email with a saved Google credential.\n const welcome = gmail.sendEmail(\u0027Welcome Email\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: hook.email,\n subject: \u0027Welcome aboard\u0027,\n body: t`Thanks for signing up from ${hook.domain}.`,\n })\n\n // Query rows, then email them as one digest. A typed column filter: { field, op, value }.\n const pending = table.queryRows(\u0027Pending Leads\u0027, {\n tableId: table(\u0027leads\u0027),\n filterInput: { all: [{ field: \u0027enriched\u0027, op: \u0027eq\u0027, value: false }] },\n limit: 100,\n })\n const digest = gmail.sendEmail(\u0027Pending Digest\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: \u0027ops@example.com\u0027,\n subject: \u0027Leads still to enrich\u0027,\n body: t`${pending.rows}`,\n })\n\n // Error path: x.onError(y) runs y when x fails, instead of failing the run; y reads the\n // failure message as x.error.\n const failed = response(\u0027Lookup Failed\u0027, {\n body: { ok: false, error: lookup.error },\n status: 502,\n })\n const done = response(\u0027Done\u0027, { body: { ok: true, row: save.row } })\n\n hook.to(lookup)\n lookup.to(save)\n lookup.onError(failed)\n save.to(welcome)\n welcome.to(done)\n weekdays.to(pending)\n pending.to(digest)\n }\n)\n"} +export const COOKBOOK_EXAMPLES: Readonly<Record<string, string>> = {"agents.ts":"// @sim-sdk 1\nimport {\n agent,\n childWorkflow,\n credential,\n customTool,\n env,\n mcp,\n parallel,\n response,\n router,\n runWorkflow,\n slack,\n t,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `lookup-order`, `linear`, `ops-slack`,\n// `enrich-lead` and ANTHROPIC_API_KEY stand in for this workspace\u0027s resources\n// (.sim/sdk/workspace.d.ts; `sim workflows source resources` lists the real keys).\nexport default workflow(\n \u0027Cookbook: agents and subflows\u0027,\n { description: \u0027Router, agent tools, a child workflow and a parallel fan-out.\u0027 },\n () => {\n const start = trigger.start(\u0027Start\u0027, {\n inputFormat: { message: \u0027string\u0027, email: \u0027string\u0027, domains: \u0027array\u0027 },\n })\n\n // Router: the model picks one route by its description; wire with triage.<route>.to(...).\n const triage = router(\u0027Triage\u0027, {\n context: start.message,\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n routes: {\n support: \u0027A customer asking about an existing order\u0027,\n sales: \u0027A prospect asking about pricing or a demo\u0027,\n },\n })\n\n // Agent tools: a workspace custom tool, one tool of an MCP server, an integration operation\n // (<service>.tool.<operation>; the params given here are fixed, the model fills the rest) and\n // another workflow.\n const support = agent(\u0027Support Agent\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Answer the customer. Use your tools when they help.\u0027 },\n { role: \u0027user\u0027, content: start.message },\n ],\n tools: [\n customTool(\u0027lookup-order\u0027),\n mcp(\u0027linear\u0027).tool(\u0027create_issue\u0027),\n slack.tool.sendMessage({ credential: credential(\u0027ops-slack\u0027), channel: \u0027support-urgent\u0027 }),\n childWorkflow(\u0027enrich-lead\u0027).asTool(),\n ],\n })\n\n // Child workflow: inputMapping is checked against the child\u0027s start inputs. Its output is\n // enrich.result (untyped JSON): the child\u0027s response block output, so a field of the child\u0027s\n // response body is enrich.result.data.<field>.\n const enrich = runWorkflow(\u0027Enrich Lead\u0027, {\n workflowId: childWorkflow(\u0027enrich-lead\u0027),\n inputMapping: { email: start.email },\n })\n\n // Parallel: one branch per item, run at once. p.currentItem is the item and p.index its\n // position; each.results lists every branch\u0027s output once all finish. (loop.forEach is the\n // sequential form.) In a function code file inside the body, the same values are\n // $ref.parallel.currentItem and $ref.parallel.index.\n const each = parallel.collection(\u0027Each Domain\u0027, { collection: start.domains }, (p) => {\n const brief = agent(\u0027Company Brief\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Describe ${p.currentItem} in two sentences.` }],\n })\n p.to(brief)\n })\n\n const answer = response(\u0027Answer\u0027, { body: { answer: support.content } })\n const briefs = response(\u0027Briefs\u0027, {\n body: { lead: t`${enrich.result}`, briefs: t`${each.results}` },\n })\n\n start.to(triage)\n triage.support.to(support)\n support.to(answer)\n triage.sales.to(enrich)\n enrich.to(each)\n each.to(briefs)\n }\n)\n","cookbook.ts":"// @sim-sdk 1\nimport {\n agent,\n condition,\n credential,\n env,\n fn,\n human,\n js,\n knowledge,\n knowledgeBase,\n loop,\n response,\n slack,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `orders`, `policies`, `ops-slack` and\n// ANTHROPIC_API_KEY stand in for this workspace\u0027s resources. `sim workflows source resources`\n// lists the real keys, and .sim/sdk/workspace.d.ts types them.\nexport default workflow(\u0027Cookbook\u0027, { description: \u0027One example of each common block.\u0027 }, () => {\n // Triggers: trigger.start (API/manual input), trigger.schedule, trigger.webhook.\n const start = trigger.start(\u0027Start\u0027, { inputFormat: { orderId: \u0027string\u0027, amount: \u0027number\u0027 } })\n const nightly = trigger.schedule(\u0027Nightly\u0027, {\n scheduleType: \u0027daily\u0027,\n dailyTime: \u002702:00\u0027,\n timezone: \u0027UTC\u0027,\n })\n\n // Agent: structured output via responseFormat; its fields become outputs (check.risk).\n // apiKey: needed on self-hosted Sim; hosted Sim supplies one.\n const check = agent(\u0027Check Order\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Rate the fraud risk of an order as low, med or high.\u0027 },\n { role: \u0027user\u0027, content: t`Order ${start.orderId} for ${start.amount}` },\n ],\n responseFormat: {\n name: \u0027risk\u0027,\n strict: true,\n schema: {\n type: \u0027object\u0027,\n properties: {\n risk: { type: \u0027string\u0027, enum: [\u0027low\u0027, \u0027med\u0027, \u0027high\u0027] },\n reason: { type: \u0027string\u0027 },\n },\n required: [\u0027risk\u0027, \u0027reason\u0027],\n additionalProperties: false,\n },\n },\n })\n\n // Condition: one branch per key, `else: true` last; wire with route.<key>.to(...).\n const route = condition(\u0027Route Risk\u0027, {\n high: js`${check.risk} === \u0027high\u0027`,\n med: js`${check.risk} === \u0027med\u0027`,\n else: true,\n })\n\n // Function: inline JavaScript; reference outputs with ${...}. Result is fn.result.\n const shape = fn(\u0027Shape\u0027, {\n code: js`return { id: ${start.orderId}, cents: Math.round(${start.amount} * 100) }`,\n })\n\n // Tables: query rows (each row is { id, data: { column } }) and insert a row.\n const recent = table.queryRows(\u0027Recent Orders\u0027, {\n tableId: table(\u0027orders\u0027),\n filterInput: { all: [{ field: \u0027status\u0027, op: \u0027eq\u0027, value: \u0027open\u0027 }] },\n limit: 50,\n })\n const save = table.insertRow(\u0027Save Order\u0027, {\n tableId: table(\u0027orders\u0027),\n data: { orderId: start.orderId, status: \u0027open\u0027, risk: check.risk },\n })\n\n // Knowledge base search: results is the list of matching chunks.\n const docs = knowledge.search(\u0027Policy Search\u0027, {\n knowledgeBaseId: knowledgeBase(\u0027policies\u0027),\n query: check.reason,\n topK: 3,\n })\n\n // Slack: a saved credential by key (see .sim/sdk/workspace.d.ts).\n const alert = slack.sendMessage(\u0027Alert Ops\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-alerts\u0027,\n messageFormat: \u0027text\u0027,\n text: t`High-risk order ${start.orderId}: ${check.reason}`,\n })\n\n // Human in the loop: inputFormat is the form; answers are approval.submission.<field>.\n const approval = human(\u0027Manager Review\u0027, {\n builderData: [{ name: \u0027orderId\u0027, type: \u0027string\u0027, value: start.orderId }],\n inputFormat: { approved: { type: \u0027boolean\u0027, required: true }, note: \u0027string\u0027 },\n })\n const decided = condition(\u0027Approved\u0027, {\n yes: js`${approval.submission.approved} === true`,\n else: true,\n })\n\n // Loop over a list: p.currentItem is one element and p.index its position; the loop\u0027s results\n // is the list of iteration outputs. In js`` code a reference is the value itself, not text\n // spliced in: write it where an expression goes, never inside quotes (\u0027${x}\u0027 would give the\n // JSON, quotes included). A function code file (code: codeFile(\u0027./functions/line.js\u0027)) reads\n // other blocks as $ref.<block key>.<path> ($ref.recentorders.rows) and, inside the loop body,\n // the iteration as $ref.loop.currentItem and $ref.loop.index.\n const each = loop.forEach(\u0027Each Order\u0027, { collection: recent.rows }, (p) => {\n const line = fn(\u0027Line\u0027, {\n code: js`return ${p.currentItem.data.orderId} + \u0027: \u0027 + ${p.currentItem.data.status}`,\n })\n p.to(line)\n })\n const report = agent(\u0027Report\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Summarize these orders in 3 lines:\\n${each.results}` }],\n })\n const post = slack.sendMessage(\u0027Post Report\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-daily\u0027,\n messageFormat: \u0027text\u0027,\n text: report.content,\n })\n\n // Response: the API reply body. Wrap an untyped value (fn.result.x) in t`` when it is text.\n const reply = response(\u0027Respond\u0027, { body: { risk: check.risk, id: t`${shape.result.id}` } })\n const rejected = response(\u0027Respond Rejected\u0027, { body: { status: \u0027rejected\u0027 } })\n\n // Wiring: a.to(b) chains; a block can fan out to several targets.\n start.to(check)\n check.to(save)\n save.to(route)\n route.high.to(alert)\n alert.to(approval)\n approval.to(decided)\n decided.yes.to(shape)\n decided.else.to(rejected)\n route.med.to(docs)\n docs.to(shape)\n route.else.to(shape)\n shape.to(reply)\n nightly.to(recent)\n recent.to(each)\n each.to(report)\n report.to(post)\n})\n","integrations.ts":"// @sim-sdk 1\nimport {\n api,\n credential,\n env,\n gmail,\n response,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `leads`, `ops-gmail`, CLEARBIT_KEY and\n// LEADS_WEBHOOK_TOKEN stand in for this workspace\u0027s resources (.sim/sdk/workspace.d.ts).\n// Integration builders are named by service, not block type: `gmail_v2` is `gmail.*`, and a\n// hidden older version lives under `legacy.*`. .sim/sdk/INDEX.md lists every builder.\nexport default workflow(\n \u0027Cookbook: integrations\u0027,\n { description: \u0027Webhook, weekday schedule, HTTP with headers, table update, Gmail, error path.\u0027 },\n () => {\n // Webhook trigger: inputFormat types the posted JSON fields (hook.leadId). With requireAuth\n // the caller must send the token; keep it in an env var.\n const hook = trigger.webhook(\u0027Lead Webhook\u0027, {\n requireAuth: true,\n token: env.LEADS_WEBHOOK_TOKEN,\n inputFormat: { leadId: \u0027string\u0027, email: \u0027string\u0027, domain: \u0027string\u0027 },\n })\n\n // Schedule: no weekday preset exists, so scheduleType \u0027custom\u0027 takes a 5-field cron.\n const weekdays = trigger.schedule(\u0027Weekdays 8am\u0027, {\n scheduleType: \u0027custom\u0027,\n cronExpression: \u00270 8 * * 1-5\u0027,\n timezone: \u0027America/New_York\u0027,\n })\n\n // HTTP request: headers (and query params) are table rows, { cells: { Key, Value } }.\n const lookup = api(\u0027Company Lookup\u0027, {\n url: t`https://company.clearbit.com/v2/companies/find?domain=${hook.domain}`,\n method: \u0027GET\u0027,\n headers: [{ cells: { Key: \u0027Authorization\u0027, Value: t`Bearer ${env.CLEARBIT_KEY}` } }],\n })\n\n // Update one row by id: data holds only the columns to change. lookup.data is untyped JSON,\n // so a text field from it is wrapped in t``.\n const save = table.updateRowById(\u0027Save Company\u0027, {\n tableId: table(\u0027leads\u0027),\n rowId: hook.leadId,\n data: { company: t`${lookup.data.name}`, enriched: true },\n })\n\n // Gmail (gmail_v2): send one email with a saved Google credential.\n const welcome = gmail.sendEmail(\u0027Welcome Email\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: hook.email,\n subject: \u0027Welcome aboard\u0027,\n body: t`Thanks for signing up from ${hook.domain}.`,\n })\n\n // Query rows, then email them as one digest. A typed column filter: { field, op, value }.\n const pending = table.queryRows(\u0027Pending Leads\u0027, {\n tableId: table(\u0027leads\u0027),\n filterInput: { all: [{ field: \u0027enriched\u0027, op: \u0027eq\u0027, value: false }] },\n limit: 100,\n })\n const digest = gmail.sendEmail(\u0027Pending Digest\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: \u0027ops@example.com\u0027,\n subject: \u0027Leads still to enrich\u0027,\n body: t`${pending.rows}`,\n })\n\n // Error path: x.onError(y) runs y when x fails, instead of failing the run; y reads the\n // failure message as x.error.\n const failed = response(\u0027Lookup Failed\u0027, {\n body: { ok: false, error: lookup.error },\n status: 502,\n })\n const done = response(\u0027Done\u0027, { body: { ok: true, row: save.row } })\n\n hook.to(lookup)\n lookup.to(save)\n lookup.onError(failed)\n save.to(welcome)\n welcome.to(done)\n weekdays.to(pending)\n pending.to(digest)\n }\n)\n"} From 7ca43a82f1bb76cf8bf442268d2e3e6f3032476c Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:46:41 -0800 Subject: [PATCH 33/59] feat(sim-cli): source get prints a primer, which opens AGENTS.md, with snippets for the builders in use The primer (under 3 KB) carries the edit loop, block identity (the lock binds a block by its const or its display name, so a rename keeps the id), $ref in code files with loop and parallel items, deletes, deploy and the exit codes, plus the params and outputs of each builder the source calls. get prints it (JSON: a primer field) so an agent needs no other read; AGENTS.md is the primer then a task-ordered reference, and the workspace guide's deletes, Git/CI and promote deep dives move to .sim/GUIDE-advanced.md. --- apps/docs/content/docs/workflows/code.mdx | 6 +- .../content/docs/workflows/workspace-code.mdx | 2 +- .../src/commands/protocol/source-primer.ts | 111 ++++++++++++++++++ .../src/commands/protocol/source-tree.ts | 57 +++++++-- .../protocol/workflow-source-apply.ts | 4 +- .../protocol/workflow-source-get.test.ts | 21 ++++ .../commands/protocol/workflow-source-get.ts | 67 ++++++----- .../commands/protocol/workspace-source-dir.ts | 58 +++++---- .../protocol/workspace-source-get.test.ts | 15 +++ .../commands/protocol/workspace-source-get.ts | 16 ++- packages/workflow-sdk/README.md | 2 +- packages/workflow-sdk/src/codegen/index.ts | 1 + packages/workflow-sdk/src/codegen/layout.ts | 29 +++-- 13 files changed, 303 insertions(+), 86 deletions(-) create mode 100644 packages/sim-cli/src/commands/protocol/source-primer.ts diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index 749a8d24a84..124660cfdc8 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -13,12 +13,14 @@ Workflows as code is in preview. When it is [enabled on your Sim deployment](/pl ## For coding agents [#for-coding-agents] -Claude Code, Cursor, Codex and other coding agents work from a directory that `get` writes. Point the agent at `.sim/AGENTS.md` in it, which walks through the loop: +Claude Code, Cursor, Codex and other coding agents work from a directory that `get` writes. `get` prints a short primer (the edit loop, how block identity and references work, deletes, deploy, exit codes, and the params of each builder the workflow already uses; in JSON output, its `primer` field), and the same primer opens `.sim/AGENTS.md`, the full guide. The loop: 1. `sim workflows source get <workflowId> --dir <dir>` writes the workflow and the guide. 2. Edit `workflow.ts`. 3. `sim workflows source check --dir <dir>` compiles it offline. 4. `sim workflows source status --dir <dir>` shows what changed and what to run next. + +To write a block the workflow does not use yet, `sim workflows source params <builder>` (for example `table.upsertRow` or `trigger.schedule`) prints its params, required ones first with their options and defaults, and its outputs, from the cached block catalog without a request. A namespace such as `gmail` lists its builders. 5. `sim workflows source apply --dir <dir> --dry-run`, then `--yes`, saves the draft. For a whole workspace, use `sim workspaces source` with the same steps; see [Workspace as code](/workflows/workspace-code). @@ -37,7 +39,7 @@ Nothing in the file runs on your machine. Sim reads it statically, checks it aga sim workflows source get <workflowId> --dir ./support-triage ``` - This writes `workflow.ts`, any function code under `functions/`, and type declarations for every block and for your workspace's tables, credentials, knowledge bases, and environment variable names, with `.sim/sdk/INDEX.md` listing every builder by block and operation. It also writes a `tsconfig.json` that resolves them and type-checks the blocks the workflow uses, and a `tsconfig.full.json` that type-checks every block, unless the directory already has files of those names, which are left as they are. For coding agents it writes `.sim/AGENTS.md`, a short guide, and example workflows under `.sim/examples/` with one correct example of each common block to copy from. The examples name placeholder resources, so the tsconfig files leave them out of the type-check. + This writes `workflow.ts`, any function code under `functions/`, and type declarations for every block and for your workspace's tables, credentials, knowledge bases, and environment variable names, with `.sim/sdk/INDEX.md` listing every builder by block and operation. It also writes a `tsconfig.json` that resolves them and type-checks the blocks the workflow uses, and a `tsconfig.full.json` that type-checks every block, unless the directory already has files of those names, which are left as they are. For coding agents it writes `.sim/AGENTS.md`, the primer `get` prints followed by a reference, and example workflows under `.sim/examples/` with one correct example of each common block to copy from. The examples name placeholder resources, so the tsconfig files leave them out of the type-check. `get` into a directory that already holds source refuses to overwrite edits you have not applied, and refuses a directory that holds a different workflow. Pass `--force` to replace them. diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index bdbbc7e1df5..ab5695b7a3e 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -48,7 +48,7 @@ const log = table.insertRow('Log Question', { }) ``` -`get` writes `.sim/AGENTS.md`, a guide for coding agents, and `.sim/examples/workspace/`, a complete example of every kind and the workflow that uses them. +`get` prints a short primer for coding agents (in JSON output, its `primer` field) and writes it atop `.sim/AGENTS.md`, the guide, with deletes in depth, Git and CI, and promoting to another workspace in `.sim/GUIDE-advanced.md`. It also writes `.sim/examples/workspace/`, a complete example of every kind and the workflow that uses them. ## Edit a workspace from the CLI diff --git a/packages/sim-cli/src/commands/protocol/source-primer.ts b/packages/sim-cli/src/commands/protocol/source-primer.ts new file mode 100644 index 00000000000..56407374728 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-primer.ts @@ -0,0 +1,111 @@ +/** + * The primer a source get prints and writes atop `.sim/AGENTS.md`: the few rules an agent needs to + * edit a source directory without reading the guide, plus a snippet for each builder the source + * already calls, so the blocks in front of it need no lookup. The guide below it is the reference. + */ +import type { BlockCatalog } from '@sim/workflow-code' +import { SOURCE_BASE_FILE, SOURCE_ENTRY_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' +import { SDK_SPECIAL_BUILDERS } from '@sim/workflow-code/sdk' +import { + builderSnippet, + describeBuilder, + referencedBuilders, + typecheckCommand, +} from '@sim/workflow-sdk/codegen' +import { + RATE_LIMITED_EXIT_CODE, + SOURCE_BUSY_EXIT_CODE, + SOURCE_DRIFT_EXIT_CODE, + SOURCE_LOCAL_CHANGES_EXIT_CODE, + SOURCE_STALE_EXIT_CODE, +} from '../../exit-codes' +import { SOURCE_AGENTS_FILE } from './source-dir' + +/** Which directory a primer is for. */ +export type PrimerKind = 'workflow' | 'workspace' + +/** The primer without its builder snippets stays within this many bytes. */ +export const PRIMER_MAX_BYTES = 3072 + +/** At most this many snippets, and this many bytes of them; the rest are named for `params`. */ +const MAX_SNIPPETS = 12 +const MAX_SNIPPET_BYTES = 4096 + +/** The command that prints any builder's params (`workflow-source-params.ts`). */ +const PARAMS_COMMAND = 'sim workflows source params' + +/** The rules every source directory shares, after the kind's own edit loop. */ +function sharedRules(group: string): string { + return `- **Identity**: a block is its entry in \`${SOURCE_LOCK_FILE}\`, which binds it by its const name or its display name. Change the display name (first argument) freely: the block keeps its id. Never change the const and the name in one apply (change one, apply, then the other). +- **References**: in \`${SOURCE_ENTRY_FILE}\` read an output as \`x.content\`, a loop or parallel item as \`p.currentItem\` (\`p.index\`), text as \`\` t\`…\` \`\`, env as \`env.NAME\`. In a function code file (\`codeFile('./functions/x.js')\`) read an output as \`$ref.<key>.<path>\`, the key being the display name lowercased with whitespace and dots removed ("Triage Agent" → \`$ref.triageagent.content\`); in a loop body \`$ref.loop.currentItem\` / \`$ref.loop.index\`, in a parallel \`$ref.parallel.currentItem\` / \`$ref.parallel.index\`; env as \`$env.NAME\`. +- **Look up a builder**: \`${PARAMS_COMMAND} <builder>\` (\`table.upsertRow\`, \`agent\`, \`trigger.schedule\`; a namespace such as \`gmail\` lists its builders), offline. +- **Exit codes**: 0 ok, 1 errors (fix what it printed), ${SOURCE_STALE_EXIT_CODE} stale (\`${group} get --dir <fresh dir>\`, redo the edit there), ${RATE_LIMITED_EXIT_CODE} rate limited, ${SOURCE_BUSY_EXIT_CODE} busy, ${SOURCE_LOCAL_CHANGES_EXIT_CODE}/${SOURCE_DRIFT_EXIT_CODE} \`status\`: your edits / the server changed. Every code: \`${SOURCE_AGENTS_FILE}\`.` +} + +const WORKFLOW_PRIMER = `# Sim workflow source: primer + +\`${SOURCE_ENTRY_FILE}\` is a Sim workflow as TypeScript; the server compiles it statically and nothing in it runs. Commands read the workflow id from \`${SOURCE_BASE_FILE}\`; run them from this directory. + +**Edit loop**: edit \`${SOURCE_ENTRY_FILE}\` (and \`functions/*\`) → \`sim workflows source check --dir .\` (offline; it also adds new blocks to \`tsconfig.json\`) → \`${typecheckCommand()}\` → \`sim workflows source apply --dir . --dry-run\` → \`--yes\`. Rerun each step until it exits 0. Apply saves the draft in canonical form: read \`${SOURCE_ENTRY_FILE}\` again before the next edit. + +- **Deletes**: removing a block's declaration deletes the block only with \`apply --allow-delete\`; without it the apply is refused (\`E_APPLY_BLOCK_DELETE\`). +- **Deploy**: apply never deploys. \`sim workflows deploy <workflowId>\` publishes the saved draft. +${sharedRules('sim workflows source')} +` + +const WORKSPACE_PRIMER = `# Sim workspace source: primer + +This directory is a Sim workspace as files (\`tables/\`, \`workflows/<key>/workflow.ts\`, …); the server reads them statically and nothing runs. Commands read the workspace id from \`${SOURCE_BASE_FILE}\`; run them from this directory. + +**Edit loop**: edit the files → \`sim workspaces source check --dir .\` (offline; regenerates \`.sim/sdk/workspace.d.ts\`, adds new blocks to \`tsconfig.json\`) → \`${typecheckCommand()}\` → \`sim workspaces source apply --dir . --dry-run\` (the plan) → \`--yes\`. Rerun each step until it exits 0. Apply rewrites the directory from what it stored: read a file again before the next edit. + +- **Deletes**: a removed file (a resource this directory managed), table column or workflow block is deleted only with \`apply --allow-delete\`; without it the apply is refused. A kind is managed only when its folder exists. Deletes cannot be undone. +- **Deploy**: \`deployments/<workflow key>.ts\` (\`defineDeployment({ live: 'draft' })\`) deploys that workflow on apply; without one, apply never deploys. +${sharedRules('sim workspaces source')} +` + +/** The primer for a `kind` directory, without builder snippets. */ +export function sourcePrimer(kind: PrimerKind): string { + return kind === 'workflow' ? WORKFLOW_PRIMER : WORKSPACE_PRIMER +} + +/** + * A snippet for each builder `sources` call ({@link builderSnippet}), integration builders before + * the core ones (`agent`, `fn`), whose shape the source shows. Over the cap, the rest are named + * for `params`. Empty when the source calls none. + */ +function builderSnippets(catalog: BlockCatalog, sources: Iterable<string>): string { + const core = (key: string) => Object.hasOwn(SDK_SPECIAL_BUILDERS, key) || key === 'agent' + const keys = referencedBuilders(catalog, sources) + const ordered = [...keys.filter((key) => !core(key)), ...keys.filter(core)] + const snippets: string[] = [] + const skipped: string[] = [] + let bytes = 0 + for (const key of ordered) { + const reference = describeBuilder(catalog, key) + if (!reference) continue + const snippet = builderSnippet(reference) + if (snippets.length >= MAX_SNIPPETS || bytes + snippet.length > MAX_SNIPPET_BYTES) { + skipped.push(key) + continue + } + snippets.push(snippet) + bytes += snippet.length + } + if (snippets.length === 0) return '' + const rest = + skipped.length > 0 + ? `\nAlso used (\`${PARAMS_COMMAND} <builder>\`): ${skipped.map((key) => `\`${key}\``).join(', ')}.\n` + : '' + return `## Builders this source uses\n\n${snippets.join('\n')}\n${rest}` +} + +/** The primer with the snippets of the builders `sources` call. */ +export function primerFor( + kind: PrimerKind, + catalog: BlockCatalog, + sources: Iterable<string> +): string { + const snippets = builderSnippets(catalog, sources) + return snippets ? `${sourcePrimer(kind)}\n${snippets}` : sourcePrimer(kind) +} diff --git a/packages/sim-cli/src/commands/protocol/source-tree.ts b/packages/sim-cli/src/commands/protocol/source-tree.ts index 9ef93cb14dc..9c85ab2e52c 100644 --- a/packages/sim-cli/src/commands/protocol/source-tree.ts +++ b/packages/sim-cli/src/commands/protocol/source-tree.ts @@ -26,7 +26,8 @@ import { sdkIndexHeader, type WorkspaceManifest, } from '@sim/workflow-sdk/codegen' -import { printError } from '#sim-cli/output/io' +import { printError, printLine } from '#sim-cli/output/io' +import type { OutputFormat } from '../../config/index' import { readJsonFile, writeJsonFile } from '../../config/json-file' import { configDir } from '../../config/paths' import { embedStore } from '../../embed-context' @@ -35,6 +36,7 @@ import type { GetSourceResourcesResponse, GetWorkflowSourceResponse } from '../. import { SimApiError } from '../../http/client' import type { OperationClient } from '../../runtime/called-operations' import { cliVersion } from '../../version' +import { printProtocolResult } from './result' import { readSourceJson, readSourceText, @@ -47,6 +49,7 @@ import { updateSourceDir, withMissingLines, } from './source-dir' +import { type PrimerKind, primerFor } from './source-primer' const TSCONFIG_FILE = 'tsconfig.json' /** Type-checks the source against every current block, for a block the default one leaves out. */ @@ -434,6 +437,8 @@ export interface SourceTree { workspaceTypes: boolean /** The agent guide to read before editing. */ guide: typeof SOURCE_AGENTS_FILE + /** The primer atop the guide (`source-primer.ts`), printed so an agent needs no other read. */ + primer: string } /** Reports a `tsconfig.json` the CLI left alone and summarizes a tree write. */ @@ -441,20 +446,46 @@ export function finishTree( dir: string, written: number, workspaceTypes: WorkspaceTypes | null, - ownsTsconfig: boolean + tree: { ownsTsconfig: boolean; primer: string } ): SourceTree { - if (!ownsTsconfig) { + if (!tree.ownsTsconfig) { printError( `note: ${TSCONFIG_FILE} in ${dir} is not the CLI's, so it was left as it is. To type-check the source, map "@sim/workflow-sdk" to ["./${SDK_DIR}/core.d.ts"] in its compilerOptions.paths.` ) } - return { files: written, workspaceTypes: workspaceTypes !== null, guide: SOURCE_AGENTS_FILE } + return { + files: written, + workspaceTypes: workspaceTypes !== null, + guide: SOURCE_AGENTS_FILE, + primer: tree.primer, + } +} + +/** + * Prints a tree write's result: in JSON or YAML one document with the primer as its `primer` + * field, otherwise the fields and then the primer as text, so it reads as written. + */ +export function printTreeResult( + format: OutputFormat, + result: Record<string, unknown> & { primer: string } +): void { + if (format === 'json' || format === 'yaml') { + printProtocolResult(format, result) + return + } + const { primer, ...fields } = result + printProtocolResult(format, fields) + printLine(`\n${primer}`) } /** What a directory's editable tree holds beside the printed source. */ export interface SourceTreeProfile { - /** `.sim/AGENTS.md`. */ + /** Which primer opens `.sim/AGENTS.md`. */ + kind: PrimerKind + /** The reference in `.sim/AGENTS.md`, below the primer. */ agents: string + /** More guides beside `.sim/AGENTS.md` (path → text), rewritten every time like it. */ + guides?: ReadonlyArray<readonly [string, string]> /** * A `tsconfig.json` the CLI writes, starting with {@link TSCONFIG_MARKER}, that type-checks the * source with the SDK declarations `sdkFiles` (directory-relative paths). @@ -584,8 +615,9 @@ export interface SourceTreeUse { } /** - * Everything beside the printed source that makes a directory editable: the agent guide and the - * example workflows (both rewritten every time and never in `.sim/written.json`), the + * Everything beside the printed source that makes a directory editable: the agent guide (the + * primer for the builders `use` calls, then the profile's reference) and the example workflows + * (both rewritten every time and never in `.sim/written.json`), the * declarations (regenerated only when the catalog or generator changed), `workspace.d.ts`, the * CLI's `tsconfig.json` and `tsconfig.full.json`, and the LF lines. `incremental` (a write that * leaves workflows on disk unread, such as a get of only some kinds) keeps the blocks @@ -598,9 +630,12 @@ export async function sourceTreeChange( profile: SourceTreeProfile, use: SourceTreeUse, options: { incremental?: boolean } = {} -): Promise<{ change: SourceDirChange; ownsTsconfig: boolean }> { +): Promise<{ change: SourceDirChange; ownsTsconfig: boolean; primer: string }> { + const sources = [...use.sources] + const primer = primerFor(profile.kind, catalog, sources) const write: Array<readonly [string, string]> = [ - [SOURCE_AGENTS_FILE, profile.agents], + [SOURCE_AGENTS_FILE, `${primer}\n${profile.agents}`], + ...(profile.guides ?? []), ...Object.entries(COOKBOOK_EXAMPLES).map( ([file, text]) => [`${SOURCE_EXAMPLES_DIR}/${file}`, text] as const ), @@ -621,7 +656,7 @@ export async function sourceTreeChange( catalog, workspaceTypes !== null, profile, - { ...use, blockTypes: [...(use.blockTypes ?? []), ...kept] }, + { sources, blockTypes: [...(use.blockTypes ?? []), ...kept] }, { tsconfig, full: tsconfigFull } ) write.push(...tsconfigs.write) @@ -642,7 +677,7 @@ export async function sourceTreeChange( } else { remove.push(WORKSPACE_MANIFEST_FILE) } - return { change: { write, remove }, ownsTsconfig } + return { change: { write, remove }, ownsTsconfig, primer } } /** A read the server refused as over the source size limits exits 7, as a refused apply does. */ diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index a2d68e9cc86..b1c41fb495d 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -34,7 +34,7 @@ import { writeBack, } from './source-dir' import { SOURCE_WRITE_EXIT_CODES_HELP } from './source-exit-codes' -import { cachedCatalogByHash } from './source-tree' +import { cachedCatalogByHash, printTreeResult } from './source-tree' import { confirm, fresh, @@ -406,7 +406,7 @@ export function attachWorkflowSourceCreate(source: Command): void { printError( `${options.dir} had no ${ENTRY_FILE}, so a starter, the SDK types for workspace ${workspaceId}, tsconfig.json, .sim/AGENTS.md and the examples in .sim/examples/ were written and nothing was created. Edit ${ENTRY_FILE}, then run \`${createCommand(options.dir, workspaceId)} --yes\`.` ) - printProtocolResult(profile.output, { created: false, scaffolded: true, ...tree }) + printTreeResult(profile.output, { created: false, scaffolded: true, ...tree }) return } requireConsent(options, 'creates a workflow', 'create') diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts index 1ff6b340033..8b3396e2e57 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts @@ -8,6 +8,7 @@ import { SimApiError } from '../../http/client' import { buildGeneratedCommands } from '../../runtime/build' import { contextMockFns, contextMockState } from '../../test/context-mock' import { digest } from './source-dir' +import { PRIMER_MAX_BYTES, sourcePrimer } from './source-primer' import { attachWorkflowSourceGet } from './workflow-source-get' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -145,6 +146,26 @@ describe('sim workflows source get --dir', () => { expect(callsTo(CATALOG_PATH)[0][1].query).toBeUndefined() }) + it('prints the primer, which opens the guide, with a snippet for each builder the source calls and no other', async () => { + serve({ + [SOURCE_PATH]: () => + source({ 'workflow.ts': "// @sim-sdk 1\nconst mail = gmail.sendEmail('Mail', {})\n" }), + }) + + await get('--dir', outDir) + + const { primer } = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') + expect(readFileSync(join(outDir, '.sim/AGENTS.md'), 'utf8').startsWith(primer)).toBe(true) + expect(primer).toContain('- `gmail.sendEmail` (Gmail: Send Email)') + expect(primer).not.toContain('`slack.sendMessage`') + }) + + it('keeps the primer without snippets within its byte budget for both directory kinds', () => { + for (const kind of ['workflow', 'workspace'] as const) { + expect(Buffer.byteLength(sourcePrimer(kind))).toBeLessThanOrEqual(PRIMER_MAX_BYTES) + } + }) + it('writes the example workflows beside the guide, outside what the tsconfig files type-check', async () => { serve() diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index fd5c5a2d915..a0e3775efc2 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -11,7 +11,6 @@ import type { GetWorkflowSourceResponse } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { callsOperations, type OperationClient } from '../../runtime/called-operations' import { buildRequest } from '../../runtime/request' -import { printProtocolResult } from './result' import { assertPrintedSourcePaths, locallyEditedFiles, @@ -31,6 +30,7 @@ import { finishTree, loadCatalog, loadWorkspaceTypes, + printTreeResult, refreshTsconfigs, type SourceTree, type SourceTreeProfile, @@ -69,31 +69,29 @@ function workflowTsconfig(sdkFiles: readonly string[]): string { )}\n` } -/** The guide an agent working in the directory reads first; the docs page has the rest. */ -const AGENTS_GUIDE = `# Sim workflow source - -This directory is a Sim workflow written as TypeScript. The server compiles \`${SOURCE_ENTRY_FILE}\` statically; nothing in it runs. The workflow id is \`workflowId\` in \`${SOURCE_BASE_FILE}\`; every command below reads it from there. - -## Start here +/** + * The reference below the primer in `.sim/AGENTS.md`, by task: the edit loop in detail, a new or + * forked workflow, the file shape, lookups, the rules and every exit code. The docs page has the + * rest. + */ +const AGENTS_GUIDE = `# Reference -1. Read the examples in \`${SOURCE_EXAMPLES_DIR}/\` before writing: \`cookbook.ts\` (agent with structured output, condition, function, table query and insert, knowledge search, Slack, human in the loop, loop, response), \`integrations.ts\` (webhook trigger, weekday cron schedule, HTTP request with headers, table update by row id, Gmail, an error path) and \`agents.ts\` (router, agent tools: custom tool, MCP tool, integration operation, child workflow; run a child workflow; parallel). Their resource keys are examples; this workspace's are in \`${WORKSPACE_TYPES_FILE}\`. -2. For any other block, find its builder in \`${SDK_DIR}/${SDK_GUIDE_FILE}\` (one line per builder: \`grep -i "gmail" ${SDK_DIR}/${SDK_GUIDE_FILE}\`). It names the builder, the file and the params type; read just that type: \`grep -n -A40 "export type GmailSendEmailParams" ${SDK_DIR}/blocks/gmail_v2.d.ts\`. Builders are named for the service, not the block type: \`gmail_v2\` is \`gmail.*\`, \`workflow_input\` is \`runWorkflow\`, the HTTP request block is \`api\`, and \`legacy.*\` builders are old versions not to write. -3. Do not read \`${SDK_DIR}/blocks/*.d.ts\` or \`${SDK_DIR}/models.d.ts\` whole: they are large. \`grep\` them for a type or a model id. +The primer above is enough to edit; read a section here when a step needs it. -## Edit loop (run from this directory) +## Edit loop in detail -1. Edit \`${SOURCE_ENTRY_FILE}\` (and \`functions/*\`). -2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. Fix and rerun until it exits 0. -3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks with the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. -4. \`sim workflows source status --dir .\` before applying: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed it since (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\`: fix every error it prints, rerun until it exits 0. -5. \`sim workflows source apply --dir . --yes\`: saves the draft (never deploys). It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out), so read the file again before the next edit rather than editing from memory. -6. Exit 5 (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get <workflowId> --dir <fresh dir>\`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare it with this one. Only when it holds your edit, run \`sim workflows source get <workflowId> --dir . --force\`; otherwise reapply your edit in the fresh directory. +1. Edit \`${SOURCE_ENTRY_FILE}\` (and \`functions/*\`). \`${SOURCE_EXAMPLES_DIR}/\` holds correct examples to copy from: \`cookbook.ts\` (agent with structured output, condition, function, table query and insert, knowledge search, Slack, human in the loop, loop, response), \`integrations.ts\` (webhook trigger, weekday cron schedule, HTTP request with headers, table update by row id, Gmail, an error path) and \`agents.ts\` (router, agent tools: custom tool, MCP tool, integration operation, child workflow; run a child workflow; parallel). Their resource keys are examples; this workspace's are in \`${WORKSPACE_TYPES_FILE}\`. +2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. +3. \`${typecheckCommand()}\`: about a second; catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It runs the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply: a builder from another block fails with \`has no exported member\` until you run step 2 (or use \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine. +4. \`sim workflows source status --dir .\`: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\` until it exits 0. +5. \`sim workflows source apply --dir . --yes\`: saves the draft (never deploys) and rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out). +6. Exit 5 (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get <workflowId> --dir <fresh dir>\`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare. Only when it holds your edit, run \`sim workflows source get <workflowId> --dir . --force\`; otherwise reapply your edit in the fresh directory. -An apply that deletes blocks (a declaration removed from \`${SOURCE_ENTRY_FILE}\`) is refused with \`E_APPLY_BLOCK_DELETE\` naming them; pass \`--allow-delete\` when the deletion is intended. Editing the \`workflow('…')\` name or description renames or redescribes the workflow; a name another workflow in its folder holds is refused with \`E_NAME_TAKEN\`. +Editing the \`workflow('…')\` name or description renames or redescribes the workflow; a name another workflow in its folder holds is refused with \`E_NAME_TAKEN\`. Lines starting \`note:\` explain something the write did differently than the source says; read them. -Lines starting \`note:\` explain something the write did differently than the source says; read them. +## A new workflow, or a fork -A new workflow: \`sim workflows source create --dir <new dir> --workspace <workspaceId>\` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, the types of that workspace, \`tsconfig.json\` and this guide, records the workspace in \`${SOURCE_STARTER_FILE}\`, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, run step 2 and 3, then \`sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run\` and the same with \`--yes\` (the starter prints both with its workspace filled in): it prints the new workflow id and writes the lock files. A create into another workspace than the starter's is refused (\`SOURCE_OTHER_WORKSPACE\`) unless \`--workspace\` names it. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir <copy> --workspace <workspaceId> --name <new name> --yes\`. +\`sim workflows source create --dir <new dir> --workspace <workspaceId>\` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, the types of that workspace, \`tsconfig.json\` and this guide, records the workspace in \`${SOURCE_STARTER_FILE}\`, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, check and type-check it, then \`sim workflows source create --dir <dir> --workspace <workspaceId> --dry-run\` and the same with \`--yes\` (the starter prints both with its workspace filled in): it prints the new workflow id and writes the lock files. A create into another workspace than the starter's is refused (\`SOURCE_OTHER_WORKSPACE\`) unless \`--workspace\` names it. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir <copy> --workspace <workspaceId> --name <new name> --yes\`. ## File shape @@ -111,28 +109,28 @@ export default workflow('Workflow name', {}, () => { The \`// @sim-sdk 1\` header must be the first line, and the file has exactly one \`export default workflow(name, options, () => { ... })\`; \`options\` holds \`description\` and \`variables\` and may be \`{}\`. -## Types +## Finding a builder and its types -- \`${SDK_DIR}/core.d.ts\`: the SDK (\`workflow\`, \`t\`, \`env\`, \`credential\`, \`table\`, \`BlockMeta\`). -- \`${SDK_DIR}/${SDK_GUIDE_FILE}\`: every builder, its block and operation, file and params type. -- \`${SDK_DIR}/blocks/<block type>.d.ts\`: one block's builders with their params and outputs. +- \`sim workflows source params <builder>\`: one builder's params (required first, enums, defaults) and outputs; a namespace (\`gmail\`) lists its builders; a miss names the closest. Builders are named for the service, not the block type: \`gmail_v2\` is \`gmail.*\`, \`workflow_input\` is \`runWorkflow\`, the HTTP request block is \`api\`, and \`legacy.*\` builders are old versions not to write. +- \`${SDK_DIR}/core.d.ts\`: the SDK (\`workflow\`, \`t\`, \`env\`, \`credential\`, \`table\`, \`loop\`, \`parallel\`, \`BlockMeta\`). +- \`${SDK_DIR}/blocks/<block type>.d.ts\`: one block's builders with their params and outputs. Do not read them, or \`${SDK_DIR}/models.d.ts\`, whole: \`params\` prints the part you need, and \`grep\` finds a model id. - \`${WORKSPACE_TYPES_FILE}\`: this workspace's tables, knowledge bases, credentials, workflows and env names. -## Diagnostics and exit codes - -Findings print as \`file:line:column severity CODE: message\`, then the code frame and a hint. \`E_*\` refuses the whole apply; \`W_*\` is saved as written and names a real problem. \`check\` prints the \`W_*\` codes step 2 lists as \`error\` on a block you added, since they fail it there; the apply still saves them as warnings. Exit codes of every \`workflows source\` command: - -${sourceExitCodesTable('sim workflows source')} - ## Rules - Beside the header and the imports from \`@sim/workflow-sdk\`, the file holds only the default-exported workflow. Its callback holds block declarations (\`const x = builder('Name', params)\`) and wiring (\`x.to(y)\`). Helpers, loops, conditionals and other imports have no stored form, so the compiler refuses them: use \`loop.forEach\`, \`condition\` and \`runWorkflow\` blocks instead. -- Logic that does not fit a block goes in a function block with \`code: codeFile('./functions/name.js')\`; in JavaScript read outputs as \`$ref.blockKey.path\` (the block's key: its name lowercased with every whitespace run and every \`.\` removed, so "Triage Agent" is \`triageagent\`; not its variable) and env vars as \`$env.NAME\`. +- Logic that does not fit a block goes in a function block with \`code: codeFile('./functions/name.js')\` (references as the primer says). - Hosted Sim supplies an agent's model key. On self-hosted Sim the agent needs \`apiKey: env.NAME\` (a secret the workspace holds); without it \`check\` reports \`W_REQUIRED_MISSING: apiKey is required\`. - A child workflow's output is \`c.result\`, untyped: its \`response\` block's \`{ data, status, headers }\`, so a field of the child's response body is \`c.result.data.field\` (\`${SDK_DIR}/${SDK_GUIDE_FILE}\` "How it runs" has the rest). - A table block's row is \`{ id, data }\`: read a cell as \`row.data.column\` (in a loop over \`query.rows\`, \`p.currentItem.data.column\`). - Keep \`${SOURCE_LOCK_FILE}\` and \`${SOURCE_BASE_FILE}\` as written: they bind each block to its stored id and make the next apply safe. +## Diagnostics and exit codes + +Findings print as \`file:line:column severity CODE: message\`, then the code frame and a hint. \`E_*\` refuses the whole apply; \`W_*\` is saved as written and names a real problem. \`check\` prints the \`W_*\` codes it fails on as \`error\` on a block you added; the apply still saves them as warnings. Exit codes of every \`workflows source\` command: + +${sourceExitCodesTable('sim workflows source')} + The full guide: ${SOURCE_DOCS_URL} ` @@ -167,11 +165,12 @@ async function writeSourceTree( previousLock, edges ? { ...tree.change, write: [...tree.change.write, edges] } : tree.change ) - return finishTree(dir, written, workspaceTypes, tree.ownsTsconfig) + return finishTree(dir, written, workspaceTypes, tree) } /** The workflow directory's tree beside the printed source. */ export const WORKFLOW_TREE: SourceTreeProfile = { + kind: 'workflow', agents: AGENTS_GUIDE, tsconfig: workflowTsconfig, } @@ -213,7 +212,7 @@ export async function writeSourceScaffold( ...tree.change.write, ] await updateSourceDir(dir, { ...tree.change, write }) - return finishTree(dir, write.length, workspaceTypes, tree.ownsTsconfig) + return finishTree(dir, write.length, workspaceTypes, tree) } /** @@ -366,7 +365,7 @@ export function attachWorkflowSourceGet(source: Command): void { workflowSource, workflowSource.blockTypes ) - printProtocolResult(profile.output, { + printTreeResult(profile.output, { workflowId: workflowSource.workflowId, dir, ...tree, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index f07abccffb4..b802870538e 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -9,7 +9,6 @@ import { mapWithConcurrency } from '@sim/utils/concurrency' import { toRecord } from '@sim/utils/object' import type { WorkflowInput } from '@sim/workflow-code' import { - SDK_GUIDE_FILE, SOURCE_COMPILER_OPTIONS, typecheckCommand, WORKSPACE_COOKBOOK, @@ -104,9 +103,18 @@ function workspaceTsconfig(sdkFiles: readonly string[]): string { /** Where the workspace example is written: `.sim/examples/workspace/<path>`. */ const WORKSPACE_EXAMPLES_DIR = '.sim/examples/workspace' -const AGENTS_GUIDE = `# Sim workspace source +/** Where the workspace guide's deep dives are written, beside `.sim/AGENTS.md`. */ +const WORKSPACE_ADVANCED_FILE = '.sim/GUIDE-advanced.md' -This directory is a Sim workspace written as files. The server reads every file statically; nothing in it runs. The workspace id is \`workspaceId\` in \`${WORKSPACE_BASE_FILE}\`. \`${WORKSPACE_EXAMPLES_DIR}/\` holds one correct example of each kind and a workflow using them, and \`.sim/examples/*.ts\` one workflow example of each common block (agent with structured output, condition, HTTP request, child workflow, Slack, loop, …): copy from them. Find any other builder in \`${SOURCE_SDK_DIR}/${SDK_GUIDE_FILE}\`. +/** + * The reference below the primer in `.sim/AGENTS.md`: the layout, the edit loop in detail, what + * to know and every exit code. + */ +const AGENTS_GUIDE = `# Reference + +The primer above is enough to edit; read a section here when a step needs it. Deletes in depth, Git and CI, and promoting to another workspace: \`${WORKSPACE_ADVANCED_FILE}\`. + +\`${WORKSPACE_EXAMPLES_DIR}/\` holds one correct example of each kind and a workflow using them, and \`.sim/examples/*.ts\` one workflow example of each common block (agent with structured output, condition, HTTP request, child workflow, Slack, loop, …): copy from them. Look up any other builder with \`sim workflows source params <builder>\`. ## Layout @@ -118,14 +126,35 @@ One apply creates resources before the workflows that use them, so a workflow ma Secrets never appear in source: a workflow reads an environment variable as \`env.NAME\` and tool code as \`$env.NAME\`, both by name, and a saved credential is \`credential('<key>')\`. Declare the ones workflows use in \`${KIND_DIRS.secret}\` and \`${KIND_DIRS.credential}\`: a workflow using a declared secret the workspace has no value for, or a declared credential key nothing binds to, is held back (exit 1, with the command that fixes it) while the rest of the apply is written. The same directory applied to another workspace binds each credential key to that workspace's own credential. -## Edit loop (run from this directory) +## Edit loop in detail 1. Edit the files. 2. \`sim workspaces source check --dir .\`: compiles every workflow offline with the server's compiler, against what this directory declares: an unknown resource key (skills included) or table column, a child workflow's inputs, and an env name \`${KIND_DIRS.secret}\` lacks (a warning); and checks each \`${KIND_DIRS.table}/<key>.rows.jsonl\` line against its table as the apply would. It fails (exit 1) on errors and on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block), regenerates \`.sim/sdk/workspace.d.ts\` and adds the blocks the workflows now use to \`tsconfig.json\`. In a workflow folder, \`sim workflows source check --dir .\` checks that one workflow the same way. Fix and rerun until it exits 0. -3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks the files with the TypeScript release the declarations are built for. \`tsconfig.json\` includes only the blocks the workflows used at the last check or get, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). \`${SOURCE_SDK_DIR}/${SDK_GUIDE_FILE}\` lists every builder with its file and params type; \`grep\` it rather than reading the declarations. The dry run is the authority: a printed file you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. +3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks the files with the TypeScript release the declarations are built for. \`tsconfig.json\` includes only the blocks the workflows used at the last check or get, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). \`sim workflows source params <builder>\` prints one builder's params and outputs; do not read the declarations whole. The dry run is the authority: a printed file you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. 4. \`sim workspaces source status --dir .\` before applying: your changes by resource (\`sim workspaces source diff --dir .\` shows the edits), what changed on the server since the get (a workflow edited in the editor included), the conflicts, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed since (\`--offline\`: skip the server). Then \`sim workspaces source apply --dir . --dry-run\`: shows the plan (\`+\` create, \`~\` update, \`=\` adopt an existing resource of the same name, \`-\` delete) and every error; fix and rerun until it exits 0. 5. \`sim workspaces source apply --dir . --yes\`: writes it and rewrites this directory from what was stored. +## Good to know + +- A column's object key stays bound to its stored column: to rename one keep the key and set \`name\` (\`email: column.string({ name: 'email_address' })\`). Changing the key drops the column and its cells. +- Renaming a skill (its \`name\`) does not rewrite the workflows that use it: each agent keeps the name it stored until its \`skills\` list is next edited. The key is what binds them. +- A workflow folder you did not edit is sent with only its base, and the server does not compare it with the stored workflow: unlike other kinds, an apply does not report that a workflow changed on the server since the get (\`status\`, which sends every folder whole, does, and the next get shows it). An edit of the folder is checked against its own \`.sim/base.json\` and refused as stale (exit 5) when the workflow changed since. +- A knowledge base's \`embeddingModel\` is fixed when it is created: an apply that changes it is refused. +- A document or workspace file over 10 MB, or whose name no directory can hold, is left out of a get and listed in \`.sim/skipped.json\`; an apply never writes or deletes it. +- A hidden file (a name starting with a dot) under a kind folder is never sent: an apply names it in a warning. + +## Exit codes + +${sourceExitCodesTable('sim workspaces source')} + +Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means that apply or someone else changed the workspace: get into a fresh directory and compare, and only when it holds your edits run \`sim workspaces source get --dir . --force\`. A get that exits 9 keeps the binary files it downloaded and fetches only the rest. + +Keep \`${WORKSPACE_BASE_FILE}\` and each workflow's \`.sim/\` files as written. +` + +/** The workspace guide's deep dives, which `.sim/AGENTS.md` points to in one line. */ +const ADVANCED_GUIDE = `# Sim workspace source: advanced + ## Deletes - A kind is managed only when its folder (or root file) exists: without \`skills/\`, no skill is touched. @@ -148,28 +177,13 @@ Secrets never appear in source: a workflow reads an environment variable as \`en ## Promote to another workspace \`sim workspaces source apply --dir . --to <workspaceId> --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name), and a change made in the target since is refused as stale (exit 5). Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w <workspaceId>\`) and connect its credentials, or the workflows using them are held back. This directory is not changed and stays bound to the workspace it was read from. Without \`--to\`, an apply refuses a directory read from another workspace (\`SOURCE_OTHER_WORKSPACE\`). - -## Good to know - -- A column's object key stays bound to its stored column: to rename one keep the key and set \`name\` (\`email: column.string({ name: 'email_address' })\`). Changing the key drops the column and its cells. -- Renaming a skill (its \`name\`) does not rewrite the workflows that use it: each agent keeps the name it stored until its \`skills\` list is next edited. The key is what binds them. -- A workflow folder you did not edit is sent with only its base, and the server does not compare it with the stored workflow: unlike other kinds, an apply does not report that a workflow changed on the server since the get (\`status\`, which sends every folder whole, does, and the next get shows it). An edit of the folder is checked against its own \`.sim/base.json\` and refused as stale (exit 5) when the workflow changed since. -- A knowledge base's \`embeddingModel\` is fixed when it is created: an apply that changes it is refused. -- A document or workspace file over 10 MB, or whose name no directory can hold, is left out of a get and listed in \`.sim/skipped.json\`; an apply never writes or deletes it. -- A hidden file (a name starting with a dot) under a kind folder is never sent: an apply names it in a warning. - -## Exit codes - -${sourceExitCodesTable('sim workspaces source')} - -Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means that apply or someone else changed the workspace: get into a fresh directory and compare, and only when it holds your edits run \`sim workspaces source get --dir . --force\`. A get that exits 9 keeps the binary files it downloaded and fetches only the rest. - -Keep \`${WORKSPACE_BASE_FILE}\` and each workflow's \`.sim/\` files as written. ` /** The workspace directory's tree beside the printed source. */ export const WORKSPACE_TREE: SourceTreeProfile = { + kind: 'workspace', agents: AGENTS_GUIDE, + guides: [[WORKSPACE_ADVANCED_FILE, ADVANCED_GUIDE]], tsconfig: workspaceTsconfig, // Workspace files and knowledge base documents are bytes, kept as they are (`-text`, after the // LF lines it overrides): a commit that rewrote their line endings would change their hashes. diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts index fb633377108..107999956e3 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts @@ -125,6 +125,21 @@ describe('workspaces source get', () => { expect(readFileSync(join(dir, 'tables/leads.ts'), 'utf8')).toBe(`${LEADS}// server\n`) }) + it('writes the workspace primer atop the guide, and the deep dives to the advanced guide it names', async () => { + serve({ 'tables/leads.ts': LEADS }) + + await get() + + const agents = readFileSync(join(dir, '.sim/AGENTS.md'), 'utf8') + expect(agents.startsWith('# Sim workspace source: primer')).toBe(true) + expect(agents).toContain('`.sim/GUIDE-advanced.md`') + expect(agents).not.toContain('## Git and CI') + const advanced = readFileSync(join(dir, '.sim/GUIDE-advanced.md'), 'utf8') + for (const section of ['## Deletes', '## Git and CI', '## Promote to another workspace']) { + expect(advanced).toContain(section) + } + }) + it('refuses to restore a file deleted since the CLI wrote it, a pending delete', async () => { serve({ 'tables/leads.ts': LEADS, 'tables/accounts.ts': ACCOUNTS }) await get() diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index 2c25ff88f9e..aa90fb6c96a 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -20,7 +20,6 @@ import type { GetWorkspaceSourceResponse } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { apiCommand } from '../../runtime/called-operations' import { removeStagingOnSignal } from './files-get' -import { printProtocolResult } from './result' import { checkedPath, editedFiles, @@ -42,6 +41,7 @@ import { finishTree, loadCatalog, loadWorkspaceTypes, + printTreeResult, sourceTreeChange, withSourceExitCode, } from './source-tree' @@ -161,7 +161,7 @@ async function writeWorkspaceSource( dir: string, source: WorkspaceSource, force: boolean -): Promise<{ written: number; removed: number; guide: string }> { +): Promise<{ written: number; removed: number; guide: string; primer: string }> { assertWorkspacePaths(Object.keys(source.files)) assertListedBlobs(source.blobs) const blobs = new Map(source.blobs.map((blob) => [blob.path, blob])) @@ -299,8 +299,13 @@ async function writeWorkspaceSource( ], }) await rm(downloads, { recursive: true, force: true }) - const summary = finishTree(dir, write.length + stale.length, workspaceTypes, tree.ownsTsconfig) - return { written: summary.files, removed: gone.size, guide: summary.guide } + const summary = finishTree(dir, write.length + stale.length, workspaceTypes, tree) + return { + written: summary.files, + removed: gone.size, + guide: summary.guide, + primer: summary.primer, + } } finally { if (partial) await rm(partial, { recursive: true, force: true }) disposeSignalCleanup() @@ -403,7 +408,7 @@ export function attachWorkspaceSourceGet(source: Command): void { throw withSourceExitCode(error) }) const result = await writeWorkspaceSource(client, options.dir, data, options.force === true) - printProtocolResult(profile.output, { + printTreeResult(profile.output, { workspaceId: data.workspaceId, dir: options.dir, kinds: data.kinds, @@ -412,6 +417,7 @@ export function attachWorkspaceSourceGet(source: Command): void { removed: result.removed, skipped: data.skipped.length, guide: result.guide, + primer: result.primer, }) }) }) diff --git a/packages/workflow-sdk/README.md b/packages/workflow-sdk/README.md index 20a1ade672b..37ad2fc9fc5 100644 --- a/packages/workflow-sdk/README.md +++ b/packages/workflow-sdk/README.md @@ -71,7 +71,7 @@ refuse with `CONFIRMATION_REQUIRED`). `--dry-run` never asks. | `.sim/sdk/workspace.d.ts` | This workspace's tables (with columns), knowledge bases, credentials, workflows (with inputs), skills, custom tools and env var names, so `table('tickets')`, `credential('support-slack')`, `skill('triage')` and `env.X` are checked. | | `tsconfig.json` | Resolves `@sim/workflow-sdk` to `.sim/sdk` and type-checks `workflow.ts` with the blocks it uses (from the source and the server's `blockTypes`) and the core builders' blocks: a few dozen files for `tsc` instead of every block. | | `tsconfig.full.json` | The same with every current block, plus any legacy-only block the workflow uses. | -| `.sim/AGENTS.md` | The authoring guide for agents, linking [the docs](https://docs.sim.ai/workflows/code). A top-level `AGENTS.md` of your own is never touched. | +| `.sim/AGENTS.md` | The authoring guide for agents: the primer `get` prints (edit loop, identity, references, deletes, deploy, exit codes, and a snippet for each builder the source calls), then a reference linking [the docs](https://docs.sim.ai/workflows/code). A top-level `AGENTS.md` of your own is never touched. | | `.sim/examples/*.ts` | Example workflows, together one correct example of each common block: `cookbook.ts` (agent, condition, function, table, knowledge, Slack, human in the loop, loop, response), `integrations.ts` (webhook, weekday cron, HTTP headers, table update by id, Gmail, `onError`), `agents.ts` (router, agent tools, child workflow, parallel). Read them before writing. They name example resources, so the tsconfig files leave them out. | | `.gitattributes` | `text eol=lf` for the source files, merged into any file already there. | diff --git a/packages/workflow-sdk/src/codegen/index.ts b/packages/workflow-sdk/src/codegen/index.ts index 910658290ee..b92eb930850 100644 --- a/packages/workflow-sdk/src/codegen/index.ts +++ b/packages/workflow-sdk/src/codegen/index.ts @@ -10,6 +10,7 @@ export { defaultBlockFiles, fullBlockFiles, referencedBlockTypes, + referencedBuilders, SDK_GUIDE_FILE, SDK_INDEX_FILE, SDK_MODELS_FILE, diff --git a/packages/workflow-sdk/src/codegen/layout.ts b/packages/workflow-sdk/src/codegen/layout.ts index ff7e523d8d1..bb990e59a6a 100644 --- a/packages/workflow-sdk/src/codegen/layout.ts +++ b/packages/workflow-sdk/src/codegen/layout.ts @@ -64,23 +64,36 @@ export function legacyOnlyBlockTypes(catalog: BlockCatalog): Set<string> { const CALL = /([A-Za-z_$][\w$]*(?:\s*\.\s*[A-Za-z_$][\w$]*)*)\s*\(/g /** - * The block types `sources` (workflow files) name through a builder: every dotted call whose path - * is a catalog builder (`slack.sendMessage(`) or its tool form (`slack.tool.sendMessage(`). A - * text scan, not a compile, so it reads any source, broken or not; a builder named in a comment - * only adds a file to the type-check. `block('type', …)` has untyped params, so needs no file. + * The catalog builders `sources` (workflow files) call, sorted: every dotted call whose path is a + * catalog builder (`slack.sendMessage(`) or its tool form (`slack.tool.sendMessage(`, read as + * `slack.sendMessage`). A text scan, not a compile, so it reads any source, broken or not; a + * builder named in a comment counts too. `block('type', …)` names no builder. */ -export function referencedBlockTypes(catalog: BlockCatalog, sources: Iterable<string>): string[] { - const types = new Set<string>() +export function referencedBuilders(catalog: BlockCatalog, sources: Iterable<string>): string[] { + const keys = new Set<string>() for (const text of sources) { for (const match of text.matchAll(CALL)) { const path = match[1].replace(/\s+/g, '').split('.') if (path.length > 1 && path[path.length - 2] === TOOL_NAMESPACE) path.splice(path.length - 2, 1) else if (path.length > 1 && path[path.length - 1] === TOOL_NAMESPACE) path.pop() - const builder = catalogBuilder(catalog, path.join('.')) - if (builder) types.add(builder.type) + const key = path.join('.') + if (catalogBuilder(catalog, key)) keys.add(key) } } + return [...keys].sort(compareStrings) +} + +/** + * The block types `sources` name through a builder ({@link referencedBuilders}): the blocks a + * directory's `tsconfig.json` type-checks. + */ +export function referencedBlockTypes(catalog: BlockCatalog, sources: Iterable<string>): string[] { + const types = new Set<string>() + for (const key of referencedBuilders(catalog, sources)) { + const builder = catalogBuilder(catalog, key) + if (builder) types.add(builder.type) + } return [...types].sort(compareStrings) } From d63aa1b35f9d3970d80699d93e666554c580595a Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 16:53:12 -0800 Subject: [PATCH 34/59] perf(sim-cli): a source directory holds only the declarations of the blocks it uses Every block's declarations (about 500 files, 14 MB) are generated once per catalog and generator into ~/.sim/cache/source-catalog/<hash>/sdk-<generator>/ and shared by every directory; tsconfig.full.json includes the cached index, so any builder still type-checks. The directory keeps core, models, an index and INDEX.md of its own blocks, and the files tsconfig.json includes: 23 files and 121 KB instead of 522 and 14.3 MB for a typical workflow, and a get into a new directory writes in 18 ms instead of 1.1 s. check writes the file of a block an edit starts using, so tsc -p . catches an unknown param on it; an embedded run, which has no cache, still writes every block. A workspace directory gitignores tsconfig.full.json, which names this machine's cache. --- apps/docs/content/docs/workflows/code.mdx | 4 +- .../src/commands/protocol/source-dir.ts | 4 +- .../src/commands/protocol/source-tree.ts | 206 ++++++++++++++---- .../protocol/workflow-source-check.test.ts | 1 - .../protocol/workflow-source-get.test.ts | 82 ++++++- .../commands/protocol/workflow-source-get.ts | 12 +- .../workflow-source-roundtrip.test.ts | 9 +- .../commands/protocol/workspace-source-dir.ts | 10 +- packages/workflow-sdk/README.md | 14 +- packages/workflow-sdk/src/codegen/generate.ts | 50 +++-- packages/workflow-sdk/src/codegen/index.ts | 1 + packages/workflow-sdk/src/codegen/layout.ts | 23 +- 12 files changed, 320 insertions(+), 96 deletions(-) diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index 124660cfdc8..b8a6aa41c1c 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -39,7 +39,7 @@ Nothing in the file runs on your machine. Sim reads it statically, checks it aga sim workflows source get <workflowId> --dir ./support-triage ``` - This writes `workflow.ts`, any function code under `functions/`, and type declarations for every block and for your workspace's tables, credentials, knowledge bases, and environment variable names, with `.sim/sdk/INDEX.md` listing every builder by block and operation. It also writes a `tsconfig.json` that resolves them and type-checks the blocks the workflow uses, and a `tsconfig.full.json` that type-checks every block, unless the directory already has files of those names, which are left as they are. For coding agents it writes `.sim/AGENTS.md`, the primer `get` prints followed by a reference, and example workflows under `.sim/examples/` with one correct example of each common block to copy from. The examples name placeholder resources, so the tsconfig files leave them out of the type-check. + This writes `workflow.ts`, any function code under `functions/`, and type declarations for the blocks the workflow uses and for your workspace's tables, credentials, knowledge bases, and environment variable names, with `.sim/sdk/INDEX.md` listing those blocks' builders. The declarations of every block are generated once per block catalog into the CLI's cache (`~/.sim/cache/source-catalog/`) and shared by every directory. It also writes a `tsconfig.json` that resolves them and type-checks the blocks the workflow uses, and a `tsconfig.full.json` that type-checks every block through that cache, unless the directory already has files of those names, which are left as they are. For coding agents it writes `.sim/AGENTS.md`, the primer `get` prints followed by a reference, and example workflows under `.sim/examples/` with one correct example of each common block to copy from. The examples name placeholder resources, so the tsconfig files leave them out of the type-check. `get` into a directory that already holds source refuses to overwrite edits you have not applied, and refuses a directory that holds a different workflow. Pass `--force` to replace them. @@ -204,7 +204,7 @@ In the file, `$ref.block.path` reads another block's output and `$env.NAME` read Run `sim workflows source apply <workflowId> --dir <dir> --dry-run` in CI to catch a broken workflow before it is saved. It exits `0` when the workflow is valid and `1` when it has errors, so a failing check blocks the merge. It exits `5` when the workflow was changed in the editor after you committed the directory; run `get` again and reapply your edit. - **Commit** `workflow.ts`, `functions/`, `.sim/lock.json`, `.sim/base.json`, `.sim/written.json`, `tsconfig.json` and `.gitattributes`. CI finds the workflow from `workflowId` in `.sim/base.json`, and `status` and `get` use `.sim/written.json` to tell your edits apart from what the CLI last wrote. -- **Optionally gitignore** `.sim/sdk/`, the generated type declarations (thousands of files); `get` and `check` write them again. A single-workflow directory gets no `.gitignore`, so add the line yourself. +- **Optionally gitignore** `.sim/sdk/`, the generated type declarations of the blocks in use (a few dozen files); `get` and `check` write them again. Gitignore `tsconfig.full.json`, which names the CLI's cache on your machine; `check` writes it again. A single-workflow directory gets no `.gitignore`, so add the lines yourself. - **A whole workspace** in one repository, with dry runs on pull requests, apply on merge and promotion from one workspace to another, is covered in [Git and CI](/workflows/workspace-code#git-and-ci) and [Promote to another workspace](/workflows/workspace-code#promote-to-another-workspace). - **Credentials:** use a personal API key (**Settings → Sim API keys**, passed as `SIM_API_KEY`) or an OAuth login, not a workspace API key. Saving and dry-running source act as a person, so a workspace key is refused with `403`. Set `SIM_WORKSPACE` unless you pass `--workspace`, and `SIM_ENDPOINT` if you use a self-hosted Sim. See [Authenticating CI](/cli/authentication#authenticating-ci). diff --git a/packages/sim-cli/src/commands/protocol/source-dir.ts b/packages/sim-cli/src/commands/protocol/source-dir.ts index 08169a37d5c..022867f9295 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.ts @@ -43,7 +43,7 @@ import { SimApiError } from '../../http/client' import { embeddedFileContent } from '../../transfer/local-file' import { removeStagingOnSignal, saveToFile } from './files-get' -/** Generated SDK declarations: every block in the workspace catalog, plus `workspace.d.ts`. */ +/** Generated SDK declarations: the blocks the source uses (every block's are cached), plus `workspace.d.ts`. */ export const SOURCE_SDK_DIR = '.sim/sdk' /** A short guide for an agent working in the directory; never a top-level `AGENTS.md`. */ export const SOURCE_AGENTS_FILE = '.sim/AGENTS.md' @@ -110,7 +110,7 @@ const GITATTRIBUTES_LINES = [ `${SOURCE_ENTRY_FILE} text eol=lf`, `${SOURCE_FUNCTIONS_DIR}/** text eol=lf`, ] -/** Files read or written at once on a local disk; the declarations run to thousands of small files. */ +/** Files read or written at once on a local disk; a catalog's declarations run to hundreds of small files. */ export const FILE_CONCURRENCY = 32 /** diff --git a/packages/sim-cli/src/commands/protocol/source-tree.ts b/packages/sim-cli/src/commands/protocol/source-tree.ts index 9c85ab2e52c..f5ba410c59f 100644 --- a/packages/sim-cli/src/commands/protocol/source-tree.ts +++ b/packages/sim-cli/src/commands/protocol/source-tree.ts @@ -6,7 +6,9 @@ */ import { createHash } from 'node:crypto' import { rmSync } from 'node:fs' -import { join } from 'node:path' +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { mapWithConcurrency } from '@sim/utils/concurrency' import { isRecordLike, toRecord } from '@sim/utils/object' import { escapeRegExp } from '@sim/utils/string' import type { BlockCatalog } from '@sim/workflow-code' @@ -15,6 +17,7 @@ import { SOURCE_BASE_FILE } from '@sim/workflow-code/limits' import { COOKBOOK_EXAMPLES, CORE_DTS, + checkedBlockTypes, defaultBlockFiles, fullBlockFiles, generateSdkTypes, @@ -38,6 +41,7 @@ import type { OperationClient } from '../../runtime/called-operations' import { cliVersion } from '../../version' import { printProtocolResult } from './result' import { + FILE_CONCURRENCY, readSourceJson, readSourceText, SOURCE_SDK_DIR as SDK_DIR, @@ -419,16 +423,72 @@ function sdkGeneratorLine(): string { return `// generated by sim ${cliVersion()} · core.d.ts ${core}` } -/** The generated declarations, with the generator line under the index's catalog header. */ -function sdkTypeFiles(catalog: BlockCatalog): Array<readonly [string, string]> { +/** Under the index's catalog header, the generator line; `blockTypes` as `generateSdkTypes` takes it. */ +function sdkTypeFiles( + catalog: BlockCatalog, + blockTypes?: ReadonlySet<string> +): Array<readonly [string, string]> { const generator = sdkGeneratorLine() - return generateSdkTypes(catalog).map((file) => { - if (file.path !== SDK_INDEX_FILE) return [`${SDK_DIR}/${file.path}`, file.text] as const + return generateSdkTypes(catalog, blockTypes ? { blockTypes } : {}).map((file) => { + if (file.path !== SDK_INDEX_FILE) return [file.path, file.text] as const const [header, ...rest] = file.text.split('\n') - return [`${SDK_DIR}/${file.path}`, [header, generator, ...rest].join('\n')] as const + return [file.path, [header, generator, ...rest].join('\n')] as const }) } +/** The head a current `index.d.ts` starts with: the catalog header and the generator line. */ +function sdkIndexHead(catalog: BlockCatalog): string { + return `${sdkIndexHeader(catalog)}\n${sdkGeneratorLine()}` +} + +/** + * Where the declarations of every block of `catalog` are cached, once per catalog and generator + * (the generator line's digest), beside the catalog itself. `tsconfig.full.json` includes the + * `index.d.ts` there, so a directory holds only the blocks it uses. Callers gate the hash with + * `CACHE_KEY` first. + */ +function sdkCacheDir(catalog: BlockCatalog): string { + const generator = createHash('sha256').update(sdkGeneratorLine(), 'utf8').digest('hex') + return join(catalogCacheDir(), catalog.hash, `sdk-${generator.slice(0, 16)}`) +} + +/** + * The cached full declarations' `index.d.ts` for `catalog` (an absolute path), generating them + * into a staging folder moved into place on the first call; later calls only check the index. + * `null` in an embedded run, which has no disk of its own, or when the cache cannot be written: + * the directory then holds every block itself, as before the cache. + */ +async function sdkCacheIndex(catalog: BlockCatalog): Promise<string | null> { + if (embedStore.getStore() || !CACHE_KEY.test(catalog.hash)) return null + const dir = sdkCacheDir(catalog) + const index = join(dir, SDK_INDEX_FILE) + const head = sdkIndexHead(catalog) + const current = await readFile(index, 'utf8').catch(() => null) + if (current?.startsWith(`${head}\n`)) return index + try { + await mkdir(dirname(dir), { recursive: true }) + const staging = await mkdtemp(`${dir}.staging-`) + try { + const files = sdkTypeFiles(catalog) + const parents = new Set(files.map(([path]) => dirname(join(staging, path)))) + for (const parent of parents) await mkdir(parent, { recursive: true }) + await mapWithConcurrency(files, FILE_CONCURRENCY, ([path, text]) => + writeFile(join(staging, path), text) + ) + await rm(dir, { recursive: true, force: true }) + // Another command that generated the same files first already moved them into place. + await rename(staging, dir).catch(async (error: unknown) => { + if (!(await readFile(index, 'utf8').catch(() => null))?.startsWith(`${head}\n`)) throw error + }) + } finally { + await rm(staging, { recursive: true, force: true }) + } + return index + } catch { + return null + } +} + /** What a tree write reports: the same fields after a get and a create scaffold. */ export interface SourceTree { /** How many files were written. */ @@ -504,13 +564,15 @@ export interface SourceTreeProfile { /** * The SDK files each written tsconfig includes for source that uses the `used` block types: * `tsconfig.json` the models, `workspace.d.ts` and only those blocks (plus the core builders'), - * so `tsc` reads a few dozen files instead of every block; `tsconfig.full.json` every current - * block for adding one the source does not use yet. + * which the directory holds, so `tsc` reads a few dozen files instead of every block; + * `tsconfig.full.json` every current block for adding one the source does not use yet, through + * `fullIndex` (the cached index, else the directory's own). */ function sdkIncludes( catalog: BlockCatalog, used: readonly string[], - hasWorkspaceTypes: boolean + hasWorkspaceTypes: boolean, + fullIndex: string ): { tsconfig: string[]; full: string[] } { const workspace = hasWorkspaceTypes ? [WORKSPACE_TYPES_FILE] : [] const inSdk = (files: readonly string[]) => files.map((file) => `${SDK_DIR}/${file}`) @@ -520,25 +582,33 @@ function sdkIncludes( ...workspace, ...inSdk(defaultBlockFiles(catalog, used)), ], - full: [...inSdk(fullBlockFiles(catalog, used)), ...workspace], + full: [ + ...fullBlockFiles(catalog, used).map((file) => + file === SDK_INDEX_FILE ? fullIndex : `${SDK_DIR}/${file}` + ), + ...workspace, + ], } } +/** The block types source that uses `use` names: the server's, and the builders it calls. */ +function usedBlockTypes(catalog: BlockCatalog, use: SourceTreeUse): string[] { + return [...new Set([...(use.blockTypes ?? []), ...referencedBlockTypes(catalog, use.sources)])] +} + /** - * The CLI's tsconfig files for source that uses `use`: each written when it is absent or still + * The CLI's tsconfig files for the `used` block types: each written when it is absent or still * starts with {@link TSCONFIG_MARKER}, so a person's own file is never replaced. */ function tsconfigWrites( catalog: BlockCatalog, hasWorkspaceTypes: boolean, profile: Pick<SourceTreeProfile, 'tsconfig'>, - use: SourceTreeUse, + used: readonly string[], + fullIndex: string, current: { tsconfig: string | null; full: string | null } ): { write: Array<readonly [string, string]>; ownsTsconfig: boolean } { - const used = [ - ...new Set([...(use.blockTypes ?? []), ...referencedBlockTypes(catalog, use.sources)]), - ] - const includes = sdkIncludes(catalog, used, hasWorkspaceTypes) + const includes = sdkIncludes(catalog, used, hasWorkspaceTypes, fullIndex) const owned = (text: string | null) => text === null || text.startsWith(TSCONFIG_MARKER) const write: Array<readonly [string, string]> = [] const ownsTsconfig = owned(current.tsconfig) @@ -548,13 +618,36 @@ function tsconfigWrites( } /** - * Rewrites the CLI's tsconfig files for the source `use` names: after an apply, which keeps the - * declarations but may have added a block `tsconfig.json` should now include, and on a check, so - * a block the edit added type-checks before anything is applied. `incremental` (a check) keeps - * the blocks the files include now, since a check scans only the source text and not the - * server's block types, and rewrites only files that exist. It reads the workspace's cached - * catalog, so it costs no request; without one nothing changes until the next get. A file whose - * text would not change is not rewritten. + * The generated declarations a directory holds for the `used` block types, under the SDK + * directory: the models, an index and `INDEX.md` of its blocks, and the file of each block + * `tsconfig.json` includes. Without a cache (`cached` false) every block, as the full config then + * reads the directory's own index. + */ +function localSdkFiles( + catalog: BlockCatalog, + used: readonly string[], + cached: boolean +): Array<readonly [string, string]> { + const blockTypes = cached ? new Set(checkedBlockTypes(catalog, used)) : undefined + return sdkTypeFiles(catalog, blockTypes).map( + ([path, text]) => [`${SDK_DIR}/${path}`, text] as const + ) +} + +/** `tsconfig.full.json`'s index: the cached one, else the directory's own. */ +function fullIndexPath(cacheIndex: string | null): string { + return cacheIndex ?? SDK_INDEX_PATH +} + +/** + * Brings the CLI's tsconfig files and the block declarations the directory holds up to date with + * the source `use` names: after an apply, which may have added a block, and on a check, so a block + * the edit added type-checks before anything is applied. `incremental` (a check) keeps the blocks + * `tsconfig.json` includes now, since a check scans only the source text and not the server's + * block types, and rewrites `tsconfig.json` only when it exists (`tsconfig.full.json`, which names + * this machine's cache, is written again when missing, as a fresh clone lacks it). It reads the + * cached catalog, so it costs no request; without one nothing changes until the next get. A file + * whose text would not change is not rewritten. */ export async function refreshTsconfigs( dir: string, @@ -565,25 +658,43 @@ export async function refreshTsconfigs( ): Promise<void> { const catalog = (await cachedSourceCatalog(dir)) ?? cachedCatalog(workspaceId) if (!catalog) return - const [tsconfig, full, workspaceTypes] = await Promise.all([ + const [tsconfig, full, workspaceTypes, index, cacheIndex] = await Promise.all([ readSourceText(dir, TSCONFIG_FILE), readSourceText(dir, TSCONFIG_FULL_FILE), readSourceText(dir, WORKSPACE_TYPES_FILE), + readSourceText(dir, SDK_INDEX_PATH), + sdkCacheIndex(catalog), ]) const kept = options.incremental ? includedBlockTypes(tsconfig) : [] + const used = usedBlockTypes(catalog, { + ...use, + blockTypes: [...(use.blockTypes ?? []), ...kept], + }) const { write } = tsconfigWrites( catalog, workspaceTypes !== null, profile, - { ...use, blockTypes: [...(use.blockTypes ?? []), ...kept] }, + used, + fullIndexPath(cacheIndex), { tsconfig, full } ) - const current: Record<string, string | null> = { - [TSCONFIG_FILE]: tsconfig, - [TSCONFIG_FULL_FILE]: full, - } - const changed = write.filter( - ([path, text]) => current[path] !== text && (!options.incremental || current[path] !== null) + const current = new Map<string, string | null>([ + [TSCONFIG_FILE, tsconfig], + [TSCONFIG_FULL_FILE, full], + ]) + // Declarations of another catalog or generator are left for the next get, which replaces them. + const declarations = + cacheIndex !== null && index?.startsWith(`${sdkIndexHead(catalog)}\n`) + ? localSdkFiles(catalog, used, true).filter(([path]) => path.startsWith(`${SDK_DIR}/blocks/`)) + : [] + const texts = await mapWithConcurrency(declarations, FILE_CONCURRENCY, ([path]) => + readSourceText(dir, path) + ) + declarations.forEach(([path], position) => current.set(path, texts[position])) + const changed = [...write, ...declarations].filter( + ([path, text]) => + current.get(path) !== text && + (!options.incremental || path !== TSCONFIG_FILE || current.get(path) !== null) ) if (changed.length > 0) await updateSourceDir(dir, { write: changed }) } @@ -617,11 +728,13 @@ export interface SourceTreeUse { /** * Everything beside the printed source that makes a directory editable: the agent guide (the * primer for the builders `use` calls, then the profile's reference) and the example workflows - * (both rewritten every time and never in `.sim/written.json`), the - * declarations (regenerated only when the catalog or generator changed), `workspace.d.ts`, the - * CLI's `tsconfig.json` and `tsconfig.full.json`, and the LF lines. `incremental` (a write that - * leaves workflows on disk unread, such as a get of only some kinds) keeps the blocks - * `tsconfig.json` includes now, as {@link refreshTsconfigs} does, adding the ones `use` needs. + * (both rewritten every time and never in `.sim/written.json`), the declarations of the blocks + * the source uses (every block's are cached once per catalog, for `tsconfig.full.json`), + * `workspace.d.ts`, the CLI's `tsconfig.json` and `tsconfig.full.json`, and the LF lines. + * Declarations of another catalog or generator are removed whole; otherwise only the block files, + * so a block the source no longer uses loses its file. `incremental` (a write that leaves + * workflows on disk unread, such as a get of only some kinds) keeps the blocks `tsconfig.json` + * includes now, as {@link refreshTsconfigs} does, adding the ones `use` needs. */ export async function sourceTreeChange( dir: string, @@ -644,19 +757,25 @@ export async function sourceTreeChange( ...(profile.sdk ?? []), ] const remove: string[] = [] - const [gitattributes, gitignore, index, tsconfig, tsconfigFull] = await Promise.all([ + const [gitattributes, gitignore, index, tsconfig, tsconfigFull, cacheIndex] = await Promise.all([ readSourceText(dir, SOURCE_GITATTRIBUTES_FILE), profile.gitignore ? readSourceText(dir, SOURCE_GITIGNORE_FILE) : null, readSourceText(dir, SDK_INDEX_PATH), readSourceText(dir, TSCONFIG_FILE), readSourceText(dir, TSCONFIG_FULL_FILE), + sdkCacheIndex(catalog), ]) const kept = options.incremental ? includedBlockTypes(tsconfig) : [] + const used = usedBlockTypes(catalog, { + sources, + blockTypes: [...(use.blockTypes ?? []), ...kept], + }) const tsconfigs = tsconfigWrites( catalog, workspaceTypes !== null, profile, - { sources, blockTypes: [...(use.blockTypes ?? []), ...kept] }, + used, + fullIndexPath(cacheIndex), { tsconfig, full: tsconfigFull } ) write.push(...tsconfigs.write) @@ -665,10 +784,13 @@ export async function sourceTreeChange( if (mergedAttributes !== null) write.push([SOURCE_GITATTRIBUTES_FILE, mergedAttributes]) const mergedIgnore = profile.gitignore ? withMissingLines(gitignore, profile.gitignore) : null if (mergedIgnore !== null) write.push([SOURCE_GITIGNORE_FILE, mergedIgnore]) - const indexHead = index?.split('\n', 2).join('\n') - if (indexHead !== `${sdkIndexHeader(catalog)}\n${sdkGeneratorLine()}`) { - remove.push(SDK_DIR) - write.push(...sdkTypeFiles(catalog)) + const current = index?.startsWith(`${sdkIndexHead(catalog)}\n`) === true + if (!current) remove.push(SDK_DIR) + if (cacheIndex !== null) { + if (current) remove.push(`${SDK_DIR}/blocks`) + write.push(...localSdkFiles(catalog, used, true)) + } else if (!current) { + write.push(...localSdkFiles(catalog, used, false)) } if (workspaceTypes === null) remove.push(WORKSPACE_TYPES_FILE) else write.push([WORKSPACE_TYPES_FILE, workspaceTypes.text]) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index 261d7f85fc6..3b91a557862 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -481,6 +481,5 @@ describe('sim workflows source check', () => { expect(includes(join(dir, 'tsconfig.json'))).toEqual( expect.arrayContaining(['.sim/sdk/blocks/slack_v2.d.ts', '.sim/sdk/blocks/gmail_v2.d.ts']) ) - expect(() => readFileSync(join(dir, 'tsconfig.full.json'))).toThrow() }) }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts index 8b3396e2e57..27cab50ad38 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts @@ -1,6 +1,7 @@ +import { execFileSync } from 'node:child_process' import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' import { COOKBOOK_EXAMPLES } from '@sim/workflow-sdk/codegen' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -9,6 +10,7 @@ import { buildGeneratedCommands } from '../../runtime/build' import { contextMockFns, contextMockState } from '../../test/context-mock' import { digest } from './source-dir' import { PRIMER_MAX_BYTES, sourcePrimer } from './source-primer' +import { attachWorkflowSourceCheck } from './workflow-source-check' import { attachWorkflowSourceGet } from './workflow-source-get' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -65,6 +67,21 @@ function program(): Command { return root } +/** Type-checks the output directory with one of its tsconfig files; returns tsc's output, or ''. */ +function typecheck(project: string): string { + try { + execFileSync( + join(__dirname, '../../../../../node_modules/.bin/tsc'), + ['--noEmit', '-p', join(outDir, project)], + { encoding: 'utf8', stdio: 'pipe' } + ) + return '' + } catch (error) { + const failure = error as { stdout?: string; stderr?: string } + return `${failure.stdout ?? ''}${failure.stderr ?? ''}` + } +} + /** The `include` list of a tsconfig the CLI wrote into the output directory. */ function tsconfigInclude(project: string): string[] { const [, ...json] = readFileSync(join(outDir, project), 'utf8').split('\n') @@ -122,7 +139,7 @@ afterEach(() => { }) describe('sim workflows source get --dir', () => { - it('writes the source, lock, base, an agent guide and declarations for every block in the workspace catalog', async () => { + it('writes the source, lock, base, an agent guide and the declarations of the blocks it uses', async () => { serve({ [SOURCE_PATH]: () => source({ 'workflow.ts': '// @sim-sdk 1\n', 'functions/score.js': 'return 1\n' }), @@ -137,7 +154,7 @@ describe('sim workflows source get --dir', () => { ) expect(existsSync(join(outDir, '.sim/sdk/core.d.ts'))).toBe(true) expect(existsSync(join(outDir, '.sim/sdk/blocks/agent.d.ts'))).toBe(true) - expect(existsSync(join(outDir, '.sim/sdk/blocks/slack_v2.d.ts'))).toBe(true) + expect(existsSync(join(outDir, '.sim/sdk/blocks/slack_v2.d.ts'))).toBe(false) const workspaceTypes = readFileSync(join(outDir, '.sim/sdk/workspace.d.ts'), 'utf8') expect(workspaceTypes).toContain("'docs': true") expect(workspaceTypes).toContain("interface WorkspaceSkills {\n 'triage': true") @@ -166,6 +183,43 @@ describe('sim workflows source get --dir', () => { } }) + it('type-checks a builder of a block the directory does not hold: through the cache at once, and locally after a check', async () => { + const workflow = [ + '// @sim-sdk 1', + "import { trigger, workflow } from '@sim/workflow-sdk'", + '', + "export default workflow('W', {}, () => {", + " const start = trigger.start('Start', {})", + '})', + '', + ].join('\n') + serve({ [SOURCE_PATH]: () => source({ 'workflow.ts': workflow }, []) }) + await get('--dir', outDir) + expect(existsSync(join(outDir, '.sim/sdk/blocks/gmail_v2.d.ts'))).toBe(false) + + writeFileSync( + join(outDir, 'workflow.ts'), + workflow + .replace('{ trigger, workflow }', '{ gmail, trigger, workflow }') + .replace( + " const start = trigger.start('Start', {})\n", + " const start = trigger.start('Start', {})\n const mail = gmail.sendEmail('Mail', { to: 'a@b.c', subject: 's', body: 'b', bogusParam: 1 })\n start.to(mail)\n" + ) + ) + const unknownParam = /'bogusParam' does not exist in type 'GmailSendEmailParams'/ + expect(typecheck('tsconfig.full.json')).toMatch(unknownParam) + + contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'json' }) + const check = new Command('source') + attachWorkflowSourceCheck(check) + check.commands.forEach((command) => command.exitOverride()) + await check.parseAsync(['node', 'source', 'check', '--dir', outDir]) + process.exitCode = undefined + + expect(existsSync(join(outDir, '.sim/sdk/blocks/gmail_v2.d.ts'))).toBe(true) + expect(typecheck('tsconfig.json')).toMatch(unknownParam) + }) + it('writes the example workflows beside the guide, outside what the tsconfig files type-check', async () => { serve() @@ -181,7 +235,7 @@ describe('sim workflows source get --dir', () => { } }) - it('type-checks only the blocks the source uses by default, and every current block in tsconfig.full.json', async () => { + it('type-checks only the blocks the source uses by default, and every current block through the cache in tsconfig.full.json', async () => { serve({ [SOURCE_PATH]: () => source( @@ -214,20 +268,26 @@ describe('sim workflows source get --dir', () => { ) expect(include).not.toContain('.sim/sdk/blocks/google_sheets_v2.d.ts') expect(include.every((path) => existsSync(join(outDir, path)))).toBe(true) - expect(tsconfigInclude('tsconfig.full.json')).toEqual([ - 'workflow.ts', - '.sim/sdk/index.d.ts', - '.sim/sdk/workspace.d.ts', - ]) + const [entry, fullIndex, workspace] = tsconfigInclude('tsconfig.full.json') + expect([entry, workspace]).toEqual(['workflow.ts', '.sim/sdk/workspace.d.ts']) + expect(fullIndex.startsWith(join(rootDir, 'config/cache/source-catalog', CATALOG.hash))).toBe( + true + ) + expect(readFileSync(fullIndex, 'utf8')).toContain("import './blocks/google_sheets_v2'") + expect(existsSync(join(dirname(fullIndex), 'blocks/google_sheets_v2.d.ts'))).toBe(true) }) - it('writes one declaration file per block, and the builder index', async () => { + it('lists the builders of the blocks it holds in the builder index, and every builder in the cached one', async () => { serve() await get('--dir', outDir) expect(existsSync(join(outDir, '.sim/sdk/operations'))).toBe(false) - expect(readFileSync(join(outDir, '.sim/sdk/INDEX.md'), 'utf8')).toContain( + const local = readFileSync(join(outDir, '.sim/sdk/INDEX.md'), 'utf8') + expect(local).toContain('- `agent` — Agent → `blocks/agent.d.ts` (`AgentParams`)') + expect(local).not.toContain('`gmail.sendEmail`') + const [, fullIndex] = tsconfigInclude('tsconfig.full.json') + expect(readFileSync(join(dirname(fullIndex), 'INDEX.md'), 'utf8')).toContain( '- `gmail.sendEmail` — Gmail: Send Email → `blocks/gmail_v2.d.ts` (`GmailSendEmailParams`)' ) }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index a0e3775efc2..06ebf6bbd7f 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -113,7 +113,7 @@ The \`// @sim-sdk 1\` header must be the first line, and the file has exactly on - \`sim workflows source params <builder>\`: one builder's params (required first, enums, defaults) and outputs; a namespace (\`gmail\`) lists its builders; a miss names the closest. Builders are named for the service, not the block type: \`gmail_v2\` is \`gmail.*\`, \`workflow_input\` is \`runWorkflow\`, the HTTP request block is \`api\`, and \`legacy.*\` builders are old versions not to write. - \`${SDK_DIR}/core.d.ts\`: the SDK (\`workflow\`, \`t\`, \`env\`, \`credential\`, \`table\`, \`loop\`, \`parallel\`, \`BlockMeta\`). -- \`${SDK_DIR}/blocks/<block type>.d.ts\`: one block's builders with their params and outputs. Do not read them, or \`${SDK_DIR}/models.d.ts\`, whole: \`params\` prints the part you need, and \`grep\` finds a model id. +- \`${SDK_DIR}/blocks/<block type>.d.ts\`: the builders of a block the source uses, with their params and outputs (check adds the file of a block an edit starts using). Do not read them, or \`${SDK_DIR}/models.d.ts\`, whole: \`params\` prints the part you need, and \`grep\` finds a model id. - \`${WORKSPACE_TYPES_FILE}\`: this workspace's tables, knowledge bases, credentials, workflows and env names. ## Rules @@ -296,11 +296,11 @@ get into another directory, or pass --force); 7 the workflow is too large to pri /** * Teaches the generated `workflows source get` leaf `--dir <dir>`: instead of printing the * response, it writes `workflow.ts`, `functions/*`, `.sim/lock.json`, `.sim/base.json`, SDK - * declarations for every block in the workspace's catalog (`.sim/sdk/**`, so a block an agent - * adds resolves too; the catalog is ETag-cached per workspace in the CLI's config directory), - * `.sim/sdk/workspace.d.ts` from the resource manifest, a `tsconfig.json` that resolves them, a - * short `.sim/AGENTS.md`, the example `.sim/examples/cookbook.ts` and the `.gitattributes` lines - * that keep the source LF. Without `--dir` the generated handler runs unchanged. Commander offers + * declarations for the blocks the source uses (`.sim/sdk/**`; every block's are generated once + * per catalog into the CLI's cache, which `tsconfig.full.json` includes, and the catalog itself is + * ETag-cached), `.sim/sdk/workspace.d.ts` from the resource manifest, the tsconfig files, the + * primer it prints atop `.sim/AGENTS.md`, the examples in `.sim/examples/` and the + * `.gitattributes` lines that keep the source LF. Without `--dir` the generated handler runs unchanged. Commander offers * no way to read the action it holds, so it is captured and delegated to, as `runs get` does. */ export function attachWorkflowSourceGet(source: Command): void { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 870b79dbfc6..b0db87ac50d 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -493,13 +493,14 @@ describe('catalog and declarations', () => { expect(existsSync(join(dir, 'workflow.ts'))).toBe(false) }) - it('leaves the declarations alone while the index on disk names the same catalog', async () => { + it('drops the file of a block the source no longer uses while the index names the same catalog', async () => { await get() - writeFileSync(join(dir, '.sim/sdk/blocks/kept_block.d.ts'), 'declare const x: 1\n') + writeFileSync(join(dir, '.sim/sdk/blocks/unused_block.d.ts'), 'declare const x: 1\n') await get() - expect(existsSync(join(dir, '.sim/sdk/blocks/kept_block.d.ts'))).toBe(true) + expect(existsSync(join(dir, '.sim/sdk/blocks/unused_block.d.ts'))).toBe(false) + expect(existsSync(join(dir, '.sim/sdk/blocks/agent.d.ts'))).toBe(true) }) it('regenerates the declarations for the same catalog when another CLI release generated them', async () => { @@ -1016,7 +1017,7 @@ describe('checking a workflow holding a type the workspace catalog withholds', ( await get() expect(readdirSync(cacheDir).sort()).toEqual( - [`${CATALOG.hash}.json`, `${WORKSPACE_CATALOG.hash}.json`, 'workspaces'].sort() + [CATALOG.hash, `${CATALOG.hash}.json`, `${WORKSPACE_CATALOG.hash}.json`, 'workspaces'].sort() ) expect(queries.filter((query) => query === WORKFLOW_ID)).toHaveLength(1) }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index b802870538e..e914afd6688 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -168,7 +168,7 @@ const ADVANCED_GUIDE = `# Sim workspace source: advanced ## Git and CI -- Commit the whole directory, \`.sim/\` included: \`${WORKSPACE_BASE_FILE}\`, each workflow's \`.sim/\` and \`.sim/written.json\` are what a later apply compares with, from any clone. The generated \`.gitignore\` leaves out only what a command keeps while it runs (\`.sim/downloads/\`, \`.sim/command.lock/\`, \`.sim-staging-*\`), and \`.gitattributes\` keeps workspace files and documents byte for byte. +- Commit the whole directory, \`.sim/\` included: \`${WORKSPACE_BASE_FILE}\`, each workflow's \`.sim/\` and \`.sim/written.json\` are what a later apply compares with, from any clone. The generated \`.gitignore\` leaves out only what a command keeps while it runs (\`.sim/downloads/\`, \`.sim/command.lock/\`, \`.sim-staging-*\`) and \`${TSCONFIG_FULL_FILE}\`, which names the CLI's cache on this machine (\`check\` writes it again), and \`.gitattributes\` keeps workspace files and documents byte for byte. - A fresh clone runs \`check\` (it downloads the block catalog once when none is cached), \`apply --dry-run\` and \`apply --yes\` with only the committed files. A workspace API key may apply tables, rows, knowledge bases, files, custom tools, MCP servers and credential bindings; workflows, skills, secrets and deployments need a member's personal API key (deployments an admin's). - Plan on every pull request (\`apply --dry-run\`), apply on merge, and commit the files the apply rewrote. Two branches that each applied: merge them, then run the dry run. No changes means the merged files match the workspace. Exit 5 means a resource changed since the base this branch holds: \`sim workspaces source get --dir <fresh dir>\`, carry the edit over, and apply from there. - \`sim workspaces source get --dir .\` refreshes the workspace \`${WORKSPACE_BASE_FILE}\` names, whatever the profile's default; a \`--workspace\` naming another is refused. @@ -195,7 +195,13 @@ export const WORKSPACE_TREE: SourceTreeProfile = { `${KIND_DIRS.file}/** -text`, `${KIND_DIRS['knowledge-base']}/*/docs/** -text`, ], - gitignore: [`${SOURCE_DOWNLOADS_DIR}/`, `${SOURCE_DIR_LOCK}/`, `${SOURCE_STAGING_PREFIX}*/`], + gitignore: [ + `${SOURCE_DOWNLOADS_DIR}/`, + `${SOURCE_DIR_LOCK}/`, + `${SOURCE_STAGING_PREFIX}*/`, + // It includes the cached declarations by this machine's absolute path; `check` writes it again. + TSCONFIG_FULL_FILE, + ], sdk: Object.entries(WORKSPACE_SDK_DTS).map( ([name, text]) => [`${WORKSPACE_SDK_DIR}/${name}`, text] as const ), diff --git a/packages/workflow-sdk/README.md b/packages/workflow-sdk/README.md index 37ad2fc9fc5..5cd3e24b00d 100644 --- a/packages/workflow-sdk/README.md +++ b/packages/workflow-sdk/README.md @@ -31,7 +31,8 @@ have the CLI confirm the directory holds that workflow. second and catches what check and the dry run do not, such as a reference to a start input the trigger does not declare. It type-checks with the TypeScript release the declarations are built for. `tsconfig.json` includes only the blocks the source used at the - last check, get or apply (fast); `tsconfig.full.json` includes every current block. In a + last check, get or apply (fast); `tsconfig.full.json` includes every current block, from the CLI's cache. A builder of a block + the directory does not hold yet type-checks once `check` has run: it writes that block's file. In a workspace directory, `sim workspaces source check --dir .` checks every workflow against the directory's own secrets, credentials, resource keys and table columns, and `sim workflows source check` run in a `workflows/<key>` folder does the same for one. @@ -65,12 +66,13 @@ refuse with `CONFIRMATION_REQUIRED`). `--dry-run` never asks. | `.sim/base.json` | The workflow, workspace, fingerprint and catalog hash the source was read at. Do not edit. | | `.sim/written.json` | The CLI's record of each file as it last wrote it, so `status`, `diff` and a later `get` can tell your unapplied edits from files it may replace. Commit it with the lock and base. Do not edit. | | `.sim/sdk/core.d.ts` | This package: `workflow`, `t`, `env`, `credential`, `table`, `BlockMeta`, ... | -| `.sim/sdk/INDEX.md` | One line per builder: `slack.sendMessage — Slack: Send Message → blocks/slack_v2.d.ts (SlackSendMessageParams)`, legacy builders apart, and the block type → builder naming rule. `grep` it to find a builder. | -| `.sim/sdk/blocks/{type}.d.ts` | One block type's builders with their params and outputs, for every block, not only the ones in use. `grep` the params type `INDEX.md` names rather than reading a file whole. | -| `.sim/sdk/index.d.ts` | Imports every current block (not the legacy-only ones); `tsconfig.full.json` includes it. | +| `.sim/sdk/INDEX.md` | The block type → builder naming rule, how a workflow runs, and one line per builder of the blocks the directory holds: `slack.sendMessage — Slack: Send Message → blocks/slack_v2.d.ts (SlackSendMessageParams)`. Any other builder: `sim workflows source params <builder>`. | +| `.sim/sdk/blocks/{type}.d.ts` | One block type's builders with their params and outputs, for the blocks the source uses and the core builders' (`check` adds a block an edit starts using). `sim workflows source params` prints one builder rather than a file whole. | +| `.sim/sdk/index.d.ts` | Names the catalog and generator the declarations come from, and imports the blocks the directory holds. | +| `~/.sim/cache/source-catalog/<hash>/sdk-<generator>/` | Every block's declarations and the full `INDEX.md`, generated once per catalog and shared by every directory; `tsconfig.full.json` includes its `index.d.ts` (every current block). | | `.sim/sdk/workspace.d.ts` | This workspace's tables (with columns), knowledge bases, credentials, workflows (with inputs), skills, custom tools and env var names, so `table('tickets')`, `credential('support-slack')`, `skill('triage')` and `env.X` are checked. | | `tsconfig.json` | Resolves `@sim/workflow-sdk` to `.sim/sdk` and type-checks `workflow.ts` with the blocks it uses (from the source and the server's `blockTypes`) and the core builders' blocks: a few dozen files for `tsc` instead of every block. | -| `tsconfig.full.json` | The same with every current block, plus any legacy-only block the workflow uses. | +| `tsconfig.full.json` | The same with every current block (the cached index, an absolute path on this machine), plus any legacy-only block the workflow uses. | | `.sim/AGENTS.md` | The authoring guide for agents: the primer `get` prints (edit loop, identity, references, deletes, deploy, exit codes, and a snippet for each builder the source calls), then a reference linking [the docs](https://docs.sim.ai/workflows/code). A top-level `AGENTS.md` of your own is never touched. | | `.sim/examples/*.ts` | Example workflows, together one correct example of each common block: `cookbook.ts` (agent, condition, function, table, knowledge, Slack, human in the loop, loop, response), `integrations.ts` (webhook, weekday cron, HTTP headers, table update by id, Gmail, `onError`), `agents.ts` (router, agent tools, child workflow, parallel). Read them before writing. They name example resources, so the tsconfig files leave them out. | | `.gitattributes` | `text eol=lf` for the source files, merged into any file already there. | @@ -165,7 +167,7 @@ files are stored verbatim. One file per block; imports between function files ar | `src/core.d.ts` | Authoring types. Exports exactly `SDK_CORE_EXPORTS` from `@sim/workflow-code`; after editing run `bun scripts/generate-embedded-text.ts`. | | `cookbook/examples/*.ts` | The examples the CLI writes into `.sim/examples/`. After editing run `bun scripts/generate-embedded-text.ts`; `apps/sim/lib/workflows/code/real-catalog.test.ts` type-checks and compiles each against the real catalog (its `COOKBOOK_WORKSPACE` lists the resources they name). | | `src/runtime.ts` | The runtime entry: every export throws `SimSdkNotExecutable`. | -| `src/codegen/` | `generateSdkTypes(catalog)` (catalog → `.d.ts` and the `.sim/sdk/INDEX.md` builder list, deterministic, sorted by path), `generateWorkspaceTypes(manifest)` (`workspace.d.ts`), and `layout.ts`: which declaration files a directory's tsconfig files include. | +| `src/codegen/` | `generateSdkTypes(catalog, { blockTypes })` (catalog → `.d.ts` and the `INDEX.md` builder list, for every block or a directory's subset, deterministic, sorted by path), `describeBuilder`/`lookupBuilder` (one builder's params and outputs, for `sim workflows source params`), `generateWorkspaceTypes(manifest)` (`workspace.d.ts`), and `layout.ts`: which declaration files a directory's tsconfig files include. | | `test/examples` | Five example workflows, also compiled against the real catalog with zero diagnostics in `apps/sim/lib/workflows/code/sdk-examples.test.ts`. | | `test/negative` | Code each generated type must reject. | diff --git a/packages/workflow-sdk/src/codegen/generate.ts b/packages/workflow-sdk/src/codegen/generate.ts index a7d11d3dafd..1b6be4cc45b 100644 --- a/packages/workflow-sdk/src/codegen/generate.ts +++ b/packages/workflow-sdk/src/codegen/generate.ts @@ -1,5 +1,6 @@ /** - * `generateSdkTypes(catalog)`: the block catalog → `.d.ts` files. + * `generateSdkTypes(catalog, options)`: the block catalog → `.d.ts` files, for every block (the + * CLI's cached copy of a catalog) or only the blocks a source directory holds. * * - `blocks/{type}.d.ts` per block type with builders: every builder's params and output types, * and the augmentation of `@sim/workflow-sdk` with the builder functions, tool forms and @@ -8,8 +9,9 @@ * - `models.d.ts`: model ids, apart so a model-list change touches one file. * - `index.d.ts`: imports the models and every current block (not the legacy-only ones); the only * file that names the catalog hash. - * - `INDEX.md`: one line per builder naming its block, operation, file and params type, so an - * agent finds a builder with one `grep` instead of reading declarations. + * - `INDEX.md`: the naming rule, how a workflow runs, and one line per builder naming its block, + * operation, file and params type (in a directory's copy, only its blocks' builders; any other + * builder is one `sim workflows source params` away). * * Output is deterministic: the same catalog gives byte-identical files in path order. */ @@ -56,7 +58,16 @@ export interface SdkTypeFile { text: string } -export type GenerateSdkTypesFn = (catalog: BlockCatalog) => SdkTypeFile[] +/** Which declarations `generateSdkTypes` writes. */ +interface SdkTypesOptions { + /** + * Only these block types' files, with `index.d.ts` importing them and `INDEX.md` listing their + * builders: a source directory's own copy. Without it, every block (the cached full catalog). + */ + blockTypes?: ReadonlySet<string> +} + +export type GenerateSdkTypesFn = (catalog: BlockCatalog, options?: SdkTypesOptions) => SdkTypeFile[] /** Names `core.d.ts` exports that generated files may reference; imported only when used. */ const CORE_TYPE_NAMES = [ @@ -729,12 +740,20 @@ const NAMING_RULE = [ '`<namespace>.tool.<operation>(params)` (`slack.tool.sendMessage({ … })`).', ] +/** The lookup command for any builder, which the directory's `INDEX.md` points to. */ +const PARAMS_LOOKUP = + 'Look up any builder, used here or not: `sim workflows source params <builder>` (`table.upsertRow`, `trigger.schedule`; a namespace such as `gmail` lists its builders) prints its params and outputs offline.' + /** * `INDEX.md`: one line per builder, `key — Block: Operation → file (ParamsType)`, current builders - * first and legacy ones in their own section, so `grep -i gmail .sim/sdk/INDEX.md` finds a builder - * and the params type to read. + * first and legacy ones in their own section. The full catalog's lists every builder; a + * directory's (`subset`) only the builders of the blocks it holds, beside the lookup command. */ -function generateSdkIndex(catalog: BlockCatalog, entries: readonly IndexEntry[]): SdkTypeFile { +function generateSdkIndex( + catalog: BlockCatalog, + entries: readonly IndexEntry[], + subset: boolean +): SdkTypeFile { const line = (entry: IndexEntry) => `- \`${entry.key}\` — ${entry.summary} → \`${blockTypesFile(entry.type)}\` (\`${entry.paramsName}\`)` const sorted = [...entries].sort((a, b) => compareStrings(a.key, b.key)) @@ -749,10 +768,10 @@ function generateSdkIndex(catalog: BlockCatalog, entries: readonly IndexEntry[]) '', ...NAMING_RULE, '', - 'Find a builder: `grep -i "gmail" .sim/sdk/INDEX.md`. Read its params and outputs: `grep -n -A40 "export type GmailSendEmailParams" .sim/sdk/blocks/gmail_v2.d.ts` (the output type follows as `GmailSendEmailOutput`).', + PARAMS_LOOKUP, '', ...RUNTIME_SEMANTICS, - '## Builders', + subset ? '## Builders of the blocks in this directory' : '## Builders', '', ...current.map(line), '', @@ -770,22 +789,27 @@ function generateSdkIndex(catalog: BlockCatalog, entries: readonly IndexEntry[]) } } -/** Generates every declaration file and `INDEX.md` for a catalog, sorted by path. */ -export const generateSdkTypes: GenerateSdkTypesFn = (catalog) => { +/** + * Generates the declaration files and `INDEX.md` for a catalog, sorted by path: every block, or + * only `options.blockTypes` (a directory's own copy, whose `index.d.ts` imports just those). + */ +export const generateSdkTypes: GenerateSdkTypesFn = (catalog, options = {}) => { const byType = new Map<string, BuilderEntry[]>() for (const entry of builderEntries(catalog)) { + if (options.blockTypes && !options.blockTypes.has(entry.builder.type)) continue const list = byType.get(entry.builder.type) if (list) list.push(entry) else byType.set(entry.builder.type, [entry]) } const types = [...byType.keys()].sort(compareStrings) - const legacyOnly = legacyOnlyBlockTypes(catalog) + // A directory's index imports every block it holds: it holds a legacy-only one only when used. + const legacyOnly = options.blockTypes ? new Set<string>() : legacyOnlyBlockTypes(catalog) const index: IndexEntry[] = [] const files: SdkTypeFile[] = [generateModelsFile(catalog)] for (const type of types) { files.push(generateBlockFile(catalog, type, byType.get(type) ?? [], index)) } - files.push(generateSdkIndex(catalog, index)) + files.push(generateSdkIndex(catalog, index, options.blockTypes !== undefined)) files.push({ path: SDK_INDEX_FILE, text: [ diff --git a/packages/workflow-sdk/src/codegen/index.ts b/packages/workflow-sdk/src/codegen/index.ts index b92eb930850..57668009355 100644 --- a/packages/workflow-sdk/src/codegen/index.ts +++ b/packages/workflow-sdk/src/codegen/index.ts @@ -7,6 +7,7 @@ export { sdkIndexHeader, } from './generate' export { + checkedBlockTypes, defaultBlockFiles, fullBlockFiles, referencedBlockTypes, diff --git a/packages/workflow-sdk/src/codegen/layout.ts b/packages/workflow-sdk/src/codegen/layout.ts index bb990e59a6a..ce65bb8d80e 100644 --- a/packages/workflow-sdk/src/codegen/layout.ts +++ b/packages/workflow-sdk/src/codegen/layout.ts @@ -1,8 +1,9 @@ /** * The layout of the generated declarations (`generateSdkTypes`) and which of them a source - * directory type-checks. A whole catalog is about 500 block files; `tsc` over all of them costs - * hundreds of megabytes, so a directory's `tsconfig.json` includes only the blocks its source - * uses, and `tsconfig.full.json` every current block for adding a new one. + * directory holds and type-checks. A whole catalog is about 500 block files (over 13 MB), so a + * directory holds and its `tsconfig.json` includes only the blocks its source uses; the whole set + * lives once per catalog in the CLI's cache, which `tsconfig.full.json` includes for adding any + * block. */ import { compareStrings } from '@sim/utils/string' @@ -98,10 +99,10 @@ export function referencedBlockTypes(catalog: BlockCatalog, sources: Iterable<st } /** - * The block files a directory's `tsconfig.json` includes, relative to the SDK directory: those of - * the `used` types that have one, plus the {@link ALWAYS_CHECKED_BUILDERS}' blocks. + * The block types a directory holds declarations for and its `tsconfig.json` type-checks: the + * `used` types that have builders, plus the {@link ALWAYS_CHECKED_BUILDERS}' blocks. */ -export function defaultBlockFiles(catalog: BlockCatalog, used: Iterable<string>): string[] { +export function checkedBlockTypes(catalog: BlockCatalog, used: Iterable<string>): string[] { const declared = declaredBlockTypes(catalog) const types = new Set<string>() for (const key of ALWAYS_CHECKED_BUILDERS) { @@ -109,7 +110,15 @@ export function defaultBlockFiles(catalog: BlockCatalog, used: Iterable<string>) if (type) types.add(type) } for (const type of used) if (declared.has(type)) types.add(type) - return [...types].sort(compareStrings).map(blockTypesFile) + return [...types].sort(compareStrings) +} + +/** + * The block files a directory's `tsconfig.json` includes, relative to the SDK directory: those of + * the {@link checkedBlockTypes}. + */ +export function defaultBlockFiles(catalog: BlockCatalog, used: Iterable<string>): string[] { + return checkedBlockTypes(catalog, used).map(blockTypesFile) } /** From 160b5d9142be4c2542a9430466d9b2b274f9fafa Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 17:05:15 -0800 Subject: [PATCH 35/59] perf(api): encode the source catalog and other large v2 source answers as brotli or gzip Staging serves the 8.7 MB block catalog uncompressed (no Content-Encoding, no Vary: Accept-Encoding: Next's own compression does not reach these answers in deployment), so the workflow and workspace source GETs, the catalog and the resource manifest opt into encoding a body over 8 KB themselves: brotli (quality 5) when the request accepts it, else gzip, on the thread pool, cached per entity tag beside the validated body. An encoded answer says Vary: Accept-Encoding and carries its tag weak; Next's middleware skips a body already encoded. The CLI asks for br, gzip on the wire (Node's fetch offers only gzip and deflate) and never through an in-process transport, which reads the handler's response undecoded. Measured locally with the CLI client: 8.67 MB plain, 922 KB gzip, 781 KB br. --- .../v2/workflows/[workflowId]/source/route.ts | 1 + .../workspaces/[workspaceId]/source/route.ts | 1 + .../workflows/source/catalog/route.ts | 4 +- .../workflows/source/resources/route.ts | 1 + .../api/server/routes/response-compression.ts | 59 ++++++++++++++++ .../api/server/routes/v2-json-route.test.ts | 65 +++++++++++++++++ .../lib/api/server/routes/v2-json-route.ts | 47 +++++++++++-- packages/sim-cli/src/http/client.test.ts | 70 +++++++++++++++++++ packages/sim-cli/src/http/client.ts | 7 ++ 9 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 apps/sim/lib/api/server/routes/response-compression.ts diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts index 14a45dafb6e..02dc40c5e77 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts @@ -33,6 +33,7 @@ export const GET = defineV2JsonRoute({ }), useCase: readWorkflowSource, present: (source) => ({ data: source }), + encodeLargeBody: true, }) /** diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts index 681a3c871dc..beb36c71761 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts @@ -36,6 +36,7 @@ export const GET = defineV2JsonRoute({ }), useCase: withSourceSlot(readWorkspaceSource, 'read'), present: (source) => ({ data: source }), + encodeLargeBody: true, }) /** diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts index 435ba9f7da9..6a180f24db6 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts @@ -9,7 +9,8 @@ export const revalidate = 0 /** * GET /api/v2/workspaces/{workspaceId}/workflows/source/catalog — the source catalog. The - * response carries `ETag: "<hash>"`; a matching `If-None-Match` answers `304` with no body. + * response carries `ETag: "<hash>"`; a matching `If-None-Match` answers `304` with no body. The + * catalog runs to megabytes, so a client accepting brotli or gzip gets it encoded. */ export const GET = defineV2JsonRoute({ contract: v2GetSourceCatalogContract, @@ -24,4 +25,5 @@ export const GET = defineV2JsonRoute({ useCase: readSourceCatalog, entityTag: (catalog) => catalog.hash, present: (catalog) => ({ data: catalog }), + encodeLargeBody: true, }) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts index fb3fc483340..66b0442456d 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts @@ -25,4 +25,5 @@ export const GET = defineV2JsonRoute({ mapInput: ({ params }) => ({ workspaceId: params.workspaceId }), useCase: readSourceResources, present: (manifest) => ({ data: manifest }), + encodeLargeBody: true, }) diff --git a/apps/sim/lib/api/server/routes/response-compression.ts b/apps/sim/lib/api/server/routes/response-compression.ts new file mode 100644 index 00000000000..ed690a2ae03 --- /dev/null +++ b/apps/sim/lib/api/server/routes/response-compression.ts @@ -0,0 +1,59 @@ +/** + * Content-encoding for the v2 JSON answers that run to megabytes (the workflow source catalog, a + * workspace's source). Next's own compression does not reach these answers in deployment, so a + * route that opts in encodes its body itself: brotli when the request accepts it, else gzip, and + * never for a request that names neither (an in-process transport sends no `Accept-Encoding`, so + * its caller reads plain JSON). Next's compression middleware skips a response that already has a + * `Content-Encoding`, so nothing is encoded twice. + */ +import { promisify } from 'node:util' +import { brotliCompress, constants, gzip } from 'node:zlib' + +/** An encoding a route may answer with. */ +export type BodyEncoding = 'br' | 'gzip' + +/** Bodies smaller than this are sent as they are: encoding them saves too little. */ +export const MIN_ENCODED_BODY_BYTES = 8 * 1024 + +/** + * Brotli quality 5: within a few percent of the best ratio on JSON at a fraction of the time + * (about 50 ms for the 9 MB block catalog, a tenth of its size). + */ +const BROTLI_QUALITY = 5 + +const brotliAsync = promisify(brotliCompress) +const gzipAsync = promisify(gzip) + +/** + * The encoding an `Accept-Encoding` header prefers among the ones a route sends: `br` over + * `gzip` unless its q-value is lower, `*` standing for either, a q-value of 0 refusing one. + * `null` when the header is absent or accepts neither. + */ +export function preferredEncoding(acceptEncoding: string | null): BodyEncoding | null { + if (!acceptEncoding) return null + const weights = new Map<string, number>() + for (const part of acceptEncoding.split(',')) { + const [name, ...params] = part.trim().toLowerCase().split(';') + if (!name) continue + const q = params.map((param) => /^\s*q=([0-9.]+)\s*$/.exec(param)?.[1]).find(Boolean) + const weight = q === undefined ? 1 : Number(q) + weights.set(name, Number.isFinite(weight) ? weight : 0) + } + const weight = (name: BodyEncoding) => weights.get(name) ?? weights.get('*') ?? 0 + const br = weight('br') + const gz = weight('gzip') + if (br <= 0 && gz <= 0) return null + return br >= gz ? 'br' : 'gzip' +} + +/** `body` encoded as `encoding`, on the thread pool so a large body never blocks the event loop. */ +export function encodeBody(body: Uint8Array, encoding: BodyEncoding): Promise<Buffer> { + return encoding === 'br' + ? brotliAsync(body, { + params: { + [constants.BROTLI_PARAM_QUALITY]: BROTLI_QUALITY, + [constants.BROTLI_PARAM_SIZE_HINT]: body.byteLength, + }, + }) + : gzipAsync(body) +} diff --git a/apps/sim/lib/api/server/routes/v2-json-route.test.ts b/apps/sim/lib/api/server/routes/v2-json-route.test.ts index 7b8a28dc381..a05cb29cd17 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.test.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.test.ts @@ -1,3 +1,4 @@ +import { brotliDecompressSync, gunzipSync } from 'node:zlib' import type { PersonalApiKeyPrincipal } from '@sim/auth/principal' import { MockV2ApiKeyUnauthenticatedError, @@ -231,6 +232,70 @@ describe('defineV2JsonRoute', () => { expect(unchanged.headers.get('Cache-Control')).toBe('private, no-cache') }) + it.each([ + { acceptEncoding: 'br, gzip', encoding: 'br' }, + { acceptEncoding: 'gzip, deflate', encoding: 'gzip' }, + { acceptEncoding: 'br;q=0, gzip', encoding: 'gzip' }, + { acceptEncoding: 'gzip;q=0.5, *', encoding: 'br' }, + { acceptEncoding: 'identity', encoding: null }, + { acceptEncoding: null, encoding: null }, + ])( + 'encodes a large body as the request accepts ($acceptEncoding → $encoding), keeping the tag weak', + async ({ acceptEncoding, encoding }) => { + const value = 'x'.repeat(64 * 1024) + const handler = defineV2JsonRoute({ + contract, + auth: v2ApiKeyAuth, + operation, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: { operation, execute: async () => ({ value }) }, + entityTag: () => 'v1', + present: (result) => ({ data: result }), + encodeLargeBody: true, + }) + const asking = request() + if (acceptEncoding) asking.headers.set('accept-encoding', acceptEncoding) + + const served = await handler(asking) + + const bytes = Buffer.from(await served.arrayBuffer()) + const decoded = + encoding === 'br' + ? brotliDecompressSync(bytes) + : encoding === 'gzip' + ? gunzipSync(bytes) + : bytes + expect(JSON.parse(decoded.toString('utf8'))).toEqual({ data: { value } }) + expect(served.headers.get('Content-Encoding')).toBe(encoding) + expect(served.headers.get('Vary')).toBe('Accept-Encoding') + expect(served.headers.get('ETag')).toBe(encoding ? 'W/"v1"' : '"v1"') + if (encoding) expect(bytes.byteLength).toBeLessThan(value.length / 10) + } + ) + + it('sends a body under the encoding threshold as it is', async () => { + const handler = defineV2JsonRoute({ + contract, + auth: v2ApiKeyAuth, + operation, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: { operation, execute: async ({ input }) => input }, + present: (result) => ({ data: result }), + encodeLargeBody: true, + }) + const asking = request() + asking.headers.set('accept-encoding', 'br, gzip') + + const served = await handler(asking) + + expect(served.headers.get('Content-Encoding')).toBeNull() + await expect(served.json()).resolves.toEqual({ data: { value: 'ok' } }) + }) + it('never serves a cached body to a request whose input presents it differently', async () => { const handler = defineV2JsonRoute({ contract, diff --git a/apps/sim/lib/api/server/routes/v2-json-route.ts b/apps/sim/lib/api/server/routes/v2-json-route.ts index adbc0f3d670..df2bbc7a7fb 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.ts @@ -18,6 +18,11 @@ import { requireHeadAuthorizableUseCase, requireJsonRouteDefinition, } from '@/lib/api/server/routes/definition' +import { + encodeBody, + MIN_ENCODED_BODY_BYTES, + preferredEncoding, +} from '@/lib/api/server/routes/response-compression' import type { JsonApiRouteContract, JsonNextRouteHandler, @@ -597,6 +602,13 @@ interface V2JsonRouteOptions<C extends JsonApiRouteContract, O extends Applicati * `present` never sees the principal, so the body is a function of exactly that key. */ entityTag?(result: NoInfer<R>): string + /** + * Encodes a success body of at least {@link MIN_ENCODED_BODY_BYTES} as brotli or gzip when the + * request's `Accept-Encoding` allows (`response-compression.ts`), for a route whose answers run + * to megabytes. Every such answer says `Vary: Accept-Encoding`; an encoded one with an entity + * tag carries it weak (`W/"…"`), as its bytes differ from the plain body's. + */ + encodeLargeBody?: true } /** @@ -630,6 +642,11 @@ export function defineV2JsonRoute< const validatedByTag = options.entityTag ? new LRUCache<string, NonNullable<unknown>>({ max: VALIDATED_BODY_CACHE_MAX }) : undefined + /** Encoded bodies by the validated body's cache key and encoding, so a repeat skips encoding. */ + const encodedByTag = + options.entityTag && options.encodeLargeBody + ? new LRUCache<string, Buffer>({ max: VALIDATED_BODY_CACHE_MAX }) + : undefined const wrapped = withRouteHandler<JsonRouteContext | undefined>( async (request, context) => { if (!methodMatchesContract(request.method, options.contract.method)) { @@ -741,11 +758,33 @@ export function defineV2JsonRoute< ) } await options.onSuccess?.({ principal: auth.principal, input, result }) - return NextResponse.json(validatedBody, { + const cacheControl = entityTag ? REVALIDATE_CACHE_CONTROL : 'private, no-store' + if (!options.encodeLargeBody) { + return NextResponse.json(validatedBody, { + status: responseStatus, + headers: { 'Cache-Control': cacheControl, ...(entityTag ? { ETag: entityTag } : {}) }, + }) + } + const json = Buffer.from(JSON.stringify(validatedBody)) + const encoding = + json.byteLength >= MIN_ENCODED_BODY_BYTES + ? preferredEncoding(request.headers.get('accept-encoding')) + : null + const encodedKey = cacheKey && encoding ? `${encoding} ${cacheKey}` : undefined + const encoded = encoding + ? ((encodedKey ? encodedByTag?.get(encodedKey) : undefined) ?? + (await encodeBody(json, encoding))) + : null + if (encodedKey && encoded) encodedByTag?.set(encodedKey, encoded) + return new NextResponse(new Uint8Array(encoded ?? json), { status: responseStatus, - headers: entityTag - ? { 'Cache-Control': REVALIDATE_CACHE_CONTROL, ETag: entityTag } - : { 'Cache-Control': 'private, no-store' }, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': cacheControl, + Vary: 'Accept-Encoding', + ...(encoding ? { 'Content-Encoding': encoding } : {}), + ...(entityTag ? { ETag: encoding ? `W/${entityTag}` : entityTag } : {}), + }, }) } catch (error) { const response = options.errorPolicy.render(error) diff --git a/packages/sim-cli/src/http/client.test.ts b/packages/sim-cli/src/http/client.test.ts index 85b7312a37d..e8ef2101888 100644 --- a/packages/sim-cli/src/http/client.test.ts +++ b/packages/sim-cli/src/http/client.test.ts @@ -1019,3 +1019,73 @@ describe('OAuth bearer credentials', () => { expect(init.headers).not.toHaveProperty('authorization') }) }) + +describe('encoded answers', () => { + function profile(overrides: Partial<ConstructorParameters<typeof SimClient>[0]>) { + return { + name: 'default', + endpoint: 'https://sim.example', + authProfile: 'default', + apiKey: 'key', + oauth: null, + workspaceId: 'ws_1', + output: 'json' as const, + sources: { + endpoint: 'default' as const, + credential: 'env' as const, + workspaceId: 'env' as const, + output: 'default' as const, + }, + ...overrides, + } + } + + /** A local server that answers brotli when asked, recording each request's Accept-Encoding. */ + async function encodingServer(body: unknown) { + const { createServer } = await import('node:http') + const { brotliCompressSync } = await import('node:zlib') + const asked: Array<string | undefined> = [] + const server = createServer((request, response) => { + asked.push(request.headers['accept-encoding']) + const json = Buffer.from(JSON.stringify(body)) + const br = /\bbr\b/.test(request.headers['accept-encoding'] ?? '') + response.writeHead(200, { + 'content-type': 'application/json', + ...(br ? { 'content-encoding': 'br' } : {}), + }) + response.end(br ? brotliCompressSync(json) : json) + }) + await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + const port = typeof address === 'object' && address ? address.port : 0 + return { endpoint: `http://127.0.0.1:${port}`, asked, close: () => server.close() } + } + + it('asks for brotli on the wire and reads the decoded answer', async () => { + const body = { data: { blocks: 'x'.repeat(50_000) } } + const server = await encodingServer(body) + try { + const wire = new SimClient(profile({ endpoint: server.endpoint })) + + await expect(wire.request('/api/v2/catalog')).resolves.toEqual(body) + expect(server.asked).toEqual(['br, gzip']) + } finally { + server.close() + } + }) + + it('never asks an in-process transport for an encoding it would hand back undecoded', async () => { + const transport = vi.fn( + async (_url: string | URL | Request, _init?: RequestInit) => + new Response(JSON.stringify({ data: 1 }), { + headers: { 'content-type': 'application/json' }, + }) + ) + const embedded = new SimClient(profile({ transport })) + + await embedded.request('/api/v2/catalog') + + const init = transport.mock.calls[0]?.[1] + expect(new Headers(init?.headers).has('accept-encoding')).toBe(false) + }) +}) diff --git a/packages/sim-cli/src/http/client.ts b/packages/sim-cli/src/http/client.ts index d3fbdf5e618..7aa7618e6cb 100644 --- a/packages/sim-cli/src/http/client.ts +++ b/packages/sim-cli/src/http/client.ts @@ -109,6 +109,9 @@ export function buildUrl( return url.toString() } +/** The encodings the CLI asks for; Node's fetch would otherwise offer only gzip and deflate. */ +const ACCEPT_ENCODING = 'br, gzip' + /** * Statuses `fetch` would otherwise follow for us, and must not. * @@ -731,6 +734,10 @@ export class SimClient { ? { authorization: `Bearer ${credential.oauth.accessToken}` } : {}), accept: 'application/json', + // Large answers (the block catalog, a workspace's source) come encoded, a tenth of + // their size; fetch decodes them. An in-process transport reads the handler's + // response as it is, so it never asks for an encoding. + ...(this.profile.transport ? {} : { 'accept-encoding': ACCEPT_ENCODING }), ...identityHeaders(), ...(hasBody ? { 'content-type': 'application/json' } : {}), ...options.headers, From 67dafefee08d0666a5665e6a0caca9112d9b83c0 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 17:14:58 -0800 Subject: [PATCH 36/59] chore(workflow-sdk): regenerate embedded workspace text after merging fix/e2e-ws --- .../src/codegen/workspace-text.generated.ts | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts index 724b21ef94e..09d12f978cd 100644 --- a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts +++ b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts @@ -1,25 +1,4 @@ // Generated by scripts/generate-embedded-text.ts from src/workspace; do not edit. /** The workspace declaration SDK, file name → text (`index.d.ts` re-exports the rest). */ -export const WORKSPACE_SDK_DTS: Readonly<Record<string, string>> = { - 'credentials.d.ts': - '/**\n * Credential declarations: `credentials.ts` at the root of a workspace source directory names the\n * saved credentials its workflows use, each by the provider it is for. Never a token: connect one\n * with `sim credentials connect <provider> --name "<display name>"`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineCredentials } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineCredentials({\n * slack: { provider: \u0027slack\u0027 },\n * gmail: { provider: \u0027google-email\u0027, name: \u0027Support inbox\u0027 },\n * })\n * ```\n *\n * A workflow uses one as `credential(\u0027slack\u0027)`, which type-checks against these keys. An apply\n * binds each key to a credential of this workspace: the one bound to it already; else the only\n * credential of the provider no other declared key holds, narrowed to the display name `name`\n * when given. A key nothing binds to, or several credentials could bind to, is reported, and a\n * workflow using it is held back while the rest of the apply goes ahead. Applying the same\n * directory to another workspace binds each key to that workspace\u0027s own credential. Removing a key\n * unbinds it only with `--allow-delete`; the credential itself is never deleted.\n */\n\n/** One declared credential. */\nexport interface CredentialDeclaration {\n /** The provider the credential is for (`slack`, `google-email`, …). */\n provider: string\n /** The display name that picks one of several credentials of the provider. */\n name?: string\n}\n\n/** Declares the workspace\u0027s credentials by key. Its default export is the whole file. */\nexport declare function defineCredentials<const T extends Record<string, CredentialDeclaration>>(\n credentials: T\n): T\n\n/**\n * The credential keys a `credentials.ts` declaration adds, each with its provider, as\n * `workspace.d.ts` merges them into `credential(\u0027…\u0027)`.\n */\nexport type DeclaredCredentials<T> = {\n [K in keyof T]: T[K] extends { provider: infer P } ? { provider: P } : never\n}\n', - 'deployment.d.ts': - '/**\n * Deployment declarations: `deployments/<workflow key>.ts` in a workspace source directory deploys\n * `workflows/<workflow key>/`, so its schedule, webhook, API and chat triggers run.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineDeployment, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineDeployment({\n * live: \u0027draft\u0027,\n * chat: {\n * identifier: \u0027support\u0027,\n * title: \u0027Support\u0027,\n * auth: { password: env.SUPPORT_CHAT_PASSWORD },\n * outputs: [{ block: \u0027Answer\u0027, path: \u0027content\u0027 }],\n * },\n * })\n * ```\n *\n * An apply deploys after it writes the workflows. Removing the file undeploys the workflow, which\n * stops its schedules, webhooks, API and chat, and needs `--allow-delete`. Deploying needs a\n * workspace admin and is refused to a workspace API key. Publishing to an MCP server is not part\n * of source.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** Who may open a chat. */\nexport type ChatAuth =\n | \u0027public\u0027\n /**\n * A password from an environment variable, by name, never the value itself; or `stored()`, the\n * password the chat already stores (valid only for a chat that has one).\n */\n | { password: EnvRef }\n /** These emails (or `@domain.com` patterns), signing in with a code. */\n | { emails: string[] }\n /** These emails (or `@domain.com` patterns), signing in through the workspace\u0027s SSO. */\n | { sso: string[] }\n\n/** One output a chat answers with. */\nexport interface ChatOutput {\n /** The block\u0027s name, as the workflow names it. */\n block: string\n /** The output path on that block (`content`, `result.text`). */\n path: string\n}\n\n/** The chat a deployment serves. */\nexport interface ChatDeployment {\n /** The chat\u0027s URL path segment, unique across the platform. */\n identifier: string\n title: string\n description?: string\n auth: ChatAuth\n outputs?: ChatOutput[]\n /** The chat\u0027s look; an omitted field takes the platform default. */\n customizations?: { primaryColor?: string; welcomeMessage?: string; imageUrl?: string }\n /** Show the model\u0027s thinking to clients that ask for it. */\n includeThinking?: boolean\n /** Show tool calls to clients that ask for them. */\n includeToolCalls?: boolean\n}\n\n/** A deployment declaration. */\nexport interface DeploymentDeclaration {\n /**\n * `\u0027draft\u0027`: an apply deploys the draft whenever it differs from the live version (after this\n * apply\u0027s workflow edits). `\u0027pinned\u0027`: an apply never redeploys, and says when the draft differs.\n */\n live: \u0027draft\u0027 | \u0027pinned\u0027\n /** Public: anyone may run the workflow\u0027s API without a key. Private when omitted. */\n api?: { public?: boolean }\n /** The chat the deployment serves; none when omitted. */\n chat?: ChatDeployment\n}\n\n/** Declares a workflow\u0027s deployment. Its default export is the whole file. */\nexport declare function defineDeployment(deployment: DeploymentDeclaration): DeploymentDeclaration\n', - 'file.d.ts': - '/**\n * Workspace files need no declaration: every file under `files/` is one workspace file, at the\n * folder path and name it has there (`files/reports/q1.pdf` is `q1.pdf` in the `reports`\n * folder), synced by content hash. Each apply that changes a file\u0027s bytes records a new version\n * of it, so an earlier one can be restored in the Files tab. Files over 10 MB, or whose name no\n * directory can hold (a reserved character, or a path that differs only in letter case from\n * another file or folder), are left out of a get (listed in `.sim/skipped.json`) and never written\n * by an apply; a file the directory no longer holds is deleted only with `--allow-delete`.\n */\nexport {}\n', - 'index.d.ts': - '/**\n * The workspace declaration SDK: `import { … } from \u0027@sim/workflow-sdk/workspace\u0027` in the\n * declaration files of a workspace source directory (`tables/<key>.ts`, …). Like the workflow\n * SDK, nothing here runs: the server reads each file statically.\n */\nexport * from \u0027./credentials\u0027\nexport * from \u0027./deployment\u0027\nexport * from \u0027./file\u0027\nexport * from \u0027./knowledge\u0027\nexport * from \u0027./mcp\u0027\nexport * from \u0027./secrets\u0027\nexport * from \u0027./skill\u0027\nexport * from \u0027./table\u0027\nexport * from \u0027./tool\u0027\n', - 'knowledge.d.ts': - '/**\n * Knowledge base declarations: `knowledge/<key>/knowledge.ts` in a workspace source directory,\n * beside `knowledge/<key>/docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload that takes the old one\u0027s tag values), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n', - 'mcp.d.ts': - '/**\n * MCP server declarations: `mcp/<key>.ts` declares one server the workspace\u0027s agents call tools\n * on.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineMcpServer, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineMcpServer({\n * name: \u0027GitHub\u0027,\n * url: \u0027https://api.githubcopilot.com/mcp/\u0027,\n * headers: { Authorization: \u0027Bearer {{GITHUB_TOKEN}}\u0027, \u0027X-Team\u0027: env.GITHUB_TEAM },\n * })\n * ```\n *\n * A workflow names the server by its key and one of its tools by name:\n * `mcp(\u0027github\u0027).tool(\u0027create_issue\u0027)`. Secrets never sit in source: a credential header\n * (`Authorization`, `Cookie`, or a name with `key`, `token`, `secret`, `auth`, …) holds env\n * references (`env.NAME`, or `{{NAME}}` in its text after a scheme such as `Bearer`) or\n * `stored()`, which keeps the value the server holds. The server\u0027s id derives from its URL, so a\n * `url` with another host or path replaces the server; a new query string updates it in place.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** A header value: text (`{{NAME}}` names an env variable), `env.NAME`, or `stored()`. */\nexport type McpHeaderValue = string | EnvRef\n\n/** How the server authenticates. */\nexport interface McpServerAuth {\n /** `headers` (the default) sends the declared headers; `oauth` signs in with an OAuth grant. */\n type: \u0027none\u0027 | \u0027headers\u0027 | \u0027oauth\u0027\n /** A pre-registered OAuth client, for a server without dynamic client registration. */\n clientId?: string\n}\n\n/** An MCP server declaration. */\nexport interface McpServerDeclaration {\n /** The display name; the key when left out. */\n name?: string\n description?: string\n /** An `http://` or `https://` URL. A new host or path replaces the server; a new query does not. */\n url: string\n transport?: \u0027streamable-http\u0027\n headers?: { [name: string]: McpHeaderValue }\n /** `{ type: \u0027headers\u0027 }` when left out. The OAuth client secret is set in the server settings. */\n auth?: McpServerAuth\n /** Milliseconds a request waits; 30000 when left out. */\n timeout?: number\n /** 3 when left out. */\n retries?: number\n /** `true` when left out. */\n enabled?: boolean\n}\n\n/** Declares an MCP server. Its default export is the whole file. */\nexport declare function defineMcpServer(server: McpServerDeclaration): McpServerDeclaration\n', - 'secrets.d.ts': - '/**\n * Secret declarations: `secrets.ts` at the root of a workspace source directory names the\n * workspace secrets its workflows read, by name and what each is for. Never a value: set one with\n * `sim secrets set <NAME> --scope workspace`, or in the workspace\u0027s secrets settings.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineSecrets } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineSecrets({\n * ANTHROPIC_API_KEY: { description: \u0027Model access for the support agent\u0027 },\n * STRIPE_KEY: {},\n * })\n * ```\n *\n * A workflow reads one as `env.NAME`, which type-checks against these names. An apply sets only\n * descriptions: a name the workspace holds no value for is reported, and a workflow using it is\n * held back (not written) while the rest of the apply goes ahead. Removing a name changes nothing\n * on the server.\n */\n\n/** One declared secret. */\nexport interface SecretDeclaration {\n /** What the secret is for, shown to teammates (at most 500 characters). */\n description?: string\n}\n\n/** Declares the workspace\u0027s secrets by name. Its default export is the whole file. */\nexport declare function defineSecrets<const T extends Record<string, SecretDeclaration>>(\n secrets: T\n): T\n\n/** The env names a `secrets.ts` declaration adds, as `workspace.d.ts` merges them into `env`. */\nexport type DeclaredSecretNames<T> = { [K in keyof T]: true }\n', - 'skill.d.ts': - '/**\n * Skills are markdown, not TypeScript: `skills/<key>.md` is a frontmatter block of `name` and\n * `description`, then the skill\u0027s content.\n *\n * ```md\n * ---\n * name: tone\n * description: "Answer in the support team\u0027s voice: short, warm, specific."\n * ---\n * Write like a person, not a policy.\n * ```\n *\n * A value is plain text on one line, or a JSON string (`"…"`) when it holds a line break,\n * surrounding spaces, quotes, `: ` or ` #`, ends in `:`, or starts with anything but a letter,\n * digit or `(`. Quote a value YAML would read as a boolean, number or date (`"yes"`, `"1e3"`) too,\n * so other markdown tools read it as text. The name is kebab-case; a workflow names the skill by\n * its key, `skill(\u0027tone\u0027)`. This module declares nothing.\n */\nexport {}\n', - 'table.d.ts': - '/**\n * Table declarations: `tables/<key>.ts` in a workspace source directory.\n *\n * ```ts\n * // @sim-sdk 1\n * import { column, defineTable } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTable({\n * name: \u0027leads\u0027,\n * columns: {\n * email: column.string({ required: true, unique: true }),\n * status: column.select({ options: [\u0027new\u0027, \u0027won\u0027, \u0027lost\u0027] }),\n * score: column.number(),\n * },\n * })\n * ```\n *\n * A column\u0027s object key is its source key and stays bound to the stored column; `name` renames\n * the stored column while keeping its cells.\n *\n * Rows are part of source only when `rows` opts in: `tables/<key>.rows.jsonl` beside the\n * declaration holds one JSON object per line, keyed by column key (a select cell by option name).\n * `{ mode: \u0027seed\u0027 }` writes them into the table when an apply creates it (or adopts it empty);\n * `{ mode: \u0027managed\u0027, key }` keeps the table\u0027s rows equal to the file, each matched by the unique\n * `key` column (removing a row needs `--allow-delete`); a stored row whose `key` cell is empty is\n * left out of the file and kept as it is.\n */\nimport type { Json } from \u0027@sim/workflow-sdk\u0027\n\ndeclare const COLUMN: unique symbol\n\n/**\n * A declared column: the cell type `T` it holds, the name `N` it is stored under, and whether\n * every row must hold a value (`R`); a cell of a column that is not required may be empty (`null`).\n */\nexport interface Column<T, N extends string | undefined = undefined, R extends boolean = false> {\n readonly [COLUMN]: { value: T; name: N; required: R }\n}\n\n/** Options every column type takes. */\nexport interface ColumnOptions<\n N extends string | undefined = string | undefined,\n R extends boolean = boolean,\n> {\n /** The stored column name, when it differs from the column\u0027s key. */\n name?: N\n /** Every row must hold a value. */\n required?: R\n /** No two rows may hold the same value. */\n unique?: boolean\n}\n\n/** The shared options a `select` column takes: never `unique`. */\ntype SelectOptions<N extends string | undefined, R extends boolean> = Omit<\n ColumnOptions<N, R>,\n \u0027unique\u0027\n>\n\n/** The column helpers: `column.string()`, `column.select({ options: [...] })`, … */\nexport declare const column: {\n string<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n number<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<number, N, R>\n /** A number shown as money; `currencyCode` is an ISO 4217 code (`USD` when omitted). */\n currency<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R> & { currencyCode?: string }\n ): Column<number, N, R>\n boolean<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<boolean, N, R>\n /** An ISO 8601 date or date-time string. */\n date<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n /** A time to live: a date-time after which the row expires. A table holds at most one. */\n ttl<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n json<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<Json, N, R>\n /** One option per cell, from `options`. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple?: false }): Column<O[number], N, R>\n /** Several options per cell. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple: true }): Column<O[number][], N, R>\n}\n\n/** Any column, whatever it holds. */\ntype AnyColumn = Column<unknown, string | undefined, boolean>\n\n/** A table declaration. */\nexport interface TableDeclaration<C extends Record<string, AnyColumn> = Record<string, AnyColumn>> {\n /** The table name: a letter or `_`, then letters, digits and `_`. */\n name: string\n description?: string\n /** The table folder path (`/reports`); the workspace root when omitted. */\n folder?: string\n columns: C\n /** Rows as source, from `tables/<key>.rows.jsonl`; omitted leaves rows out of source. */\n rows?: TableRows<keyof C & string>\n}\n\n/**\n * How a table\u0027s rows are source: `seed` rows go into the table once, when an apply creates it;\n * `managed` rows are the whole table, matched to stored rows by the unique `key` column.\n */\nexport type TableRows<K extends string = string> = { mode: \u0027seed\u0027 } | { mode: \u0027managed\u0027; key: K }\n\n/** Declares a table. Its default export is the whole file. */\nexport declare function defineTable<const C extends Record<string, AnyColumn>>(\n table: TableDeclaration<C>\n): TableDeclaration<C>\n\n/** The stored name of column `K`: its `name` option, else its key. */\ntype StoredName<K, C> = C extends Column<unknown, infer N, boolean>\n ? [N] extends [string]\n ? N\n : K\n : K\n\n/** A cell of column `C`: its value, or `null` when the column is not required. */\ntype CellOf<C> = C extends Column<infer V, infer _N, infer R>\n ? R extends true\n ? V\n : V | null\n : never\n\n/**\n * A declared table\u0027s row, keyed by stored column name: `RowOf<typeof leads>`. A cell of a column\n * that is not required may be `null`.\n */\nexport type RowOf<T> = T extends TableDeclaration<infer C>\n ? { [K in keyof C as StoredName<K, C[K]>]: CellOf<C[K]> }\n : never\n', - 'tool.d.ts': - '/**\n * Custom tool declarations: `tools/<key>.ts`, with the tool\u0027s code beside it in `tools/<key>.js`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { codeFile, defineTool } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTool({\n * title: \u0027Lookup order\u0027,\n * schema: {\n * type: \u0027function\u0027,\n * function: {\n * name: \u0027lookup_order\u0027,\n * description: \u0027Find an order by id.\u0027,\n * parameters: {\n * type: \u0027object\u0027,\n * properties: { orderId: { type: \u0027string\u0027 } },\n * required: [\u0027orderId\u0027],\n * },\n * },\n * },\n * code: codeFile(\u0027./lookup-order.js\u0027),\n * })\n * ```\n *\n * The code file runs with each parameter as a variable; `$env.NAME` reads an environment variable\n * by name. The key is the tool\u0027s identity: a workflow names it as `customTool(\u0027lookup-order\u0027)`.\n * Code holding a literal credential prints as `code: stored()`, which keeps the code the server\n * holds without showing it.\n */\nimport type { CodeFileRef, Json } from \u0027@sim/workflow-sdk\u0027\n\nexport { codeFile, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** JSON as an object literal writes it: an optional member may be absent. */\ntype SchemaJson = Json | readonly SchemaJson[] | { readonly [key: string]: SchemaJson | undefined }\n\n/** The function declaration a model calls the tool by, stored verbatim. */\nexport interface ToolSchema {\n type: \u0027function\u0027\n function: {\n /** What the model calls: letters, digits, `_` and `-`, at most 64 characters. */\n name: string\n description?: string\n /** A JSON Schema object describing the arguments. */\n parameters: {\n type: \u0027object\u0027\n properties: { [name: string]: Json }\n required?: string[]\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n}\n\n/** A custom tool declaration. */\nexport interface ToolDeclaration {\n /** The tool\u0027s display title, unique in the workspace. */\n title: string\n schema: ToolSchema\n /** `codeFile(\u0027./<key>.js\u0027)`, the code inline as a string, or `stored()` to keep the stored code. */\n code: CodeFileRef | string\n}\n\n/** Declares a custom tool. Its default export is the whole file. */\nexport declare function defineTool(tool: ToolDeclaration): ToolDeclaration\n', -} +export const WORKSPACE_SDK_DTS: Readonly<Record<string, string>> = {"credentials.d.ts":"/**\n * Credential declarations: `credentials.ts` at the root of a workspace source directory names the\n * saved credentials its workflows use, each by the provider it is for. Never a token: connect one\n * with `sim credentials connect <provider> --name \"<display name>\"`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineCredentials } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineCredentials({\n * slack: { provider: \u0027slack\u0027 },\n * gmail: { provider: \u0027google-email\u0027, name: \u0027Support inbox\u0027 },\n * })\n * ```\n *\n * A workflow uses one as `credential(\u0027slack\u0027)`, which type-checks against these keys. An apply\n * binds each key to a credential of this workspace: the one bound to it already; else the only\n * credential of the provider no other declared key holds, narrowed to the display name `name`\n * when given. A key nothing binds to, or several credentials could bind to, is reported, and a\n * workflow using it is held back while the rest of the apply goes ahead. Applying the same\n * directory to another workspace binds each key to that workspace\u0027s own credential. Removing a key\n * unbinds it only with `--allow-delete`; the credential itself is never deleted.\n */\n\n/** One declared credential. */\nexport interface CredentialDeclaration {\n /** The provider the credential is for (`slack`, `google-email`, …). */\n provider: string\n /** The display name that picks one of several credentials of the provider. */\n name?: string\n}\n\n/** Declares the workspace\u0027s credentials by key. Its default export is the whole file. */\nexport declare function defineCredentials<const T extends Record<string, CredentialDeclaration>>(\n credentials: T\n): T\n\n/**\n * The credential keys a `credentials.ts` declaration adds, each with its provider, as\n * `workspace.d.ts` merges them into `credential(\u0027…\u0027)`.\n */\nexport type DeclaredCredentials<T> = {\n [K in keyof T]: T[K] extends { provider: infer P } ? { provider: P } : never\n}\n","deployment.d.ts":"/**\n * Deployment declarations: `deployments/<workflow key>.ts` in a workspace source directory deploys\n * `workflows/<workflow key>/`, so its schedule, webhook, API and chat triggers run.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineDeployment, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineDeployment({\n * live: \u0027draft\u0027,\n * chat: {\n * identifier: \u0027support\u0027,\n * title: \u0027Support\u0027,\n * auth: { password: env.SUPPORT_CHAT_PASSWORD },\n * outputs: [{ block: \u0027Answer\u0027, path: \u0027content\u0027 }],\n * },\n * })\n * ```\n *\n * An apply deploys after it writes the workflows. Removing the file undeploys the workflow, which\n * stops its schedules, webhooks, API and chat, and needs `--allow-delete`. Deploying needs a\n * workspace admin and is refused to a workspace API key. Publishing to an MCP server is not part\n * of source.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** Who may open a chat. */\nexport type ChatAuth =\n | \u0027public\u0027\n /**\n * A password from an environment variable, by name, never the value itself; or `stored()`, the\n * password the chat already stores (valid only for a chat that has one).\n */\n | { password: EnvRef }\n /** These emails (or `@domain.com` patterns), signing in with a code. */\n | { emails: string[] }\n /** These emails (or `@domain.com` patterns), signing in through the workspace\u0027s SSO. */\n | { sso: string[] }\n\n/** One output a chat answers with. */\nexport interface ChatOutput {\n /** The block\u0027s name, as the workflow names it. */\n block: string\n /** The output path on that block (`content`, `result.text`). */\n path: string\n}\n\n/** The chat a deployment serves. */\nexport interface ChatDeployment {\n /** The chat\u0027s URL path segment, unique across the platform. */\n identifier: string\n title: string\n description?: string\n auth: ChatAuth\n outputs?: ChatOutput[]\n /** The chat\u0027s look; an omitted field takes the platform default. */\n customizations?: { primaryColor?: string; welcomeMessage?: string; imageUrl?: string }\n /** Show the model\u0027s thinking to clients that ask for it. */\n includeThinking?: boolean\n /** Show tool calls to clients that ask for them. */\n includeToolCalls?: boolean\n}\n\n/** A deployment declaration. */\nexport interface DeploymentDeclaration {\n /**\n * `\u0027draft\u0027`: an apply deploys the draft whenever it differs from the live version (after this\n * apply\u0027s workflow edits). `\u0027pinned\u0027`: an apply never redeploys, and says when the draft differs.\n */\n live: \u0027draft\u0027 | \u0027pinned\u0027\n /** Public: anyone may run the workflow\u0027s API without a key. Private when omitted. */\n api?: { public?: boolean }\n /** The chat the deployment serves; none when omitted. */\n chat?: ChatDeployment\n}\n\n/** Declares a workflow\u0027s deployment. Its default export is the whole file. */\nexport declare function defineDeployment(deployment: DeploymentDeclaration): DeploymentDeclaration\n","file.d.ts":"/**\n * Workspace files need no declaration: every file under `files/` is one workspace file, at the\n * folder path and name it has there (`files/reports/q1.pdf` is `q1.pdf` in the `reports`\n * folder), synced by content hash. Each apply that changes a file\u0027s bytes records a new version\n * of it, so an earlier one can be restored in the Files tab. Files over 10 MB, or whose name no\n * directory can hold (a reserved character, or a path that differs only in letter case from\n * another file or folder), are left out of a get (listed in `.sim/skipped.json`) and never written\n * by an apply; a file the directory no longer holds is deleted only with `--allow-delete`.\n */\nexport {}\n","index.d.ts":"/**\n * The workspace declaration SDK: `import { … } from \u0027@sim/workflow-sdk/workspace\u0027` in the\n * declaration files of a workspace source directory (`tables/<key>.ts`, …). Like the workflow\n * SDK, nothing here runs: the server reads each file statically.\n */\nexport * from \u0027./credentials\u0027\nexport * from \u0027./deployment\u0027\nexport * from \u0027./file\u0027\nexport * from \u0027./knowledge\u0027\nexport * from \u0027./mcp\u0027\nexport * from \u0027./secrets\u0027\nexport * from \u0027./skill\u0027\nexport * from \u0027./table\u0027\nexport * from \u0027./tool\u0027\n","knowledge.d.ts":"/**\n * Knowledge base declarations: `knowledge/<key>/knowledge.ts` in a workspace source directory,\n * beside `knowledge/<key>/docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload that takes the old one\u0027s tag values), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n","mcp.d.ts":"/**\n * MCP server declarations: `mcp/<key>.ts` declares one server the workspace\u0027s agents call tools\n * on.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineMcpServer, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineMcpServer({\n * name: \u0027GitHub\u0027,\n * url: \u0027https://api.githubcopilot.com/mcp/\u0027,\n * headers: { Authorization: \u0027Bearer {{GITHUB_TOKEN}}\u0027, \u0027X-Team\u0027: env.GITHUB_TEAM },\n * })\n * ```\n *\n * A workflow names the server by its key and one of its tools by name:\n * `mcp(\u0027github\u0027).tool(\u0027create_issue\u0027)`. Secrets never sit in source: a credential header\n * (`Authorization`, `Cookie`, or a name with `key`, `token`, `secret`, `auth`, …) holds env\n * references (`env.NAME`, or `{{NAME}}` in its text after a scheme such as `Bearer`) or\n * `stored()`, which keeps the value the server holds. The server\u0027s id derives from its URL, so a\n * `url` with another host or path replaces the server; a new query string updates it in place.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** A header value: text (`{{NAME}}` names an env variable), `env.NAME`, or `stored()`. */\nexport type McpHeaderValue = string | EnvRef\n\n/** How the server authenticates. */\nexport interface McpServerAuth {\n /** `headers` (the default) sends the declared headers; `oauth` signs in with an OAuth grant. */\n type: \u0027none\u0027 | \u0027headers\u0027 | \u0027oauth\u0027\n /** A pre-registered OAuth client, for a server without dynamic client registration. */\n clientId?: string\n}\n\n/** An MCP server declaration. */\nexport interface McpServerDeclaration {\n /** The display name; the key when left out. */\n name?: string\n description?: string\n /** An `http://` or `https://` URL. A new host or path replaces the server; a new query does not. */\n url: string\n transport?: \u0027streamable-http\u0027\n headers?: { [name: string]: McpHeaderValue }\n /** `{ type: \u0027headers\u0027 }` when left out. The OAuth client secret is set in the server settings. */\n auth?: McpServerAuth\n /** Milliseconds a request waits; 30000 when left out. */\n timeout?: number\n /** 3 when left out. */\n retries?: number\n /** `true` when left out. */\n enabled?: boolean\n}\n\n/** Declares an MCP server. Its default export is the whole file. */\nexport declare function defineMcpServer(server: McpServerDeclaration): McpServerDeclaration\n","secrets.d.ts":"/**\n * Secret declarations: `secrets.ts` at the root of a workspace source directory names the\n * workspace secrets its workflows read, by name and what each is for. Never a value: set one with\n * `sim secrets set <NAME> --scope workspace`, or in the workspace\u0027s secrets settings.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineSecrets } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineSecrets({\n * ANTHROPIC_API_KEY: { description: \u0027Model access for the support agent\u0027 },\n * STRIPE_KEY: {},\n * })\n * ```\n *\n * A workflow reads one as `env.NAME`, which type-checks against these names. An apply sets only\n * descriptions: a name the workspace holds no value for is reported, and a workflow using it is\n * held back (not written) while the rest of the apply goes ahead. Removing a name changes nothing\n * on the server.\n */\n\n/** One declared secret. */\nexport interface SecretDeclaration {\n /** What the secret is for, shown to teammates (at most 500 characters). */\n description?: string\n}\n\n/** Declares the workspace\u0027s secrets by name. Its default export is the whole file. */\nexport declare function defineSecrets<const T extends Record<string, SecretDeclaration>>(\n secrets: T\n): T\n\n/** The env names a `secrets.ts` declaration adds, as `workspace.d.ts` merges them into `env`. */\nexport type DeclaredSecretNames<T> = { [K in keyof T]: true }\n","skill.d.ts":"/**\n * Skills are markdown, not TypeScript: `skills/<key>.md` is a frontmatter block of `name` and\n * `description`, then the skill\u0027s content.\n *\n * ```md\n * ---\n * name: tone\n * description: \"Answer in the support team\u0027s voice: short, warm, specific.\"\n * ---\n * Write like a person, not a policy.\n * ```\n *\n * A value is plain text on one line, or a JSON string (`\"…\"`) when it holds a line break,\n * surrounding spaces, quotes, `: ` or ` #`, ends in `:`, or starts with anything but a letter,\n * digit or `(`. Quote a value YAML would read as a boolean, number or date (`\"yes\"`, `\"1e3\"`) too,\n * so other markdown tools read it as text. The name is kebab-case; a workflow names the skill by\n * its key, `skill(\u0027tone\u0027)`. This module declares nothing.\n */\nexport {}\n","table.d.ts":"/**\n * Table declarations: `tables/<key>.ts` in a workspace source directory.\n *\n * ```ts\n * // @sim-sdk 1\n * import { column, defineTable } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTable({\n * name: \u0027leads\u0027,\n * columns: {\n * email: column.string({ required: true, unique: true }),\n * status: column.select({ options: [\u0027new\u0027, \u0027won\u0027, \u0027lost\u0027] }),\n * score: column.number(),\n * },\n * })\n * ```\n *\n * A column\u0027s object key is its source key and stays bound to the stored column; `name` renames\n * the stored column while keeping its cells.\n *\n * Rows are part of source only when `rows` opts in: `tables/<key>.rows.jsonl` beside the\n * declaration holds one JSON object per line, keyed by column key (a select cell by option name).\n * `{ mode: \u0027seed\u0027 }` writes them into the table when an apply creates it (or adopts it empty);\n * `{ mode: \u0027managed\u0027, key }` keeps the table\u0027s rows equal to the file, each matched by the unique\n * `key` column (removing a row needs `--allow-delete`); a stored row whose `key` cell is empty is\n * left out of the file and kept as it is.\n */\nimport type { Json } from \u0027@sim/workflow-sdk\u0027\n\ndeclare const COLUMN: unique symbol\n\n/**\n * A declared column: the cell type `T` it holds, the name `N` it is stored under, and whether\n * every row must hold a value (`R`); a cell of a column that is not required may be empty (`null`).\n */\nexport interface Column<T, N extends string | undefined = undefined, R extends boolean = false> {\n readonly [COLUMN]: { value: T; name: N; required: R }\n}\n\n/** Options every column type takes. */\nexport interface ColumnOptions<\n N extends string | undefined = string | undefined,\n R extends boolean = boolean,\n> {\n /** The stored column name, when it differs from the column\u0027s key. */\n name?: N\n /** Every row must hold a value. */\n required?: R\n /** No two rows may hold the same value. */\n unique?: boolean\n}\n\n/** The shared options a `select` column takes: never `unique`. */\ntype SelectOptions<N extends string | undefined, R extends boolean> = Omit<\n ColumnOptions<N, R>,\n \u0027unique\u0027\n>\n\n/** The column helpers: `column.string()`, `column.select({ options: [...] })`, … */\nexport declare const column: {\n string<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n number<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<number, N, R>\n /** A number shown as money; `currencyCode` is an ISO 4217 code (`USD` when omitted). */\n currency<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R> & { currencyCode?: string }\n ): Column<number, N, R>\n boolean<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<boolean, N, R>\n /** An ISO 8601 date or date-time string. */\n date<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n /** A time to live: a date-time after which the row expires. A table holds at most one. */\n ttl<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n json<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<Json, N, R>\n /** One option per cell, from `options`. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple?: false }): Column<O[number], N, R>\n /** Several options per cell. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple: true }): Column<O[number][], N, R>\n}\n\n/** Any column, whatever it holds. */\ntype AnyColumn = Column<unknown, string | undefined, boolean>\n\n/** A table declaration. */\nexport interface TableDeclaration<C extends Record<string, AnyColumn> = Record<string, AnyColumn>> {\n /** The table name: a letter or `_`, then letters, digits and `_`. */\n name: string\n description?: string\n /** The table folder path (`/reports`); the workspace root when omitted. */\n folder?: string\n columns: C\n /** Rows as source, from `tables/<key>.rows.jsonl`; omitted leaves rows out of source. */\n rows?: TableRows<keyof C & string>\n}\n\n/**\n * How a table\u0027s rows are source: `seed` rows go into the table once, when an apply creates it;\n * `managed` rows are the whole table, matched to stored rows by the unique `key` column.\n */\nexport type TableRows<K extends string = string> = { mode: \u0027seed\u0027 } | { mode: \u0027managed\u0027; key: K }\n\n/** Declares a table. Its default export is the whole file. */\nexport declare function defineTable<const C extends Record<string, AnyColumn>>(\n table: TableDeclaration<C>\n): TableDeclaration<C>\n\n/** The stored name of column `K`: its `name` option, else its key. */\ntype StoredName<K, C> = C extends Column<unknown, infer N, boolean>\n ? [N] extends [string]\n ? N\n : K\n : K\n\n/** A cell of column `C`: its value, or `null` when the column is not required. */\ntype CellOf<C> = C extends Column<infer V, infer _N, infer R>\n ? R extends true\n ? V\n : V | null\n : never\n\n/**\n * A declared table\u0027s row, keyed by stored column name: `RowOf<typeof leads>`. A cell of a column\n * that is not required may be `null`.\n */\nexport type RowOf<T> = T extends TableDeclaration<infer C>\n ? { [K in keyof C as StoredName<K, C[K]>]: CellOf<C[K]> }\n : never\n","tool.d.ts":"/**\n * Custom tool declarations: `tools/<key>.ts`, with the tool\u0027s code beside it in `tools/<key>.js`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { codeFile, defineTool } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTool({\n * title: \u0027Lookup order\u0027,\n * schema: {\n * type: \u0027function\u0027,\n * function: {\n * name: \u0027lookup_order\u0027,\n * description: \u0027Find an order by id.\u0027,\n * parameters: {\n * type: \u0027object\u0027,\n * properties: { orderId: { type: \u0027string\u0027 } },\n * required: [\u0027orderId\u0027],\n * },\n * },\n * },\n * code: codeFile(\u0027./lookup-order.js\u0027),\n * })\n * ```\n *\n * The code file runs with each parameter as a variable; `$env.NAME` reads an environment variable\n * by name. The key is the tool\u0027s identity: a workflow names it as `customTool(\u0027lookup-order\u0027)`.\n * Code holding a literal credential prints as `code: stored()`, which keeps the code the server\n * holds without showing it.\n */\nimport type { CodeFileRef, Json } from \u0027@sim/workflow-sdk\u0027\n\nexport { codeFile, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** JSON as an object literal writes it: an optional member may be absent. */\ntype SchemaJson = Json | readonly SchemaJson[] | { readonly [key: string]: SchemaJson | undefined }\n\n/** The function declaration a model calls the tool by, stored verbatim. */\nexport interface ToolSchema {\n type: \u0027function\u0027\n function: {\n /** What the model calls: letters, digits, `_` and `-`, at most 64 characters. */\n name: string\n description?: string\n /** A JSON Schema object describing the arguments. */\n parameters: {\n type: \u0027object\u0027\n properties: { [name: string]: Json }\n required?: string[]\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n}\n\n/** A custom tool declaration. */\nexport interface ToolDeclaration {\n /** The tool\u0027s display title, unique in the workspace. */\n title: string\n schema: ToolSchema\n /** `codeFile(\u0027./<key>.js\u0027)`, the code inline as a string, or `stored()` to keep the stored code. */\n code: CodeFileRef | string\n}\n\n/** Declares a custom tool. Its default export is the whole file. */\nexport declare function defineTool(tool: ToolDeclaration): ToolDeclaration\n"} From 6d77cb1abfb13d7768a7917371ce77453f8f4e2d Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 17:15:08 -0800 Subject: [PATCH 37/59] fix(workflow-code): infer a name-and-const rename only on a strong identity signal A new block takes a vacated lock entry's id only when exactly one pair of its type and parent shows equal settings, mostly shared connections, or the references to it renamed with it; a trigger only on equal settings. An unrelated delete and add stays a delete, refused without --allow-delete, whose hint now says to keep the const name to keep the id or pass --allow-delete to replace the block. --- .../apply-workflow-source.integration.ts | 91 ++++++++++- .../workflows/application/source-evaluate.ts | 8 +- packages/workflow-code/DESIGN.md | 12 +- packages/workflow-code/src/compile/SYNTAX.md | 2 +- .../workflow-code/src/materialize/identity.ts | 142 +++++++++++++----- 5 files changed, 206 insertions(+), 49 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index cf58125805c..9be934a0bd2 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -38,7 +38,7 @@ import { realtimeNotifyMock, realtimeNotifyMockFns } from '@sim/testing/mocks/re import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { isRecordLike } from '@sim/utils/object' +import { isRecordLike, omit } from '@sim/utils/object' import { SOURCE_LIMITS } from '@sim/workflow-code/limits' import { saveWorkflowToNormalizedTables } from '@sim/workflow-persistence/save' import { CONTAINER_DIMENSIONS } from '@sim/workflow-renderer/dimensions' @@ -2335,7 +2335,7 @@ describe('renames in source', () => { }) check( - 'renames nothing ties to the stored blocks are refused with how to keep the ids, not --allow-delete', + 'renames nothing ties to the stored blocks are refused with how to keep the ids', async () => { const seeded = await seedChain() const source = await getSource(seeded.workflowId) @@ -2347,6 +2347,11 @@ describe('renames in source', () => { files = edit(renameIdent(files, ident, newIdent), `fn('${name}'`, `fn('${newName}'`) } files = edit(edit(files, 'return 1', 'return 10'), 'return 2', 'return 20') + files = edit( + files, + ' start.to(bang)\n bang.to(total)\n total.to(report)', + ' start.to(report)\n report.to(bang)\n report.to(total)' + ) const dryRun = await apply(source, files, { dryRun: true }) @@ -2354,7 +2359,9 @@ describe('renames in source', () => { expect(refusal?.message).toBe( 'This apply deletes 2 blocks the source no longer declares: Boom, Tally. It adds Bang, Total of the same type: if those are the same blocks renamed, both their name and const changed, so nothing ties them to the stored blocks and they would get new ids.' ) - expect(refusal?.hint).not.toContain('allow-delete') + expect(refusal?.hint).toBe( + 'To rename a block and keep its id, keep its const name in this apply (change it in the next), or rename the references to it with it; to replace it with a new block, pass --allow-delete.' + ) const allowed = await apply(source, files, { dryRun: true, allowDelete: true }) expect(allowed.diff.blocks.removed.map(({ key, type }) => `${key} ${type}`)).toEqual([ @@ -2376,6 +2383,84 @@ describe('renames in source', () => { } ) + check( + 'an unrelated delete and add of one type is refused without --allow-delete, keeping no id', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = omit<Record<string, string>, string>( + { + ...edit( + edit( + edit( + source.files, + " const tally = fn('Tally', { language: null, code: codeFile('./functions/tally.js') })\n", + " const fresh = fn('Fresh', { code: js`return 'unrelated'` })\n" + ), + ' boom.to(tally)\n tally.to(report)', + ' boom.to(report)\n start.to(fresh)' + ), + 'import { codeFile, fn,', + 'import { codeFile, fn, js,' + ), + 'functions/report.js': 'return $ref.boom.result', + }, + ['functions/tally.js'] + ) + + const dryRun = await apply(source, files, { dryRun: true }) + + expect(codes(dryRun.diagnostics)).toContain('E_APPLY_BLOCK_DELETE') + expect(codes(dryRun.diagnostics)).not.toContain('W_MAT_INFERRED_RENAME') + const allowed = await apply(source, files, { allowDelete: true }) + expect(allowed.diff.blocks.removed.map((block) => block.id)).toEqual([seeded.ids.tally]) + } + ) + + check( + 'a rename of name and const that rewires, changes its code and renames its references keeps its id', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const renamed = edit(renameIdent(source.files, 'tally', 'total'), "fn('Tally'", "fn('Total'") + const files = { + ...edit(renamed, ' boom.to(total)\n', ' start.to(total)\n'), + 'functions/tally.js': 'return 3', + 'functions/report.js': 'return $ref.boom.result + $ref.total.result', + } + + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed).toEqual([ + { id: seeded.ids.tally, from: 'tally', to: 'total' }, + ]) + expect(result.diagnostics.find((d) => d.code === 'W_MAT_INFERRED_RENAME')?.message).toContain( + 'the references to it renamed with it' + ) + } + ) + + check('a trigger renamed in name and const with changed settings is refused', async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = edit( + edit( + source.files, + "const start = trigger.start('Start', {})", + "const begin = trigger.start('Begin', { inputFormat: { question: 'string' } })" + ), + ' start.to(boom)', + ' begin.to(boom)' + ) + + const dryRun = await apply(source, files, { dryRun: true }) + + expect(codes(dryRun.diagnostics)).toContain('E_APPLY_BLOCK_DELETE') + expect(codes(dryRun.diagnostics)).not.toContain('W_MAT_INFERRED_RENAME') + }) + check( 'a $ref to a missing block in a function file is the same warning in check and dry run, at the file', async () => { diff --git a/apps/sim/lib/workflows/application/source-evaluate.ts b/apps/sim/lib/workflows/application/source-evaluate.ts index 76330f6e9e0..d794cb00cc3 100644 --- a/apps/sim/lib/workflows/application/source-evaluate.ts +++ b/apps/sim/lib/workflows/application/source-evaluate.ts @@ -308,9 +308,9 @@ function listNames(names: readonly string[]): string { /** * `E_APPLY_BLOCK_DELETE` for the stored blocks an apply removes. When the source adds blocks of - * a removed block's type, they may be renames whose name and const both changed, which no rule - * could tie to the stored blocks (several alike, and none reads as the one renamed): the finding - * names them and says how to keep the ids, instead of offering to delete. + * a removed block's type, they may be renames whose name and const both changed with no identity + * signal tying them to the stored blocks (settings, connections or renamed references): the + * finding names them and says how to keep the ids, or to pass `--allow-delete` to replace them. */ /** * Each removed block with the added block of the same type and parent it most probably became: @@ -345,7 +345,7 @@ function blockDeleteRefusal( ...(lookalikes.length > 0 ? { message: `${deletes} It adds ${listNames(lookalikes.map((block) => block.name))} of the same type: if those are the same blocks renamed, both their name and const changed, so nothing ties them to the stored blocks and they would get new ids.`, - hint: 'To rename a block and keep its id, change its name or its const in one apply and the other in the next (the lock binds a block by either); or declare the removed blocks again.', + hint: 'To rename a block and keep its id, keep its const name in this apply (change it in the next), or rename the references to it with it; to replace it with a new block, pass --allow-delete.', } : { message: deletes, diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index 07f5ca29f84..1007c3d7a15 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -200,10 +200,14 @@ key-keyed lock; `mergeSourceMetaFromLock` stores a write's hints back by id. 4. `inferred`: exactly one unmatched prev block and one unmatched compiled block with the same type and parent and agreeing edge neighbourhoods, so renaming the variable and the display name together keeps the id; failing that, a prev block the lock binds (the source dropped its - key and its ident) pairs with the one new block of its type and parent whatever its wiring, - or, when several were renamed at once, with the one new block whose params agree with it - (`W_MAT_INFERRED_RENAME`; to get a new block instead, delete the old one in one apply and add the new one in the next). Renames nothing ties - are refused as `E_APPLY_BLOCK_DELETE`, naming the look-alike new blocks and how to keep the ids; + key and its ident) pairs with a new block of its type and parent only when exactly one such + pair shows a strong identity signal: equal settings (references to the old key read as the + new), most bound neighbours shared, or every block that referenced the old key now referencing + the new one. A trigger (webhook paths, schedules) pairs only on equal settings, under either + rule (`W_MAT_INFERRED_RENAME`; to get a new block instead, delete the old one in one apply and + add the new one in the next). A delete and an add with no signal stay a delete, refused as + `E_APPLY_BLOCK_DELETE` (naming the look-alike new blocks and how to keep the ids) unless the + caller passes `allowDelete`; 5. otherwise created: the lock's id when unclaimed, else minted (`idSource`). A match across types is a delete plus a create (`W_MAT_RETYPE_RECREATED`), except a catalog diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index 9ada4de1e5f..dc7ec3cfa03 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -514,7 +514,7 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `W_APPLY_TRIGGER_REMOVED` | A deleted or retyped block still serves a live webhook, schedule, chat output or MCP tool until the next deploy. | | `W_APPLY_CATALOG_MOVED` | Compile failed and the block catalog changed since the read: get the source again. | | `E_APPLY_LINT`, `W_APPLY_LINT` | A workflow lint finding, mapped to a source line. | -| `E_APPLY_BLOCK_DELETE` | The apply deletes blocks the source no longer declares; pass `--allow-delete`. When it also adds blocks of a deleted one's type, it names them as possible renames and says how to keep the ids instead. | +| `E_APPLY_BLOCK_DELETE` | The apply deletes blocks the source no longer declares; pass `--allow-delete`. When it also adds blocks of a deleted one's type, it names them as possible renames and says how to keep the ids. | | `W_APPLY_PROBABLE_RENAME` | With `--allow-delete`, a deleted block and an added one share a type and parent: a probable rename that gets a new id. | | `E_NAME_TAKEN` | The `workflow('…')` name renames the workflow to one another workflow in its folder holds. | | `E_WORKFLOW_META_INVALID` | The `workflow('…')` name is empty or over 255 characters, or its description is over 50,000 characters: limits the workflow API holds every workflow to. | diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 4b1e36b9366..77c76a2b0e0 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -5,7 +5,7 @@ import { catalogBlock } from '../catalog/view' import type { MaterializeDiagnosticCode } from '../diagnostics/codes' import type { Diagnostic } from '../diagnostics/frame' import { pointer, type WorkflowLockBlock, type WorkflowLockV1 } from '../lock' -import type { BlockKey, CanonicalWorkflow } from '../model' +import type { BlockKey, CanonicalBlock, CanonicalWorkflow } from '../model' import { deepEqual, getOwn } from '../records' import type { MaterializeHooks } from './context' @@ -248,6 +248,7 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { match, settled, diagnostics, + catalog, lockedIds: new Set( Object.entries(lock?.blocks ?? {}).flatMap(([key, entry]) => duplicates.has(key) ? [] : [entry.id] @@ -329,6 +330,7 @@ interface InferenceState { * unmatched only when the source dropped both its key and its ident. */ lockedIds: ReadonlySet<string> + catalog: BlockCatalog } /** `in:<id>` / `out:<id>` for every already-bound neighbour of a block. */ @@ -355,44 +357,79 @@ function sameSet(left: Set<string>, right: Set<string>): boolean { return left.size === right.size && [...left].every((item) => right.has(item)) } +/** A block's params with every `<from.` / `<from>` reference token rewritten to `to`. */ +function withKeyRenamed(params: CanonicalBlock['params'], from: BlockKey, to: BlockKey) { + const rewrite = (value: unknown): unknown => { + if (typeof value === 'string') { + return value.split(`<${from}.`).join(`<${to}.`).split(`<${from}>`).join(`<${to}>`) + } + if (Array.isArray(value)) return value.map(rewrite) + if (value === null || typeof value !== 'object') return value + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewrite(item)])) + } + return rewrite(params) +} + /** - * Whether two blocks' params agree: equal on every param both hold, and holding at least one in - * common. A param only one side holds is a default compile filled in or canvas state, which says - * nothing about identity. + * Whether two blocks' settings are equal, references to the renamed block's old key read as its + * new one: equal on every param both hold, and holding at least one in common. A param only one + * side holds is a default compile filled in or canvas state, which says nothing about identity. */ -function paramsAgree( - left: CanonicalWorkflow['blocks'][string]['params'], - right: CanonicalWorkflow['blocks'][string]['params'] +function settingsAgree( + target: CanonicalBlock, + prev: CanonicalBlock, + key: BlockKey, + prevKey: BlockKey ): boolean { + const left = target.params + const right = withKeyRenamed(prev.params, prevKey, key) as CanonicalBlock['params'] const shared = Object.keys(left).filter((id) => Object.hasOwn(right, id)) return shared.length > 0 && shared.every((id) => deepEqual(left[id], right[id])) } +/** Whether `text` (a block's params, serialized) holds a reference token to `key`. */ +function referencesKey(params: CanonicalBlock['params'], key: BlockKey): boolean { + const text = JSON.stringify(params) + return text.includes(`<${key}.`) || text.includes(`<${key}>`) +} + /** - * How rule 4 pairs a group: `connections`, a single unmatched block on each side whose bound - * neighbours agree; `vacated`, a single unmatched block on each side where the stored one is a - * block the lock binds (the source dropped its key and its ident: a rename of both, with its - * wiring free to change); `params`, the one vacated stored block whose params agree with a - * new block's, when several of one type were renamed at once. + * The strong identity signal that ties a new block to a vacated stored one, if any: `settings`, + * equal settings (references to the old key read as the new); `connections`, most of their + * already-bound neighbours shared; `references`, every bound block that referenced the old key + * now references the new one (the author renamed the references). */ -type InferenceBasis = 'connections' | 'vacated' | 'params' +type InferenceBasis = 'connections' | 'settings' | 'references' const INFERENCE_REASON: Record<InferenceBasis, string> = { connections: 'same type, parent and connections', - vacated: 'same type and parent, and the only block of them the source dropped', - params: 'same type, parent and params', + settings: 'same type, parent and settings', + references: 'same type and parent, and the references to it renamed with it', +} + +/** + * Whether a block acts as a trigger: its id serves webhook paths, schedules and run history, so + * rule 4 binds one only on equal settings. + */ +function isTriggerLike(block: CanonicalBlock, catalog: BlockCatalog): boolean { + const declared = catalogBlock(catalog, block.type) + return ( + block.triggerMode || declared?.category === 'triggers' || declared?.graph.canBeTarget === false + ) } /** - * Identity rule 4, repeated until it binds nothing new so a renamed container lets its - * renamed children pair next. A group is `(type, parent id)`. A group with exactly one unmatched - * block on each side pairs when both have the same already-bound neighbours in the same - * directions; failing that, and only once no group pairs that way, when the stored block is one - * the lock binds (its key and ident both left the source), or, for several such blocks, the one - * whose params agree with a new block's. Each pairing is flagged with `W_MAT_INFERRED_RENAME`. + * Identity rule 4, repeated until it binds nothing new so a renamed container lets its renamed + * children pair next. A group is `(type, parent id)`. A group with exactly one unmatched block on + * each side pairs when both have the same already-bound neighbours in the same directions; then, + * once no group pairs that way, a new block pairs with a stored block the lock binds (its key and + * ident both left the source) when exactly one such block of its group, and no other new block, + * shows a strong identity signal ({@link InferenceBasis}). A trigger pairs only on equal + * settings. Each pairing is flagged with `W_MAT_INFERRED_RENAME`; a delete and an add that show + * no signal stay a delete, which `--allow-delete` guards. */ function inferRenames(state: InferenceState): void { - const { workflow, prev, resolved, claimed, lockedIds } = state + const { workflow, prev, resolved, claimed, lockedIds, catalog } = state const prevCanonical = prev.canonical.workflow const prevIdentity = prev.canonical.identity.blocks /** Prev blocks a settled key accounts for: a retype's own prev, else the prev under its key. */ @@ -447,37 +484,68 @@ function inferRenames(state: InferenceState): void { return true } + const neighbours = (key: BlockKey, prevKey: BlockKey) => ({ + target: neighbourhood(workflow.edges, key, idOfTarget), + prev: neighbourhood(prevCanonical.edges, prevKey, idOfPrev), + }) + const byConnections = (): boolean => { let progressed = false for (const { targets, prevKeys } of groups()) { if (targets.length !== 1 || prevKeys.length !== 1) continue const [key] = targets const [prevKey] = prevKeys - const agree = sameSet( - neighbourhood(workflow.edges, key, idOfTarget), - neighbourhood(prevCanonical.edges, prevKey, idOfPrev) - ) - if (agree && pair(key, prevKey, 'connections')) progressed = true + const sides = neighbours(key, prevKey) + if (!sameSet(sides.target, sides.prev)) continue + const triggerHeld = + isTriggerLike(workflow.blocks[key], catalog) && + !settingsAgree(workflow.blocks[key], prevCanonical.blocks[prevKey], key, prevKey) + if (!triggerHeld && pair(key, prevKey, 'connections')) progressed = true } return progressed } + /** The strong signal tying new block `key` to vacated stored block `prevKey`, if any. */ + const signal = (key: BlockKey, prevKey: BlockKey): InferenceBasis | null => { + const target = workflow.blocks[key] + if (settingsAgree(target, prevCanonical.blocks[prevKey], key, prevKey)) return 'settings' + if (isTriggerLike(target, catalog)) return null + const sides = neighbours(key, prevKey) + const shared = [...sides.target].filter((item) => sides.prev.has(item)).length + const union = new Set([...sides.target, ...sides.prev]).size + if (shared > 0 && shared * 2 > union) return 'connections' + const referrers = Object.keys(prevCanonical.blocks).filter( + (other) => other !== prevKey && referencesKey(prevCanonical.blocks[other].params, prevKey) + ) + const renamedAtEverySite = + referrers.length > 0 && + referrers.every((other) => { + const id = idOfPrev(other) + const now = id === undefined ? undefined : [...resolved].find(([, b]) => b.id === id)?.[0] + return now !== undefined && referencesKey(workflow.blocks[now].params, key) + }) + return renamedAtEverySite ? 'references' : null + } + const byVacatedLock = (): boolean => { let progressed = false for (const { targets, prevKeys } of groups()) { const vacated = prevKeys.filter((prevKey) => lockedIds.has(prevIdentity[prevKey].id)) if (vacated.length === 0) continue - if (targets.length === 1 && prevKeys.length === 1 && vacated.length === 1) { - if (pair(targets[0], vacated[0], 'vacated')) progressed = true - continue - } - const equalParams = (key: BlockKey, prevKey: BlockKey) => - paramsAgree(workflow.blocks[key].params, prevCanonical.blocks[prevKey].params) + const signals = new Map<BlockKey, Map<BlockKey, InferenceBasis>>() for (const key of targets) { - const [only, ...more] = vacated.filter((prevKey) => equalParams(key, prevKey)) - if (only === undefined || more.length > 0) continue - if (targets.filter((other) => equalParams(other, only)).length !== 1) continue - if (pair(key, only, 'params')) progressed = true + const found = new Map<BlockKey, InferenceBasis>() + for (const prevKey of vacated) { + const basis = signal(key, prevKey) + if (basis) found.set(prevKey, basis) + } + signals.set(key, found) + } + for (const [key, found] of signals) { + if (found.size !== 1) continue + const [[prevKey, basis]] = [...found] + const rivals = [...signals.values()].filter((other) => other.has(prevKey)).length + if (rivals === 1 && pair(key, prevKey, basis)) progressed = true } } return progressed From 95c6ee1b0c3f293558df8605a74abd479cf69340 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 17:32:44 -0800 Subject: [PATCH 38/59] chore(docs): record the source params code example in the docs baseline --- apps/docs/scripts/fixtures/content-baseline.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/docs/scripts/fixtures/content-baseline.json b/apps/docs/scripts/fixtures/content-baseline.json index 6b9edc3c27b..54cc31249cc 100644 --- a/apps/docs/scripts/fixtures/content-baseline.json +++ b/apps/docs/scripts/fixtures/content-baseline.json @@ -13189,6 +13189,13 @@ "bytes": null, "line": null }, + { + "lang": "bash", + "meta": null, + "sha256": "2b6c40dcef94f6188188d6369383e982f5a6dc88654b8ab76664c189f9af8ab0", + "bytes": null, + "line": null + }, { "lang": "bash", "meta": null, @@ -17768,6 +17775,13 @@ "bytes": null, "line": null }, + { + "lang": "bash", + "meta": null, + "sha256": "2b6c40dcef94f6188188d6369383e982f5a6dc88654b8ab76664c189f9af8ab0", + "bytes": null, + "line": null + }, { "lang": "bash", "meta": null, From 410082f0b58e622fa16a6b2e250b876a20b110f1 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 17:33:19 -0800 Subject: [PATCH 39/59] style(workflow-sdk): format the regenerated embedded workspace text --- .../src/codegen/workspace-text.generated.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts index 09d12f978cd..724b21ef94e 100644 --- a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts +++ b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts @@ -1,4 +1,25 @@ // Generated by scripts/generate-embedded-text.ts from src/workspace; do not edit. /** The workspace declaration SDK, file name → text (`index.d.ts` re-exports the rest). */ -export const WORKSPACE_SDK_DTS: Readonly<Record<string, string>> = {"credentials.d.ts":"/**\n * Credential declarations: `credentials.ts` at the root of a workspace source directory names the\n * saved credentials its workflows use, each by the provider it is for. Never a token: connect one\n * with `sim credentials connect <provider> --name \"<display name>\"`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineCredentials } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineCredentials({\n * slack: { provider: \u0027slack\u0027 },\n * gmail: { provider: \u0027google-email\u0027, name: \u0027Support inbox\u0027 },\n * })\n * ```\n *\n * A workflow uses one as `credential(\u0027slack\u0027)`, which type-checks against these keys. An apply\n * binds each key to a credential of this workspace: the one bound to it already; else the only\n * credential of the provider no other declared key holds, narrowed to the display name `name`\n * when given. A key nothing binds to, or several credentials could bind to, is reported, and a\n * workflow using it is held back while the rest of the apply goes ahead. Applying the same\n * directory to another workspace binds each key to that workspace\u0027s own credential. Removing a key\n * unbinds it only with `--allow-delete`; the credential itself is never deleted.\n */\n\n/** One declared credential. */\nexport interface CredentialDeclaration {\n /** The provider the credential is for (`slack`, `google-email`, …). */\n provider: string\n /** The display name that picks one of several credentials of the provider. */\n name?: string\n}\n\n/** Declares the workspace\u0027s credentials by key. Its default export is the whole file. */\nexport declare function defineCredentials<const T extends Record<string, CredentialDeclaration>>(\n credentials: T\n): T\n\n/**\n * The credential keys a `credentials.ts` declaration adds, each with its provider, as\n * `workspace.d.ts` merges them into `credential(\u0027…\u0027)`.\n */\nexport type DeclaredCredentials<T> = {\n [K in keyof T]: T[K] extends { provider: infer P } ? { provider: P } : never\n}\n","deployment.d.ts":"/**\n * Deployment declarations: `deployments/<workflow key>.ts` in a workspace source directory deploys\n * `workflows/<workflow key>/`, so its schedule, webhook, API and chat triggers run.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineDeployment, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineDeployment({\n * live: \u0027draft\u0027,\n * chat: {\n * identifier: \u0027support\u0027,\n * title: \u0027Support\u0027,\n * auth: { password: env.SUPPORT_CHAT_PASSWORD },\n * outputs: [{ block: \u0027Answer\u0027, path: \u0027content\u0027 }],\n * },\n * })\n * ```\n *\n * An apply deploys after it writes the workflows. Removing the file undeploys the workflow, which\n * stops its schedules, webhooks, API and chat, and needs `--allow-delete`. Deploying needs a\n * workspace admin and is refused to a workspace API key. Publishing to an MCP server is not part\n * of source.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** Who may open a chat. */\nexport type ChatAuth =\n | \u0027public\u0027\n /**\n * A password from an environment variable, by name, never the value itself; or `stored()`, the\n * password the chat already stores (valid only for a chat that has one).\n */\n | { password: EnvRef }\n /** These emails (or `@domain.com` patterns), signing in with a code. */\n | { emails: string[] }\n /** These emails (or `@domain.com` patterns), signing in through the workspace\u0027s SSO. */\n | { sso: string[] }\n\n/** One output a chat answers with. */\nexport interface ChatOutput {\n /** The block\u0027s name, as the workflow names it. */\n block: string\n /** The output path on that block (`content`, `result.text`). */\n path: string\n}\n\n/** The chat a deployment serves. */\nexport interface ChatDeployment {\n /** The chat\u0027s URL path segment, unique across the platform. */\n identifier: string\n title: string\n description?: string\n auth: ChatAuth\n outputs?: ChatOutput[]\n /** The chat\u0027s look; an omitted field takes the platform default. */\n customizations?: { primaryColor?: string; welcomeMessage?: string; imageUrl?: string }\n /** Show the model\u0027s thinking to clients that ask for it. */\n includeThinking?: boolean\n /** Show tool calls to clients that ask for them. */\n includeToolCalls?: boolean\n}\n\n/** A deployment declaration. */\nexport interface DeploymentDeclaration {\n /**\n * `\u0027draft\u0027`: an apply deploys the draft whenever it differs from the live version (after this\n * apply\u0027s workflow edits). `\u0027pinned\u0027`: an apply never redeploys, and says when the draft differs.\n */\n live: \u0027draft\u0027 | \u0027pinned\u0027\n /** Public: anyone may run the workflow\u0027s API without a key. Private when omitted. */\n api?: { public?: boolean }\n /** The chat the deployment serves; none when omitted. */\n chat?: ChatDeployment\n}\n\n/** Declares a workflow\u0027s deployment. Its default export is the whole file. */\nexport declare function defineDeployment(deployment: DeploymentDeclaration): DeploymentDeclaration\n","file.d.ts":"/**\n * Workspace files need no declaration: every file under `files/` is one workspace file, at the\n * folder path and name it has there (`files/reports/q1.pdf` is `q1.pdf` in the `reports`\n * folder), synced by content hash. Each apply that changes a file\u0027s bytes records a new version\n * of it, so an earlier one can be restored in the Files tab. Files over 10 MB, or whose name no\n * directory can hold (a reserved character, or a path that differs only in letter case from\n * another file or folder), are left out of a get (listed in `.sim/skipped.json`) and never written\n * by an apply; a file the directory no longer holds is deleted only with `--allow-delete`.\n */\nexport {}\n","index.d.ts":"/**\n * The workspace declaration SDK: `import { … } from \u0027@sim/workflow-sdk/workspace\u0027` in the\n * declaration files of a workspace source directory (`tables/<key>.ts`, …). Like the workflow\n * SDK, nothing here runs: the server reads each file statically.\n */\nexport * from \u0027./credentials\u0027\nexport * from \u0027./deployment\u0027\nexport * from \u0027./file\u0027\nexport * from \u0027./knowledge\u0027\nexport * from \u0027./mcp\u0027\nexport * from \u0027./secrets\u0027\nexport * from \u0027./skill\u0027\nexport * from \u0027./table\u0027\nexport * from \u0027./tool\u0027\n","knowledge.d.ts":"/**\n * Knowledge base declarations: `knowledge/<key>/knowledge.ts` in a workspace source directory,\n * beside `knowledge/<key>/docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload that takes the old one\u0027s tag values), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n","mcp.d.ts":"/**\n * MCP server declarations: `mcp/<key>.ts` declares one server the workspace\u0027s agents call tools\n * on.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineMcpServer, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineMcpServer({\n * name: \u0027GitHub\u0027,\n * url: \u0027https://api.githubcopilot.com/mcp/\u0027,\n * headers: { Authorization: \u0027Bearer {{GITHUB_TOKEN}}\u0027, \u0027X-Team\u0027: env.GITHUB_TEAM },\n * })\n * ```\n *\n * A workflow names the server by its key and one of its tools by name:\n * `mcp(\u0027github\u0027).tool(\u0027create_issue\u0027)`. Secrets never sit in source: a credential header\n * (`Authorization`, `Cookie`, or a name with `key`, `token`, `secret`, `auth`, …) holds env\n * references (`env.NAME`, or `{{NAME}}` in its text after a scheme such as `Bearer`) or\n * `stored()`, which keeps the value the server holds. The server\u0027s id derives from its URL, so a\n * `url` with another host or path replaces the server; a new query string updates it in place.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** A header value: text (`{{NAME}}` names an env variable), `env.NAME`, or `stored()`. */\nexport type McpHeaderValue = string | EnvRef\n\n/** How the server authenticates. */\nexport interface McpServerAuth {\n /** `headers` (the default) sends the declared headers; `oauth` signs in with an OAuth grant. */\n type: \u0027none\u0027 | \u0027headers\u0027 | \u0027oauth\u0027\n /** A pre-registered OAuth client, for a server without dynamic client registration. */\n clientId?: string\n}\n\n/** An MCP server declaration. */\nexport interface McpServerDeclaration {\n /** The display name; the key when left out. */\n name?: string\n description?: string\n /** An `http://` or `https://` URL. A new host or path replaces the server; a new query does not. */\n url: string\n transport?: \u0027streamable-http\u0027\n headers?: { [name: string]: McpHeaderValue }\n /** `{ type: \u0027headers\u0027 }` when left out. The OAuth client secret is set in the server settings. */\n auth?: McpServerAuth\n /** Milliseconds a request waits; 30000 when left out. */\n timeout?: number\n /** 3 when left out. */\n retries?: number\n /** `true` when left out. */\n enabled?: boolean\n}\n\n/** Declares an MCP server. Its default export is the whole file. */\nexport declare function defineMcpServer(server: McpServerDeclaration): McpServerDeclaration\n","secrets.d.ts":"/**\n * Secret declarations: `secrets.ts` at the root of a workspace source directory names the\n * workspace secrets its workflows read, by name and what each is for. Never a value: set one with\n * `sim secrets set <NAME> --scope workspace`, or in the workspace\u0027s secrets settings.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineSecrets } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineSecrets({\n * ANTHROPIC_API_KEY: { description: \u0027Model access for the support agent\u0027 },\n * STRIPE_KEY: {},\n * })\n * ```\n *\n * A workflow reads one as `env.NAME`, which type-checks against these names. An apply sets only\n * descriptions: a name the workspace holds no value for is reported, and a workflow using it is\n * held back (not written) while the rest of the apply goes ahead. Removing a name changes nothing\n * on the server.\n */\n\n/** One declared secret. */\nexport interface SecretDeclaration {\n /** What the secret is for, shown to teammates (at most 500 characters). */\n description?: string\n}\n\n/** Declares the workspace\u0027s secrets by name. Its default export is the whole file. */\nexport declare function defineSecrets<const T extends Record<string, SecretDeclaration>>(\n secrets: T\n): T\n\n/** The env names a `secrets.ts` declaration adds, as `workspace.d.ts` merges them into `env`. */\nexport type DeclaredSecretNames<T> = { [K in keyof T]: true }\n","skill.d.ts":"/**\n * Skills are markdown, not TypeScript: `skills/<key>.md` is a frontmatter block of `name` and\n * `description`, then the skill\u0027s content.\n *\n * ```md\n * ---\n * name: tone\n * description: \"Answer in the support team\u0027s voice: short, warm, specific.\"\n * ---\n * Write like a person, not a policy.\n * ```\n *\n * A value is plain text on one line, or a JSON string (`\"…\"`) when it holds a line break,\n * surrounding spaces, quotes, `: ` or ` #`, ends in `:`, or starts with anything but a letter,\n * digit or `(`. Quote a value YAML would read as a boolean, number or date (`\"yes\"`, `\"1e3\"`) too,\n * so other markdown tools read it as text. The name is kebab-case; a workflow names the skill by\n * its key, `skill(\u0027tone\u0027)`. This module declares nothing.\n */\nexport {}\n","table.d.ts":"/**\n * Table declarations: `tables/<key>.ts` in a workspace source directory.\n *\n * ```ts\n * // @sim-sdk 1\n * import { column, defineTable } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTable({\n * name: \u0027leads\u0027,\n * columns: {\n * email: column.string({ required: true, unique: true }),\n * status: column.select({ options: [\u0027new\u0027, \u0027won\u0027, \u0027lost\u0027] }),\n * score: column.number(),\n * },\n * })\n * ```\n *\n * A column\u0027s object key is its source key and stays bound to the stored column; `name` renames\n * the stored column while keeping its cells.\n *\n * Rows are part of source only when `rows` opts in: `tables/<key>.rows.jsonl` beside the\n * declaration holds one JSON object per line, keyed by column key (a select cell by option name).\n * `{ mode: \u0027seed\u0027 }` writes them into the table when an apply creates it (or adopts it empty);\n * `{ mode: \u0027managed\u0027, key }` keeps the table\u0027s rows equal to the file, each matched by the unique\n * `key` column (removing a row needs `--allow-delete`); a stored row whose `key` cell is empty is\n * left out of the file and kept as it is.\n */\nimport type { Json } from \u0027@sim/workflow-sdk\u0027\n\ndeclare const COLUMN: unique symbol\n\n/**\n * A declared column: the cell type `T` it holds, the name `N` it is stored under, and whether\n * every row must hold a value (`R`); a cell of a column that is not required may be empty (`null`).\n */\nexport interface Column<T, N extends string | undefined = undefined, R extends boolean = false> {\n readonly [COLUMN]: { value: T; name: N; required: R }\n}\n\n/** Options every column type takes. */\nexport interface ColumnOptions<\n N extends string | undefined = string | undefined,\n R extends boolean = boolean,\n> {\n /** The stored column name, when it differs from the column\u0027s key. */\n name?: N\n /** Every row must hold a value. */\n required?: R\n /** No two rows may hold the same value. */\n unique?: boolean\n}\n\n/** The shared options a `select` column takes: never `unique`. */\ntype SelectOptions<N extends string | undefined, R extends boolean> = Omit<\n ColumnOptions<N, R>,\n \u0027unique\u0027\n>\n\n/** The column helpers: `column.string()`, `column.select({ options: [...] })`, … */\nexport declare const column: {\n string<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n number<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<number, N, R>\n /** A number shown as money; `currencyCode` is an ISO 4217 code (`USD` when omitted). */\n currency<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R> & { currencyCode?: string }\n ): Column<number, N, R>\n boolean<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<boolean, N, R>\n /** An ISO 8601 date or date-time string. */\n date<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n /** A time to live: a date-time after which the row expires. A table holds at most one. */\n ttl<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n json<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<Json, N, R>\n /** One option per cell, from `options`. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple?: false }): Column<O[number], N, R>\n /** Several options per cell. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple: true }): Column<O[number][], N, R>\n}\n\n/** Any column, whatever it holds. */\ntype AnyColumn = Column<unknown, string | undefined, boolean>\n\n/** A table declaration. */\nexport interface TableDeclaration<C extends Record<string, AnyColumn> = Record<string, AnyColumn>> {\n /** The table name: a letter or `_`, then letters, digits and `_`. */\n name: string\n description?: string\n /** The table folder path (`/reports`); the workspace root when omitted. */\n folder?: string\n columns: C\n /** Rows as source, from `tables/<key>.rows.jsonl`; omitted leaves rows out of source. */\n rows?: TableRows<keyof C & string>\n}\n\n/**\n * How a table\u0027s rows are source: `seed` rows go into the table once, when an apply creates it;\n * `managed` rows are the whole table, matched to stored rows by the unique `key` column.\n */\nexport type TableRows<K extends string = string> = { mode: \u0027seed\u0027 } | { mode: \u0027managed\u0027; key: K }\n\n/** Declares a table. Its default export is the whole file. */\nexport declare function defineTable<const C extends Record<string, AnyColumn>>(\n table: TableDeclaration<C>\n): TableDeclaration<C>\n\n/** The stored name of column `K`: its `name` option, else its key. */\ntype StoredName<K, C> = C extends Column<unknown, infer N, boolean>\n ? [N] extends [string]\n ? N\n : K\n : K\n\n/** A cell of column `C`: its value, or `null` when the column is not required. */\ntype CellOf<C> = C extends Column<infer V, infer _N, infer R>\n ? R extends true\n ? V\n : V | null\n : never\n\n/**\n * A declared table\u0027s row, keyed by stored column name: `RowOf<typeof leads>`. A cell of a column\n * that is not required may be `null`.\n */\nexport type RowOf<T> = T extends TableDeclaration<infer C>\n ? { [K in keyof C as StoredName<K, C[K]>]: CellOf<C[K]> }\n : never\n","tool.d.ts":"/**\n * Custom tool declarations: `tools/<key>.ts`, with the tool\u0027s code beside it in `tools/<key>.js`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { codeFile, defineTool } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTool({\n * title: \u0027Lookup order\u0027,\n * schema: {\n * type: \u0027function\u0027,\n * function: {\n * name: \u0027lookup_order\u0027,\n * description: \u0027Find an order by id.\u0027,\n * parameters: {\n * type: \u0027object\u0027,\n * properties: { orderId: { type: \u0027string\u0027 } },\n * required: [\u0027orderId\u0027],\n * },\n * },\n * },\n * code: codeFile(\u0027./lookup-order.js\u0027),\n * })\n * ```\n *\n * The code file runs with each parameter as a variable; `$env.NAME` reads an environment variable\n * by name. The key is the tool\u0027s identity: a workflow names it as `customTool(\u0027lookup-order\u0027)`.\n * Code holding a literal credential prints as `code: stored()`, which keeps the code the server\n * holds without showing it.\n */\nimport type { CodeFileRef, Json } from \u0027@sim/workflow-sdk\u0027\n\nexport { codeFile, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** JSON as an object literal writes it: an optional member may be absent. */\ntype SchemaJson = Json | readonly SchemaJson[] | { readonly [key: string]: SchemaJson | undefined }\n\n/** The function declaration a model calls the tool by, stored verbatim. */\nexport interface ToolSchema {\n type: \u0027function\u0027\n function: {\n /** What the model calls: letters, digits, `_` and `-`, at most 64 characters. */\n name: string\n description?: string\n /** A JSON Schema object describing the arguments. */\n parameters: {\n type: \u0027object\u0027\n properties: { [name: string]: Json }\n required?: string[]\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n}\n\n/** A custom tool declaration. */\nexport interface ToolDeclaration {\n /** The tool\u0027s display title, unique in the workspace. */\n title: string\n schema: ToolSchema\n /** `codeFile(\u0027./<key>.js\u0027)`, the code inline as a string, or `stored()` to keep the stored code. */\n code: CodeFileRef | string\n}\n\n/** Declares a custom tool. Its default export is the whole file. */\nexport declare function defineTool(tool: ToolDeclaration): ToolDeclaration\n"} +export const WORKSPACE_SDK_DTS: Readonly<Record<string, string>> = { + 'credentials.d.ts': + '/**\n * Credential declarations: `credentials.ts` at the root of a workspace source directory names the\n * saved credentials its workflows use, each by the provider it is for. Never a token: connect one\n * with `sim credentials connect <provider> --name "<display name>"`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineCredentials } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineCredentials({\n * slack: { provider: \u0027slack\u0027 },\n * gmail: { provider: \u0027google-email\u0027, name: \u0027Support inbox\u0027 },\n * })\n * ```\n *\n * A workflow uses one as `credential(\u0027slack\u0027)`, which type-checks against these keys. An apply\n * binds each key to a credential of this workspace: the one bound to it already; else the only\n * credential of the provider no other declared key holds, narrowed to the display name `name`\n * when given. A key nothing binds to, or several credentials could bind to, is reported, and a\n * workflow using it is held back while the rest of the apply goes ahead. Applying the same\n * directory to another workspace binds each key to that workspace\u0027s own credential. Removing a key\n * unbinds it only with `--allow-delete`; the credential itself is never deleted.\n */\n\n/** One declared credential. */\nexport interface CredentialDeclaration {\n /** The provider the credential is for (`slack`, `google-email`, …). */\n provider: string\n /** The display name that picks one of several credentials of the provider. */\n name?: string\n}\n\n/** Declares the workspace\u0027s credentials by key. Its default export is the whole file. */\nexport declare function defineCredentials<const T extends Record<string, CredentialDeclaration>>(\n credentials: T\n): T\n\n/**\n * The credential keys a `credentials.ts` declaration adds, each with its provider, as\n * `workspace.d.ts` merges them into `credential(\u0027…\u0027)`.\n */\nexport type DeclaredCredentials<T> = {\n [K in keyof T]: T[K] extends { provider: infer P } ? { provider: P } : never\n}\n', + 'deployment.d.ts': + '/**\n * Deployment declarations: `deployments/<workflow key>.ts` in a workspace source directory deploys\n * `workflows/<workflow key>/`, so its schedule, webhook, API and chat triggers run.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineDeployment, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineDeployment({\n * live: \u0027draft\u0027,\n * chat: {\n * identifier: \u0027support\u0027,\n * title: \u0027Support\u0027,\n * auth: { password: env.SUPPORT_CHAT_PASSWORD },\n * outputs: [{ block: \u0027Answer\u0027, path: \u0027content\u0027 }],\n * },\n * })\n * ```\n *\n * An apply deploys after it writes the workflows. Removing the file undeploys the workflow, which\n * stops its schedules, webhooks, API and chat, and needs `--allow-delete`. Deploying needs a\n * workspace admin and is refused to a workspace API key. Publishing to an MCP server is not part\n * of source.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** Who may open a chat. */\nexport type ChatAuth =\n | \u0027public\u0027\n /**\n * A password from an environment variable, by name, never the value itself; or `stored()`, the\n * password the chat already stores (valid only for a chat that has one).\n */\n | { password: EnvRef }\n /** These emails (or `@domain.com` patterns), signing in with a code. */\n | { emails: string[] }\n /** These emails (or `@domain.com` patterns), signing in through the workspace\u0027s SSO. */\n | { sso: string[] }\n\n/** One output a chat answers with. */\nexport interface ChatOutput {\n /** The block\u0027s name, as the workflow names it. */\n block: string\n /** The output path on that block (`content`, `result.text`). */\n path: string\n}\n\n/** The chat a deployment serves. */\nexport interface ChatDeployment {\n /** The chat\u0027s URL path segment, unique across the platform. */\n identifier: string\n title: string\n description?: string\n auth: ChatAuth\n outputs?: ChatOutput[]\n /** The chat\u0027s look; an omitted field takes the platform default. */\n customizations?: { primaryColor?: string; welcomeMessage?: string; imageUrl?: string }\n /** Show the model\u0027s thinking to clients that ask for it. */\n includeThinking?: boolean\n /** Show tool calls to clients that ask for them. */\n includeToolCalls?: boolean\n}\n\n/** A deployment declaration. */\nexport interface DeploymentDeclaration {\n /**\n * `\u0027draft\u0027`: an apply deploys the draft whenever it differs from the live version (after this\n * apply\u0027s workflow edits). `\u0027pinned\u0027`: an apply never redeploys, and says when the draft differs.\n */\n live: \u0027draft\u0027 | \u0027pinned\u0027\n /** Public: anyone may run the workflow\u0027s API without a key. Private when omitted. */\n api?: { public?: boolean }\n /** The chat the deployment serves; none when omitted. */\n chat?: ChatDeployment\n}\n\n/** Declares a workflow\u0027s deployment. Its default export is the whole file. */\nexport declare function defineDeployment(deployment: DeploymentDeclaration): DeploymentDeclaration\n', + 'file.d.ts': + '/**\n * Workspace files need no declaration: every file under `files/` is one workspace file, at the\n * folder path and name it has there (`files/reports/q1.pdf` is `q1.pdf` in the `reports`\n * folder), synced by content hash. Each apply that changes a file\u0027s bytes records a new version\n * of it, so an earlier one can be restored in the Files tab. Files over 10 MB, or whose name no\n * directory can hold (a reserved character, or a path that differs only in letter case from\n * another file or folder), are left out of a get (listed in `.sim/skipped.json`) and never written\n * by an apply; a file the directory no longer holds is deleted only with `--allow-delete`.\n */\nexport {}\n', + 'index.d.ts': + '/**\n * The workspace declaration SDK: `import { … } from \u0027@sim/workflow-sdk/workspace\u0027` in the\n * declaration files of a workspace source directory (`tables/<key>.ts`, …). Like the workflow\n * SDK, nothing here runs: the server reads each file statically.\n */\nexport * from \u0027./credentials\u0027\nexport * from \u0027./deployment\u0027\nexport * from \u0027./file\u0027\nexport * from \u0027./knowledge\u0027\nexport * from \u0027./mcp\u0027\nexport * from \u0027./secrets\u0027\nexport * from \u0027./skill\u0027\nexport * from \u0027./table\u0027\nexport * from \u0027./tool\u0027\n', + 'knowledge.d.ts': + '/**\n * Knowledge base declarations: `knowledge/<key>/knowledge.ts` in a workspace source directory,\n * beside `knowledge/<key>/docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload that takes the old one\u0027s tag values), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n', + 'mcp.d.ts': + '/**\n * MCP server declarations: `mcp/<key>.ts` declares one server the workspace\u0027s agents call tools\n * on.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineMcpServer, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineMcpServer({\n * name: \u0027GitHub\u0027,\n * url: \u0027https://api.githubcopilot.com/mcp/\u0027,\n * headers: { Authorization: \u0027Bearer {{GITHUB_TOKEN}}\u0027, \u0027X-Team\u0027: env.GITHUB_TEAM },\n * })\n * ```\n *\n * A workflow names the server by its key and one of its tools by name:\n * `mcp(\u0027github\u0027).tool(\u0027create_issue\u0027)`. Secrets never sit in source: a credential header\n * (`Authorization`, `Cookie`, or a name with `key`, `token`, `secret`, `auth`, …) holds env\n * references (`env.NAME`, or `{{NAME}}` in its text after a scheme such as `Bearer`) or\n * `stored()`, which keeps the value the server holds. The server\u0027s id derives from its URL, so a\n * `url` with another host or path replaces the server; a new query string updates it in place.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** A header value: text (`{{NAME}}` names an env variable), `env.NAME`, or `stored()`. */\nexport type McpHeaderValue = string | EnvRef\n\n/** How the server authenticates. */\nexport interface McpServerAuth {\n /** `headers` (the default) sends the declared headers; `oauth` signs in with an OAuth grant. */\n type: \u0027none\u0027 | \u0027headers\u0027 | \u0027oauth\u0027\n /** A pre-registered OAuth client, for a server without dynamic client registration. */\n clientId?: string\n}\n\n/** An MCP server declaration. */\nexport interface McpServerDeclaration {\n /** The display name; the key when left out. */\n name?: string\n description?: string\n /** An `http://` or `https://` URL. A new host or path replaces the server; a new query does not. */\n url: string\n transport?: \u0027streamable-http\u0027\n headers?: { [name: string]: McpHeaderValue }\n /** `{ type: \u0027headers\u0027 }` when left out. The OAuth client secret is set in the server settings. */\n auth?: McpServerAuth\n /** Milliseconds a request waits; 30000 when left out. */\n timeout?: number\n /** 3 when left out. */\n retries?: number\n /** `true` when left out. */\n enabled?: boolean\n}\n\n/** Declares an MCP server. Its default export is the whole file. */\nexport declare function defineMcpServer(server: McpServerDeclaration): McpServerDeclaration\n', + 'secrets.d.ts': + '/**\n * Secret declarations: `secrets.ts` at the root of a workspace source directory names the\n * workspace secrets its workflows read, by name and what each is for. Never a value: set one with\n * `sim secrets set <NAME> --scope workspace`, or in the workspace\u0027s secrets settings.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineSecrets } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineSecrets({\n * ANTHROPIC_API_KEY: { description: \u0027Model access for the support agent\u0027 },\n * STRIPE_KEY: {},\n * })\n * ```\n *\n * A workflow reads one as `env.NAME`, which type-checks against these names. An apply sets only\n * descriptions: a name the workspace holds no value for is reported, and a workflow using it is\n * held back (not written) while the rest of the apply goes ahead. Removing a name changes nothing\n * on the server.\n */\n\n/** One declared secret. */\nexport interface SecretDeclaration {\n /** What the secret is for, shown to teammates (at most 500 characters). */\n description?: string\n}\n\n/** Declares the workspace\u0027s secrets by name. Its default export is the whole file. */\nexport declare function defineSecrets<const T extends Record<string, SecretDeclaration>>(\n secrets: T\n): T\n\n/** The env names a `secrets.ts` declaration adds, as `workspace.d.ts` merges them into `env`. */\nexport type DeclaredSecretNames<T> = { [K in keyof T]: true }\n', + 'skill.d.ts': + '/**\n * Skills are markdown, not TypeScript: `skills/<key>.md` is a frontmatter block of `name` and\n * `description`, then the skill\u0027s content.\n *\n * ```md\n * ---\n * name: tone\n * description: "Answer in the support team\u0027s voice: short, warm, specific."\n * ---\n * Write like a person, not a policy.\n * ```\n *\n * A value is plain text on one line, or a JSON string (`"…"`) when it holds a line break,\n * surrounding spaces, quotes, `: ` or ` #`, ends in `:`, or starts with anything but a letter,\n * digit or `(`. Quote a value YAML would read as a boolean, number or date (`"yes"`, `"1e3"`) too,\n * so other markdown tools read it as text. The name is kebab-case; a workflow names the skill by\n * its key, `skill(\u0027tone\u0027)`. This module declares nothing.\n */\nexport {}\n', + 'table.d.ts': + '/**\n * Table declarations: `tables/<key>.ts` in a workspace source directory.\n *\n * ```ts\n * // @sim-sdk 1\n * import { column, defineTable } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTable({\n * name: \u0027leads\u0027,\n * columns: {\n * email: column.string({ required: true, unique: true }),\n * status: column.select({ options: [\u0027new\u0027, \u0027won\u0027, \u0027lost\u0027] }),\n * score: column.number(),\n * },\n * })\n * ```\n *\n * A column\u0027s object key is its source key and stays bound to the stored column; `name` renames\n * the stored column while keeping its cells.\n *\n * Rows are part of source only when `rows` opts in: `tables/<key>.rows.jsonl` beside the\n * declaration holds one JSON object per line, keyed by column key (a select cell by option name).\n * `{ mode: \u0027seed\u0027 }` writes them into the table when an apply creates it (or adopts it empty);\n * `{ mode: \u0027managed\u0027, key }` keeps the table\u0027s rows equal to the file, each matched by the unique\n * `key` column (removing a row needs `--allow-delete`); a stored row whose `key` cell is empty is\n * left out of the file and kept as it is.\n */\nimport type { Json } from \u0027@sim/workflow-sdk\u0027\n\ndeclare const COLUMN: unique symbol\n\n/**\n * A declared column: the cell type `T` it holds, the name `N` it is stored under, and whether\n * every row must hold a value (`R`); a cell of a column that is not required may be empty (`null`).\n */\nexport interface Column<T, N extends string | undefined = undefined, R extends boolean = false> {\n readonly [COLUMN]: { value: T; name: N; required: R }\n}\n\n/** Options every column type takes. */\nexport interface ColumnOptions<\n N extends string | undefined = string | undefined,\n R extends boolean = boolean,\n> {\n /** The stored column name, when it differs from the column\u0027s key. */\n name?: N\n /** Every row must hold a value. */\n required?: R\n /** No two rows may hold the same value. */\n unique?: boolean\n}\n\n/** The shared options a `select` column takes: never `unique`. */\ntype SelectOptions<N extends string | undefined, R extends boolean> = Omit<\n ColumnOptions<N, R>,\n \u0027unique\u0027\n>\n\n/** The column helpers: `column.string()`, `column.select({ options: [...] })`, … */\nexport declare const column: {\n string<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n number<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<number, N, R>\n /** A number shown as money; `currencyCode` is an ISO 4217 code (`USD` when omitted). */\n currency<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R> & { currencyCode?: string }\n ): Column<number, N, R>\n boolean<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<boolean, N, R>\n /** An ISO 8601 date or date-time string. */\n date<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n /** A time to live: a date-time after which the row expires. A table holds at most one. */\n ttl<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<string, N, R>\n json<const N extends string | undefined = undefined, const R extends boolean = false>(\n options?: ColumnOptions<N, R>\n ): Column<Json, N, R>\n /** One option per cell, from `options`. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple?: false }): Column<O[number], N, R>\n /** Several options per cell. */\n select<\n const O extends readonly [string, ...string[]],\n const N extends string | undefined = undefined,\n const R extends boolean = false,\n >(options: SelectOptions<N, R> & { options: O; multiple: true }): Column<O[number][], N, R>\n}\n\n/** Any column, whatever it holds. */\ntype AnyColumn = Column<unknown, string | undefined, boolean>\n\n/** A table declaration. */\nexport interface TableDeclaration<C extends Record<string, AnyColumn> = Record<string, AnyColumn>> {\n /** The table name: a letter or `_`, then letters, digits and `_`. */\n name: string\n description?: string\n /** The table folder path (`/reports`); the workspace root when omitted. */\n folder?: string\n columns: C\n /** Rows as source, from `tables/<key>.rows.jsonl`; omitted leaves rows out of source. */\n rows?: TableRows<keyof C & string>\n}\n\n/**\n * How a table\u0027s rows are source: `seed` rows go into the table once, when an apply creates it;\n * `managed` rows are the whole table, matched to stored rows by the unique `key` column.\n */\nexport type TableRows<K extends string = string> = { mode: \u0027seed\u0027 } | { mode: \u0027managed\u0027; key: K }\n\n/** Declares a table. Its default export is the whole file. */\nexport declare function defineTable<const C extends Record<string, AnyColumn>>(\n table: TableDeclaration<C>\n): TableDeclaration<C>\n\n/** The stored name of column `K`: its `name` option, else its key. */\ntype StoredName<K, C> = C extends Column<unknown, infer N, boolean>\n ? [N] extends [string]\n ? N\n : K\n : K\n\n/** A cell of column `C`: its value, or `null` when the column is not required. */\ntype CellOf<C> = C extends Column<infer V, infer _N, infer R>\n ? R extends true\n ? V\n : V | null\n : never\n\n/**\n * A declared table\u0027s row, keyed by stored column name: `RowOf<typeof leads>`. A cell of a column\n * that is not required may be `null`.\n */\nexport type RowOf<T> = T extends TableDeclaration<infer C>\n ? { [K in keyof C as StoredName<K, C[K]>]: CellOf<C[K]> }\n : never\n', + 'tool.d.ts': + '/**\n * Custom tool declarations: `tools/<key>.ts`, with the tool\u0027s code beside it in `tools/<key>.js`.\n *\n * ```ts\n * // @sim-sdk 1\n * import { codeFile, defineTool } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineTool({\n * title: \u0027Lookup order\u0027,\n * schema: {\n * type: \u0027function\u0027,\n * function: {\n * name: \u0027lookup_order\u0027,\n * description: \u0027Find an order by id.\u0027,\n * parameters: {\n * type: \u0027object\u0027,\n * properties: { orderId: { type: \u0027string\u0027 } },\n * required: [\u0027orderId\u0027],\n * },\n * },\n * },\n * code: codeFile(\u0027./lookup-order.js\u0027),\n * })\n * ```\n *\n * The code file runs with each parameter as a variable; `$env.NAME` reads an environment variable\n * by name. The key is the tool\u0027s identity: a workflow names it as `customTool(\u0027lookup-order\u0027)`.\n * Code holding a literal credential prints as `code: stored()`, which keeps the code the server\n * holds without showing it.\n */\nimport type { CodeFileRef, Json } from \u0027@sim/workflow-sdk\u0027\n\nexport { codeFile, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** JSON as an object literal writes it: an optional member may be absent. */\ntype SchemaJson = Json | readonly SchemaJson[] | { readonly [key: string]: SchemaJson | undefined }\n\n/** The function declaration a model calls the tool by, stored verbatim. */\nexport interface ToolSchema {\n type: \u0027function\u0027\n function: {\n /** What the model calls: letters, digits, `_` and `-`, at most 64 characters. */\n name: string\n description?: string\n /** A JSON Schema object describing the arguments. */\n parameters: {\n type: \u0027object\u0027\n properties: { [name: string]: Json }\n required?: string[]\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n }\n readonly [extra: string]: SchemaJson | undefined\n}\n\n/** A custom tool declaration. */\nexport interface ToolDeclaration {\n /** The tool\u0027s display title, unique in the workspace. */\n title: string\n schema: ToolSchema\n /** `codeFile(\u0027./<key>.js\u0027)`, the code inline as a string, or `stored()` to keep the stored code. */\n code: CodeFileRef | string\n}\n\n/** Declares a custom tool. Its default export is the whole file. */\nexport declare function defineTool(tool: ToolDeclaration): ToolDeclaration\n', +} From 47b220c9b7579f9cd974fdd7e87e78d782bc6a12 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:10:18 -0800 Subject: [PATCH 40/59] fix(sim-cli): save a workspace apply before its best-effort stored-edges records --- .../src/commands/protocol/stored-edges.ts | 48 +++++++++++++--- .../protocol/workspace-source-apply.test.ts | 56 +++++++++++++++++++ .../protocol/workspace-source-apply.ts | 12 +++- .../commands/protocol/workspace-source-get.ts | 11 +++- 4 files changed, 114 insertions(+), 13 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/stored-edges.ts b/packages/sim-cli/src/commands/protocol/stored-edges.ts index 888c553e1b8..04306d11abb 100644 --- a/packages/sim-cli/src/commands/protocol/stored-edges.ts +++ b/packages/sim-cli/src/commands/protocol/stored-edges.ts @@ -75,18 +75,27 @@ function parseJson(text: string): unknown { } } +/** The stored-edges records of a workspace answer, and the folders left without one. */ +export interface WorkflowFolderRecords { + records: Array<readonly [string, string]> + /** Folder keys whose catalog could not be fetched: their record is skipped, not failed. */ + unrecorded: string[] +} + /** * The `.sim/stored-edges.json` write of each workflow folder a workspace get or apply answered * with, each compiled against the catalog its base names. A workflow holding a type the workspace * catalog lacks was printed with its own catalog: one not cached yet is fetched, once per distinct - * hash, and cached, so an offline check compiles that folder against it too. + * hash, and cached, so an offline check compiles that folder against it too. The records are a + * cache fill, never part of the write: a catalog fetch that fails skips the folders printed with + * that catalog (named in `unrecorded`) rather than failing a write the server already committed. */ export async function workflowFolderRecords( client: OperationClient<'getSourceCatalog'>, workspaceId: string, workspaceCatalog: BlockCatalog, files: Record<string, string> -): Promise<Array<readonly [string, string]>> { +): Promise<WorkflowFolderRecords> { const folders = printedWorkflowFolders(files) const uncached = new Map<string, string>() for (const { base } of folders.values()) { @@ -95,16 +104,39 @@ export async function workflowFolderRecords( if (catalogHash === workspaceCatalog.hash || cachedCatalogByHash(catalogHash)) continue if (!uncached.has(catalogHash)) uncached.set(catalogHash, workflowId) } - for (const workflowId of uncached.values()) { - await loadWorkflowCatalog(client, workspaceId, workflowId) + const fetched = new Map<string, BlockCatalog>() + for (const [catalogHash, workflowId] of uncached) { + const catalog = await loadWorkflowCatalog(client, workspaceId, workflowId).catch(() => null) + if (catalog) fetched.set(catalogHash, catalog) } - return [...folders].flatMap(([key, folder]) => { + const records: Array<readonly [string, string]> = [] + const unrecorded: string[] = [] + for (const [key, folder] of folders) { const { catalogHash } = toRecord(folder.base) const catalog = - catalogHash === workspaceCatalog.hash ? workspaceCatalog : cachedCatalogByHash(catalogHash) + catalogHash === workspaceCatalog.hash + ? workspaceCatalog + : typeof catalogHash === 'string' + ? (fetched.get(catalogHash) ?? cachedCatalogByHash(catalogHash)) + : null + if (typeof catalogHash === 'string' && uncached.has(catalogHash) && !catalog) { + unrecorded.push(key) + continue + } const write = storedEdgesWrite(folder, catalog, `${KIND_DIRS.workflow}/${key}/`) - return write ? [write] : [] - }) + if (write) records.push(write) + } + return { records, unrecorded } +} + +/** + * The warning for workflow folders {@link workflowFolderRecords} could not record, or `null`. + * Their edges are unrecorded, so an offline check of each one compiles without its stored edges + * until the next get records them. + */ +export function unrecordedFoldersWarning(unrecorded: readonly string[]): string | null { + if (unrecorded.length === 0) return null + return `Warning: could not fetch the block catalog of ${unrecorded.map((key) => `${KIND_DIRS.workflow}/${key}`).join(', ')}, so its stored edges are not recorded; everything else was saved. The next \`sim workspaces source get\` records them.` } function isStoredEdge(value: unknown): value is StoredEdgeRef { diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index f634e463fab..0a253be9147 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -18,6 +18,7 @@ import { basename, dirname, join } from 'node:path' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' +import { cacheCatalogText } from '../../test/catalog-cache' import { contextMockFns, contextMockState } from '../../test/context-mock' import { runWorkspaceSource } from '../../test/workspace-source-command' import { digest } from './source-dir' @@ -1213,4 +1214,59 @@ describe('workspaces source apply', () => { .join('\n') expect(warned).toContain('Held back, not written: workflow ingest') }) + + it('saves the applied answer when the catalog of a workflow it reprinted cannot be fetched, and says its edges are unrecorded', async () => { + const configDir = mkdtempSync(join(tmpdir(), 'sim-workspace-apply-config-')) + vi.stubEnv('SIM_CONFIG_DIR', configDir) + cacheCatalogText( + configDir, + 'ws_local', + readFileSync(join(__dirname, '../../../../workflow-sdk/test/fixtures/catalog.json'), 'utf8') + ) + const storedBase = JSON.stringify({ + version: 1, + workflowId: 'wf_1', + workspaceId: 'ws_local', + fingerprint: 'sf1:b', + catalogHash: 'workflow-own-catalog', + }) + request.mockImplementation( + async (path: string, options: { query?: { workflowId?: string } }) => { + if (path.endsWith('/source/catalog') && options.query?.workflowId) { + throw new SimApiError('Service unavailable', 503, 'UNAVAILABLE') + } + return answer({ + plan: [{ kind: 'workflow', key: 'ingest', action: 'update' }], + applied: [{ kind: 'workflow', key: 'ingest', action: 'update', id: 'wf_1' }], + files: { + 'workflows/ingest/workflow.ts': `${INGEST}// stored\n`, + 'workflows/ingest/.sim/base.json': storedBase, + }, + base: { ...BASE, resources: { 'workflow:ingest': { id: 'wf_1' } } }, + }) + } + ) + + try { + await apply('--yes') + } finally { + vi.unstubAllEnvs() + rmSync(configDir, { recursive: true, force: true }) + } + + expect(process.exitCode).toBeUndefined() + expect(readFileSync(join(dir, 'workflows/ingest/workflow.ts'), 'utf8')).toBe( + `${INGEST}// stored\n` + ) + expect(readFileSync(join(dir, 'workflows/ingest/.sim/base.json'), 'utf8')).toBe(storedBase) + expect(JSON.parse(readFileSync(join(dir, '.sim/base.json'), 'utf8')).resources).toEqual({ + 'workflow:ingest': { id: 'wf_1' }, + }) + expect(existsSync(join(dir, 'workflows/ingest/.sim/stored-edges.json'))).toBe(false) + const warned = vi + .mocked(console.error) + .mock.calls.map(([line]) => String(line)) + .join('\n') + expect(warned).toContain('could not fetch the block catalog of workflows/ingest') + }) }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 6811de95e52..65567c7007c 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -61,7 +61,11 @@ import { verboseFlag, writeMode, } from './source-ui' -import { removedWorkflowRecords, workflowFolderRecords } from './stored-edges' +import { + removedWorkflowRecords, + unrecordedFoldersWarning, + workflowFolderRecords, +} from './stored-edges' import { appliedBlobDigests, assertBlobsFetched, @@ -616,10 +620,12 @@ export function attachWorkspaceSourceApply(source: Command): void { await savePromotion(options.dir, options.to, promotion, answer) } else { const workspaceCatalog = cachedCatalog(workspaceId) - const records = workspaceCatalog + const { records, unrecorded } = workspaceCatalog ? await workflowFolderRecords(client, workspaceId, workspaceCatalog, answer.files) - : [] + : { records: [], unrecorded: [] } await saveApplied(options.dir, answer, submission, written, records) + const unrecordedWarning = unrecordedFoldersWarning(unrecorded) + if (unrecordedWarning) printError(unrecordedWarning) } return answer }, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index edd2f3c9260..6cd5c22d924 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -16,6 +16,7 @@ import { WORKSPACE_SKIPPED_FILE, } from '@sim/workspace-code/layout' import type { Command } from 'commander' +import { printError } from '#sim-cli/output/io' import type { GetWorkspaceSourceResponse } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { apiCommand } from '../../runtime/called-operations' @@ -47,7 +48,11 @@ import { sourceTreeChange, withSourceExitCode, } from './source-tree' -import { removedWorkflowRecords, workflowFolderRecords } from './stored-edges' +import { + removedWorkflowRecords, + unrecordedFoldersWarning, + workflowFolderRecords, +} from './stored-edges' import { assertListedBlobs, blobDigests, @@ -212,7 +217,7 @@ async function writeWorkspaceSource( loadWorkspaceTypes(client, source.workspaceId, localDeclarations(source.files)), ]) assertSdkMajor(catalog) - const workflowRecords = await workflowFolderRecords( + const { records: workflowRecords, unrecorded } = await workflowFolderRecords( client, source.workspaceId, catalog, @@ -304,6 +309,8 @@ async function writeWorkspaceSource( ], }) await rm(downloads, { recursive: true, force: true }) + const unrecordedWarning = unrecordedFoldersWarning(unrecorded) + if (unrecordedWarning) printError(unrecordedWarning) const summary = finishTree(dir, write.length + stale.length, workspaceTypes, tree) return { written: summary.files, From ae4a2f5f24ba57687879d70aa94acd23b0e43a5d Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:13:25 -0800 Subject: [PATCH 41/59] fix(workspace-code): keep a resource deleted in the target since the last promotion deleted --- .../application/promote.integration.ts | 51 +++++++++++++++++++ packages/workspace-code/src/promote.ts | 39 ++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/apps/sim/lib/workspace-source/application/promote.integration.ts b/apps/sim/lib/workspace-source/application/promote.integration.ts index a97775eb8aa..2813a116021 100644 --- a/apps/sim/lib/workspace-source/application/promote.integration.ts +++ b/apps/sim/lib/workspace-source/application/promote.integration.ts @@ -76,6 +76,14 @@ description: House tone Be brief. ` +const NOTIFY = `// @sim-sdk 1 +import { trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('Notify', {}, () => { + trigger.start('Start', { inputFormat: { email: 'string' } }) +}) +` + const DEV: Record<string, string> = { 'tables/leads.ts': LEADS, 'skills/tone.md': TONE, @@ -277,6 +285,49 @@ describe('workspace source promotion', () => { } ) + check( + 'a resource deleted in the target since the last promotion stays deleted when the directory left it alone, and an edit of it since is refused as stale', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: { ...DEV, 'workflows/notify/workflow.ts': NOTIFY } }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + expect(first.result.failed).toBeUndefined() + + const prodRead = await runner.get(prod) + const removed = await runner.apply(prod, { + files: Object.fromEntries( + Object.entries(prodRead.files).filter( + ([path]) => path !== 'skills/tone.md' && !path.startsWith('workflows/notify/') + ) + ), + base: prodRead.base, + allowDelete: true, + }) + expect( + removed.applied.map((change) => `${change.action} ${change.kind}:${change.key}`).sort() + ).toEqual(['delete skill:tone', 'delete workflow:notify']) + + const untouched = await promoteRecorded(await runner.get(dev), prod, first.record) + expect(untouched.result.applied).toEqual([]) + expect(codes(untouched.result.diagnostics, 'info').sort()).toEqual([ + 'I_SERVER_DELETED', + 'I_SERVER_DELETED', + ]) + expect((await liveWorkflows(prod)).map((row) => row.name)).toEqual(['Ingest']) + expect((await runner.get(prod)).files['skills/tone.md']).toBeUndefined() + + const edited = await editDev(dev, { + 'skills/tone.md': TONE.replace('Be brief.', 'Be very brief.'), + 'workflows/notify/workflow.ts': NOTIFY.replace("'Notify'", "'Notify v2'"), + }) + const stale = await refusal(promoteRecorded(edited, prod, untouched.record)) + expect(stale.stale).toBe(true) + expect(codes(stale.diagnostics)).toEqual(['E_APPLY_STALE', 'E_APPLY_STALE']) + expect((await liveWorkflows(prod)).map((row) => row.name)).toEqual(['Ingest']) + } + ) + check( 'with the record, a promotion that changed nothing in the target since converges, and the next edit updates it in place', async () => { diff --git a/packages/workspace-code/src/promote.ts b/packages/workspace-code/src/promote.ts index 4013c5dda91..a66370cff73 100644 --- a/packages/workspace-code/src/promote.ts +++ b/packages/workspace-code/src/promote.ts @@ -130,6 +130,25 @@ function withFingerprint(text: string, fingerprint: string): string | null { return `${JSON.stringify({ ...parsed.base, fingerprint }, null, 2)}\n` } +/** + * The folder base of a workflow the last promotion left in target `workspaceId`, from its record: + * it names that workflow and the fingerprint the promotion left. The catalog it was printed with + * is not recorded, and a base naming a workflow the target no longer holds is never compiled. + */ +function recordedFolderBase(promoted: PromotedWorkflow, workspaceId: string): string { + return `${JSON.stringify( + { + version: 1, + workflowId: promoted.workflowId, + workspaceId, + fingerprint: promoted.fingerprint, + catalogHash: '', + }, + null, + 2 + )}\n` +} + /** The workflow a folder base's text names, or `null` if unreadable. */ function folderWorkflow( text: string | undefined @@ -200,6 +219,18 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission const targetBase = input.target.files[basePath] const live = folderWorkflow(targetBase) const promoted = input.record?.workflows[key] + if (targetBase === undefined && promoted) { + // Deleted in the target since the last promotion: the recorded base names the workflow it + // left there, so the apply keeps the deletion, or refuses an edit since as stale. + const unchanged = promoted.digest === folderDigest(key, input.files) + if (unchanged) { + for (const path of Object.keys(files)) { + if (path.startsWith(`${folder}/`)) delete files[path] + } + } + files[basePath] = recordedFolderBase(promoted, input.target.base.workspaceId) + continue + } if (targetBase === undefined || !live || promoted?.workflowId !== live.workflowId) { for (const path of [basePath, lockPath]) { const text = input.target.files[path] @@ -229,6 +260,14 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission return [[address, recorded?.id === live.id ? recorded : live] as const] }) ) + // A resource the last promotion left in the target that the target no longer holds was deleted + // there: its recorded entry stays in the base, so the apply keeps the deletion when the + // directory left it alone and refuses an edit since as stale, rather than creating it again. + for (const [address, recorded] of Object.entries(input.record?.resources ?? {})) { + if (Object.hasOwn(input.target.base.resources, address)) continue + const parsed = parseResourceAddress(address) + if (parsed && declared.get(parsed.kind)?.has(parsed.key)) resources[address] = recorded + } return { files, base: { ...input.target.base, resources } } } From c3a86c9606d47b4bbeabdd01fa6e12ec1f218210 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:15:34 -0800 Subject: [PATCH 42/59] fix(workflow-code): infer Function result kinds soundly and re-warn when an unchanged reference's Function changed kind --- .../src/compile/function-returns.ts | 74 +++++++++++++++---- packages/workflow-code/src/compile/index.ts | 15 +++- .../workflow-code/src/json-fields.test.ts | 33 ++++++++- 3 files changed, 103 insertions(+), 19 deletions(-) diff --git a/packages/workflow-code/src/compile/function-returns.ts b/packages/workflow-code/src/compile/function-returns.ts index 4307e2080a9..38af616f1a3 100644 --- a/packages/workflow-code/src/compile/function-returns.ts +++ b/packages/workflow-code/src/compile/function-returns.ts @@ -5,7 +5,8 @@ * * The reading is deliberately narrow: every `return` outside a nested function must return an * object literal, and a field counts only when each of them sets it to an expression whose kind - * is certain (a literal, arithmetic, a comparison, `String(…)`, `.length`, a local variable never + * is certain (a literal, arithmetic, a comparison, `String(…)`, the `.length` of a string or an + * array, a local variable never * reassigned to another kind, …). Anything else stays unknown, and the reference keeps its * warning: a wrong "typed" would store a string bare and break the JSON at run time. */ @@ -172,6 +173,16 @@ function collectBindings(program: Node): Bindings { list.push({ operator: '=', value: null }) writes.set(n, list) }) + } else if ( + (node.type === 'ForOfStatement' || node.type === 'ForInStatement') && + node.left.type !== 'VariableDeclaration' + ) { + // `for (x of …)` reassigns an existing `x` each pass, with no assignment expression. + walkPatternNames(node.left, (n) => { + const list = writes.get(n) ?? [] + list.push({ operator: '=', value: null }) + writes.set(n, list) + }) } else if (node.type === 'UpdateExpression' && node.argument.type === 'Identifier') { const list = writes.get(node.argument.name) ?? [] list.push({ operator: '++', value: null }) @@ -242,7 +253,10 @@ class Inference { return 'number' case 'MemberExpression': case 'OptionalMemberExpression': - return memberName(node) === 'length' ? 'number' : undefined + return memberName(node) === 'length' && + (this.kind(node.object) === 'string' || this.isArray(node.object)) + ? 'number' + : undefined case 'CallExpression': case 'OptionalCallExpression': return this.call(node.callee) @@ -283,6 +297,32 @@ class Inference { return undefined } + /** + * Whether `node` is certainly an array: a literal, an array method's result, `Object.keys` + * (`values`, `entries`), or a local variable set to one and never reassigned. Any other object + * can hold anything under `length`. + */ + private isArray(node: Node): boolean { + if (node.type === 'ArrayExpression') return true + if (node.type === 'CallExpression' || node.type === 'OptionalCallExpression') { + const method = memberName(node.callee) + if (method === null || !('object' in node.callee)) return false + if (node.callee.object.type === 'Identifier' && node.callee.object.name === 'Object') { + return method === 'keys' || method === 'values' || method === 'entries' + } + return ARRAY_METHODS.has(method) + } + if (node.type !== 'Identifier') return false + const init = this.bindings.initial.get(node.name) + if (!init || this.bindings.writes.has(node.name) || this.resolving.has(node.name)) return false + this.resolving.add(node.name) + try { + return this.isArray(init) + } finally { + this.resolving.delete(node.name) + } + } + /** * A local variable's kind: its initializer's, kept only when every later assignment keeps it * (`+=` of a number to a number, of anything to a string; `++`; `=` of the same kind). @@ -317,26 +357,32 @@ class Inference { } } -/** The fields one returned object literal sets, by name; a spread makes earlier ones unknown. */ +/** + * The fields one returned object literal sets, by name. A spread, or a key not known statically + * (computed, or any other key syntax), may replace any field before it, so it makes earlier ones + * unknown; a method or accessor makes its own field unknown. + */ function returnedFields( object: ObjectExpression, inference: Inference ): Map<string, JsKind | undefined> { const fields = new Map<string, JsKind | undefined>() for (const property of object.properties) { - if (property.type === 'SpreadElement') { - for (const key of fields.keys()) fields.set(key, undefined) + const key = + property.type === 'SpreadElement' || property.computed + ? null + : property.key.type === 'Identifier' + ? property.key.name + : property.key.type === 'StringLiteral' + ? property.key.value + : property.key.type === 'NumericLiteral' + ? String(property.key.value) + : null + if (key === null) { + for (const known of fields.keys()) fields.set(known, undefined) continue } - if (property.type !== 'ObjectProperty' || property.computed) continue - const key = - property.key.type === 'Identifier' - ? property.key.name - : property.key.type === 'StringLiteral' - ? property.key.value - : null - if (key === null) continue - fields.set(key, inference.kind(property.value)) + fields.set(key, property.type === 'ObjectProperty' ? inference.kind(property.value) : undefined) } return fields } diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 88e1c60db85..06024dd4426 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -667,8 +667,19 @@ function compileEntry( setOwn(params, id, { kind: 'value', value }) for (const ref of refs) { if (refValueKind(ref.token, refTypes) !== 'untyped') continue - if (referencedResultKind(ref.token, refTypes.block) === 'typed') continue - if (storedHolds(input.stored, ref.blockKey, id, value)) continue + const returned = referencedResultKind(ref.token, refTypes.block) + if (returned === 'typed') continue + // An unchanged field stays quiet only while what it reads is too: a Function that now + // returns text where it returned a number makes the same bare token break the JSON. + if ( + storedHolds(input.stored, ref.blockKey, id, value) && + referencedResultKind( + ref.token, + (key) => input.stored && getOwn(input.stored.blocks, key) + ) === returned + ) { + continue + } reporter.report( 'W_JSON_REF_UNTYPED', `Nothing declares the type of ${ref.token}; it is stored bare, so a string value would break the JSON.`, diff --git a/packages/workflow-code/src/json-fields.test.ts b/packages/workflow-code/src/json-fields.test.ts index ac042748fc1..80f76840906 100644 --- a/packages/workflow-code/src/json-fields.test.ts +++ b/packages/workflow-code/src/json-fields.test.ts @@ -174,9 +174,10 @@ describe('W_JSON_REF_UNTYPED on a Function block output', () => { 'const tax = Math.round(items.length * 10) / 100', 'let score = $ref.start.total > 2000 ? 90 : 20', "if ($ref.start.tier === 'new') score += 10", - 'return { tax, score, ok: score >= 70, count: items.length, list: items.map((i) => i.sku), meta: {} }', + "const names = 'a,b'.split(',')", + 'return { tax, score, ok: score >= 70, count: names.length, chars: "ab".length, list: items.map((i) => i.sku), meta: {} }', ].join('\n') - for (const field of ['tax', 'score', 'ok', 'count', 'list', 'meta']) { + for (const field of ['tax', 'score', 'ok', 'count', 'chars', 'list', 'meta']) { expect(untyped(code, field), field).toEqual([]) } }) @@ -198,8 +199,26 @@ describe('W_JSON_REF_UNTYPED on a Function block output', () => { expect(untyped('return { v: 1 ', 'v')).toHaveLength(1) }) + it('keeps the warning when a computed key, a loop or an object length may make the field text', () => { + // A computed key or a getter of the same name replaces the field written before it. + expect(untyped("const key = 'v'\nreturn { v: 1, [key]: 'text' }", 'v')).toHaveLength(1) + expect(untyped("return { v: 1, get v() { return 'text' } }", 'v')).toHaveLength(1) + // A for-of or for-in loop over an existing variable reassigns it without an assignment. + expect(untyped("let v = 1\nfor (v of ['a']) {}\nreturn { v }", 'v')).toHaveLength(1) + expect(untyped('let v = 1\nfor (v in { a: 1 }) {}\nreturn { v }', 'v')).toHaveLength(1) + expect(untyped("let v = 1\nfor ([v] of [['a']]) {}\nreturn { v }", 'v')).toHaveLength(1) + // `length` is a number only on a string or an array: an object can hold anything there. + expect(untyped("return { v: ({ length: 'oops' }).length }", 'v')).toHaveLength(1) + expect( + untyped("let list = []\nlist = { length: 'oops' }\nreturn { v: list.length }", 'v') + ).toHaveLength(1) + expect( + untyped('const items = $ref.start.items || []\nreturn { v: items.length }', 'v') + ).toHaveLength(1) + }) + it('reads only the returns of the body, not those of a callback inside it', () => { - const code = "const f = (x) => { return { v: 'text' } }\nreturn { v: f(1).length }" + const code = "const f = (x) => { return { v: 'text' } }\nreturn { v: 1 }" expect(untyped(code, 'v')).toEqual([]) }) @@ -211,4 +230,12 @@ describe('W_JSON_REF_UNTYPED on a Function block output', () => { expect(untyped(code, 'v', printed)).toEqual([]) expect(untyped(code, 'w', printed)).toHaveLength(1) }) + + it('on an apply, warns on an unchanged JSON field when the Function it reads now returns text there', () => { + const printed = compile('return { v: 1 }', 'v').workflow + if (!printed) throw new Error('the source did not compile') + + expect(untyped('return { v: 2 }', 'v', printed)).toEqual([]) + expect(untyped("return { v: 'one' }", 'v', printed)).toHaveLength(1) + }) }) From f417f49b5a015c3f0e914d57904c2db243a050eb Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:16:28 -0800 Subject: [PATCH 43/59] fix(workflow-code): name the visibleWhen clause that hides a field --- .../workflow-code/src/compile/checks.test.ts | 34 +++++++++++ .../src/compile/validate-catalog.ts | 56 +++++++++++-------- 2 files changed, 68 insertions(+), 22 deletions(-) diff --git a/packages/workflow-code/src/compile/checks.test.ts b/packages/workflow-code/src/compile/checks.test.ts index eb60ba48dd9..c433b00a8c2 100644 --- a/packages/workflow-code/src/compile/checks.test.ts +++ b/packages/workflow-code/src/compile/checks.test.ts @@ -191,6 +191,40 @@ describe("a field hidden under the block's values", () => { ) expect(written('claude-sonnet-4-5')).toEqual([]) }) + + it('names the clause an opaque condition still asks for the operation, before the model', () => { + const slackBlock = TEST_CATALOG.blocks.slack as CatalogBlock + const gated: BlockCatalog = { + ...TEST_CATALOG, + blocks: { + ...TEST_CATALOG.blocks, + slack: { + ...slackBlock, + subBlocks: slackBlock.subBlocks.map((sub) => + sub.id === 'limit' + ? { + ...sub, + condition: { opaque: true }, + visibleFor: { send: true, list_channels: true }, + visibleWhen: { send: { field: 'authMethod', value: 'bot_token' } }, + } + : sub + ), + }, + }, + } + const [hidden] = findings( + run( + "const s = slack.sendMessage('S', { authMethod: 'oauth', text: 'x', limit: 3 })", + undefined, + gated + ), + 'W_FIELD_NOT_APPLICABLE' + ) + expect(hidden?.message).toBe( + `limit is hidden (it shows only when authMethod is bot_token), ${DROPPED}` + ) + }) }) describe('references into an agent responseFormat', () => { diff --git a/packages/workflow-code/src/compile/validate-catalog.ts b/packages/workflow-code/src/compile/validate-catalog.ts index b129699fa8b..6b9429b0919 100644 --- a/packages/workflow-code/src/compile/validate-catalog.ts +++ b/packages/workflow-code/src/compile/validate-catalog.ts @@ -1,7 +1,13 @@ -import { type ConditionLookup, evaluateClause, evaluateCondition } from '../catalog/conditions' +import { + type ConditionClause, + type ConditionLookup, + evaluateClause, + evaluateCondition, +} from '../catalog/conditions' import type { CatalogField, FieldIndex } from '../catalog/surface' import type { CatalogSubBlock } from '../catalog/types' import { isPlainRecord, type SubBlockId, type Value } from '../model' +import { getOwn } from '../records' /** The field a written param name targets: its written name or, when aliased, its own name. */ export function fieldFor(index: FieldIndex, key: string): CatalogField | undefined { @@ -30,35 +36,41 @@ export function hiddenReason( sub: CatalogSubBlock, values: ReadonlyMap<string, Value> ): string | null { - const { condition, visibleFor, visibleForModel } = sub + const { condition, visibleFor, visibleWhen, visibleForModel } = sub const rejected = (field: string) => { const value = values.get(field) return `${field} ${value === undefined || value === null ? 'unset' : String(value)} does not use it` } if (!condition) return null - if ('opaque' in condition) { - const operation = values.get('operation') - if (typeof operation === 'string' && visibleFor?.[operation] === false) { - return rejected('operation') + const lookup = paramLookup(values) + /** The part of `clause` that hides the field, described; `null` when none does. */ + const failingPart = (clause: ConditionClause): string | null => { + const failing = [clause, clause.and].find( + (part): part is NonNullable<typeof part> => + part !== undefined && + evaluateClause({ field: part.field, value: part.value, not: part.not }, lookup) === false + ) + if (!failing) return null + if (Array.isArray(failing.value) && failing.value.length > LISTED_VALUES) { + return rejected(failing.field) } - const model = values.get('model') - if (typeof model === 'string' && visibleForModel?.[model] === false) return rejected('model') - return null + const wanted = Array.isArray(failing.value) + ? failing.value.map(String).join(' or ') + : String(failing.value) + return `it shows only when ${failing.field} is ${failing.not ? 'not ' : ''}${wanted}` } - const lookup = paramLookup(values) - const failing = [condition, condition.and].find( - (part): part is NonNullable<typeof part> => - part !== undefined && - evaluateClause({ field: part.field, value: part.value, not: part.not }, lookup) === false - ) - if (!failing) return null - if (Array.isArray(failing.value) && failing.value.length > LISTED_VALUES) { - return rejected(failing.field) + if (!('opaque' in condition)) return failingPart(condition) + // The order `evaluateCondition` decides an opaque condition in: the operation, then the clause + // `visibleWhen` still asks for it, and the model only for an operation `visibleFor` omits. + const operation = values.get('operation') + if (typeof operation === 'string' && visibleFor && Object.hasOwn(visibleFor, operation)) { + if (!visibleFor[operation]) return rejected('operation') + const asked = visibleWhen && getOwn(visibleWhen, operation) + return asked ? failingPart(asked) : null } - const wanted = Array.isArray(failing.value) - ? failing.value.map(String).join(' or ') - : String(failing.value) - return `it shows only when ${failing.field} is ${failing.not ? 'not ' : ''}${wanted}` + const model = values.get('model') + if (typeof model === 'string' && visibleForModel?.[model] === false) return rejected('model') + return null } /** From 8b7bd98556d05e223040ff59bda71303a28d8fd9 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:18:21 -0800 Subject: [PATCH 44/59] fix(workflow-sdk): print a special builder's core.d.ts signature and fill Value<string> params in snippets --- packages/workflow-sdk/src/codegen/generate.ts | 3 - .../src/codegen/reference.test.ts | 37 +++++++++++ .../workflow-sdk/src/codegen/reference.ts | 65 ++++++++++++++----- 3 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 packages/workflow-sdk/src/codegen/reference.test.ts diff --git a/packages/workflow-sdk/src/codegen/generate.ts b/packages/workflow-sdk/src/codegen/generate.ts index 1b6be4cc45b..9cc59794e5d 100644 --- a/packages/workflow-sdk/src/codegen/generate.ts +++ b/packages/workflow-sdk/src/codegen/generate.ts @@ -238,8 +238,6 @@ export interface BuilderTypes { paramsName: string /** The params as facts. */ facts: ParamFacts - /** What a call returns: `Block<…>` of the output type. */ - returns: string } /** @@ -524,7 +522,6 @@ export function builderTypes(catalog: BlockCatalog, entry: BuilderEntry): Builde summary: [block.name, label].filter(Boolean).join(': '), paramsName, facts: paramFacts(params, gate, candidates, context.pinned), - returns: `Block<${outputType}>`, } } diff --git a/packages/workflow-sdk/src/codegen/reference.test.ts b/packages/workflow-sdk/src/codegen/reference.test.ts new file mode 100644 index 00000000000..ef6f88fcb7d --- /dev/null +++ b/packages/workflow-sdk/src/codegen/reference.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import type { BlockCatalog } from '@sim/workflow-code' +import { catalogBuilderPaths } from '@sim/workflow-code/catalog/view' +import { describe, expect, it } from 'vitest' +import { builderSnippet, describeBuilder } from './reference' + +/** A real catalog projected from registry blocks, shared with the SDK's own type checks. */ +const CATALOG = JSON.parse( + readFileSync(join(__dirname, '../../test/fixtures/catalog.json'), 'utf8') +) as BlockCatalog + +describe('builder reference', () => { + it('gives each special builder the call core.d.ts declares, with its own params and return type', () => { + expect(describeBuilder(CATALOG, 'condition')?.signature).toBe( + "condition<const B extends Record<string, BranchExpression>>(name: string, branches: B & ConditionBranches<B>, meta?: BlockMeta): Block<SpecialOutput<'condition'>> & Branches<ConditionKeys<B>>" + ) + expect(describeBuilder(CATALOG, 'note')?.signature).toBe( + "note(name: string, params: SpecialParams<'note'>, meta?: BlockMeta): Annotation" + ) + expect(describeBuilder(CATALOG, 'response')?.signature.split('\n')).toEqual([ + "response(name: string, params: ResponseBody, meta?: BlockMeta): Block<SpecialOutput<'response'>>", + "response(name: string, params: SpecialParams<'response'>, meta?: BlockMeta): Block<SpecialOutput<'response'>>", + ]) + }) + + it('fills a text, number or boolean param of a snippet with an example value, not its type', () => { + const snippets = catalogBuilderPaths(CATALOG).flatMap((key) => { + const reference = describeBuilder(CATALOG, key) + return reference ? [builderSnippet(reference)] : [] + }) + expect(snippets.join('\n')).toContain("'…'") + for (const snippet of snippets) { + expect(snippet).not.toContain('<Value<') + } + }) +}) diff --git a/packages/workflow-sdk/src/codegen/reference.ts b/packages/workflow-sdk/src/codegen/reference.ts index 15cabede343..8cc540022b6 100644 --- a/packages/workflow-sdk/src/codegen/reference.ts +++ b/packages/workflow-sdk/src/codegen/reference.ts @@ -73,10 +73,9 @@ export function describeBuilder(catalog: BlockCatalog, key: string): BuilderRefe const entry = builderEntry(catalog, key) if (!entry) return null const types = builderTypes(catalog, entry) - const call = entry.special ? key : (types.signature.at(-1) ?? '') const signature = entry.special - ? `${call}(name: string, params: ${types.paramsName}, meta?: BlockMeta): ${types.returns}` - : `${entry.path.slice(0, -1).join('.')}.${call.replace(/^export function /, '')}`.replace( + ? coreSignatures(key) + : `${entry.path.slice(0, -1).join('.')}.${(types.signature.at(-1) ?? '').replace(/^export function /, '')}`.replace( /^\./, '' ) @@ -126,22 +125,15 @@ export function lookupBuilder(catalog: BlockCatalog, query: string): BuilderLook const MAX_CORE_LINES = 80 -/** - * Every top-level declaration of `name` in `core.d.ts` (overloads included) with the doc comment - * above it, as the file holds it: what a construct the catalog does not generate takes. - */ -function coreDeclaration(name: string): string { +/** Each top-level declaration of `name` in `core.d.ts` (overloads included): its line range. */ +function coreDeclarationRanges(name: string): Array<{ start: number; end: number }> { const lines = CORE_DTS.split('\n') const start = new RegExp( `^export (?:declare )?(?:const|function|namespace|interface|type) ${name}\\b` ) - const out: string[] = [] + const declarations: Array<{ start: number; end: number }> = [] for (let index = 0; index < lines.length; index++) { if (!start.test(lines[index])) continue - let first = index - if (lines[index - 1]?.trim().endsWith('*/')) { - while (first > 0 && !lines[first].trim().startsWith('/**')) first-- - } let depth = 0 let end = index for (; end < lines.length; end++) { @@ -152,10 +144,47 @@ function coreDeclaration(name: string): string { const continues = /[=:,|&]\s*$/.test(lines[end]) || /^\s+[?:|&]/.test(lines[end + 1] ?? '') if (depth <= 0 && !continues) break } - if (out.length > 0) out.push('') - out.push(...lines.slice(first, end + 1)) + declarations.push({ start: index, end: end + 1 }) index = end } + return declarations +} + +/** + * A special builder's call as `core.d.ts` declares it (`condition` takes its branches, `note` + * returns an `Annotation`), one line per overload. + */ +function coreSignatures(name: string): string { + const lines = CORE_DTS.split('\n') + return coreDeclarationRanges(name) + .map(({ start, end }) => + lines + .slice(start, end) + .join(' ') + .replace(/^export (?:declare )?function /, '') + .replace(/\s+/g, ' ') + .replace(/([(<]) /g, '$1') + .replace(/,? ([)>])/g, '$1') + .trim() + ) + .join('\n') +} + +/** + * Every top-level declaration of `name` in `core.d.ts` (overloads included) with the doc comment + * above it, as the file holds it: what a construct the catalog does not generate takes. + */ +function coreDeclaration(name: string): string { + const lines = CORE_DTS.split('\n') + const out: string[] = [] + for (const { start, end } of coreDeclarationRanges(name)) { + let first = start + if (lines[start - 1]?.trim().endsWith('*/')) { + while (first > 0 && !lines[first].trim().startsWith('/**')) first-- + } + if (out.length > 0) out.push('') + out.push(...lines.slice(first, end)) + } return out.slice(0, MAX_CORE_LINES).join('\n') } @@ -177,9 +206,9 @@ function exampleValue(type: string): string { if (/^TableData\b/.test(type)) return "{ column: '…' }" if (/^KnowledgeBaseRef\b/.test(type)) return "knowledgeBase('<key>')" if (/^WorkflowRef\b/.test(type)) return "childWorkflow('<key>')" - if (/^(?:Template|string|Value<string>)\b/.test(type)) return "'…'" - if (/^(?:number|Value<number>)\b/.test(type)) return '1' - if (/^(?:boolean|Value<boolean>)\b/.test(type)) return 'true' + if (/^(?:Template|string|Value<string)\b/.test(type)) return "'…'" + if (/^(?:number|Value<number)\b/.test(type)) return '1' + if (/^(?:boolean|Value<boolean)\b/.test(type)) return 'true' if (/^(?:Array<|[\w<>]+\[\])/.test(type)) return '[]' return `<${shortType(type.split(' | ')[0] ?? type)}>` } From 8e5aed6749465f8457b41f65f13f0f488adbc75a Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:19:19 -0800 Subject: [PATCH 45/59] fix(providers): bound a long tool id to the provider name limit --- apps/sim/providers/tool-identity.test.ts | 29 ++++++++++++++++++++++-- apps/sim/providers/tool-identity.ts | 8 +++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/apps/sim/providers/tool-identity.test.ts b/apps/sim/providers/tool-identity.test.ts index e2fc55f17cb..e425df15cd9 100644 --- a/apps/sim/providers/tool-identity.test.ts +++ b/apps/sim/providers/tool-identity.test.ts @@ -78,9 +78,10 @@ describe('provider tool identities', () => { const longId = `tool_${'a'.repeat(80)}` const tools = [providerTool(longId, 'a'), providerTool(longId, 'b')] - assignProviderToolIdentities(tools) + const identities = assignProviderToolIdentities(tools) - expect(tools[0].id).toBe(longId) + expect(tools[0].id).toBe(longId.slice(0, 64)) + expect(identities.toolIdByWireId.get(tools[0].id)).toBe(longId) expect(tools[1].id).toHaveLength(64) expect(tools[1].id).toMatch(/__sim_2$/) }) @@ -212,4 +213,28 @@ describe('provider tool identities', () => { assignProviderToolIdentities(tools) expect(tools.map((tool) => tool.id)).toEqual(wireIds) }) + + it('bounds a long custom tool title to the provider name limit, keeping tools that share a prefix distinct', () => { + const PROVIDER_NAME = /^[a-zA-Z0-9_-]{1,64}$/ + const prefix = `custom_${'Look up the customer account '.repeat(3)}` + const tools = [ + providerTool(`${prefix}by email`, 'a'), + providerTool(`${prefix}by phone`, 'b'), + providerTool(`custom_${'x'.repeat(80)}`, 'c'), + ] + + const identities = assignProviderToolIdentities(tools) + const wireIds = tools.map((tool) => tool.id) + + for (const wireId of wireIds) expect(wireId).toMatch(PROVIDER_NAME) + expect(new Set(wireIds).size).toBe(3) + expect(wireIds.map((wireId) => identities.toolIdByWireId.get(wireId))).toEqual([ + `${prefix}by email`, + `${prefix}by phone`, + `custom_${'x'.repeat(80)}`, + ]) + + assignProviderToolIdentities(tools) + expect(tools.map((tool) => tool.id)).toEqual(wireIds) + }) }) diff --git a/apps/sim/providers/tool-identity.ts b/apps/sim/providers/tool-identity.ts index be9cb7392cb..1be25e13209 100644 --- a/apps/sim/providers/tool-identity.ts +++ b/apps/sim/providers/tool-identity.ts @@ -33,8 +33,8 @@ function buildProviderAlias(toolId: string, occurrence: number, attempt: number) * * The first occurrence and every already-unique tool with a provider-safe id keep it for backwards * compatibility. An id with characters no provider accepts (a custom tool titled `Get Weather`) is - * sent with them replaced by `_`, and when that collides with another tool's id it is aliased like - * a later occurrence. Later occurrences receive opaque ordinal aliases; resource and credential ids + * sent with them replaced by `_`, and one longer than the strictest provider limit is cut to it; + * when either collides with another tool's id it is aliased like a later occurrence. Later occurrences receive opaque ordinal aliases; resource and credential ids * never enter the provider-visible name. Tool objects are updated in place so their instance-bound * params and secret provenance remain attached to the exact object selected by provider adapters. */ @@ -60,8 +60,8 @@ export function assignProviderToolIdentities< occurrences.set(canonicalId, occurrence) const safeId = toProviderSafeToolId(canonicalId) - let wireId = safeId - if (usedWireIds.has(wireId) || (safeId !== canonicalId && reservedIds.has(safeId))) { + let wireId = safeId.slice(0, MAX_PROVIDER_TOOL_ID_LENGTH) + if (usedWireIds.has(wireId) || (wireId !== canonicalId && reservedIds.has(wireId))) { let attempt = 0 do { wireId = buildProviderAlias(safeId, occurrence, attempt) From bd20481dd35100fcd3debe75fb07a146115f8aca Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:21:28 -0800 Subject: [PATCH 46/59] fix(workflow-code): infer a rename from references in branches and subflows, read case-insensitively --- .../workflow-code/src/materialize/identity.ts | 35 +++++++++++++---- .../src/materialize/index.test.ts | 39 +++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 77c76a2b0e0..0feb02e8e76 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -1,7 +1,9 @@ -import type { BlockState } from '@sim/workflow-types/workflow' +import { findWorkflowReferenceTokens } from '@sim/utils/workflow-references' +import { type BlockState, normalizeWorkflowBlockName } from '@sim/workflow-types/workflow' import type { CanonicalizeResult } from '../canonicalize' import type { BlockCatalog } from '../catalog/types' import { catalogBlock } from '../catalog/view' +import { parseStoredPath } from '../code-files/ref-path' import type { MaterializeDiagnosticCode } from '../diagnostics/codes' import type { Diagnostic } from '../diagnostics/frame' import { pointer, type WorkflowLockBlock, type WorkflowLockV1 } from '../lock' @@ -387,10 +389,29 @@ function settingsAgree( return shared.length > 0 && shared.every((id) => deepEqual(left[id], right[id])) } -/** Whether `text` (a block's params, serialized) holds a reference token to `key`. */ -function referencesKey(params: CanonicalBlock['params'], key: BlockKey): boolean { - const text = JSON.stringify(params) - return text.includes(`<${key}.`) || text.includes(`<${key}>`) +/** Every string under `value`, depth first. */ +function* strings(value: unknown): Generator<string> { + if (typeof value === 'string') yield value + else if (Array.isArray(value)) for (const item of value) yield* strings(item) + else if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) yield* strings(item) + } +} + +/** + * Whether a block holds a reference to `key` in any field a reference can sit in (its params, a + * condition or router's branches, a loop or parallel's collection), its head read as the resolver + * reads it: normalized, so `<Fetch.data>` references `fetch`. + */ +function referencesKey(block: CanonicalBlock, key: BlockKey): boolean { + for (const text of strings([block.params, block.branches, block.subflow])) { + for (const token of findWorkflowReferenceTokens(text)) { + if (token.kind !== 'workflow') continue + const head = parseStoredPath(token.value.slice(1, -1))?.[0] + if (head?.kind === 'name' && normalizeWorkflowBlockName(head.name) === key) return true + } + } + return false } /** @@ -515,14 +536,14 @@ function inferRenames(state: InferenceState): void { const union = new Set([...sides.target, ...sides.prev]).size if (shared > 0 && shared * 2 > union) return 'connections' const referrers = Object.keys(prevCanonical.blocks).filter( - (other) => other !== prevKey && referencesKey(prevCanonical.blocks[other].params, prevKey) + (other) => other !== prevKey && referencesKey(prevCanonical.blocks[other], prevKey) ) const renamedAtEverySite = referrers.length > 0 && referrers.every((other) => { const id = idOfPrev(other) const now = id === undefined ? undefined : [...resolved].find(([, b]) => b.id === id)?.[0] - return now !== undefined && referencesKey(workflow.blocks[now].params, key) + return now !== undefined && referencesKey(workflow.blocks[now], key) }) return renamedAtEverySite ? 'references' : null } diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 3d5f89ba0c9..14ad4a7e684 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -566,6 +566,45 @@ describe('materialize: identity', () => { expect(codes(result)).not.toContain('W_MAT_INFERRED_RENAME') }) + it.each([ + ['<one.result> > 1', '<uno.result> > 1'], + ['<One.result> > 1', '<Uno.result> > 1'], + ])( + 'infers a rename whose every reference (%s in a condition branch) was renamed with it', + (before, after) => { + const prev = workflowState( + [ + blockState('a1', 'function', 'One', { code: 'return 1' }), + blockState('c1', 'condition', 'Gate', { + conditions: JSON.stringify([ + { id: 'c1-if', title: 'if', value: before }, + { id: 'c1-else', title: 'else', value: '' }, + ]), + }), + blockState('u1', 'function', 'Use', { code: 'return 0' }), + ], + { edges: [edge('e1', 'u1', 'a1')] } + ) + // Neither settings nor connections tie the two: only the renamed references do. + const target = edit(canonical(prev).workflow, (draft) => { + draft.edges = [] + draft.blocks.uno = { + ...draft.blocks.one, + key: 'uno', + name: 'Uno', + params: { code: { kind: 'value', value: 'return 2' } }, + } + draft.blocks = omit(draft.blocks, ['one']) + const rows = draft.blocks.gate.branches?.rows ?? [] + rows[0].value = after + }) + const result = run(target, lockFor(prev), { prev, lockFreshness: 'fresh' }) + expect(result.entries).toContainEqual( + expect.objectContaining({ status: 'matched', key: 'uno', id: 'a1', via: 'inferred' }) + ) + } + ) + it('infers a rename under a fresh lock when the ident changed too', () => { const prev = base() const lock = lockFor(prev) From dae45baa9e897ce070c375989f5bde5fc124f48b Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:22:05 -0800 Subject: [PATCH 47/59] fix(sim-cli): sanitize a result field's display override --- .../src/commands/protocol/result.test.ts | 32 +++++++++++++++++++ .../sim-cli/src/commands/protocol/result.ts | 7 ++-- 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 packages/sim-cli/src/commands/protocol/result.test.ts diff --git a/packages/sim-cli/src/commands/protocol/result.test.ts b/packages/sim-cli/src/commands/protocol/result.test.ts new file mode 100644 index 00000000000..9e8dd331e65 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/result.test.ts @@ -0,0 +1,32 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { printProtocolResult } from './result' + +const ESC = String.fromCharCode(27) +const BEL = String.fromCharCode(7) + +let logged: string[] + +beforeEach(() => { + logged = [] + vi.spyOn(console, 'log').mockImplementation((line: string) => { + logged.push(line) + }) +}) + +describe('printProtocolResult', () => { + it.each(['text', 'table'] as const)( + 'sanitizes a display override built from server text in %s output', + (format) => { + const key = `leads${ESC}]0;pwned${BEL}${ESC}[2A` + printProtocolResult( + format, + { created: [{ kind: 'table', key }] }, + { created: `table ${key}` } + ) + + expect(logged.join('\n')).not.toContain(ESC) + expect(logged.join('\n')).not.toContain(BEL) + expect(logged.join('\n')).toContain('table leads') + } + ) +}) diff --git a/packages/sim-cli/src/commands/protocol/result.ts b/packages/sim-cli/src/commands/protocol/result.ts index 5f0de86ea88..d7d39852fc8 100644 --- a/packages/sim-cli/src/commands/protocol/result.ts +++ b/packages/sim-cli/src/commands/protocol/result.ts @@ -1,6 +1,6 @@ import { isRecordLike } from '@sim/utils/object' import type { OutputFormat } from '../../config/index' -import { printRecord, text } from '../../output/render' +import { printRecord, sanitize, text } from '../../output/render' /** One result value as a field: a record, or a list holding one, as compact JSON. */ function fieldText(value: unknown): string { @@ -11,7 +11,8 @@ function fieldText(value: unknown): string { /** * Prints a protocol command's result: the record itself in a machine format, else one line per * field, with `display` replacing the text of the fields it names (a list a person reads as a - * sentence rather than as JSON). + * sentence rather than as JSON). An override is built from server text (resource keys, ids), so it + * is sanitized like every other field. */ export function printProtocolResult( format: OutputFormat, @@ -20,7 +21,7 @@ export function printProtocolResult( ): void { const fields = Object.entries(result).map<[string, string]>(([key, value]) => [ key, - display[key] ?? fieldText(value), + display[key] === undefined ? fieldText(value) : sanitize(display[key]), ]) printRecord(format, fields, result) } From 1e61d0e26fde5ff3e52b464afdb34b1a5c11f3f0 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:23:13 -0800 Subject: [PATCH 48/59] fix(sim-cli): keep a collapsed warning's own message, and honor --verbose in a refusal --- .../src/commands/protocol/source-ui.ts | 34 ++++++++++++----- .../protocol/workflow-source-apply.test.ts | 37 ++++++++++++++++++- .../protocol/workflow-source-apply.ts | 5 ++- .../protocol/workspace-source-apply.ts | 11 +++--- 4 files changed, 70 insertions(+), 17 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-ui.ts b/packages/sim-cli/src/commands/protocol/source-ui.ts index b652b8e4ad0..c39e7eb14f2 100644 --- a/packages/sim-cli/src/commands/protocol/source-ui.ts +++ b/packages/sim-cli/src/commands/protocol/source-ui.ts @@ -89,8 +89,9 @@ function locationOf(diagnostic: Diagnostic): string { /** * Prints a write's findings on stderr for a person: every error with its location, frame and * hint; the first warning of each code the same way, and the rest of that code on one line - * listing where they are (one fix usually serves them all, and twenty frames of the same warning - * buried the errors); each info code folded into one line. + * listing where they are, each with its own message when it differs from the first (one fix + * usually serves them all, and twenty frames of the same warning buried the errors); each info + * code folded into one line. */ function printDiagnostics(diagnostics: readonly Diagnostic[]): void { const groups = new Map<string, Diagnostic[]>() @@ -115,7 +116,15 @@ function printDiagnostics(diagnostics: readonly Diagnostic[]): void { } printError(formatDiagnostic(first)) if (rest.length === 0) continue - const listed = rest.slice(0, COLLAPSED_LOCATIONS).map(locationOf).filter(Boolean) + // A collapsed finding saying something else than the first keeps its words beside its place. + const listed = rest + .slice(0, COLLAPSED_LOCATIONS) + .map((diagnostic) => { + const location = locationOf(diagnostic) + if (diagnostic.message === first.message) return location + return location ? `${location} (${diagnostic.message})` : diagnostic.message + }) + .filter(Boolean) const unlisted = rest.length - listed.length printError( ` ${rest.length} more ${first.code} like it${listed.length > 0 ? `: ${listed.join(', ')}` : ''}${unlisted > 0 && listed.length > 0 ? `, and ${unlisted} more` : ''}` @@ -281,10 +290,16 @@ const DETAILS_REFERENCE = / See details\.diagnostics\.$/ * Re-throws an API refusal after printing its diagnostics, so each shows its source line, with * the exit status a script branches on. In a machine `format` the error document keeps the * refusal's details with each finding as a success result carries it ({@link resultDiagnostics}); - * a display format prints the findings once, above the error. `ifAnswerLost` is the next step for - * a write whose answer never came back, appended to its message. + * a display format prints the findings once, above the error, and `verbose` keeps info findings in + * the machine document as it does in a result. `ifAnswerLost` is the next step for a write whose + * answer never came back, appended to its message. */ -export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost?: string): never { +export function printRefusal( + error: unknown, + format: OutputFormat, + options: { ifAnswerLost?: string; verbose?: boolean } = {} +): never { + const { ifAnswerLost, verbose = false } = options if (!(error instanceof SimApiError)) throw error const details = toRecord(error.details) const diagnostics = details.diagnostics @@ -304,7 +319,7 @@ export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost? hint ? `${message} ${hint}` : message, error.status, error.code, - printed ? refusalDetails(format, details, diagnostics) : error.details, + printed ? refusalDetails(format, details, diagnostics, verbose) : error.details, refusalExitCode(error, details.code), error.requestId ) @@ -314,9 +329,10 @@ export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost? function refusalDetails( format: OutputFormat, details: Record<string, unknown>, - diagnostics: Diagnostic[] + diagnostics: Diagnostic[], + verbose: boolean ): Record<string, unknown> | undefined { - const flattened = resultDiagnostics(format, diagnostics) + const flattened = resultDiagnostics(format, diagnostics, verbose) return flattened.diagnostics ? { ...details, ...flattened } : undefined } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index fea17ea9144..e1c2b6cdd95 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -1,6 +1,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { hostname, tmpdir } from 'node:os' import { join } from 'node:path' +import { toRecord } from '@sim/utils/object' import { compileWorkflow } from '@sim/workflow-code' import { fingerprintOf } from '@sim/workspace-code' import { Command } from 'commander' @@ -701,10 +702,44 @@ describe('sim workflows source apply', () => { const lines = stderrLines() expect(lines.filter((line) => line.includes('> '))).toHaveLength(1) expect(lines).toContain( - ' 3 more W_JSON_REF_UNTYPED like it: workflow.ts:5:7, workflow.ts:6:7, workflow.ts:7:7' + ' 3 more W_JSON_REF_UNTYPED like it: workflow.ts:5:7 (Nothing declares the type of <score.result.n5>.), workflow.ts:6:7 (Nothing declares the type of <score.result.n6>.), workflow.ts:7:7 (Nothing declares the type of <score.result.n7>.)' ) }) + it('in a display format, names each collapsed warning by location alone when they say the same', async () => { + setTerminal(false) + contextMockState.output = 'text' + request.mockResolvedValue( + outcome({ + dryRun: true, + diagnostics: [4, 5].map((line) => ({ ...untyped(line), message: 'Same words.' })), + }) + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stderrLines()).toContain(' 1 more W_JSON_REF_UNTYPED like it: workflow.ts:5:7') + }) + + it('with --output json and --verbose, keeps info findings in a refusal too', async () => { + setTerminal(false) + request.mockRejectedValue( + new SimApiError('Refused. See details.diagnostics.', 422, 'SOURCE_REJECTED', { + diagnostics: [layout, STALE], + }) + ) + + const refused = await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--verbose').then( + () => null, + (error: unknown) => error as SimApiError + ) + + expect(toRecord(refused?.details).diagnostics).toEqual([ + expect.objectContaining({ code: 'I_MAT_NEEDS_LAYOUT' }), + expect.objectContaining({ code: 'E_APPLY_STALE' }), + ]) + }) + it('lists the workflow and blocks a create made, with their ids', async () => { setTerminal(false) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index efa19adda82..29c3ac2752a 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -283,7 +283,10 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { }) return response.data } catch (error) { - return printRefusal(error, args.output, dryRun ? undefined : args.ifAnswerLost) + return printRefusal(error, args.output, { + ...(dryRun ? {} : { ifAnswerLost: args.ifAnswerLost }), + verbose: options.verbose === true, + }) } } const findings = findingsOutput(args.output, options.verbose) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 65567c7007c..77619db4da0 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -525,7 +525,7 @@ export function attachWorkspaceSourceApply(source: Command): void { try { staged = await stageBlobs(client, workspaceId, options.dir, submission.blobs, paths) } catch (error) { - return printRefusal(error, profile.output) + return printRefusal(error, profile.output, { verbose: options.verbose === true }) } if (staged.count > 0) { printError(`Uploaded ${staged.count} binary file(s), ${staged.bytes} bytes.`) @@ -551,11 +551,10 @@ export function attachWorkspaceSourceApply(source: Command): void { refusal = error } } - return printRefusal( - refusal, - profile.output, - dryRun ? undefined : answerLostHint(options.dir) - ) + return printRefusal(refusal, profile.output, { + ...(dryRun ? {} : { ifAnswerLost: answerLostHint(options.dir) }), + verbose: options.verbose === true, + }) } await withSourceDirLock( From 96a51e6d4b4d4943f0fc6af7a4b1b556ab6939b3 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:24:15 -0800 Subject: [PATCH 49/59] fix(sim-cli): exit 130 at Ctrl-C on a line prompt --- .../sim-cli/src/terminal/line-input.test.ts | 44 +++++++++++++++++++ packages/sim-cli/src/terminal/line-input.ts | 16 ++++++- 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 packages/sim-cli/src/terminal/line-input.test.ts diff --git a/packages/sim-cli/src/terminal/line-input.test.ts b/packages/sim-cli/src/terminal/line-input.test.ts new file mode 100644 index 00000000000..8072bca9b6a --- /dev/null +++ b/packages/sim-cli/src/terminal/line-input.test.ts @@ -0,0 +1,44 @@ +import { PassThrough } from 'node:stream' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { askLine } from './line-input' + +/** A terminal's stdin, so readline reads keys (Ctrl-C, Ctrl-D) rather than plain text. */ +function terminalInput(): PassThrough { + const input = new PassThrough() + Object.assign(input, { isTTY: true, setRawMode: () => input }) + return input +} + +let input: PassThrough + +beforeEach(() => { + input = terminalInput() + vi.spyOn(process, 'stdin', 'get').mockReturnValue(input as unknown as typeof process.stdin) + vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + Object.defineProperty(process.stderr, 'isTTY', { value: true, configurable: true }) +}) + +afterEach(() => { + Object.defineProperty(process.stderr, 'isTTY', { value: undefined, configurable: true }) + vi.restoreAllMocks() +}) + +describe('askLine', () => { + it('resolves the line typed', async () => { + const answer = askLine('Name? ') + input.write('sim\r') + await expect(answer).resolves.toBe('sim') + }) + + it('cancels with the interrupt status 130 at Ctrl-C', async () => { + const answer = askLine('Name? ') + input.write('\u0003') + await expect(answer).rejects.toMatchObject({ code: 'CANCELLED', exitCode: 130 }) + }) + + it('resolves null at Ctrl-D, for the caller to cancel as it chooses', async () => { + const answer = askLine('Name? ') + input.write('\u0004') + await expect(answer).resolves.toBeNull() + }) +}) diff --git a/packages/sim-cli/src/terminal/line-input.ts b/packages/sim-cli/src/terminal/line-input.ts index 991864b55ca..5fc53f647a5 100644 --- a/packages/sim-cli/src/terminal/line-input.ts +++ b/packages/sim-cli/src/terminal/line-input.ts @@ -1,6 +1,10 @@ import { createInterface } from 'node:readline/promises' +import { SimApiError } from '#sim-cli/http/client' import { writeStderr } from '#sim-cli/output/io' +/** The shell's status for a command the user interrupted with Ctrl-C. */ +const INTERRUPTED_EXIT_CODE = 130 + /** * Whether a pending question failed because its input ended: Bun rejects it with an * `AbortError` at Ctrl-D, where Node leaves it pending and closes the interface instead. @@ -15,16 +19,24 @@ function isInputEnded(error: unknown): boolean { /** * Asks `question` on stderr and resolves the line typed, or `null` when the input ends first * (Ctrl-D, or a stdin closed under the prompt), so a caller cancels cleanly instead of hanging - * or printing a runtime's stack trace. + * or printing a runtime's stack trace. Ctrl-C at a terminal, which readline reads as a key rather + * than delivering the signal, rejects with a `CANCELLED` error that exits 130, as an interrupt + * anywhere else does. */ export async function askLine(question: string): Promise<string | null> { const prompt = createInterface({ input: process.stdin, output: process.stderr }) const ended = new Promise<null>((resolve) => prompt.once('close', () => resolve(null))) + const interrupted = new Promise<never>((_resolve, reject) => + prompt.once('SIGINT', () => + reject(new SimApiError('Interrupted.', 0, 'CANCELLED', undefined, INTERRUPTED_EXIT_CODE)) + ) + ) try { - const answer = await Promise.race([prompt.question(question), ended]) + const answer = await Promise.race([prompt.question(question), ended, interrupted]) if (answer === null) writeStderr('\n') return answer } catch (error) { + if (error instanceof SimApiError) writeStderr('\n') if (!isInputEnded(error)) throw error writeStderr('\n') return null From 8348009e2a822f116d1b63c583eb77a2d6b3600b Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:25:28 -0800 Subject: [PATCH 50/59] fix(sim-cli): drain stdout and stderr before re-raising a deferred interrupt --- .../sim-cli/src/terminal/interrupts.test.ts | 48 +++++++++++++++++++ packages/sim-cli/src/terminal/interrupts.ts | 14 ++++-- 2 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 packages/sim-cli/src/terminal/interrupts.test.ts diff --git a/packages/sim-cli/src/terminal/interrupts.test.ts b/packages/sim-cli/src/terminal/interrupts.test.ts new file mode 100644 index 00000000000..ad0117a66fa --- /dev/null +++ b/packages/sim-cli/src/terminal/interrupts.test.ts @@ -0,0 +1,48 @@ +import { spawn } from 'node:child_process' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' + +const OUTPUT_BYTES = 4_000_000 + +/** + * Runs a deferred write in a real process that prints {@link OUTPUT_BYTES} on stdout after a + * Ctrl-C arrived, with the reader holding stdout back for a moment, as a slow pipe does. + */ +function interruptedWithBackedUpPipe(): Promise<{ bytes: number; signal: string | null }> { + const interrupts = join(import.meta.dirname, 'interrupts.ts') + const io = join(import.meta.dirname, '../output/io.ts') + const script = ` + const { deferInterrupts } = await import(${JSON.stringify(interrupts)}) + const { writeStdout } = await import(${JSON.stringify(io)}) + await deferInterrupts(async () => { + process.stderr.write('ready\\n') + await new Promise((resolve) => setTimeout(resolve, 300)) + writeStdout('x'.repeat(${OUTPUT_BYTES})) + }, { waiting: 'waiting', abandoned: 'abandoned' }) + ` + const child = spawn('bun', ['-e', script], { stdio: ['ignore', 'pipe', 'pipe'] }) + let bytes = 0 + child.stdout.pause() + child.stdout.on('data', (chunk: Buffer) => { + bytes += chunk.length + }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + if (chunk.includes('ready')) { + child.kill('SIGINT') + setTimeout(() => child.stdout.resume(), 1_000) + } + }) + return new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (_code, signal) => resolve({ bytes, signal })) + }) +} + +describe('deferInterrupts', () => { + it('ends by the held interrupt only after what the write printed reached the pipe', async () => { + const run = await interruptedWithBackedUpPipe() + + expect(run.bytes).toBe(OUTPUT_BYTES) + expect(run.signal).toBe('SIGINT') + }, 15_000) +}) diff --git a/packages/sim-cli/src/terminal/interrupts.ts b/packages/sim-cli/src/terminal/interrupts.ts index de14710f2b3..a7d4d0908f0 100644 --- a/packages/sim-cli/src/terminal/interrupts.ts +++ b/packages/sim-cli/src/terminal/interrupts.ts @@ -1,5 +1,5 @@ import { embedStore } from '#sim-cli/embed-context' -import { printError } from '#sim-cli/output/io' +import { flushOutput, printError } from '#sim-cli/output/io' const INTERRUPT_SIGNALS: readonly NodeJS.Signals[] = ['SIGINT', 'SIGTERM'] @@ -49,10 +49,18 @@ export async function deferInterrupts<T>( clearTimeout(bound) if (active === deferral) active = null } + /** + * Ends the process by `signal` once stdout and stderr have drained, as every other exit does, + * so a backed-up pipe still receives what was printed. The handler is already released: an + * interrupt while they drain ends the process at once. + */ + const reraise = (signal: NodeJS.Signals) => { + void flushOutput().finally(() => process.kill(process.pid, signal)) + } const abandon = (signal: NodeJS.Signals) => { release() printError(options.abandoned) - process.kill(process.pid, signal) + reraise(signal) } function onSignal(signal: NodeJS.Signals) { if (deferral.pending) { @@ -71,6 +79,6 @@ export async function deferInterrupts<T>( } finally { const held = active === deferral ? deferral.pending : null release() - if (held) process.kill(process.pid, held) + if (held) reraise(held) } } From 6f42bde899d447c5023668f02872dd65a05730d4 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:26:25 -0800 Subject: [PATCH 51/59] fix(sim-cli): degrade source status to the local view on a refused credential --- .../src/commands/protocol/source-status.ts | 28 +++++++++++-------- .../protocol/workflow-source-status.test.ts | 22 +++++++++++++++ .../protocol/workspace-source-check.ts | 10 ++----- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/source-status.ts b/packages/sim-cli/src/commands/protocol/source-status.ts index 51b2b8776c5..820dbcc78e1 100644 --- a/packages/sim-cli/src/commands/protocol/source-status.ts +++ b/packages/sim-cli/src/commands/protocol/source-status.ts @@ -75,28 +75,34 @@ export function statusExitCode(state: SourceStatus['state'], login: string | nul /** Gateway statuses that mean the server was not reached, rather than that it refused. */ const UNREACHABLE_STATUSES = new Set([502, 503, 504]) +/** + * Whether `error` says this profile cannot authenticate: the CLI's own refusal of a profile with + * no login, or a credential the server refused (`401`, which exits 13 like it). + */ +export function credentialRefused(error: unknown): boolean { + return ( + error instanceof SimApiError && + ((error.status === 0 && error.code === AUTH_REQUIRED_CODE) || error.status === 401) + ) +} + /** * Why the server could not be reached (no connection, a timeout, a gateway that gave up, no login - * on the profile), as a sentence, or `null` for any other failure (a cancelled request included), - * which a status reports rather than hides. + * on the profile or a credential it refused), as a sentence, or `null` for any other failure (a + * cancelled request included), which a status reports rather than hides. */ export function unreachableReason(error: unknown): string | null { if (!(error instanceof SimApiError)) return null - const offline = - noAnswer(error, UNREACHABLE_STATUSES) || - (error.status === 0 && error.code === AUTH_REQUIRED_CODE) - if (!offline) return null + if (!noAnswer(error, UNREACHABLE_STATUSES) && !credentialRefused(error)) return null return /[.!?]$/.test(error.message) ? error.message : `${error.message}.` } /** - * The login a status recommends when `error` is the CLI's own refusal of a profile with no login - * (which a status treats as an unreachable server), or `null` for any other failure. + * The login a status recommends when `error` says the profile cannot authenticate (which a status + * treats as an unreachable server), or `null` for any other failure. */ export function loginCommand(error: unknown, authProfile: string): string | null { - return error instanceof SimApiError && error.status === 0 && error.code === AUTH_REQUIRED_CODE - ? `sim login --profile ${authProfile}` - : null + return credentialRefused(error) ? `sim login --profile ${authProfile}` : null } const SERVER_WORDS = { diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts index 72cec73a101..1f210b95193 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts @@ -269,6 +269,28 @@ describe('workflows source status', () => { expect(process.exitCode).toBe(13) }) + it('degrades to the local view when the server refuses the credential, naming the login', async () => { + write(root, 'workflow.ts', `${ENTRY}// edited\n`) + mockRequest.mockRejectedValue( + new SimApiError( + 'Invalid API key — run sim login --profile default', + 401, + 'UNAUTHORIZED', + undefined, + 13 + ) + ) + + await run(root, 'status') + + expect(printed()).toMatchObject({ + server: 'unreachable', + state: 'local', + next: { command: 'sim login --profile default' }, + }) + expect(process.exitCode).toBe(13) + }) + it('fails on a cancelled request instead of reporting the server unreachable', async () => { write(root, 'workflow.ts', `${ENTRY}// edited\n`) mockRequest.mockRejectedValue(new SimApiError('Request cancelled.', 0, REQUEST_CANCELLED)) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts index 2417042b82c..c16c3caa9ca 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts @@ -4,7 +4,6 @@ import { KIND_DIRS } from '@sim/workspace-code' import { WORKSPACE_BASE_FILE } from '@sim/workspace-code/layout' import type { Command } from 'commander' import { profileFrom } from '../../context' -import { AUTH_REQUIRED_CODE } from '../../exit-codes' import { SimApiError } from '../../http/client' import { apiCommand, type Connection } from '../../runtime/called-operations' import { @@ -14,7 +13,7 @@ import { SOURCE_DIR_BUSY_HELP, withSourceDirLock, } from './source-dir' -import { unreachableReason } from './source-status' +import { credentialRefused, unreachableReason } from './source-status' import { cachedCatalog, loadCatalog, refreshWorkspaceTypes } from './source-tree' import { verboseFlag } from './source-ui' import { @@ -74,12 +73,9 @@ async function fetchedCatalog( try { return await loadCatalog(connection.client, { workspaceId, blockTypes: [] }) } catch (error) { - // No login keeps its own refusal (exit 13, naming `sim login`); a server that did not answer + // No login, or a refused credential, keeps its own refusal (exit 13, naming `sim login`); a server that did not answer // leaves the catalog uncached, which the get that caches it fixes once the server answers. - const reason = - error instanceof SimApiError && error.code !== AUTH_REQUIRED_CODE - ? unreachableReason(error) - : null + const reason = credentialRefused(error) ? null : unreachableReason(error) if (reason === null) throw error throw new SimApiError( `No block catalog is cached for workspace ${workspaceId}, and downloading it failed: ${reason} Run \`sim workspaces source get --dir ${dir}\` once the server can be reached, then check again.`, From 099950cce41e81f89fd2efb6be4fcfaf68d0f412 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:26:57 -0800 Subject: [PATCH 52/59] fix(sim-cli): print a saved write's result when the --deploy after it fails --- .../protocol/workflow-source-apply.test.ts | 2 ++ .../protocol/workflow-source-apply.ts | 21 +++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index e1c2b6cdd95..5a34c301403 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -870,6 +870,8 @@ describe('sim workflows source apply', () => { expect((error as Error).message).toContain(`The apply was saved into ${dir}`) expect((error as Error).message).toContain(`sim workflows deploy ${WORKFLOW_ID}`) expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(`${ENTRY}// stored\n`) + // The saved write's result still prints, before the deploy's failure. + expect(stdoutJson()).toMatchObject({ workflowId: WORKFLOW_ID, applied: true }) }) }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 29c3ac2752a..ee3233cefd7 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -344,12 +344,21 @@ async function runSourceWrite<Operation extends SourceWriteOperation>(args: { const stored = !data.dryRun && !refused && (data.applied || !creating) if (mode === 'write' && stored) printError(diffSummary(data)) - report( - data, - stored - ? await deployStep(args.client, data, { ...options, verb, machine: findings.machine }) - : {} - ) + let deployment: DeployOutcome = {} + if (stored) { + try { + deployment = await deployStep(args.client, data, { + ...options, + verb, + machine: findings.machine, + }) + } catch (error) { + // The write stands: its result prints before the deploy's failure. + report(data) + throw error + } + } + report(data, deployment) } /** What a write adds to its result about deploying: the deploy it ran, or the one to run. */ From f23b95806079830eb104c78f317996d0abc7e1f2 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:28:30 -0800 Subject: [PATCH 53/59] docs(sim-cli): say an interrupted write waits up to 60s for its answer --- apps/docs/content/docs/cli/reference.mdx | 6 +++--- apps/docs/content/docs/cli/scripting.mdx | 4 ++-- apps/docs/content/docs/cli/workflows.mdx | 4 ++-- apps/docs/content/docs/cli/workspaces.mdx | 2 +- .../src/commands/protocol/source-exit-codes.ts | 11 ++++++++--- packages/workflow-sdk/README.md | 2 +- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 0e158a09706..8957d0e13fe 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8582,7 +8582,7 @@ sim workflows source apply [workflowId] [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). @@ -8610,7 +8610,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). @@ -9988,7 +9988,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). diff --git a/apps/docs/content/docs/cli/scripting.mdx b/apps/docs/content/docs/cli/scripting.mdx index abebe45341c..1214f561331 100644 --- a/apps/docs/content/docs/cli/scripting.mdx +++ b/apps/docs/content/docs/cli/scripting.mdx @@ -176,7 +176,7 @@ The CLI resends a request the API answers `429` only when sending it twice is sa | `10` | `--dir` is a workflow folder of a workspace directory another `sim` command is using (`SOURCE_DIR_BUSY`); nothing was written: run it again once the other finished | | `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; nothing was sent | | `14` | Run without `--yes` and with no terminal to ask: the plan was printed with no errors and nothing was written; run the same command with `--yes` | -| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check | +| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check | | Code | `sim workflows source status` or `sim workspaces source status` | | --- | --- | @@ -210,7 +210,7 @@ The CLI resends a request the API answers `429` only when sending it twice is sa | `9` | Still rate limited (`429`) after the CLI's retries; nothing was written | | `10` | Busy, retry: another apply is writing the workspace (`409 SOURCE_BUSY`), or another `sim workspaces source` command is using the same directory (`SOURCE_DIR_BUSY`); nothing was written: run the command again once the other finished | | `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; nothing was sent | -| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check | +| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check | Both waits default to 3600 seconds; `--wait-timeout 0` waits indefinitely. `SIM_TIMEOUT_SECONDS` independently bounds one API request. A timeout does not cancel the underlying run or operation. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 82929ac33e1..f90bb57ad7e 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1363,7 +1363,7 @@ sim workflows source create [options] </CommandTable> -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). @@ -1398,7 +1398,7 @@ sim workflows source apply [workflowId] [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 3bb69cbd23f..7143068365b 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -761,7 +761,7 @@ sim workspaces source apply [options] </CommandTable> -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written and how to check. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts index 04260a4727d..8bf0508facb 100644 --- a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts @@ -10,10 +10,14 @@ import { SOURCE_STALE_EXIT_CODE, SOURCE_TOO_LARGE_EXIT_CODE, } from '../../exit-codes' +import { DEFERRED_INTERRUPT_BOUND_MS } from '#sim-cli/terminal/interrupts' /** The status a shell reports for a command Ctrl-C ended (128 + SIGINT). */ const INTERRUPTED_EXIT_CODE = 130 +/** How long an interrupted write waits for its answer before it stops without it. */ +const DEFERRED_SECONDS = DEFERRED_INTERRUPT_BOUND_MS / 1000 + /** * Every exit code of the `workflows source` and `workspaces source` commands, as the Markdown * table both directory guides (`.sim/AGENTS.md`) print, built from the codes the CLI exits with so @@ -81,7 +85,7 @@ export function sourceExitCodesTable( ], [ INTERRUPTED_EXIT_CODE, - 'Interrupted (Ctrl-C). An apply or create already sent is finished and saved into the directory first; a second Ctrl-C stops at once and prints that it may have been written', + `Interrupted (Ctrl-C). An apply or create already sent is finished and saved into the directory first, waiting up to ${DEFERRED_SECONDS}s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written`, `\`${prefix} status --dir .\``, ], ] @@ -96,5 +100,6 @@ export function sourceExitCodesTable( export const SOURCE_WRITE_EXIT_CODES_HELP = ` Exit ${AUTH_REQUIRED_EXIT_CODE}: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run \`sim login\`, or fix or unset SIM_API_KEY when the key comes from it. Exit ${INTERRUPTED_EXIT_CODE}: interrupted -(Ctrl-C). A write already sent is finished and saved into the directory first; a second Ctrl-C -stops at once and prints that it may have been written and how to check.` +(Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to +${DEFERRED_SECONDS}s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may +have been written and how to check.` diff --git a/packages/workflow-sdk/README.md b/packages/workflow-sdk/README.md index 3e3b5388d82..c2106c43538 100644 --- a/packages/workflow-sdk/README.md +++ b/packages/workflow-sdk/README.md @@ -121,7 +121,7 @@ Every `sim workflows source` and `sim workspaces source` command exits with one | 12 | `status`: the server changed since the get (drift, or a conflict with your edits) | Follow the `Next:` line it prints | | 13 | Not logged in, or the server refused the credential (`AUTH_REQUIRED`); nothing was sent | `sim login`, or fix or unset `SIM_API_KEY` | | 14 | `apply` or `create` without `--yes` and no terminal to ask: it printed the plan, found no errors, and wrote nothing | Run the same command with `--yes` | -| 130 | Interrupted (Ctrl-C). An apply or create already sent is finished and saved first; a second Ctrl-C stops at once and says it may have been written | `sim workflows source status --dir ./wf` | +| 130 | Interrupted (Ctrl-C). An apply or create already sent is finished and saved first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and says it may have been written | `sim workflows source status --dir ./wf` | ## Write declarations, not programs From 1741ed98e5f7d80967bb6c46e6d58583e2df854d Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 18:59:42 -0800 Subject: [PATCH 54/59] fix(workspace-source): tell a full staging area apart from a body over the limit, and word a chunked body's 413 --- apps/docs/openapi-v2-workflows.json | 2 +- .../[workspaceId]/source/blobs/route.test.ts | 42 +++++++++++++++++++ .../[workspaceId]/source/blobs/route.ts | 13 +++++- .../contracts/v2/openapi/workspace-source.ts | 2 +- .../workspace-source/api/route-policies.ts | 7 +++- .../sim/lib/workspace-source/staging/store.ts | 18 ++++++-- .../commands/protocol/source-exit-codes.ts | 2 +- .../protocol/workspace-source-apply.test.ts | 20 +++++++++ .../protocol/workspace-source-blobs.ts | 5 ++- 9 files changed, 101 insertions(+), 10 deletions(-) diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 83aabfa35a7..e5bf63509a0 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -8987,7 +8987,7 @@ "post": { "operationId": "stageWorkspaceSourceBlobs", "summary": "Stage workspace source blobs", - "description": "Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most 1000 files and 8000000 bytes per request, 10000000 bytes per file; a larger file goes in a request of its own. A body over 10001024 bytes answers `413` (`PAYLOAD_TOO_LARGE`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (`400`), nothing staged. Staged files are scoped to the workspace and the caller, capped at 20000 files and 4000000000 bytes (`413` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:write`.", + "description": "Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most 1000 files and 8000000 bytes per request, 10000000 bytes per file; a larger file goes in a request of its own. A body over 10001024 bytes answers `413` (`PAYLOAD_TOO_LARGE`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (`400`), nothing staged. Staged files are scoped to the workspace and the caller, capped at 20000 files and 4000000000 bytes (`413` with `details.code` `SOURCE_STAGING_FULL` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:write`.", "x-sim-operation": "workspaces.source.blobs.stage", "x-oauth-scope": "api:write", "tags": ["Workflows"], diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts index 10c48131826..a025ec5f6d0 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts @@ -21,6 +21,7 @@ vi.mock('@/lib/workspace-source/application/stage-blobs', () => ({ stageSourceBlobs: { operation: { id: 'workspaces.source.blobs.stage' }, execute: mocks.stage }, })) +import { SourceStagingFullError } from '@/lib/workspace-source/staging/store' import { POST } from '@/app/api/v2/workspaces/[workspaceId]/source/blobs/route' const WORKSPACE_ID = '11111111-2222-4333-8444-555555555555' @@ -76,6 +77,47 @@ describe('POST /api/v2/workspaces/{workspaceId}/source/blobs', () => { expect(mocks.stage).not.toHaveBeenCalled() }) + it('answers a chunked body past the limit with 413 saying it carries more, not a count it never read', async () => { + const declared = await stagingRequest([Buffer.alloc(6_000_000, 1), Buffer.alloc(6_000_000, 2)]) + const body = Buffer.from(await declared.arrayBuffer()) + const headers = new Headers(declared.headers) + headers.delete('content-length') + const chunked = new NextRequest(declared.url, { + method: 'POST', + headers, + body: new ReadableStream({ + start(controller) { + for (let at = 0; at < body.byteLength; at += 1_000_000) { + controller.enqueue(new Uint8Array(body.subarray(at, at + 1_000_000))) + } + controller.close() + }, + }), + duplex: 'half', + } as ConstructorParameters<typeof NextRequest>[1]) + + const response = await POST(chunked, context) + + expect(response.status).toBe(413) + const { error } = await response.json() + expect(error.message).toMatch(/at most (\d+) bytes; this one carries more than \1\./) + expect(mocks.stage).not.toHaveBeenCalled() + }) + + it('answers a caller whose staging area is full with 413 SOURCE_STAGING_FULL, apart from a body over the limit', async () => { + mocks.admit.mockRejectedValue( + new SourceStagingFullError( + 'Staging 1 more binary files would hold 5001, over the 5000-file limit.' + ) + ) + + const response = await POST(await stagingRequest([Buffer.from('a')]), context) + + expect(response.status).toBe(413) + const { error } = await response.json() + expect(error.details).toEqual({ code: 'SOURCE_STAGING_FULL' }) + }) + it('stages a single file of the largest size a directory holds', async () => { const response = await POST(await stagingRequest([Buffer.alloc(10_000_000, 3)]), context) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts index 2a172009630..f0db210cc38 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts @@ -23,6 +23,17 @@ export const revalidate = 0 const SHA256 = /^[0-9a-f]{64}$/ +/** + * What a body over the limit carried, for its refusal: the declared length, or for a chunked body + * (read only until it passed the limit) that it carried more than the limit. + */ +function carried(request: Request, observedBytes: number | undefined): string { + if (request.headers.get('content-length')) { + return observedBytes === undefined ? '' : `; this one carries ${observedBytes}` + } + return `; this one carries more than ${WORKSPACE_SOURCE_STAGE_MAX_BYTES}` +} + /** * POST /api/v2/workspaces/[workspaceId]/source/blobs — stages the binary files of a workspace * source apply: a multipart form with one file part per blob, named by its SHA-256. The caller's @@ -51,7 +62,7 @@ export const POST = defineV2BodyLifecycleRoute({ if (isPayloadSizeLimitError(error)) { throw new OrchestrationError( 'payload_too_large', - `A staging request body carries at most ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes${error.observedBytes === undefined ? '' : `; this one carries ${error.observedBytes}`}. Send fewer files per request, a file over ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes in a request of its own.` + `A staging request body carries at most ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes${carried(request, error.observedBytes)}. Send fewer files per request, a file over ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes in a request of its own.` ) } if (isMultipartFieldValidationError(error)) { diff --git a/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts b/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts index b970de08982..ad5182a0343 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts @@ -195,7 +195,7 @@ export const workspaceSourceOpenApiRoutes = [ applicationOperation: workspaceSourceOperations.stageBlobs, operationId: 'stageWorkspaceSourceBlobs', summary: 'Stage workspace source blobs', - description: `Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, ${WORKSPACE_SOURCE_LIMITS.maxBlobBytes} bytes per file; a larger file goes in a request of its own. A body over ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes answers \`413\` (\`PAYLOAD_TOO_LARGE\`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (\`400\`), nothing staged. Staged files are scoped to the workspace and the caller, capped at ${WORKSPACE_SOURCE_LIMITS.maxStagedBlobs} files and ${WORKSPACE_SOURCE_LIMITS.maxStagedBytes} bytes (\`413\` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns \`403\` while workflows as code is not enabled.`, + description: `Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, ${WORKSPACE_SOURCE_LIMITS.maxBlobBytes} bytes per file; a larger file goes in a request of its own. A body over ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes answers \`413\` (\`PAYLOAD_TOO_LARGE\`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (\`400\`), nothing staged. Staged files are scoped to the workspace and the caller, capped at ${WORKSPACE_SOURCE_LIMITS.maxStagedBlobs} files and ${WORKSPACE_SOURCE_LIMITS.maxStagedBytes} bytes (\`413\` with \`details.code\` \`SOURCE_STAGING_FULL\` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns \`403\` while workflows as code is not enabled.`, tags: ['Workflows'], errors: [...RESOURCE_MUTATION_ERRORS, 'PayloadTooLarge'], success: { description: 'The binary files staged.', headers: RATE_LIMIT_HEADERS }, diff --git a/apps/sim/lib/workspace-source/api/route-policies.ts b/apps/sim/lib/workspace-source/api/route-policies.ts index 3722b7c8273..ca9dfc24e18 100644 --- a/apps/sim/lib/workspace-source/api/route-policies.ts +++ b/apps/sim/lib/workspace-source/api/route-policies.ts @@ -9,6 +9,7 @@ import { createV2ResourceConcealmentPolicy } from '@/lib/api/server/routes' import type { ApplicationOperation, AuthorizingUseCase } from '@/lib/core/application' import { WorkspaceSourceRejectedError } from '@/lib/workspace-source/application/errors' import { WorkspaceSourceBusyError } from '@/lib/workspace-source/lease/store' +import { SourceStagingFullError } from '@/lib/workspace-source/staging/store' import { v2CaughtOrchestrationError, v2Error, @@ -66,7 +67,8 @@ export function withSourceSlot<O extends ApplicationOperation, I, R>( * status (`400`, `409` stale, `423` locked, `413` over a limit) with `details.diagnostics`; * `details.code` is `SOURCE_STALE` when a resource changed since the directory was read, else * `SOURCE_REJECTED`. An apply while another one writes the workspace answers `409` with - * `details.code` `SOURCE_BUSY`. + * `details.code` `SOURCE_BUSY`. Staging past the caller's staging caps answers `413` with + * `details.code` `SOURCE_STAGING_FULL`, apart from a body over the request limit. */ export const v2WorkspaceSourceErrorPolicy = createV2ResourceConcealmentPolicy({ notFoundMessage: 'Workspace not found', @@ -85,6 +87,9 @@ export const v2WorkspaceSourceErrorPolicy = createV2ResourceConcealmentPolicy({ if (error instanceof WorkspaceSourceBusyError) { return v2ErrorForOrchestration(error.code, error.message, { code: 'SOURCE_BUSY' }) } + if (error instanceof SourceStagingFullError) { + return v2ErrorForOrchestration(error.code, error.message, { code: 'SOURCE_STAGING_FULL' }) + } return v2CaughtOrchestrationError(error) }, }) diff --git a/apps/sim/lib/workspace-source/staging/store.ts b/apps/sim/lib/workspace-source/staging/store.ts index da607b45774..d2bd7a7548d 100644 --- a/apps/sim/lib/workspace-source/staging/store.ts +++ b/apps/sim/lib/workspace-source/staging/store.ts @@ -100,6 +100,18 @@ export async function stagedUsage( return { count: Number(usage?.count ?? 0), bytes: Number(usage?.bytes ?? 0) } } +/** + * A caller's staging area has no room for what it stages: `413` like a body over the limit, told + * apart by `details.code` `SOURCE_STAGING_FULL`, since freeing staged files (or waiting for them + * to expire) fixes it and sending smaller requests does not. + */ +export class SourceStagingFullError extends OrchestrationError { + constructor(message: string) { + super('payload_too_large', message) + this.name = 'SourceStagingFullError' + } +} + /** Refuses staging that would take `target` past its blob or byte cap. */ export function assertStagingFits( usage: { count: number; bytes: number }, @@ -107,14 +119,12 @@ export function assertStagingFits( ): void { const { maxStagedBlobs, maxStagedBytes } = WORKSPACE_SOURCE_LIMITS if (usage.count + adding.count > maxStagedBlobs) { - throw new OrchestrationError( - 'payload_too_large', + throw new SourceStagingFullError( `Staging ${adding.count} more binary files would hold ${usage.count + adding.count}, over the ${maxStagedBlobs}-file limit. Staged files expire a day after their last use.` ) } if (usage.bytes + adding.bytes > maxStagedBytes) { - throw new OrchestrationError( - 'payload_too_large', + throw new SourceStagingFullError( `Staging ${adding.bytes} more bytes would hold ${usage.bytes + adding.bytes}, over the ${maxStagedBytes}-byte limit. Staged files expire a day after their last use.` ) } diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts index 8bf0508facb..9a4fed61347 100644 --- a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts @@ -1,3 +1,4 @@ +import { DEFERRED_INTERRUPT_BOUND_MS } from '#sim-cli/terminal/interrupts' import { AUTH_REQUIRED_EXIT_CODE, RATE_LIMITED_EXIT_CODE, @@ -10,7 +11,6 @@ import { SOURCE_STALE_EXIT_CODE, SOURCE_TOO_LARGE_EXIT_CODE, } from '../../exit-codes' -import { DEFERRED_INTERRUPT_BOUND_MS } from '#sim-cli/terminal/interrupts' /** The status a shell reports for a command Ctrl-C ended (128 + SIGINT). */ const INTERRUPTED_EXIT_CODE = 130 diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 0a253be9147..1cbc484917f 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -743,6 +743,26 @@ describe('workspaces source apply', () => { expect(request).not.toHaveBeenCalled() }) + it('keeps the server’s refusal of a staging area with no room, which no CLI update or proxy fixes', async () => { + write('files/a.bin', Buffer.alloc(3_000, 1)) + const full = + 'Staging 3000 more bytes would hold 2000000001, over the 2000000000-byte limit. Staged files expire a day after their last use.' + mockRequest.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path.endsWith('/source/blobs')) { + throw new SimApiError(full, 413, 'PAYLOAD_TOO_LARGE', { code: 'SOURCE_STAGING_FULL' }) + } + return path.includes('/source/blobs') ? stagingAnswer(path, options) : request(path, options) + }) + + const refused = await apply('--yes').then( + () => null, + (error: unknown) => error + ) + + expect(refused).toMatchObject({ status: 413, message: full }) + expect(request).not.toHaveBeenCalled() + }) + it('uploads two files that would overflow one request body in two requests', async () => { write('files/a.bin', Buffer.alloc(6_000_000, 1)) write('files/b.bin', Buffer.alloc(6_000_000, 2)) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts b/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts index 8ef45a4ecfd..ed1a1d94e12 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts @@ -12,6 +12,7 @@ import { Readable, Transform } from 'node:stream' import { pipeline } from 'node:stream/promises' import { mapWithConcurrency } from '@sim/utils/concurrency' import { generateShortId } from '@sim/utils/id' +import { toRecord } from '@sim/utils/object' import { APPLY_ORDER, type BlobEntry, @@ -537,10 +538,12 @@ export async function stageBlobs( * A staging request the server refused as too large (`413`), re-raised naming what it carried and * what to do: nothing was applied, and since this CLI batches within the limits it shares with the * server, a refusal means the server runs other limits (another version, or a proxy in front of - * it with a smaller body limit). Any other error passes through. + * it with a smaller body limit). A staging area with no room (`SOURCE_STAGING_FULL`), which only + * freeing staged files fixes, and any other error pass through. */ function stagingTooLarge(error: unknown, batch: readonly BlobEntry[]): unknown { if (!(error instanceof SimApiError) || error.status !== 413) return error + if (toRecord(error.details).code === 'SOURCE_STAGING_FULL') return error const bytes = batch.reduce((sum, blob) => sum + blob.size, 0) const largest = batch.reduce((top, blob) => (blob.size > top.size ? blob : top), batch[0]) return new SimApiError( From b8af83715e21c36fc9b88410532e088d90847f57 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 19:00:15 -0800 Subject: [PATCH 55/59] fix(workspace-code): keep a knowledge base's document errors ahead of truncated warnings --- .../workspace-code/src/kinds/knowledge.test.ts | 15 +++++++++++++++ packages/workspace-code/src/kinds/knowledge.ts | 7 +++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/workspace-code/src/kinds/knowledge.test.ts b/packages/workspace-code/src/kinds/knowledge.test.ts index 1df52328d46..f3fed3f8001 100644 --- a/packages/workspace-code/src/kinds/knowledge.test.ts +++ b/packages/workspace-code/src/kinds/knowledge.test.ts @@ -192,6 +192,21 @@ describe('knowledgeCodec', () => { ]) }) + it('still refuses when the error is past the first hundred document findings', () => { + const suspicious = Array.from({ length: 120 }, (_, index) => ({ + path: `knowledge/policies/docs/a${index}\u200b.md`, + sha256: 'a'.repeat(64), + size: 1, + })) + const parsed = knowledgeCodec.parse('policies', declaration("{ name: 'Policies' }"), [ + ...suspicious, + { path: 'knowledge/policies/docs/z.md', sha256: 'abc', size: 1 }, + ]) + + expect(parsed.decl).toBeUndefined() + expect(parsed.diagnostics.map((finding) => finding.code)).toContain('E_DECL_SHAPE') + }) + it('caps document findings without spending the declaration file its own findings', () => { const bad = Array.from({ length: 150 }, (_, index) => ({ path: `knowledge/policies/docs/d${index}.md`, diff --git a/packages/workspace-code/src/kinds/knowledge.ts b/packages/workspace-code/src/kinds/knowledge.ts index 8626244fbf2..96717afe5e2 100644 --- a/packages/workspace-code/src/kinds/knowledge.ts +++ b/packages/workspace-code/src/kinds/knowledge.ts @@ -239,12 +239,15 @@ function fileFinding( /** * `findings` about a knowledge base's document files, past {@link MAX_FINDINGS_PER_FILE} folded * into one `I_FINDINGS_TRUNCATED`: a folder of bad documents costs no more than its first - * hundred, and refuses for what those hundred are, not as a size limit. + * hundred, and refuses for what those hundred are, not as a size limit. Errors are kept before + * warnings, so a refusal is never folded away behind a hundred warnings. */ function cappedFileFindings(key: string, findings: WorkspaceDiagnostic[]): WorkspaceDiagnostic[] { if (findings.length <= MAX_FINDINGS_PER_FILE) return findings + const errors = findings.filter((finding) => finding.severity === 'error') + const rest = findings.filter((finding) => finding.severity !== 'error') return [ - ...findings.slice(0, MAX_FINDINGS_PER_FILE), + ...[...errors, ...rest].slice(0, MAX_FINDINGS_PER_FILE), diagnostic( 'I_FINDINGS_TRUNCATED', `Stopped after ${MAX_FINDINGS_PER_FILE} findings about the files in knowledge/${key}/.`, From 92b6812741165479b71d7536026e438ee3e959d3 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 19:01:28 -0800 Subject: [PATCH 56/59] fix(workspace-source): count a finished replacement among the documents keeping their chunks --- .../application/kinds/knowledge/diff.ts | 4 ++- .../application/knowledge.integration.ts | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts b/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts index 60bc716bf2f..3a82b53167e 100644 --- a/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts +++ b/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts @@ -363,7 +363,9 @@ export async function diffKnowledgeBase(input: { ...changes.finishes.map((finish) => finish.replaces.id), ...changes.deletes.map((doc) => doc.id), ]) - const keepingChunks = [...byName.values()].filter((doc) => !reindexed.has(doc.id)).length + // A finished replacement is already stored and indexed, so it keeps its chunks too. + const keepingChunks = + [...byName.values()].filter((doc) => !reindexed.has(doc.id)).length + changes.finishes.length changes.diagnostics.push( ...indexingFindings(key, changes.uploads, chunkingChanged, keepingChunks) ) diff --git a/apps/sim/lib/workspace-source/application/knowledge.integration.ts b/apps/sim/lib/workspace-source/application/knowledge.integration.ts index d857bda8383..6161edc554c 100644 --- a/apps/sim/lib/workspace-source/application/knowledge.integration.ts +++ b/apps/sim/lib/workspace-source/application/knowledge.integration.ts @@ -1048,6 +1048,38 @@ describe('knowledge bases as workspace source', () => { } ) + check( + 'a chunking change counts the replacement a stopped replace left among the documents keeping their chunks', + async () => { + const { workspaceId, read, dir } = await setUp() + const edited = withDoc(dir, REFUNDS, '# Refunds\nWithin 60 days.\n') + const failing = vi + .spyOn(deleteKnowledgeDocument, 'execute') + .mockRejectedValueOnce(new Error('storage unavailable')) + .mockRejectedValueOnce(new Error('storage unavailable')) + const first = await apply(workspaceId, edited, { base: read.base }).finally(() => + failing.mockRestore() + ) + expect(first.failed).toMatchObject({ key: 'policies' }) + + const planned = await apply( + workspaceId, + { + ...edited, + files: { + ...edited.files, + [POLICIES_FILE]: declaration('Policies', '\n chunking: { maxSize: 512 },'), + }, + }, + { base: first.base, dryRun: true, carry: 'none' } + ) + + // The finished replacement and shipping.txt keep their chunks; the replaced refunds.md goes. + const chunking = planned.diagnostics.find((finding) => finding.code === 'I_KB_CHUNKING') + expect(chunking?.message).toContain('2 stored document(s) keep their chunks') + } + ) + check('a create that fails partway is adopted and finished by the next apply', async () => { const workspaceId = await newWorkspace() const failing = failUpload(2) From da088b271de6ef997b0c7fc6d50dac8f664558cc Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 19:04:14 -0800 Subject: [PATCH 57/59] fix(workspace-source): warn on a column rename the live version of a deployed workflow still uses --- .../application/deployments.integration.ts | 55 ++++++++++ .../workspace-source/application/in-use.ts | 2 +- .../application/kinds/table/renames.ts | 101 +++++++++++++----- 3 files changed, 129 insertions(+), 29 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/deployments.integration.ts b/apps/sim/lib/workspace-source/application/deployments.integration.ts index 9b58488697c..592d012ead2 100644 --- a/apps/sim/lib/workspace-source/application/deployments.integration.ts +++ b/apps/sim/lib/workspace-source/application/deployments.integration.ts @@ -1055,6 +1055,61 @@ describe('workspace source: deployments', () => { } ) + check( + 'warns when a column rename breaks the live version of a workflow whose draft writes the new name, unless the apply redeploys', + async () => { + const workspaceId = await workspaces.create({ name: 'Deployed column user', ownerId }) + const kinds: WorkspaceKind[] = ['table', 'workflow', 'deployment'] + const created = await apply( + workspaceId, + { + 'tables/leads.ts': LEADS_TABLE, + 'workflows/ingest/workflow.ts': INGEST_LEADS, + 'deployments/ingest.ts': DRAFT, + }, + { kinds } + ) + expect(created.failed).toBeUndefined() + const read = await getAll(workspaceId, kinds) + // The table renames email, and the draft writes the new name; the live version still writes email. + const renamed: Files = { + ...read.files, + 'tables/leads.ts': LEADS_TABLE.replace( + 'email: column.string(),', + "email: column.string({ name: 'email_address' })," + ), + 'workflows/ingest/workflow.ts': INGEST_LEADS.replace( + 'data: { email: start.email }', + 'data: { email_address: start.email }' + ), + } + + const { 'deployments/ingest.ts': _deployment, ...undeclaredDeployment } = renamed + const kept = await apply(workspaceId, undeclaredDeployment, { + kinds: ['table', 'workflow'], + base: read.base, + dryRun: true, + }) + expect(codes(kept.diagnostics)).toEqual([]) + const warning = kept.diagnostics.find( + (finding) => finding.code === 'W_TABLE_COLUMN_RENAMED_IN_USE' + ) + expect(warning).toMatchObject({ kind: 'table', key: 'leads' }) + expect(warning?.message).toContain('live version') + + const redeployed = await apply(workspaceId, renamed, { + kinds, + base: read.base, + dryRun: true, + }) + expect(codes(redeployed.diagnostics)).toEqual([]) + expect(actions(redeployed.plan)).toContain('update deployment:ingest') + expect(codes(redeployed.diagnostics, 'warning')).not.toContain( + 'W_TABLE_COLUMN_RENAMED_IN_USE' + ) + } + ) + check('refuses a deployment of a workflow the workspace does not hold', async () => { const workspaceId = await workspaces.create({ name: 'Deploy nothing', ownerId }) const refused = await refusal(apply(workspaceId, { 'deployments/missing.ts': DRAFT })) diff --git a/apps/sim/lib/workspace-source/application/in-use.ts b/apps/sim/lib/workspace-source/application/in-use.ts index 2e6314363c4..98b99bb09bc 100644 --- a/apps/sim/lib/workspace-source/application/in-use.ts +++ b/apps/sim/lib/workspace-source/application/in-use.ts @@ -136,7 +136,7 @@ export function inUseUncheckedFinding(change: Pick<PlannedChange, 'kind' | 'key' * The workflows whose live version this plan replaces or stops before its deletes run: a * deployment it writes that publishes the draft, or one it undeploys. Held changes write nothing. */ -function liveVersionsReplaced(changes: readonly PlannedChange[]): Set<string> { +export function liveVersionsReplaced(changes: readonly PlannedChange[]): Set<string> { const replaced = new Set<string>() for (const change of changes) { if (change.kind !== 'deployment' || change.held) continue diff --git a/apps/sim/lib/workspace-source/application/kinds/table/renames.ts b/apps/sim/lib/workspace-source/application/kinds/table/renames.ts index b2a10fbfa0f..790ee72e3b9 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/renames.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/renames.ts @@ -11,9 +11,13 @@ import { SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' import { workflowFolder } from '@sim/workspace-code/kinds/workflow' import { readWorkflowSource } from '@/lib/workflows/application/read-workflow-source' import { collectTableBlockFieldIssues } from '@/lib/workflows/editing/lint' -import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' +import { + loadDeployedWorkflowState, + loadWorkflowFromNormalizedTables, + NoActiveDeploymentError, +} from '@/lib/workflows/persistence/utils' import { finding, where } from '@/lib/workspace-source/application/findings' -import { workflowsUsing } from '@/lib/workspace-source/application/in-use' +import { liveVersionsReplaced, workflowsUsing } from '@/lib/workspace-source/application/in-use' import type { TablePlan } from '@/lib/workspace-source/application/kinds/table/adapter' import type { PlanContext, PlannedChange } from '@/lib/workspace-source/application/types' @@ -67,10 +71,22 @@ function locateWrite( return { line: declaration + 1, column: 1 } } +/** The live version of `workflowId`'s deployment, or `null` when it has none. */ +async function liveBlocks(workflowId: string) { + try { + return (await loadDeployedWorkflowState(workflowId)).blocks + } catch (error) { + if (error instanceof NoActiveDeploymentError) return null + throw error + } +} + /** * Warns `W_TABLE_COLUMN_RENAMED_IN_USE` on each table change that renames a column some stored * workflow's Table block still writes, filters or sorts by the old name: one finding per block, - * naming the workflow and the block, pinned to the declared workflow's file and line. + * naming the workflow and the block, pinned to the declared workflow's file and line. A deployed + * workflow's live version runs apart from its draft, so it is checked too (an evaluated draft + * included), unless the apply publishes the draft or undeploys it. */ export async function checkColumnRenamesInUse( changes: readonly PlannedChange[], @@ -89,45 +105,74 @@ export async function checkColumnRenamesInUse( if (change.kind !== 'workflow' || change.storedId === undefined) continue declared.set(change.storedId, { key: change.key, evaluated: change.refs !== undefined }) } + const replacedLive = liveVersionsReplaced(changes) for (const table of tables) { + const columns = new Map([ + [table.tableId, { name: table.tableName, columnNames: table.columnNames }], + ]) for (const workflow of using.get(table.tableId) ?? []) { - if (!workflow.draft) continue const folder = declared.get(workflow.id) if ( - folder?.evaluated || changes.some( (c) => c.kind === 'workflow' && c.storedId === workflow.id && c.action === 'delete' ) - ) + ) { continue - const loaded = await loadWorkflowFromNormalizedTables(workflow.id) - if (!loaded) continue - const issues = collectTableBlockFieldIssues( - loaded.blocks, - new Map([[table.tableId, { name: table.tableName, columnNames: table.columnNames }]]) - ).filter((issue) => table.renamed.has(issue.field)) - if (issues.length === 0) continue - const text = folder - ? ( - await readWorkflowSource.execute({ - principal: ctx.principal, - input: { workflowId: workflow.id }, - }) - ).files[SOURCE_ENTRY_FILE] - : undefined - for (const issue of issues) { + } + /** `blockId:field` of each finding already made from the draft. */ + const reported = new Set<string>() + if (workflow.draft && !folder?.evaluated) { + const loaded = await loadWorkflowFromNormalizedTables(workflow.id) + const issues = loaded + ? collectTableBlockFieldIssues(loaded.blocks, columns).filter((issue) => + table.renamed.has(issue.field) + ) + : [] + const text = + folder && issues.length > 0 + ? ( + await readWorkflowSource.execute({ + principal: ctx.principal, + input: { workflowId: workflow.id }, + }) + ).files[SOURCE_ENTRY_FILE] + : undefined + for (const issue of issues) { + reported.add(`${issue.blockId}:${issue.field}`) + const renamedTo = table.renamed.get(issue.field) + const blockName = issue.blockName ?? issue.blockId + const at = text !== undefined ? locateWrite(text, blockName, issue.field) : undefined + table.change.diagnostics.push( + finding( + 'W_TABLE_COLUMN_RENAMED_IN_USE', + `${where('table', table.change.key)} renames column "${issue.field}" to "${renamedTo}", but block "${blockName}" of workflow "${workflow.name}" still ${issue.rowWrite ? 'writes' : 'reads'} "${issue.field}"; the run drops it.`, + { + hint: `Write "${renamedTo}" in that block${folder ? ` (${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE})` : ''}, in this apply or right after it.`, + ...(folder + ? { file: `${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE}`, ...(at ?? {}) } + : {}), + kind: 'table', + key: table.change.key, + } + ) + ) + } + } + if (!workflow.live || replacedLive.has(workflow.id)) continue + const live = await liveBlocks(workflow.id) + if (!live) continue + for (const issue of collectTableBlockFieldIssues(live, columns)) { + if (!table.renamed.has(issue.field) || reported.has(`${issue.blockId}:${issue.field}`)) { + continue + } const renamedTo = table.renamed.get(issue.field) const blockName = issue.blockName ?? issue.blockId - const at = text !== undefined ? locateWrite(text, blockName, issue.field) : undefined table.change.diagnostics.push( finding( 'W_TABLE_COLUMN_RENAMED_IN_USE', - `${where('table', table.change.key)} renames column "${issue.field}" to "${renamedTo}", but block "${blockName}" of workflow "${workflow.name}" still ${issue.rowWrite ? 'writes' : 'reads'} "${issue.field}"; the run drops it.`, + `${where('table', table.change.key)} renames column "${issue.field}" to "${renamedTo}", but block "${blockName}" of workflow "${workflow.name}" (live version) still ${issue.rowWrite ? 'writes' : 'reads'} "${issue.field}"; its runs drop it until the workflow is deployed again.`, { - hint: `Write "${renamedTo}" in that block${folder ? ` (${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE})` : ''}, in this apply or right after it.`, - ...(folder - ? { file: `${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE}`, ...(at ?? {}) } - : {}), + hint: `Write "${renamedTo}" in that block and deploy the workflow (\`deployments/<key>.ts\` with live: 'draft'), in this apply or right after it.`, kind: 'table', key: table.change.key, } From d2cf60aaef887e9af0530988724fba3f7dfd8208 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 19:07:09 -0800 Subject: [PATCH 58/59] fix(api): stop encoding large v2 source answers in the route, which Next strips the Content-Encoding of Next's sendResponse drops a route handler's Content-Encoding header (FORBIDDEN_HEADERS in send-response.js), so a production server sent brotli bytes as plain JSON and every client failed to parse them (the workflow source e2e: JSON Parse error). The routes answer plain JSON again; the CLI still offers br and gzip for a server or proxy that compresses. --- .../v2/workflows/[workflowId]/source/route.ts | 1 - .../workspaces/[workspaceId]/source/route.ts | 1 - .../workflows/source/catalog/route.ts | 4 +- .../workflows/source/resources/route.ts | 1 - .../api/server/routes/response-compression.ts | 59 ----------------- .../api/server/routes/v2-json-route.test.ts | 65 ------------------- .../lib/api/server/routes/v2-json-route.ts | 47 ++------------ packages/sim-cli/src/http/client.ts | 6 +- 8 files changed, 8 insertions(+), 176 deletions(-) delete mode 100644 apps/sim/lib/api/server/routes/response-compression.ts diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts index 02dc40c5e77..14a45dafb6e 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.ts @@ -33,7 +33,6 @@ export const GET = defineV2JsonRoute({ }), useCase: readWorkflowSource, present: (source) => ({ data: source }), - encodeLargeBody: true, }) /** diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts index beb36c71761..681a3c871dc 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts @@ -36,7 +36,6 @@ export const GET = defineV2JsonRoute({ }), useCase: withSourceSlot(readWorkspaceSource, 'read'), present: (source) => ({ data: source }), - encodeLargeBody: true, }) /** diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts index 6a180f24db6..435ba9f7da9 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/catalog/route.ts @@ -9,8 +9,7 @@ export const revalidate = 0 /** * GET /api/v2/workspaces/{workspaceId}/workflows/source/catalog — the source catalog. The - * response carries `ETag: "<hash>"`; a matching `If-None-Match` answers `304` with no body. The - * catalog runs to megabytes, so a client accepting brotli or gzip gets it encoded. + * response carries `ETag: "<hash>"`; a matching `If-None-Match` answers `304` with no body. */ export const GET = defineV2JsonRoute({ contract: v2GetSourceCatalogContract, @@ -25,5 +24,4 @@ export const GET = defineV2JsonRoute({ useCase: readSourceCatalog, entityTag: (catalog) => catalog.hash, present: (catalog) => ({ data: catalog }), - encodeLargeBody: true, }) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts index 66b0442456d..fb3fc483340 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/resources/route.ts @@ -25,5 +25,4 @@ export const GET = defineV2JsonRoute({ mapInput: ({ params }) => ({ workspaceId: params.workspaceId }), useCase: readSourceResources, present: (manifest) => ({ data: manifest }), - encodeLargeBody: true, }) diff --git a/apps/sim/lib/api/server/routes/response-compression.ts b/apps/sim/lib/api/server/routes/response-compression.ts deleted file mode 100644 index ed690a2ae03..00000000000 --- a/apps/sim/lib/api/server/routes/response-compression.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Content-encoding for the v2 JSON answers that run to megabytes (the workflow source catalog, a - * workspace's source). Next's own compression does not reach these answers in deployment, so a - * route that opts in encodes its body itself: brotli when the request accepts it, else gzip, and - * never for a request that names neither (an in-process transport sends no `Accept-Encoding`, so - * its caller reads plain JSON). Next's compression middleware skips a response that already has a - * `Content-Encoding`, so nothing is encoded twice. - */ -import { promisify } from 'node:util' -import { brotliCompress, constants, gzip } from 'node:zlib' - -/** An encoding a route may answer with. */ -export type BodyEncoding = 'br' | 'gzip' - -/** Bodies smaller than this are sent as they are: encoding them saves too little. */ -export const MIN_ENCODED_BODY_BYTES = 8 * 1024 - -/** - * Brotli quality 5: within a few percent of the best ratio on JSON at a fraction of the time - * (about 50 ms for the 9 MB block catalog, a tenth of its size). - */ -const BROTLI_QUALITY = 5 - -const brotliAsync = promisify(brotliCompress) -const gzipAsync = promisify(gzip) - -/** - * The encoding an `Accept-Encoding` header prefers among the ones a route sends: `br` over - * `gzip` unless its q-value is lower, `*` standing for either, a q-value of 0 refusing one. - * `null` when the header is absent or accepts neither. - */ -export function preferredEncoding(acceptEncoding: string | null): BodyEncoding | null { - if (!acceptEncoding) return null - const weights = new Map<string, number>() - for (const part of acceptEncoding.split(',')) { - const [name, ...params] = part.trim().toLowerCase().split(';') - if (!name) continue - const q = params.map((param) => /^\s*q=([0-9.]+)\s*$/.exec(param)?.[1]).find(Boolean) - const weight = q === undefined ? 1 : Number(q) - weights.set(name, Number.isFinite(weight) ? weight : 0) - } - const weight = (name: BodyEncoding) => weights.get(name) ?? weights.get('*') ?? 0 - const br = weight('br') - const gz = weight('gzip') - if (br <= 0 && gz <= 0) return null - return br >= gz ? 'br' : 'gzip' -} - -/** `body` encoded as `encoding`, on the thread pool so a large body never blocks the event loop. */ -export function encodeBody(body: Uint8Array, encoding: BodyEncoding): Promise<Buffer> { - return encoding === 'br' - ? brotliAsync(body, { - params: { - [constants.BROTLI_PARAM_QUALITY]: BROTLI_QUALITY, - [constants.BROTLI_PARAM_SIZE_HINT]: body.byteLength, - }, - }) - : gzipAsync(body) -} diff --git a/apps/sim/lib/api/server/routes/v2-json-route.test.ts b/apps/sim/lib/api/server/routes/v2-json-route.test.ts index a05cb29cd17..7b8a28dc381 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.test.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.test.ts @@ -1,4 +1,3 @@ -import { brotliDecompressSync, gunzipSync } from 'node:zlib' import type { PersonalApiKeyPrincipal } from '@sim/auth/principal' import { MockV2ApiKeyUnauthenticatedError, @@ -232,70 +231,6 @@ describe('defineV2JsonRoute', () => { expect(unchanged.headers.get('Cache-Control')).toBe('private, no-cache') }) - it.each([ - { acceptEncoding: 'br, gzip', encoding: 'br' }, - { acceptEncoding: 'gzip, deflate', encoding: 'gzip' }, - { acceptEncoding: 'br;q=0, gzip', encoding: 'gzip' }, - { acceptEncoding: 'gzip;q=0.5, *', encoding: 'br' }, - { acceptEncoding: 'identity', encoding: null }, - { acceptEncoding: null, encoding: null }, - ])( - 'encodes a large body as the request accepts ($acceptEncoding → $encoding), keeping the tag weak', - async ({ acceptEncoding, encoding }) => { - const value = 'x'.repeat(64 * 1024) - const handler = defineV2JsonRoute({ - contract, - auth: v2ApiKeyAuth, - operation, - rateLimit: v2RateLimits.publicApi, - errorPolicy: v2OrchestrationErrorPolicy, - mapInput: ({ body }) => body, - useCase: { operation, execute: async () => ({ value }) }, - entityTag: () => 'v1', - present: (result) => ({ data: result }), - encodeLargeBody: true, - }) - const asking = request() - if (acceptEncoding) asking.headers.set('accept-encoding', acceptEncoding) - - const served = await handler(asking) - - const bytes = Buffer.from(await served.arrayBuffer()) - const decoded = - encoding === 'br' - ? brotliDecompressSync(bytes) - : encoding === 'gzip' - ? gunzipSync(bytes) - : bytes - expect(JSON.parse(decoded.toString('utf8'))).toEqual({ data: { value } }) - expect(served.headers.get('Content-Encoding')).toBe(encoding) - expect(served.headers.get('Vary')).toBe('Accept-Encoding') - expect(served.headers.get('ETag')).toBe(encoding ? 'W/"v1"' : '"v1"') - if (encoding) expect(bytes.byteLength).toBeLessThan(value.length / 10) - } - ) - - it('sends a body under the encoding threshold as it is', async () => { - const handler = defineV2JsonRoute({ - contract, - auth: v2ApiKeyAuth, - operation, - rateLimit: v2RateLimits.publicApi, - errorPolicy: v2OrchestrationErrorPolicy, - mapInput: ({ body }) => body, - useCase: { operation, execute: async ({ input }) => input }, - present: (result) => ({ data: result }), - encodeLargeBody: true, - }) - const asking = request() - asking.headers.set('accept-encoding', 'br, gzip') - - const served = await handler(asking) - - expect(served.headers.get('Content-Encoding')).toBeNull() - await expect(served.json()).resolves.toEqual({ data: { value: 'ok' } }) - }) - it('never serves a cached body to a request whose input presents it differently', async () => { const handler = defineV2JsonRoute({ contract, diff --git a/apps/sim/lib/api/server/routes/v2-json-route.ts b/apps/sim/lib/api/server/routes/v2-json-route.ts index df2bbc7a7fb..adbc0f3d670 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.ts @@ -18,11 +18,6 @@ import { requireHeadAuthorizableUseCase, requireJsonRouteDefinition, } from '@/lib/api/server/routes/definition' -import { - encodeBody, - MIN_ENCODED_BODY_BYTES, - preferredEncoding, -} from '@/lib/api/server/routes/response-compression' import type { JsonApiRouteContract, JsonNextRouteHandler, @@ -602,13 +597,6 @@ interface V2JsonRouteOptions<C extends JsonApiRouteContract, O extends Applicati * `present` never sees the principal, so the body is a function of exactly that key. */ entityTag?(result: NoInfer<R>): string - /** - * Encodes a success body of at least {@link MIN_ENCODED_BODY_BYTES} as brotli or gzip when the - * request's `Accept-Encoding` allows (`response-compression.ts`), for a route whose answers run - * to megabytes. Every such answer says `Vary: Accept-Encoding`; an encoded one with an entity - * tag carries it weak (`W/"…"`), as its bytes differ from the plain body's. - */ - encodeLargeBody?: true } /** @@ -642,11 +630,6 @@ export function defineV2JsonRoute< const validatedByTag = options.entityTag ? new LRUCache<string, NonNullable<unknown>>({ max: VALIDATED_BODY_CACHE_MAX }) : undefined - /** Encoded bodies by the validated body's cache key and encoding, so a repeat skips encoding. */ - const encodedByTag = - options.entityTag && options.encodeLargeBody - ? new LRUCache<string, Buffer>({ max: VALIDATED_BODY_CACHE_MAX }) - : undefined const wrapped = withRouteHandler<JsonRouteContext | undefined>( async (request, context) => { if (!methodMatchesContract(request.method, options.contract.method)) { @@ -758,33 +741,11 @@ export function defineV2JsonRoute< ) } await options.onSuccess?.({ principal: auth.principal, input, result }) - const cacheControl = entityTag ? REVALIDATE_CACHE_CONTROL : 'private, no-store' - if (!options.encodeLargeBody) { - return NextResponse.json(validatedBody, { - status: responseStatus, - headers: { 'Cache-Control': cacheControl, ...(entityTag ? { ETag: entityTag } : {}) }, - }) - } - const json = Buffer.from(JSON.stringify(validatedBody)) - const encoding = - json.byteLength >= MIN_ENCODED_BODY_BYTES - ? preferredEncoding(request.headers.get('accept-encoding')) - : null - const encodedKey = cacheKey && encoding ? `${encoding} ${cacheKey}` : undefined - const encoded = encoding - ? ((encodedKey ? encodedByTag?.get(encodedKey) : undefined) ?? - (await encodeBody(json, encoding))) - : null - if (encodedKey && encoded) encodedByTag?.set(encodedKey, encoded) - return new NextResponse(new Uint8Array(encoded ?? json), { + return NextResponse.json(validatedBody, { status: responseStatus, - headers: { - 'Content-Type': 'application/json', - 'Cache-Control': cacheControl, - Vary: 'Accept-Encoding', - ...(encoding ? { 'Content-Encoding': encoding } : {}), - ...(entityTag ? { ETag: encoding ? `W/${entityTag}` : entityTag } : {}), - }, + headers: entityTag + ? { 'Cache-Control': REVALIDATE_CACHE_CONTROL, ETag: entityTag } + : { 'Cache-Control': 'private, no-store' }, }) } catch (error) { const response = options.errorPolicy.render(error) diff --git a/packages/sim-cli/src/http/client.ts b/packages/sim-cli/src/http/client.ts index 7aa7618e6cb..5990b79db07 100644 --- a/packages/sim-cli/src/http/client.ts +++ b/packages/sim-cli/src/http/client.ts @@ -734,9 +734,9 @@ export class SimClient { ? { authorization: `Bearer ${credential.oauth.accessToken}` } : {}), accept: 'application/json', - // Large answers (the block catalog, a workspace's source) come encoded, a tenth of - // their size; fetch decodes them. An in-process transport reads the handler's - // response as it is, so it never asks for an encoding. + // A server or proxy that compresses answers may send the large ones (the block catalog, + // a workspace's source) as brotli or gzip; fetch decodes them. An in-process transport + // reads the handler's response as it is, so it never asks for an encoding. ...(this.profile.transport ? {} : { 'accept-encoding': ACCEPT_ENCODING }), ...identityHeaders(), ...(hasBody ? { 'content-type': 'application/json' } : {}), From 51738fe510dc874805a2e1e182b19afa960bb710 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Thu, 19 Nov 2026 19:20:21 -0800 Subject: [PATCH 59/59] test(sim-cli): use the shared sleep and test hooks in the terminal tests --- packages/sim-cli/src/terminal/interrupts.test.ts | 8 ++++++-- packages/sim-cli/src/terminal/line-input.test.ts | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/sim-cli/src/terminal/interrupts.test.ts b/packages/sim-cli/src/terminal/interrupts.test.ts index ad0117a66fa..f9dccc313c6 100644 --- a/packages/sim-cli/src/terminal/interrupts.test.ts +++ b/packages/sim-cli/src/terminal/interrupts.test.ts @@ -14,13 +14,17 @@ function interruptedWithBackedUpPipe(): Promise<{ bytes: number; signal: string const script = ` const { deferInterrupts } = await import(${JSON.stringify(interrupts)}) const { writeStdout } = await import(${JSON.stringify(io)}) + const { sleep } = await import('@sim/utils/helpers') await deferInterrupts(async () => { process.stderr.write('ready\\n') - await new Promise((resolve) => setTimeout(resolve, 300)) + await sleep(300) writeStdout('x'.repeat(${OUTPUT_BYTES})) }, { waiting: 'waiting', abandoned: 'abandoned' }) ` - const child = spawn('bun', ['-e', script], { stdio: ['ignore', 'pipe', 'pipe'] }) + const child = spawn('bun', ['-e', script], { + cwd: join(import.meta.dirname, '../..'), + stdio: ['ignore', 'pipe', 'pipe'], + }) let bytes = 0 child.stdout.pause() child.stdout.on('data', (chunk: Buffer) => { diff --git a/packages/sim-cli/src/terminal/line-input.test.ts b/packages/sim-cli/src/terminal/line-input.test.ts index 8072bca9b6a..acf59604c02 100644 --- a/packages/sim-cli/src/terminal/line-input.test.ts +++ b/packages/sim-cli/src/terminal/line-input.test.ts @@ -20,7 +20,6 @@ beforeEach(() => { afterEach(() => { Object.defineProperty(process.stderr, 'isTTY', { value: undefined, configurable: true }) - vi.restoreAllMocks() }) describe('askLine', () => {