diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index c66a09de33b..8957d0e13fe 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8573,16 +8573,18 @@ sim workflows source apply [workflowId] [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir ` | Yes | Source directory: workflow.ts, functions/*, .sim/lock.json and .sim/base.json. | | `--dry-run` | No | Check the source and show the outcome without saving or writing files. | | `-y, --yes` | No | Apply without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy ` does). | | `--allow-delete` | No | Delete blocks the source no longer declares; without it, an apply that deletes a block is refused. | -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### sim workflows source create @@ -8598,17 +8600,19 @@ sim workflows source create [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir ` | Yes | Source directory: workflow.ts and functions/*. Without workflow.ts, writes a starter and the types instead (an error with --dry-run or --yes). | | `--dry-run` | No | Check the source and show the outcome without creating anything. | | `--name ` | No | The workflow name (default: the source's). | | `--folder ` | No | The folder to create it in (default: the root). | | `-y, --yes` | No | Create without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy ` does). | -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### sim workflows source check @@ -8627,6 +8631,7 @@ sim workflows source check [options] | `--dir ` | No | Source directory: workflow.ts, functions/* and .sim/. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | @@ -8634,7 +8639,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -8656,7 +8661,7 @@ sim workflows source status [options] -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). @@ -8694,6 +8699,38 @@ Without --server, each file created, edited or deleted since the directory was r API request and response schemas: [getWorkflowSource](/api-reference/workflows/getWorkflowSource). +### sim workflows source params + +Print a builder's params and outputs from the cached block catalog (offline) + +```bash +sim workflows source params [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `builder` | Yes | Builder path (table.upsertRow, agent, trigger.schedule) or namespace | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--dir ` | No | Source directory whose catalog to read. Defaults to `.`. | + + + +Prints a builder's params (required first, with enums and defaults), the gate field that picks which others apply, and its outputs, from the block catalog the last get, create or check cached. No request is made. <builder> is the dotted builder path as source writes it: table.upsertRow, agent, trigger.schedule, slack.sendMessage (slack.tool.sendMessage names the same builder). A namespace (gmail, trigger.gmail) lists its builders; a construct core.d.ts declares (loop, parallel, t, env) prints its declaration. Run it from a workflow or workspace source directory, or pass --dir. + +Exit codes: 0 printed; 1 no builder or namespace of that name (the closest one is named), or no cached catalog (run a get once). + ### sim workflows get Get workflow @@ -9941,18 +9978,19 @@ sim workspaces source apply [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir ` | Yes | Workspace source directory written by `workspaces source get`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | | `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | | `--only ` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | -| `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place; the directory is not changed. | +| `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource). +API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). ### sim workspaces source check @@ -9971,6 +10009,7 @@ sim workspaces source check [options] | `--dir ` | No | Workspace source directory: the kind folders and .sim/base.json. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | @@ -9978,7 +10017,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. @@ -10005,7 +10044,7 @@ sim workspaces source status [options] -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/scripting.mdx b/apps/docs/content/docs/cli/scripting.mdx index e3e8540d3b0..1214f561331 100644 --- a/apps/docs/content/docs/cli/scripting.mdx +++ b/apps/docs/content/docs/cli/scripting.mdx @@ -170,10 +170,22 @@ The CLI resends a request the API answers `429` only when sending it twice is sa | `0` | Applied or created, or a dry run with no errors | | `1` | The source has errors (each printed as `file:line:column` with its frame and hint), or command/request failure | | `5` | The workflow changed since the directory was read (`409 SOURCE_STALE`): get it again, reapply the edit, and retry | -| `6` | The workflow is locked (`423`) | +| `6` | The workflow or its folder is locked (`423`, `E_APPLY_LOCKED`); a locked block exits `1` (`E_APPLY_LOCKED_BLOCK`) | | `7` | The source is over the size limits (`413`) | | `9` | Still rate limited (`429`) after the CLI's retries; nothing was written | | `10` | `--dir` is a workflow folder of a workspace directory another `sim` command is using (`SOURCE_DIR_BUSY`); nothing was written: run it again once the other finished | +| `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; nothing was sent | +| `14` | Run without `--yes` and with no terminal to ask: the plan was printed with no errors and nothing was written; run the same command with `--yes` | +| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check | + +| Code | `sim workflows source status` or `sim workspaces source status` | +| --- | --- | +| `0` | Clean: nothing to apply, and the server changed nothing since the directory was read | +| `1` | Command/request failure | +| `7` | The source is too large for the server to read in one dry run | +| `11` | Local changes only: applying writes them | +| `12` | The server changed since the directory was read (drift, or a conflict with local changes) | +| `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; the local changes are still shown | | Code | `sim workflows source check` or `sim workspaces source check` | | --- | --- | @@ -192,11 +204,13 @@ The CLI resends a request the API answers `429` only when sending it twice is sa | `0` | Applied, or a dry run with no errors | | `1` | The source has errors, changes were held back for a secret or credential the workspace cannot satisfy, or command/request failure | | `5` | A resource changed since the directory was read (`409 SOURCE_STALE`): get it again into a fresh directory, reapply the edit, and retry | -| `6` | A resource is locked (`423`) | +| `6` | A resource (a workflow or its folder, a table) is locked (`423`); a locked block exits `1` (`E_APPLY_LOCKED_BLOCK`) | | `7` | The directory is over the size limits (`413`) | | `8` | The apply stopped part-way: what was written is saved into the directory; apply again | | `9` | Still rate limited (`429`) after the CLI's retries; nothing was written | | `10` | Busy, retry: another apply is writing the workspace (`409 SOURCE_BUSY`), or another `sim workspaces source` command is using the same directory (`SOURCE_DIR_BUSY`); nothing was written: run the command again once the other finished | +| `13` | `AUTH_REQUIRED`: not logged in, or the server refused the credential; nothing was sent | +| `130` | Interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check | Both waits default to 3600 seconds; `--wait-timeout 0` waits indefinitely. `SIM_TIMEOUT_SECONDS` independently bounds one API request. A timeout does not cancel the underlying run or operation. @@ -403,12 +417,14 @@ jobs: `check` runs first because it is offline and fast, and it regenerates `.sim/sdk/workspace.d.ts`, so the `tsc` step after it types resources the pull request adds. A workspace API key can apply tables, rows, knowledge bases, files, custom tools, MCP servers and credential bindings. Workflows, skills, secrets and deployments need a member's personal API key, and deployments need an admin. Apply rewrites files under the directory (each `.sim/base.json`, ids it allocated, values the server normalized), and the next apply compares with them, so the last step commits them back to the branch; `[skip ci]` keeps that commit from running the job again. Where the branch is protected, run `sim workspaces source get --dir workspace` on a new branch instead and open a pull request. After merging two branches that each applied, a dry run with no changes means the files match the workspace. Exit `5` means a resource changed since this branch's base: get the workspace into a fresh directory and carry the edit over. Exit `10` means another apply is writing the workspace (or another command is using the directory), so retry; the `concurrency` key keeps two runs of this job from overlapping. -To promote the same directory to a second workspace, add a step with `--to`: +To promote the same directory to a second workspace, add a step with `--to` before the step that commits the files back: ```bash sim workspaces source apply --dir workspace --to "$PROD_WORKSPACE_ID" --yes ``` +The promotion writes `workspace/.sim/targets/.json`, what it left in that workspace. Committing it lets the next run tell a change made in that workspace since (kept, or exit `5` when the directory changed the same resource) from one to promote. + ## Related docs - [Output formats](/cli/output) diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 318297758b1..f90bb57ad7e 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1337,7 +1337,7 @@ sim workflows source status [options] -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). @@ -1353,17 +1353,19 @@ sim workflows source create [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir ` | Yes | Source directory: workflow.ts and functions/*. Without workflow.ts, writes a starter and the types instead (an error with --dry-run or --yes). | | `--dry-run` | No | Check the source and show the outcome without creating anything. | | `--name ` | No | The workflow name (default: the source's). | | `--folder ` | No | The folder to create it in (default: the root). | | `-y, --yes` | No | Create without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy ` does). | -Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed; 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### Apply a workflow source directory to a workflow draft @@ -1387,16 +1389,18 @@ sim workflows source apply [workflowId] [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir ` | Yes | Source directory: workflow.ts, functions/*, .sim/lock.json and .sim/base.json. | | `--dry-run` | No | Check the source and show the outcome without saving or writing files. | | `-y, --yes` | No | Apply without asking for confirmation. | +| `--deploy` | No | After the write is saved, deploy the workflow (as `sim workflows deploy ` does). | | `--allow-delete` | No | Delete blocks the source no longer declares; without it, an apply that deletes a block is refused. | -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again. Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 10 (busy, retry): --dir is a workflow folder of a workspace directory another sim command is using (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow). ### Show the block catalog workflow source is checked against @@ -1431,6 +1435,7 @@ sim workflows source check [options] | `--dir ` | No | Source directory: workflow.ts, functions/* and .sim/. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | @@ -1438,7 +1443,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory @@ -1472,6 +1477,38 @@ Without --server, each file created, edited or deleted since the directory was r API request and response schemas: [getWorkflowSource](/api-reference/workflows/getWorkflowSource). +### Print a builder's params and outputs from the cached block catalog + +Offline. + +```bash +sim workflows source params [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `builder` | Yes | Builder path (table.upsertRow, agent, trigger.schedule) or namespace | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--dir ` | No | Source directory whose catalog to read. Defaults to `.`. | + + + +Prints a builder's params (required first, with enums and defaults), the gate field that picks which others apply, and its outputs, from the block catalog the last get, create or check cached. No request is made. <builder> is the dotted builder path as source writes it: table.upsertRow, agent, trigger.schedule, slack.sendMessage (slack.tool.sendMessage names the same builder). A namespace (gmail, trigger.gmail) lists its builders; a construct core.d.ts declares (loop, parallel, t, env) prints its declaration. Run it from a workflow or workspace source directory, or pass --dir. + +Exit codes: 0 printed; 1 no builder or namespace of that name (the closest one is named), or no cached catalog (run a get once). + ### List the tables, knowledge bases, credentials, workflows, skills, custom tools, MCP servers and env names source can name ```bash diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 3cbe8c01746..7143068365b 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -733,7 +733,7 @@ sim workspaces source status [options] -Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). +Exit codes: 0 clean (nothing to apply, and the server changed nothing since the directory was read); 11 local changes only (applying writes them); 12 the server changed since the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; 7 the source is too large for the server to read in one dry run; 1 the command failed; 13 not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the local changes only, then run `sim login`. A server that cannot be reached is not a failure: the status shows the local changes only and says so (exit 0 or 11). Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. @@ -751,18 +751,19 @@ sim workspaces source apply [options] | Option | Required | Description | | --- | --- | --- | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | | `--dir ` | Yes | Workspace source directory written by `workspaces source get`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | | `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | | `--only ` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | -| `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place; the directory is not changed. | +| `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | -Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to to promote it), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished. +Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or promote it with --to; without --workspace an apply targets the workspace the directory was read from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read (SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory (SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource). +API request and response schemas: [applyWorkspaceSource](/api-reference/workflows/applyWorkspaceSource), [findMissingWorkspaceSourceBlobs](/api-reference/workflows/findMissingWorkspaceSourceBlobs), [stageWorkspaceSourceBlobs](/api-reference/workflows/stageWorkspaceSourceBlobs), [getWorkspaceSource](/api-reference/workflows/getWorkspaceSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog). ### Compile every workflow of a workspace source directory locally, without the server @@ -779,6 +780,7 @@ sim workspaces source check [options] | `--dir ` | No | Workspace source directory: the kind folders and .sim/base.json. Defaults to `.`. | | `--strict` | No | Fail on the warnings listed below in every block, not only new ones. | | `--no-strict` | No | Never fail on a warning. | +| `--verbose` | No | With --output json, also list info findings (such as blocks placed by autolayout). | @@ -786,7 +788,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never fails on a warning. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index 8366500512f..a46903e0f7a 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -13,12 +13,14 @@ Workflows as code is in preview. When it is [enabled on your Sim deployment](/pl ## For coding agents [#for-coding-agents] -Claude Code, Cursor, Codex and other coding agents work from a directory that `get` writes. Point the agent at `.sim/AGENTS.md` in it, which walks through the loop: +Claude Code, Cursor, Codex and other coding agents work from a directory that `get` writes. `get` prints a short primer (the edit loop, how block identity and references work, deletes, deploy, exit codes, and the params of each builder the workflow already uses; in JSON output, its `primer` field), and the same primer opens `.sim/AGENTS.md`, the full guide. The loop: 1. `sim workflows source get --dir ` writes the workflow and the guide. 2. Edit `workflow.ts`. 3. `sim workflows source check --dir ` compiles it offline. 4. `sim workflows source status --dir ` shows what changed and what to run next. + +To write a block the workflow does not use yet, `sim workflows source params ` (for example `table.upsertRow` or `trigger.schedule`) prints its params, required ones first with their options and defaults, and its outputs, from the cached block catalog without a request. A namespace such as `gmail` lists its builders. 5. `sim workflows source apply --dir --dry-run`, then `--yes`, saves the draft. For a whole workspace, use `sim workspaces source` with the same steps; see [Workspace as code](/workflows/workspace-code). @@ -37,7 +39,7 @@ Nothing in the file runs on your machine. Sim reads it statically, checks it aga sim workflows source get --dir ./support-triage ``` - This writes `workflow.ts`, any function code under `functions/`, and type declarations for every block and for your workspace's tables, credentials, knowledge bases, and environment variable names, with `.sim/sdk/INDEX.md` listing every builder by block and operation. It also writes a `tsconfig.json` that resolves them and type-checks the blocks the workflow uses, and a `tsconfig.full.json` that type-checks every block, unless the directory already has files of those names, which are left as they are. For coding agents it writes `.sim/AGENTS.md`, a short guide, and example workflows under `.sim/examples/` with one correct example of each common block to copy from. The examples name placeholder resources, so the tsconfig files leave them out of the type-check. + This writes `workflow.ts`, any function code under `functions/`, and type declarations for the blocks the workflow uses and for your workspace's tables, credentials, knowledge bases, and environment variable names, with `.sim/sdk/INDEX.md` listing those blocks' builders. The declarations of every block are generated once per block catalog into the CLI's cache (`~/.sim/cache/source-catalog/`) and shared by every directory. It also writes a `tsconfig.json` that resolves them and type-checks the blocks the workflow uses, and a `tsconfig.full.json` that type-checks every block through that cache, unless the directory already has files of those names, which are left as they are. For coding agents it writes `.sim/AGENTS.md`, the primer `get` prints followed by a reference, and example workflows under `.sim/examples/` with one correct example of each common block to copy from. The examples name placeholder resources, so the tsconfig files leave them out of the type-check. `get` into a directory that already holds source refuses to overwrite edits you have not applied, and refuses a directory that holds a different workflow. Pass `--force` to replace them. @@ -84,13 +86,13 @@ Nothing in the file runs on your machine. Sim reads it statically, checks it aga sim workflows source apply --dir ./support-triage --yes ``` - The workflow builder updates for anyone who has the workflow open. Deploy when you are ready. The save rewrites `workflow.ts` in canonical form, so read it again before the next edit. + The workflow builder updates for anyone who has the workflow open. Deploy when you are ready, or add `--deploy` to deploy right after the save; without it, a deployed workflow whose draft now differs prints the `sim workflows deploy` command to run. `--yes` refuses everything a dry run would, so step 4 is optional: without `--yes`, and with no terminal to ask, `apply` prints the plan, writes nothing and exits `14`. The save rewrites `workflow.ts` in canonical form, so read it again before the next edit. Editing the name or description in `workflow('…')` renames or redescribes the workflow; a name another workflow in its folder already has is refused with `E_NAME_TAKEN`. Lines starting with `note:` report something the save did beyond what the file says; read them. A save that removes a block (its declaration is gone from `workflow.ts`) is refused with `E_APPLY_BLOCK_DELETE`, naming the blocks. Pass `--allow-delete` when the deletion is intended. -To create a new workflow, run `sim workflows source create --dir ` on an empty directory: without `--dry-run` or `--yes` it creates nothing and writes a starter `workflow.ts` with the type declarations, `tsconfig.json` and `.sim/AGENTS.md`. Edit the starter, check it, then run `sim workflows source create --dir --dry-run` and `--yes`. Until the workflow is created the directory has no `.sim/base.json`, so `status`, `diff` and `apply` refuse it. The file starts with the `// @sim-sdk 1` line and default-exports one `workflow('Name', {}, () => { ... })`. The command prints the new workflow ID and writes the same type declarations, `tsconfig.json`, and `.sim/` files that `get` writes, so you can continue editing with `apply`. If another workflow already has that name, the command exits `1`; pass `--name` to choose another. +To create a new workflow, run `sim workflows source create --dir --workspace ` on an empty directory: without `--dry-run` or `--yes` it creates nothing and writes a starter `workflow.ts` with that workspace's type declarations, `tsconfig.json` and `.sim/AGENTS.md`, and records the workspace in `.sim/starter.json`. Edit the starter, check it, then run `sim workflows source create --dir --workspace --dry-run` and `--yes` (the starter prints both). A create goes to the starter's workspace whatever your default workspace, unless `--workspace` names another. Until the workflow is created the directory has no `.sim/base.json`, so `status`, `diff` and `apply` refuse it. The file starts with the `// @sim-sdk 1` line and default-exports one `workflow('Name', {}, () => { ... })`. The command prints the new workflow ID and writes the same type declarations, `tsconfig.json`, and `.sim/` files that `get` writes, so you can continue editing with `apply`. If another workflow already has that name, the command exits `1`; pass `--name` to choose another. Keep the `.sim/` folder the commands write. `.sim/lock.json` ties each block in the file to the stored block, so renaming a block keeps the same block, with its webhook URL, schedules and run history. Changing its name string keeps it through the lock; renaming the variable too keeps it when the block's type, parent and connections are unchanged, and the dry run reports that as `W_MAT_INFERRED_RENAME`. Renaming two blocks of the same type in the same place in one save is ambiguous, so do those one save at a time. `.sim/base.json` records the version you read. @@ -195,14 +197,14 @@ Code that does not fit a block goes in a Function block, kept in its own file: const score = fn('Score lead', { code: codeFile('./functions/score-lead.js') }) ``` -In the file, `$ref.block.path` reads another block's output and `$env.NAME` reads an environment variable. `block` is the block's key: its name lowercased with all whitespace and every `.` removed (`$ref.triageagent.content` for "Triage Agent", `$ref.v2result.text` for "V2. Result"). A `$ref` that names no block is a warning, because it would not resolve when the workflow runs. Python and shell files are saved as written. +In the file, `$ref.block.path` reads another block's output and `$env.NAME` reads an environment variable. `block` is the block's key: its name lowercased with all whitespace and every `.` removed (`$ref.triageagent.content` for "Triage Agent", `$ref.v2result.text` for "V2. Result"). Inside a loop body, `$ref.loop.currentItem` and `$ref.loop.index` read the iteration's item and position; inside a parallel body, `$ref.parallel.currentItem` and `$ref.parallel.index`. A `$ref` that names no block is a warning, because it would not resolve when the workflow runs. Python and shell files are saved as written. ## Use it in CI Run `sim workflows source apply --dir --dry-run` in CI to catch a broken workflow before it is saved. It exits `0` when the workflow is valid and `1` when it has errors, so a failing check blocks the merge. It exits `5` when the workflow was changed in the editor after you committed the directory; run `get` again and reapply your edit. - **Commit** `workflow.ts`, `functions/`, `.sim/lock.json`, `.sim/base.json`, `.sim/written.json`, `tsconfig.json` and `.gitattributes`. CI finds the workflow from `workflowId` in `.sim/base.json`, and `status` and `get` use `.sim/written.json` to tell your edits apart from what the CLI last wrote. -- **Optionally gitignore** `.sim/sdk/`, the generated type declarations (thousands of files); `get` and `check` write them again. A single-workflow directory gets no `.gitignore`, so add the line yourself. +- **Optionally gitignore** `.sim/sdk/`, the generated type declarations of the blocks in use (a few dozen files); `get` and `check` write them again. Gitignore `tsconfig.full.json`, which names the CLI's cache on your machine; `check` writes it again. A single-workflow directory gets no `.gitignore`, so add the lines yourself. - **A whole workspace** in one repository, with dry runs on pull requests, apply on merge and promotion from one workspace to another, is covered in [Git and CI](/workflows/workspace-code#git-and-ci) and [Promote to another workspace](/workflows/workspace-code#promote-to-another-workspace). - **Credentials:** use a personal API key (**Settings → Sim API keys**, passed as `SIM_API_KEY`) or an OAuth login, not a workspace API key. Saving and dry-running source act as a person, so a workspace key is refused with `403`. Set `SIM_WORKSPACE` unless you pass `--workspace`, and `SIM_ENDPOINT` if you use a self-hosted Sim. See [Authenticating CI](/cli/authentication#authenticating-ci). diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index bdbbc7e1df5..f5b580f4740 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -48,7 +48,7 @@ const log = table.insertRow('Log Question', { }) ``` -`get` writes `.sim/AGENTS.md`, a guide for coding agents, and `.sim/examples/workspace/`, a complete example of every kind and the workflow that uses them. +`get` prints a short primer for coding agents (in JSON output, its `primer` field) and writes it atop `.sim/AGENTS.md`, the guide, with deletes in depth, Git and CI, and promoting to another workspace in `.sim/GUIDE-advanced.md`. It also writes `.sim/examples/workspace/`, a complete example of every kind and the workflow that uses them. ## Edit a workspace from the CLI @@ -79,6 +79,7 @@ To apply the rest with a workspace key, such as in CI, narrow the apply to the k - A resource the directory never managed is kept, and the apply names it in a warning. - Something deleted in Sim is not re-created by an unchanged directory, and an edit to it is refused as stale. - Table rows are part of the directory only when the table's `rows` opts in (`tables/.rows.jsonl`). Connector-synced documents and run history never are. +- Seed rows (`rows: { mode: 'seed' }`) belong to the directory: `get` keeps the declaration and the rows file beside it, but Sim does not store them, so a get into a new directory has the declaration without the file. - A delete cannot be undone by applying an older copy of the directory, such as a reverted commit. The resource stays deleted, and its rows, documents, run history, webhook URLs and schedules are gone. The dry run marks each change that deletes data and counts them as `destructive`. ## Promote to another workspace @@ -90,7 +91,15 @@ sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --dry sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --yes ``` -Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`. The directory is not changed and stays bound to the workspace it was read from. Without `--to`, an apply refuses a directory read from another workspace. +Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`. The directory stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. + +Each promotion records what it left in the target in `.sim/targets/.json`, and the next promotion compares with that record: + +- A resource changed in the target since the last promotion, such as a hotfix made in Sim, is kept when the directory left it alone. +- When the directory changed it too, the promotion is refused as stale and exits `5`. Carry the target's change into the source workspace, or undo it in the target, then promote again. +- The first promotion, and any resource the record does not hold (one created in the target since), updates the target to match the directory. Review the dry run first. + +Commit the record with the rest of the directory, so a promotion from CI or another clone compares with it. Deleting it makes the next promotion take the target as it stands. ## Git and CI @@ -102,7 +111,7 @@ Commit the whole directory, including `.sim/`: its base files are what an apply - Renaming a skill does not rewrite the workflows that use it: each agent keeps the name it stored until its skills list is next edited. - One apply sends at most 100 edited workflows and makes at most 1,000 changes; a directory holds at most 500 resources of a kind, and a `tables/.rows.jsonl` at most 10,000 rows. An apply over a limit is refused before anything is written. - A document or file over 10 MB, or whose name a directory cannot hold, is left out of `get` and listed in `.sim/skipped.json`. An apply never writes or deletes it. -- Hidden files (a name starting with a dot, such as `files/.env`) are never sent. An apply names them in a warning. +- Hidden files (a name starting with a dot, such as `files/.env`) and names with a character no workspace path holds (such as `:` or `?`) are never sent. An apply names each one in a `W_PATH_NOT_SENT` warning. - A knowledge base's `embeddingModel` is fixed when it is created. An apply that changes it is refused. - An apply that stops part-way exits `8` and saves what it wrote into the directory. Run the same apply again until it exits `0`. - An apply exits `5` when a resource changed in Sim after you read it. Run `get` into a new directory and reapply your edit. diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 847540cbd2a..e5bf63509a0 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -6343,7 +6343,8 @@ "added": [ { "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "key": "start" + "key": "start", + "type": "start_trigger" } ], "removed": [], @@ -8986,7 +8987,7 @@ "post": { "operationId": "stageWorkspaceSourceBlobs", "summary": "Stage workspace source blobs", - "description": "Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most 1000 files and 24000000 bytes per request, 10000000 bytes per file. A part whose bytes hash to another SHA-256 refuses the whole request (`400`), nothing staged. Staged files are scoped to the workspace and the caller, capped at 20000 files and 4000000000 bytes (`413` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:write`.", + "description": "Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most 1000 files and 8000000 bytes per request, 10000000 bytes per file; a larger file goes in a request of its own. A body over 10001024 bytes answers `413` (`PAYLOAD_TOO_LARGE`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (`400`), nothing staged. Staged files are scoped to the workspace and the caller, capped at 20000 files and 4000000000 bytes (`413` with `details.code` `SOURCE_STAGING_FULL` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:write`.", "x-sim-operation": "workspaces.source.blobs.stage", "x-oauth-scope": "api:write", "tags": ["Workflows"], @@ -18915,6 +18916,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -19060,11 +19062,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "New blocks." }, @@ -19080,11 +19086,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "Deleted blocks." }, @@ -19350,6 +19360,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -20012,11 +20023,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "New blocks." }, @@ -20032,11 +20047,15 @@ "key": { "type": "string", "description": "Block key in the source." + }, + "type": { + "type": "string", + "description": "Block type." } }, - "required": ["id", "key"], + "required": ["id", "key", "type"], "additionalProperties": false, - "description": "A block by id and source key." + "description": "An added or removed block by id, source key and type." }, "description": "Deleted blocks." }, @@ -20302,6 +20321,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -20623,7 +20643,8 @@ "added": [ { "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "key": "start" + "key": "start", + "type": "start_trigger" } ], "removed": [], @@ -25117,6 +25138,7 @@ "E_DECL_SHAPE", "E_DECL_UNKNOWN_FIELD", "E_DECL_UNKNOWN_FILE", + "W_PATH_NOT_SENT", "E_BLOB_MANIFEST", "E_BLOB_MISSING", "E_LIMIT_EXCEEDED", @@ -25191,6 +25213,7 @@ "E_PLAN_REF_DELETED", "E_PLAN_REF_HELD", "W_RESOURCE_IN_USE", + "W_TABLE_COLUMN_RENAMED_IN_USE", "E_NAME_TAKEN", "E_WORKFLOW_FILES_MISSING", "E_WORKFLOW_FOLDER_COPIED", @@ -25340,6 +25363,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -25680,6 +25704,7 @@ "E_DECL_SHAPE", "E_DECL_UNKNOWN_FIELD", "E_DECL_UNKNOWN_FILE", + "W_PATH_NOT_SENT", "E_BLOB_MANIFEST", "E_BLOB_MISSING", "E_LIMIT_EXCEEDED", @@ -25754,6 +25779,7 @@ "E_PLAN_REF_DELETED", "E_PLAN_REF_HELD", "W_RESOURCE_IN_USE", + "W_TABLE_COLUMN_RENAMED_IN_USE", "E_NAME_TAKEN", "E_WORKFLOW_FILES_MISSING", "E_WORKFLOW_FOLDER_COPIED", @@ -25903,6 +25929,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", @@ -26086,6 +26113,7 @@ "E_DECL_SHAPE", "E_DECL_UNKNOWN_FIELD", "E_DECL_UNKNOWN_FILE", + "W_PATH_NOT_SENT", "E_BLOB_MANIFEST", "E_BLOB_MISSING", "E_LIMIT_EXCEEDED", @@ -26160,6 +26188,7 @@ "E_PLAN_REF_DELETED", "E_PLAN_REF_HELD", "W_RESOURCE_IN_USE", + "W_TABLE_COLUMN_RENAMED_IN_USE", "E_NAME_TAKEN", "E_WORKFLOW_FILES_MISSING", "E_WORKFLOW_FOLDER_COPIED", @@ -26309,6 +26338,7 @@ "W_APPLY_TYPE_CHANGED", "W_APPLY_TRIGGER_REMOVED", "W_APPLY_CATALOG_MOVED", + "W_APPLY_PROBABLE_RENAME", "E_APPLY_LINT", "W_APPLY_LINT", "W_TABLE_UNKNOWN_COLUMN", diff --git a/apps/docs/scripts/fixtures/content-baseline.json b/apps/docs/scripts/fixtures/content-baseline.json index 6b9edc3c27b..54cc31249cc 100644 --- a/apps/docs/scripts/fixtures/content-baseline.json +++ b/apps/docs/scripts/fixtures/content-baseline.json @@ -13189,6 +13189,13 @@ "bytes": null, "line": null }, + { + "lang": "bash", + "meta": null, + "sha256": "2b6c40dcef94f6188188d6369383e982f5a6dc88654b8ab76664c189f9af8ab0", + "bytes": null, + "line": null + }, { "lang": "bash", "meta": null, @@ -17768,6 +17775,13 @@ "bytes": null, "line": null }, + { + "lang": "bash", + "meta": null, + "sha256": "2b6c40dcef94f6188188d6369383e982f5a6dc88654b8ab76664c189f9af8ab0", + "bytes": null, + "line": null + }, { "lang": "bash", "meta": null, diff --git a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts index f1b1b2ca24d..741d8ffe85e 100644 --- a/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts +++ b/apps/sim/app/api/v2/workflows/[workflowId]/source/route.test.ts @@ -215,7 +215,7 @@ describe('POST /api/v2/workflows/{workflowId}/source', () => { it('answers 409 with the current fingerprint when the draft moved', async () => { const current = `sf1:${'f'.repeat(64)}` - mocks.apply.mockRejectedValue(new WorkflowSourceStaleError(current)) + mocks.apply.mockRejectedValue(new WorkflowSourceStaleError(current, 'Lead Scoring')) const response = await POST(applyRequest(applyBody), context) const body = await response.json() diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts new file mode 100644 index 00000000000..a025ec5f6d0 --- /dev/null +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.test.ts @@ -0,0 +1,127 @@ +import { createHash } from 'node:crypto' +import { + V2_OPERATION_RATE_LIMIT_ALLOWED, + V2_PREAUTH_RATE_LIMIT_ALLOWED, + v2ApiKeyAuthModuleMock, + v2RateLimiterModuleMock, + v2RouteMocks, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ admit: vi.fn(), stage: vi.fn() })) + +vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) +vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) +vi.mock('@/lib/workspace-source/application/stage-blobs', () => ({ + admitSourceBlobStage: { + operation: { id: 'workspaces.source.blobs.stage' }, + execute: mocks.admit, + }, + stageSourceBlobs: { operation: { id: 'workspaces.source.blobs.stage' }, execute: mocks.stage }, +})) + +import { SourceStagingFullError } from '@/lib/workspace-source/staging/store' +import { POST } from '@/app/api/v2/workspaces/[workspaceId]/source/blobs/route' + +const WORKSPACE_ID = '11111111-2222-4333-8444-555555555555' + +const auth = { + principal: { kind: 'personal_api_key' as const, userId: 'user-1', keyId: 'key-2' }, + rateLimitSubjectIds: ['api-key:key-2'] as const, + rateLimitSubscription: null, + keyType: 'personal' as const, +} + +const context = { params: Promise.resolve({ workspaceId: WORKSPACE_ID }) } + +/** A staging request carrying one part per buffer, as the CLI builds it. */ +async function stagingRequest(buffers: Buffer[]): Promise { + const form = new FormData() + for (const bytes of buffers) { + const sha256 = createHash('sha256').update(bytes).digest('hex') + form.append(sha256, new Blob([new Uint8Array(bytes)]), sha256) + } + const encoded = new Response(form) + const body = Buffer.from(await encoded.arrayBuffer()) + return new NextRequest(`http://localhost:3000/api/v2/workspaces/${WORKSPACE_ID}/source/blobs`, { + method: 'POST', + headers: { + 'x-api-key': 'key', + 'content-type': encoded.headers.get('content-type') ?? '', + 'content-length': String(body.byteLength), + }, + body, + }) +} + +describe('POST /api/v2/workspaces/{workspaceId}/source/blobs', () => { + beforeEach(() => { + v2RouteMocks.authenticate.mockResolvedValue(auth) + v2RouteMocks.preauthRate.mockResolvedValue(V2_PREAUTH_RATE_LIMIT_ALLOWED) + v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) + mocks.admit.mockReset().mockResolvedValue({ workspaceId: WORKSPACE_ID }) + mocks.stage.mockReset().mockResolvedValue({ workspaceId: WORKSPACE_ID, staged: [] }) + }) + + it('answers a body the proxy would cut off with 413 PAYLOAD_TOO_LARGE naming the limit, before reading it', async () => { + const response = await POST( + await stagingRequest([Buffer.alloc(6_000_000, 1), Buffer.alloc(6_000_000, 2)]), + context + ) + + expect(response.status).toBe(413) + const { error } = await response.json() + expect(error.code).toBe('PAYLOAD_TOO_LARGE') + expect(error.message).toMatch(/at most \d+ bytes; this one carries \d+/) + expect(mocks.stage).not.toHaveBeenCalled() + }) + + it('answers a chunked body past the limit with 413 saying it carries more, not a count it never read', async () => { + const declared = await stagingRequest([Buffer.alloc(6_000_000, 1), Buffer.alloc(6_000_000, 2)]) + const body = Buffer.from(await declared.arrayBuffer()) + const headers = new Headers(declared.headers) + headers.delete('content-length') + const chunked = new NextRequest(declared.url, { + method: 'POST', + headers, + body: new ReadableStream({ + start(controller) { + for (let at = 0; at < body.byteLength; at += 1_000_000) { + controller.enqueue(new Uint8Array(body.subarray(at, at + 1_000_000))) + } + controller.close() + }, + }), + duplex: 'half', + } as ConstructorParameters[1]) + + const response = await POST(chunked, context) + + expect(response.status).toBe(413) + const { error } = await response.json() + expect(error.message).toMatch(/at most (\d+) bytes; this one carries more than \1\./) + expect(mocks.stage).not.toHaveBeenCalled() + }) + + it('answers a caller whose staging area is full with 413 SOURCE_STAGING_FULL, apart from a body over the limit', async () => { + mocks.admit.mockRejectedValue( + new SourceStagingFullError( + 'Staging 1 more binary files would hold 5001, over the 5000-file limit.' + ) + ) + + const response = await POST(await stagingRequest([Buffer.from('a')]), context) + + expect(response.status).toBe(413) + const { error } = await response.json() + expect(error.details).toEqual({ code: 'SOURCE_STAGING_FULL' }) + }) + + it('stages a single file of the largest size a directory holds', async () => { + const response = await POST(await stagingRequest([Buffer.alloc(10_000_000, 3)]), context) + + expect(response.status).toBe(200) + expect(mocks.stage.mock.calls[0][0].input.blobs).toHaveLength(1) + }) +}) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts index 6c8845c5a0e..f0db210cc38 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/blobs/route.ts @@ -1,4 +1,7 @@ -import { WORKSPACE_SOURCE_STAGE_MAX_BYTES } from '@sim/workspace-code/limits' +import { + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' import { v2StageWorkspaceSourceBlobsContract } from '@/lib/api/contracts/v2/workspace-source' import { defineV2BodyLifecycleRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -20,11 +23,23 @@ export const revalidate = 0 const SHA256 = /^[0-9a-f]{64}$/ +/** + * What a body over the limit carried, for its refusal: the declared length, or for a chunked body + * (read only until it passed the limit) that it carried more than the limit. + */ +function carried(request: Request, observedBytes: number | undefined): string { + if (request.headers.get('content-length')) { + return observedBytes === undefined ? '' : `; this one carries ${observedBytes}` + } + return `; this one carries more than ${WORKSPACE_SOURCE_STAGE_MAX_BYTES}` +} + /** * POST /api/v2/workspaces/[workspaceId]/source/blobs — stages the binary files of a workspace * source apply: a multipart form with one file part per blob, named by its SHA-256. The caller's - * write access and staging room are checked before the body is read; a part whose bytes hash to - * another SHA-256 refuses the batch. Draws on its own staging budget, not the source-write one. + * write access and staging room are checked before the body is read; a body over + * `WORKSPACE_SOURCE_STAGE_MAX_BYTES` (a declared length, or one the proxy truncated at its ceiling) + * answers `413 PAYLOAD_TOO_LARGE`, and a part whose bytes hash to another SHA-256 refuses the batch. Draws on its own staging budget, not the source-write one. */ export const POST = defineV2BodyLifecycleRoute({ contract: v2StageWorkspaceSourceBlobsContract, @@ -44,7 +59,12 @@ export const POST = defineV2BodyLifecycleRoute({ label: 'workspace source staging body', }) } catch (error) { - if (isPayloadSizeLimitError(error)) throw error + if (isPayloadSizeLimitError(error)) { + throw new OrchestrationError( + 'payload_too_large', + `A staging request body carries at most ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes${carried(request, error.observedBytes)}. Send fewer files per request, a file over ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes in a request of its own.` + ) + } if (isMultipartFieldValidationError(error)) { throw new OrchestrationError('validation', error.message) } @@ -61,7 +81,7 @@ export const POST = defineV2BodyLifecycleRoute({ blobs.push({ sha256: name, content: await readFileToBufferWithLimit(value, { - maxBytes: WORKSPACE_SOURCE_STAGE_MAX_BYTES, + maxBytes: WORKSPACE_SOURCE_LIMITS.maxBlobBytes, label: 'workspace source blob', }), }) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts index 229dfc3afc3..f82f20c20d1 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/workflows/source/route.test.ts @@ -49,7 +49,12 @@ const created = { applied: true, diff: { noop: false, - blocks: { added: [{ id: 'b-1', key: 'start' }], removed: [], renamed: [], modified: [] }, + blocks: { + added: [{ id: 'b-1', key: 'start', type: 'start_trigger' }], + removed: [], + renamed: [], + modified: [], + }, edges: { added: 0, removed: 0 }, variables: { added: [], removed: [], modified: [] }, }, diff --git a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts index 0c5da604194..6a25d2449a0 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts @@ -181,7 +181,7 @@ export const workflowSourceOpenApiRoutes = [ diff: { noop: false, blocks: { - added: [{ id: BLOCK_ID, key: 'start' }], + added: [{ id: BLOCK_ID, key: 'start', type: 'start_trigger' }], removed: [], renamed: [], modified: [], diff --git a/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts b/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts index f1b12e7500a..ad5182a0343 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/workspace-source.ts @@ -1,4 +1,7 @@ -import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' +import { + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' import { documentedSchema, RATE_LIMIT_HEADERS, @@ -192,7 +195,7 @@ export const workspaceSourceOpenApiRoutes = [ applicationOperation: workspaceSourceOperations.stageBlobs, operationId: 'stageWorkspaceSourceBlobs', summary: 'Stage workspace source blobs', - description: `Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, ${WORKSPACE_SOURCE_LIMITS.maxBlobBytes} bytes per file. A part whose bytes hash to another SHA-256 refuses the whole request (\`400\`), nothing staged. Staged files are scoped to the workspace and the caller, capped at ${WORKSPACE_SOURCE_LIMITS.maxStagedBlobs} files and ${WORKSPACE_SOURCE_LIMITS.maxStagedBytes} bytes (\`413\` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns \`403\` while workflows as code is not enabled.`, + description: `Stage the bytes of binary files (workspace files, knowledge documents) for a workspace source apply, as multipart form data: one file part per blob, its field name the SHA-256 of its bytes. At most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, ${WORKSPACE_SOURCE_LIMITS.maxBlobBytes} bytes per file; a larger file goes in a request of its own. A body over ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes answers \`413\` (\`PAYLOAD_TOO_LARGE\`) before it is read. A part whose bytes hash to another SHA-256 refuses the whole request (\`400\`), nothing staged. Staged files are scoped to the workspace and the caller, capped at ${WORKSPACE_SOURCE_LIMITS.maxStagedBlobs} files and ${WORKSPACE_SOURCE_LIMITS.maxStagedBytes} bytes (\`413\` with \`details.code\` \`SOURCE_STAGING_FULL\` past either), and expire a day after their last use (staged, asked for, or applied); an apply does not remove them, so applying the same bytes again uploads nothing. Staging the same bytes again only renews them. Requires workspace write access. Returns \`403\` while workflows as code is not enabled.`, tags: ['Workflows'], errors: [...RESOURCE_MUTATION_ERRORS, 'PayloadTooLarge'], success: { description: 'The binary files staged.', headers: RATE_LIMIT_HEADERS }, diff --git a/apps/sim/lib/api/contracts/v2/workflow-source.ts b/apps/sim/lib/api/contracts/v2/workflow-source.ts index 003f57b94a7..42b34189693 100644 --- a/apps/sim/lib/api/contracts/v2/workflow-source.ts +++ b/apps/sim/lib/api/contracts/v2/workflow-source.ts @@ -123,6 +123,10 @@ const blockRefSchema = z }) .describe('A block by id and source key.') +const typedBlockRefSchema = blockRefSchema + .extend({ type: z.string().describe('Block type.') }) + .describe('An added or removed block by id, source key and type.') + const sourceDiffSchema = z .object({ noop: z.boolean().describe('True when the source matches the saved draft.'), @@ -132,8 +136,8 @@ const sourceDiffSchema = z .describe('The workflow name and description fields the source changes.'), blocks: z .object({ - added: z.array(blockRefSchema).describe('New blocks.'), - removed: z.array(blockRefSchema).describe('Deleted blocks.'), + added: z.array(typedBlockRefSchema).describe('New blocks.'), + removed: z.array(typedBlockRefSchema).describe('Deleted blocks.'), renamed: z .array( z diff --git a/apps/sim/lib/api/contracts/v2/workspace-source.ts b/apps/sim/lib/api/contracts/v2/workspace-source.ts index f21adf6277e..4a6199e535a 100644 --- a/apps/sim/lib/api/contracts/v2/workspace-source.ts +++ b/apps/sim/lib/api/contracts/v2/workspace-source.ts @@ -354,7 +354,7 @@ export const v2StageWorkspaceSourceBlobsFormSchema = z z.file().max(WORKSPACE_SOURCE_LIMITS.maxBlobBytes).describe('The bytes of one binary file.') ) .describe( - `One file part per binary file, named by its SHA-256: at most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request.` + `One file part per binary file, named by its SHA-256: at most ${WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest} files and ${WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest} bytes per request, a larger file alone.` ) /** `POST /api/v2/workspaces/{workspaceId}/source/blobs`: stage binary files for an apply. */ diff --git a/apps/sim/lib/table/application/reshape.ts b/apps/sim/lib/table/application/reshape.ts index a97cff3853f..f61f12172e4 100644 --- a/apps/sim/lib/table/application/reshape.ts +++ b/apps/sim/lib/table/application/reshape.ts @@ -79,7 +79,12 @@ export interface ReshapeTableInput { * Marks the rows as managed by workspace source, matched by the column named `keyColumn` as the * reshape leaves the columns (`null` clears the mark); absent leaves it alone. */ - sourceRows?: { mode: 'managed'; keyColumn: string } | null + sourceRows?: { mode: 'managed'; keyColumn: string } | { mode: 'seed' } | null + /** + * Records each source column's key, by column name as the reshape leaves the columns + * (`metadata.sourceColumnKeys`, replaced whole); absent leaves the record alone. + */ + sourceColumnKeys?: Record /** * The row version the caller read the rows at. Checked under the table's lock before anything * changes: a row written since refuses the reshape (`TableRowsChangedError`), so `rows` @@ -127,6 +132,7 @@ function sourceRowsMark( idByName: ReadonlyMap ): TableSourceRows | null { if (!input) return null + if (input.mode === 'seed') return input const keyColumnId = idByName.get(input.keyColumn) if (!keyColumnId) { throw new OrchestrationError('validation', `Column "${input.keyColumn}" does not exist`) @@ -143,6 +149,20 @@ function withSourceRows( return sourceRows ? { ...rest, sourceRows } : rest } +/** Source column keys by column name, as `metadata.sourceColumnKeys` holds them: by column id. */ +function sourceColumnKeysById( + keysByName: Readonly>, + idByName: ReadonlyMap +): Record { + const byId: Record = {} + for (const [name, key] of Object.entries(keysByName)) { + const id = idByName.get(name) + if (!id) throw new OrchestrationError('validation', `Column "${name}" does not exist`) + byId[id] = key + } + return byId +} + /** A column name no column of `table` holds, to park a column under during a rename cycle. */ function freeColumnName(table: TableDefinition): string { const taken = new Set(table.schema.columns.map((column) => column.name.toLowerCase())) @@ -376,11 +396,17 @@ export const reshapeTableUseCase = defineAuthorizedTableUseCase({ }) } const idByName = buildIdByName(table.schema) - if (input.sourceRows !== undefined) { - const metadata = withSourceRows( - table.metadata, - sourceRowsMark(input.sourceRows, idByName) - ) + if (input.sourceRows !== undefined || input.sourceColumnKeys !== undefined) { + let metadata: TableMetadata = table.metadata ?? {} + if (input.sourceRows !== undefined) { + metadata = withSourceRows(metadata, sourceRowsMark(input.sourceRows, idByName)) + } + if (input.sourceColumnKeys !== undefined) { + metadata = { + ...metadata, + sourceColumnKeys: sourceColumnKeysById(input.sourceColumnKeys, idByName), + } + } await tx.trx .update(userTableDefinitions) .set({ metadata }) diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 9e1abd5533c..7ad7a797cf5 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -299,18 +299,33 @@ export interface TableMetadata { */ hiddenColumns?: string[] /** - * Set while workspace source manages the table's rows (`rows: { mode: 'managed' }`), so reading - * the workspace as source prints them. Written only by the reshape that applies them. + * Set while a workspace source declaration opts the table's rows in (`rows: { mode: 'managed' }` + * or `{ mode: 'seed' }`), so reading the workspace as source prints them, or for seed rows (which + * the directory keeps) the declaration that keeps its rows file. Written only by the reshape that + * applies them. */ sourceRows?: TableSourceRows + /** + * Each source column's key (`columns: { : ... }` in workspace source) by **column id**, as + * the workspace source write that last wrote the table declared it, so reading the table as + * source prints a column under its key even after a rename. Written only by that write; a column + * without an entry is keyed by its name. + */ + sourceColumnKeys?: Record } -/** Rows workspace source manages: the whole table, each row matched by its key column. */ -export interface TableSourceRows { - mode: 'managed' - /** The id (`getColumnId`) of the unique column each row is matched by. */ - keyColumnId: string -} +/** + * Rows a workspace source declaration opts in: `managed`, the whole table, each row matched by its + * key column; `seed`, rows written into the table when source created it, which the directory + * keeps and the server does not. + */ +export type TableSourceRows = + | { + mode: 'managed' + /** The id (`getColumnId`) of the unique column each row is matched by. */ + keyColumnId: string + } + | { mode: 'seed' } /** * The saved shape of a table view: everything `TableMetadata` covers plus the diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts index 95f68dbb334..83472ee5cbf 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.fidelity.integration.ts @@ -483,6 +483,53 @@ check( } ) +check( + 'a field the model hides, added in source, is reported as dropped and apply keeps the stored value', + async () => { + const workflowId = await insertWorkflow('Fidelity hidden field') + const start = generateId() + const agentId = generateId() + await putState(workflowId, { + blocks: { + [start]: block(start, 'start_trigger', 'Start'), + [agentId]: { + ...block(agentId, 'agent', 'Answer', { + model: { id: 'model', type: 'combobox', value: 'gpt-5.4' }, + }), + advancedMode: true, + }, + }, + edges: [ + { + id: generateId(), + source: start, + target: agentId, + sourceHandle: 'source', + targetHandle: 'target', + }, + ], + loops: {}, + parallels: {}, + variables: {}, + }) + const source = await getSource(workflowId) + expect(source.files['workflow.ts']).not.toContain('temperature') + const before = await snapshot(workflowId) + + const result = await applySource( + source, + editEntry(source.files, "model: 'gpt-5.4' }", "model: 'gpt-5.4', temperature: 0.5 }") + ) + const dropped = result.diagnostics.filter((d) => d.code === 'W_FIELD_NOT_APPLICABLE') + expect(dropped.map((d) => d.message)).toEqual([ + 'temperature is hidden (model gpt-5.4 does not use it), so it never runs: an apply does not save it over the stored value, and the source it writes back drops this line.', + ]) + expect(result.diff.noop).toBe(true) + expect(result.files['workflow.ts']).not.toContain('temperature') + expect(await snapshot(workflowId)).toEqual(before) + } +) + check( 'a workflow built by editing operations and one created from its source read the same through the JSON API', async () => { @@ -530,7 +577,63 @@ check( const canvas = byName(await readGraph(canvasId)) const fromSource = byName(await readGraph(created.workflowId)) - expect(findDifferences(canvas, fromSource, 20)).toEqual([]) + // The loop's `iterations`/`loopType` subBlocks the editing operations leave beside + // `data.count` never run, so source does not carry them to the created workflow. + const differences = findDifferences(canvas, fromSource, 20).map((d) => d.path) + expect(differences.sort()).toEqual([ + '/blocks/Retry/subBlocks/iterations', + '/blocks/Retry/subBlocks/loopType', + ]) + } +) + +check( + 'a loop the editing operations built prints no superseded subBlocks, and a no-op apply keeps them stored', + async () => { + const workflowId = await insertWorkflow('Fidelity operations loop') + const start = generateId() + const loop = generateId() + await applyOperations(workflowId, [ + { operation_type: 'add', block_id: start, params: { type: 'start_trigger', name: 'Start' } }, + { + operation_type: 'add', + block_id: loop, + params: { type: 'loop', name: 'Retry', inputs: { loopType: 'for', iterations: 3 } }, + }, + { + operation_type: 'insert_into_subflow', + block_id: generateId(), + params: { + subflowId: loop, + type: 'function', + name: 'Attempt', + inputs: { code: 'return 1' }, + }, + }, + { operation_type: 'edit', block_id: start, params: { connections: { source: loop } } }, + ]) + const stored = (await readGraph(workflowId)).blocks[loop] + expect(Object.keys(stored.subBlocks ?? {}).sort()).toEqual(['iterations', 'loopType']) + expect(stored.data?.count).toBe(3) + + const source = await getSource(workflowId) + expect(source.files['workflow.ts']).toContain("loop.for('Retry', { count: 3 }") + expect(source.files['workflow.ts']).not.toContain('iterations') + expect(source.files['workflow.ts']).not.toContain('extra') + + const before = await snapshot(workflowId) + const unchanged = await applySource(source, source.files) + expect(unchanged.diff.noop).toBe(true) + expect(await snapshot(workflowId)).toEqual(before) + + const counted = await applySource( + source, + editEntry(source.files, '{ count: 3 }', '{ count: 5 }') + ) + expect(counted.applied).toBe(true) + const after = (await readGraph(workflowId)).blocks[loop] + expect(after.data?.count).toBe(5) + expect(after.subBlocks).toEqual(stored.subBlocks) } ) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index e2d516f70a7..9be934a0bd2 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -38,7 +38,7 @@ import { realtimeNotifyMock, realtimeNotifyMockFns } from '@sim/testing/mocks/re import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { isRecordLike } from '@sim/utils/object' +import { isRecordLike, omit } from '@sim/utils/object' import { SOURCE_LIMITS } from '@sim/workflow-code/limits' import { saveWorkflowToNormalizedTables } from '@sim/workflow-persistence/save' import { CONTAINER_DIMENSIONS } from '@sim/workflow-renderer/dimensions' @@ -174,6 +174,14 @@ function apply( ) } +/** The 1-based line and column where `needle` first starts in `text`. */ +function positionOf(text: string, needle: string): { line: number; column: number } { + const index = text.indexOf(needle) + expect(index, `the text holds "${needle}"`).toBeGreaterThanOrEqual(0) + const before = text.slice(0, index) + return { line: before.split('\n').length, column: index - before.lastIndexOf('\n') } +} + /** Replaces the one occurrence of `from` across the source files. */ function edit(files: Record, from: string, to: string): Record { const hits = Object.entries(files).filter(([, text]) => text.split(from).length === 2) @@ -1605,7 +1613,7 @@ describe('source apply edge cases', () => { ) check( - 'a workflow renamed in source to a name its folder holds is refused at the header', + 'a workflow renamed in source to a name its folder holds is refused at the name', async () => { const seeded = await seedWorkflow() const other = await seedWorkflow() @@ -1614,13 +1622,11 @@ describe('source apply edge cases', () => { const taken = await storedName(other.workflowId) const before = await snapshot(seeded.workflowId) - const error = await expectRejected( - apply(source, edit(source.files, `'${name}'`, `'${taken}'`)), - 'E_NAME_TAKEN' - ) + const files = edit(source.files, `'${name}'`, `'${taken}'`) + const error = await expectRejected(apply(source, files), 'E_NAME_TAKEN') expect(error.diagnostics.find((d) => d.code === 'E_NAME_TAKEN')).toMatchObject({ file: 'workflow.ts', - line: expect.any(Number), + ...positionOf(files['workflow.ts'], `'${taken}'`), }) expect(await storedName(seeded.workflowId)).toBe(name) expect(await snapshot(seeded.workflowId)).toEqual(before) @@ -1789,9 +1795,11 @@ export default workflow('', {}, () => { } const error = await expectRejected(apply(source, emptied), 'E_APPLY_BLOCK_DELETE') - expect(error.diagnostics.find((d) => d.code === 'E_APPLY_BLOCK_DELETE')?.message).toBe( - 'This apply deletes 2 blocks the source no longer declares: Start, Score.' - ) + expect(error.diagnostics.find((d) => d.code === 'E_APPLY_BLOCK_DELETE')).toMatchObject({ + message: 'This apply deletes 2 blocks the source no longer declares: Start, Score.', + file: 'workflow.ts', + ...positionOf(emptied['workflow.ts'], "'Source apply edge"), + }) expect(await snapshot(workflowId)).toEqual(before) const dryRun = await apply(source, emptied, { dryRun: true }) @@ -1971,7 +1979,11 @@ export default workflow('', {}, () => { expect(result.dryRun).toBe(true) expect(result.applied).toBe(false) expect(result.diagnostics).toContainEqual( - expect.objectContaining({ code: 'E_APPLY_STALE', severity: 'error' }) + expect.objectContaining({ + code: 'E_APPLY_STALE', + severity: 'error', + message: `Workflow "Source apply ${seeded.workflowId}" changed since this source was read. Get the source again, reapply your edit and retry.`, + }) ) expect(result.base.fingerprint).toBe(source.base.fingerprint) expect(await snapshot(seeded.workflowId)).toEqual(before) @@ -2199,3 +2211,268 @@ export default workflow('', {}, () => { expect(await snapshot(seeded.workflowId)).toEqual(before) }) }) + +describe('renames in source', () => { + /** Start → Boom → Tally → Report, each a function; Report's code reads Boom and Tally by $ref. */ + async function seedChain() { + const workflowId = generateId() + const now = new Date() + await db.insert(workflow).values({ + id: workflowId, + userId: ownerId, + workspaceId, + name: `Source renames ${workflowId}`, + lastSynced: now, + createdAt: now, + updatedAt: now, + }) + const fn = (name: string, code: string): BlockState => ({ + id: generateId(), + type: 'function', + name, + position: { x: 0, y: 0 }, + subBlocks: { code: { id: 'code', type: 'code', value: code } }, + outputs: {}, + enabled: true, + }) + const start: BlockState = { ...fn('Start', ''), type: 'start_trigger', subBlocks: {} } + const boom = fn('Boom', 'return 1') + const tally = fn('Tally', 'return 2') + const report = fn('Report', 'return + ') + const chain = [start, boom, tally, report] + const saved = await saveWorkflowToNormalizedTables(workflowId, { + blocks: Object.fromEntries(chain.map((block) => [block.id, block])), + edges: chain.slice(1).map((block, index) => ({ + id: generateId(), + source: chain[index].id, + target: block.id, + })), + loops: {}, + parallels: {}, + }) + if (!saved.success) throw new Error(`Seeding the workflow failed: ${saved.error}`) + return { workflowId, ids: { boom: boom.id, tally: tally.id, report: report.id } } + } + + const codes = (diagnostics: ReadonlyArray<{ code: string }>) => diagnostics.map((d) => d.code) + + /** Replaces every whole-word `from` in `workflow.ts`. */ + function renameIdent(files: Record, from: string, to: string) { + return { + ...files, + 'workflow.ts': files['workflow.ts'].replace(new RegExp(`\\b${from}\\b(?!\\.js)`, 'g'), to), + } + } + + check('two display-name renames in one save keep both ids and resolve their $refs', async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = edit(edit(source.files, "fn('Boom'", "fn('Bang'"), "fn('Tally'", "fn('Total'") + + const dryRun = await apply(source, files, { dryRun: true }) + expect(codes(dryRun.diagnostics)).toEqual([]) + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed).toEqual([ + { id: seeded.ids.boom, from: 'boom', to: 'bang' }, + { id: seeded.ids.tally, from: 'tally', to: 'total' }, + ]) + const after = await snapshot(seeded.workflowId) + expect(after.state?.blocks[seeded.ids.report]?.subBlocks.code?.value).toBe( + 'return + ' + ) + expect(result.files['functions/report.js']).toBe('return $ref.bang.result + $ref.total.result') + }) + + check( + 'a block whose name and const both change while its wiring changes keeps its id', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = edit( + edit(renameIdent(source.files, 'boom', 'bang'), "fn('Boom'", "fn('Bang'"), + ' tally.to(report)', + ' tally.to(report)\n bang.to(report)' + ) + + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed).toEqual([ + { id: seeded.ids.boom, from: 'boom', to: 'bang' }, + ]) + expect(codes(result.diagnostics)).toContain('W_MAT_INFERRED_RENAME') + } + ) + + check('three blocks of one type renamed name and const in one save keep their ids', async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + let files = source.files + for (const [ident, name, newIdent, newName] of [ + ['boom', 'Boom', 'bang', 'Bang'], + ['tally', 'Tally', 'total', 'Total'], + ['report', 'Report', 'summary', 'Summary'], + ]) { + files = edit(renameIdent(files, ident, newIdent), `fn('${name}'`, `fn('${newName}'`) + } + + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed.map((r) => r.id).sort()).toEqual( + [seeded.ids.boom, seeded.ids.tally, seeded.ids.report].sort() + ) + expect(codes(result.diagnostics)).not.toContain('W_UNRESOLVED_REF') + const after = await snapshot(seeded.workflowId) + expect(after.state?.blocks[seeded.ids.report]?.subBlocks.code?.value).toBe( + 'return + ' + ) + }) + + check( + 'renames nothing ties to the stored blocks are refused with how to keep the ids', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + let files = source.files + for (const [ident, name, newIdent, newName] of [ + ['boom', 'Boom', 'bang', 'Bang'], + ['tally', 'Tally', 'total', 'Total'], + ]) { + files = edit(renameIdent(files, ident, newIdent), `fn('${name}'`, `fn('${newName}'`) + } + files = edit(edit(files, 'return 1', 'return 10'), 'return 2', 'return 20') + files = edit( + files, + ' start.to(bang)\n bang.to(total)\n total.to(report)', + ' start.to(report)\n report.to(bang)\n report.to(total)' + ) + + const dryRun = await apply(source, files, { dryRun: true }) + + const refusal = dryRun.diagnostics.find((d) => d.code === 'E_APPLY_BLOCK_DELETE') + expect(refusal?.message).toBe( + 'This apply deletes 2 blocks the source no longer declares: Boom, Tally. It adds Bang, Total of the same type: if those are the same blocks renamed, both their name and const changed, so nothing ties them to the stored blocks and they would get new ids.' + ) + expect(refusal?.hint).toBe( + 'To rename a block and keep its id, keep its const name in this apply (change it in the next), or rename the references to it with it; to replace it with a new block, pass --allow-delete.' + ) + + const allowed = await apply(source, files, { dryRun: true, allowDelete: true }) + expect(allowed.diff.blocks.removed.map(({ key, type }) => `${key} ${type}`)).toEqual([ + 'boom function', + 'tally function', + ]) + expect(allowed.diff.blocks.added.map(({ key, type }) => `${key} ${type}`)).toEqual([ + 'bang function', + 'total function', + ]) + expect( + allowed.diagnostics + .filter((d) => d.code === 'W_APPLY_PROBABLE_RENAME') + .map((d) => d.message) + ).toEqual([ + 'Boom (function) is deleted and Bang (function) added: probably a rename, but its name and const both changed, so it gets a new id and loses its run history and webhook URL.', + 'Tally (function) is deleted and Total (function) added: probably a rename, but its name and const both changed, so it gets a new id and loses its run history and webhook URL.', + ]) + } + ) + + check( + 'an unrelated delete and add of one type is refused without --allow-delete, keeping no id', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = omit, string>( + { + ...edit( + edit( + edit( + source.files, + " const tally = fn('Tally', { language: null, code: codeFile('./functions/tally.js') })\n", + " const fresh = fn('Fresh', { code: js`return 'unrelated'` })\n" + ), + ' boom.to(tally)\n tally.to(report)', + ' boom.to(report)\n start.to(fresh)' + ), + 'import { codeFile, fn,', + 'import { codeFile, fn, js,' + ), + 'functions/report.js': 'return $ref.boom.result', + }, + ['functions/tally.js'] + ) + + const dryRun = await apply(source, files, { dryRun: true }) + + expect(codes(dryRun.diagnostics)).toContain('E_APPLY_BLOCK_DELETE') + expect(codes(dryRun.diagnostics)).not.toContain('W_MAT_INFERRED_RENAME') + const allowed = await apply(source, files, { allowDelete: true }) + expect(allowed.diff.blocks.removed.map((block) => block.id)).toEqual([seeded.ids.tally]) + } + ) + + check( + 'a rename of name and const that rewires, changes its code and renames its references keeps its id', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const renamed = edit(renameIdent(source.files, 'tally', 'total'), "fn('Tally'", "fn('Total'") + const files = { + ...edit(renamed, ' boom.to(total)\n', ' start.to(total)\n'), + 'functions/tally.js': 'return 3', + 'functions/report.js': 'return $ref.boom.result + $ref.total.result', + } + + const result = await apply(source, files) + + expect(result.applied).toBe(true) + expect(result.diff.blocks.removed).toEqual([]) + expect(result.diff.blocks.renamed).toEqual([ + { id: seeded.ids.tally, from: 'tally', to: 'total' }, + ]) + expect(result.diagnostics.find((d) => d.code === 'W_MAT_INFERRED_RENAME')?.message).toContain( + 'the references to it renamed with it' + ) + } + ) + + check('a trigger renamed in name and const with changed settings is refused', async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = edit( + edit( + source.files, + "const start = trigger.start('Start', {})", + "const begin = trigger.start('Begin', { inputFormat: { question: 'string' } })" + ), + ' start.to(boom)', + ' begin.to(boom)' + ) + + const dryRun = await apply(source, files, { dryRun: true }) + + expect(codes(dryRun.diagnostics)).toContain('E_APPLY_BLOCK_DELETE') + expect(codes(dryRun.diagnostics)).not.toContain('W_MAT_INFERRED_RENAME') + }) + + check( + 'a $ref to a missing block in a function file is the same warning in check and dry run, at the file', + async () => { + const seeded = await seedChain() + const source = await getSource(seeded.workflowId) + const files = { ...source.files, 'functions/boom.js': 'return $ref.nosuchblock.value\n' } + + const dryRun = await apply(source, files, { dryRun: true }) + + expect( + dryRun.diagnostics.map((d) => ({ code: d.code, severity: d.severity, file: d.file })) + ).toEqual([{ code: 'W_UNRESOLVED_REF', severity: 'warning', file: 'functions/boom.js' }]) + } + ) +}) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.ts b/apps/sim/lib/workflows/application/apply-workflow-source.ts index a8f7f34280d..428edf4a5e5 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.ts @@ -49,10 +49,10 @@ import { } from '@/lib/workflows/application/source-commit' import { emptyDiffSummary, type SourceDiffSummary } from '@/lib/workflows/application/source-diff' import { - WORKFLOW_SOURCE_STALE_MESSAGE, type WorkflowSourceDiagnostic, WorkflowSourceRejectedError, WorkflowSourceStaleError, + workflowSourceStaleMessage, } from '@/lib/workflows/application/source-error' import { evaluateSource, @@ -559,12 +559,12 @@ async function runSourceApply( ]) } if (target.kind === 'existing' && storedFingerprint !== target.base.fingerprint) { - if (!dryRun) throw new WorkflowSourceStaleError(storedFingerprint) + if (!dryRun) throw new WorkflowSourceStaleError(storedFingerprint, stored.meta.name) return refuse([ { code: 'E_APPLY_STALE', severity: 'error', - message: WORKFLOW_SOURCE_STALE_MESSAGE, + message: workflowSourceStaleMessage(stored.meta.name), }, ]) } diff --git a/apps/sim/lib/workflows/application/block-visibility-writes.integration.ts b/apps/sim/lib/workflows/application/block-visibility-writes.integration.ts new file mode 100644 index 00000000000..d42554fe47a --- /dev/null +++ b/apps/sim/lib/workflows/application/block-visibility-writes.integration.ts @@ -0,0 +1,402 @@ +/** + * Every write that places a block, against real PostgreSQL, under a `block-visibility` document + * that kill-switches `api` for everyone but platform admins and one allowlisted user. A caller who + * cannot see a type cannot add it (source apply, editing operations, state replace), a block + * already of that type stays editable, and a caller the rule reveals it to adds it. The document + * goes through the real evaluation and the admin flag through the real user row; only the + * AppConfig fetch is replaced. + */ +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { db } from '@sim/db' +import { member, organization, permissions, user, workflow, workspace } from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { envFlagsMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { saveWorkflowToNormalizedTables } from '@sim/workflow-persistence/save' +import type { BlockState } from '@sim/workflow-types/workflow' +import { eq, inArray } from 'drizzle-orm' +import { afterAll, beforeAll, expect, it, vi } from 'vitest' + +const { mockFetchProfile } = vi.hoisted(() => ({ mockFetchProfile: vi.fn() })) + +vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) +vi.mock('@/lib/core/config/appconfig', () => ({ fetchAppConfigProfile: mockFetchProfile })) +vi.mock('@/lib/realtime/notify', () => realtimeNotifyMock) + +import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' +import { applyWorkflowOperations } from '@/lib/workflows/application/apply-workflow-operations' +import { applyWorkflowSource } from '@/lib/workflows/application/apply-workflow-source' +import { readWorkflowSource } from '@/lib/workflows/application/read-workflow-source' +import { replaceWorkflowState } from '@/lib/workflows/application/replace-workflow-state' +import { WorkflowSourceRejectedError } from '@/lib/workflows/application/source-error' +import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' + +const organizationId = generateId() +const adminId = generateId() +const memberId = generateId() +const revealedId = generateId() +const workspaceId = generateId() +const userIds = [adminId, memberId, revealedId] +const admin = createSessionPrincipal({ userId: adminId, sessionId: generateId() }) +const hiddenFrom = createSessionPrincipal({ userId: memberId, sessionId: generateId() }) +const revealedTo = createSessionPrincipal({ userId: revealedId, sessionId: generateId() }) + +type Principal = typeof admin + +const VISIBILITY_DOCUMENT = { + api: { enabled: false, adminEnabled: true, userIds: [revealedId] }, +} + +const checks: { name: string; status: string; durationMs: number; error?: string }[] = [] + +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +function seedBlock( + type: string, + name: string, + subBlocks: Record = {} +): BlockState { + return { + id: generateId(), + type, + name, + position: { x: 0, y: 0 }, + subBlocks: Object.fromEntries( + Object.entries(subBlocks).map(([key, [subType, value]]) => [ + key, + { id: key, type: subType, value }, + ]) + ) as BlockState['subBlocks'], + outputs: {}, + enabled: true, + } +} + +function seedApi(url: string): BlockState { + return seedBlock('api', 'Fetch', { + url: ['short-input', url], + method: ['dropdown', 'GET'], + }) +} + +/** A workflow whose first block feeds every other block. */ +async function seedGraph(blocks: BlockState[]): Promise { + const workflowId = generateId() + const now = new Date() + await db.insert(workflow).values({ + id: workflowId, + userId: adminId, + workspaceId, + name: `Block visibility ${workflowId}`, + lastSynced: now, + createdAt: now, + updatedAt: now, + }) + const [first, ...rest] = blocks + const saved = await saveWorkflowToNormalizedTables(workflowId, { + blocks: Object.fromEntries(blocks.map((block) => [block.id, block])), + edges: rest.map((block) => ({ id: generateId(), source: first.id, target: block.id })), + loops: {}, + parallels: {}, + }) + if (!saved.success) throw new Error(`Seeding the workflow failed: ${saved.error}`) + return workflowId +} + +function seedPlainGraph(): Promise { + return seedGraph([ + seedBlock('start_trigger', 'Start'), + seedBlock('function', 'Score', { code: ['code', 'return 1'] }), + ]) +} + +function getSource(principal: Principal, workflowId: string) { + return withPermissionGroupScope(() => + readWorkflowSource.execute({ principal, input: { workflowId, functionCode: 'inline' } }) + ) +} + +function applySource( + principal: Principal, + source: Awaited>, + files: Record +) { + return withPermissionGroupScope(() => + applyWorkflowSource.execute({ + principal, + input: { workflowId: source.workflowId, files, lock: source.lock, base: source.base }, + }) + ) +} + +/** The source with a generic `api` block declared just before the workflow body closes. */ +function withAddedApi(files: Record): Record { + const text = files['workflow.ts'].replace(/import \{ /, 'import { block, ') + const close = text.lastIndexOf('})') + expect(close, 'workflow.ts closes its workflow body').toBeGreaterThan(0) + const added = + " block('api', 'Added fetch', { url: 'https://example.com/added', method: 'GET' })\n" + return { ...files, 'workflow.ts': `${text.slice(0, close)}${added}${text.slice(close)}` } +} + +function edit(files: Record, from: string, to: string): Record { + const text = files['workflow.ts'] + expect(text.split(from), `workflow.ts holds "${from}" once`).toHaveLength(2) + return { ...files, 'workflow.ts': text.replace(from, to) } +} + +function addApiOperation(principal: Principal, workflowId: string) { + return withPermissionGroupScope(() => + applyWorkflowOperations.execute({ + principal, + input: { + workflowId, + layout: 'none', + operations: [ + { + operation_type: 'add', + block_id: generateId(), + params: { + type: 'api', + name: 'Added fetch', + inputs: { url: 'https://example.com/added', method: 'GET' }, + }, + }, + ], + }, + }) + ) +} + +function replaceState( + principal: Principal, + workflowId: string, + stored: NonNullable>>, + blocks: Record +) { + return withPermissionGroupScope(() => + replaceWorkflowState.execute({ + principal, + input: { workflowId, blocks, edges: stored.edges }, + }) + ) +} + +async function storedTypes(workflowId: string): Promise { + const stored = await loadWorkflowFromNormalizedTables(workflowId) + return Object.values(stored?.blocks ?? {}).map((block) => block.type) +} + +async function loadStored(workflowId: string) { + const stored = await loadWorkflowFromNormalizedTables(workflowId) + if (!stored) throw new Error('The seeded workflow has no stored graph') + return stored +} + +beforeAll(async () => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation( + async (flag: string) => flag === 'workflow-code' + ) + setEnvFlags({ isAppConfigEnabled: true }) + mockFetchProfile.mockImplementation( + async (ids: { profile: string }, parse: (doc: unknown) => unknown) => + parse(ids.profile === 'block-visibility' ? VISIBILITY_DOCUMENT : {}) + ) + const now = new Date() + await db.insert(user).values( + userIds.map((id) => ({ + id, + name: 'Block visibility fixture', + email: `${id}@block-visibility.invalid`, + emailVerified: true, + role: id === adminId ? 'admin' : 'user', + createdAt: now, + updatedAt: now, + })) + ) + await db.insert(organization).values({ + id: organizationId, + name: 'Block visibility fixture', + slug: organizationId, + createdAt: now, + updatedAt: now, + }) + await db.insert(member).values( + userIds.map((userId) => ({ + id: generateId(), + userId, + organizationId, + role: userId === adminId ? 'owner' : 'member', + createdAt: now, + })) + ) + await insertWorkspaceFixture(db, [ + { + id: workspaceId, + name: 'Block visibility fixture', + ownerId: adminId, + billedAccountUserId: adminId, + organizationId, + workspaceMode: 'organization' as const, + }, + ]) + await db.insert(permissions).values( + userIds.map((userId) => ({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: userId === adminId ? ('admin' as const) : ('write' as const), + })) + ) +}) + +afterAll(async () => { + const reportPath = + process.env.BLOCK_VISIBILITY_WRITES_REPORT_PATH ?? + resolve('test-results/block-visibility-writes.json') + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ checks }, null, 2)) + resetEnvFlagsMock() + await deleteWorkspaceFixture(db, eq(workspace.organizationId, organizationId)) + await db.delete(organization).where(eq(organization.id, organizationId)) + await db.delete(user).where(inArray(user.id, userIds)) +}) + +check('source apply refuses adding a kill-switched type the caller cannot see', async () => { + const workflowId = await seedPlainGraph() + const source = await getSource(hiddenFrom, workflowId) + const before = await loadWorkflowFromNormalizedTables(workflowId) + + const refused = await applySource(hiddenFrom, source, withAddedApi(source.files)).then( + () => null, + (error: unknown) => error + ) + + expect(refused).toBeInstanceOf(WorkflowSourceRejectedError) + expect((refused as WorkflowSourceRejectedError).diagnostics).toContainEqual( + expect.objectContaining({ + code: 'E_APPLY_BLOCK_NOT_ALLOWED', + message: expect.stringContaining('Block type "api" is not available to you'), + }) + ) + expect(await loadWorkflowFromNormalizedTables(workflowId)).toEqual(before) +}) + +check('editing operations withhold adding a kill-switched type the caller cannot see', async () => { + const workflowId = await seedPlainGraph() + + const result = await addApiOperation(hiddenFrom, workflowId) + + expect(result.skipped).toContainEqual( + expect.objectContaining({ + type: 'block_not_allowed', + reason: expect.stringContaining('Block type "api" is not available to you'), + }) + ) + expect(await storedTypes(workflowId)).not.toContain('api') +}) + +check( + 'state replace refuses a graph adding a kill-switched type the caller cannot see', + async () => { + const workflowId = await seedPlainGraph() + const stored = await loadStored(workflowId) + const added = seedApi('https://example.com/added') + + await expect( + replaceState(hiddenFrom, workflowId, stored, { ...stored.blocks, [added.id]: added }) + ).rejects.toMatchObject({ + code: 'forbidden', + message: expect.stringContaining('Block type "api" is not available to you'), + }) + expect(await storedTypes(workflowId)).not.toContain('api') + } +) + +check('a placed block of a hidden type stays editable on every write path', async () => { + const placed = seedApi('https://example.com/before') + const workflowId = await seedGraph([seedBlock('start_trigger', 'Start'), placed]) + + const source = await getSource(hiddenFrom, workflowId) + const applied = await applySource( + hiddenFrom, + source, + edit(source.files, 'https://example.com/before', 'https://example.com/source') + ) + expect(applied.applied).toBe(true) + + const operated = await withPermissionGroupScope(() => + applyWorkflowOperations.execute({ + principal: hiddenFrom, + input: { + workflowId, + layout: 'none', + operations: [ + { + operation_type: 'edit', + block_id: placed.id, + params: { type: 'api', inputs: { url: 'https://example.com/operations' } }, + }, + ], + }, + }) + ) + expect(operated.skipped).toEqual([]) + expect((await loadStored(workflowId)).blocks[placed.id]?.subBlocks.url?.value).toBe( + 'https://example.com/operations' + ) + + const stored = await loadStored(workflowId) + const replaced = structuredClone(stored.blocks) + replaced[placed.id].subBlocks.url.value = 'https://example.com/state' + await replaceState(hiddenFrom, workflowId, stored, replaced) + expect((await loadStored(workflowId)).blocks[placed.id]?.subBlocks.url?.value).toBe( + 'https://example.com/state' + ) +}) + +check('a platform admin the rule reveals the type to adds it through source', async () => { + const workflowId = await seedPlainGraph() + const source = await getSource(admin, workflowId) + + const result = await applySource(admin, source, withAddedApi(source.files)) + + expect(result.applied).toBe(true) + expect(await storedTypes(workflowId)).toContain('api') +}) + +check('an allowlisted user adds the type through operations and state replace', async () => { + const workflowId = await seedPlainGraph() + + const result = await addApiOperation(revealedTo, workflowId) + expect(result.skipped).toEqual([]) + expect(await storedTypes(workflowId)).toContain('api') + + const stored = await loadStored(workflowId) + const added = seedApi('https://example.com/second') + await replaceState(revealedTo, workflowId, stored, { ...stored.blocks, [added.id]: added }) + expect((await storedTypes(workflowId)).filter((type) => type === 'api')).toHaveLength(2) +}) diff --git a/apps/sim/lib/workflows/application/replace-workflow-state.ts b/apps/sim/lib/workflows/application/replace-workflow-state.ts index 4bcafb107b9..2fd779f4487 100644 --- a/apps/sim/lib/workflows/application/replace-workflow-state.ts +++ b/apps/sim/lib/workflows/application/replace-workflow-state.ts @@ -10,6 +10,7 @@ import { createLogger } from '@sim/logger' import { evaluateSubBlockCondition } from '@sim/workflow-code/catalog/canonical-index' import type { BlockState, WorkflowState } from '@sim/workflow-types/workflow' import { principalAuditSource } from '@/lib/core/application' +import { getBlockVisibility } from '@/lib/core/config/block-visibility' import { OrchestrationError } from '@/lib/core/orchestration/types' import { notifyWorkflowUpdated } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' @@ -36,6 +37,7 @@ import { validateToolBindingAuthoring, } from '@/lib/workflows/tool-input/authoring' import { getBlock } from '@/blocks/registry' +import { isHiddenUnder } from '@/blocks/visibility/context' const logger = createLogger('ReplaceWorkflowState') @@ -61,6 +63,36 @@ function humanSubjectUserId(principal: Principal): string | null { } } +/** + * Refuses a graph that places a block type the acting user cannot see (an unrevealed `preview` + * block or a kill-switched one): on a block the stored graph lacks, or one it holds with another + * type. The editing operations and source apply refuse the same placement through their + * visibility gate; a block the stored graph already holds with that type stays writable, so a + * workflow keeps saving after its type is hidden. The stored graph is read only when the body + * names a hidden type at all. + */ +async function assertNoHiddenPlacement( + blocks: Record, + context: { workflowId: string; workspaceOrganizationId: string | null }, + subjectUserId: string | null, + previous: Awaited> | undefined +): Promise { + const visibility = await getBlockVisibility({ + ...(subjectUserId ? { userId: subjectUserId } : {}), + ...(context.workspaceOrganizationId ? { orgId: context.workspaceOrganizationId } : {}), + }) + const hidden = Object.values(blocks).filter((block) => { + const config = getBlock(block.type) + return config !== undefined && isHiddenUnder(visibility, config) + }) + if (hidden.length === 0) return + const stored = previous ?? (await loadWorkflowFromNormalizedTables(context.workflowId)) + const placed = hidden.find((block) => stored?.blocks[block.id]?.type !== block.type) + if (placed) { + throw new OrchestrationError('forbidden', `Block type "${placed.type}" is not available to you`) + } +} + export interface ReplaceWorkflowStateInput { workflowId: string assertedWorkspaceId?: string @@ -177,6 +209,9 @@ export const replaceWorkflowState = defineAuthorizedWorkflowUseCase({ } } + const subjectUserId = humanSubjectUserId(principal) + await assertNoHiddenPlacement(blocks, context, subjectUserId, previous) + const graph = { blocks, edges: sanitized.edges as WorkflowState['edges'], @@ -189,8 +224,6 @@ export const replaceWorkflowState = defineAuthorizedWorkflowUseCase({ * the reference pass is skipped for them rather than resolved against the * billing owner. See {@link buildWorkflowLintReport}. */ - const subjectUserId = humanSubjectUserId(principal) - const lint = await buildWorkflowLintReport(graph, { workflowId: context.workflowId, workspaceId: context.workspaceId, diff --git a/apps/sim/lib/workflows/application/source-analytics.test.ts b/apps/sim/lib/workflows/application/source-analytics.test.ts index 742fdd95586..29edc1a30ea 100644 --- a/apps/sim/lib/workflows/application/source-analytics.test.ts +++ b/apps/sim/lib/workflows/application/source-analytics.test.ts @@ -35,7 +35,7 @@ describe('recordSourceRefused', () => { beforeEach(() => capture.mockClear()) it.each([ - ['stale', new WorkflowSourceStaleError('sf1:x')], + ['stale', new WorkflowSourceStaleError('sf1:x', 'Lead Scoring')], ['rejected', new WorkflowSourceRejectedError([])], ['locked', new OrchestrationError('locked', 'Workflow is locked')], ])('records a %s refusal as ok:false with its error_kind', (kind, error) => { @@ -74,7 +74,7 @@ describe('recordSourceRefused', () => { principal: createWorkspaceApiKeyPrincipal({ workspaceId: 'ws-1' }), workflowId: 'wf-1', }, - new WorkflowSourceStaleError('sf1:x') + new WorkflowSourceStaleError('sf1:x', 'Lead Scoring') ) expect(capture).not.toHaveBeenCalled() @@ -86,7 +86,7 @@ describe('recordSourceWritten', () => { it('counts a create from source as a workflow creation, attributed to the Copilot subject', () => { const diff = emptyDiffSummary() - diff.blocks.added.push({ id: 'b-1', key: 'start' }) + diff.blocks.added.push({ id: 'b-1', key: 'start', type: 'start_trigger' }) recordSourceWritten( { @@ -125,7 +125,7 @@ describe('recordSourceDryRun', () => { it('records a clean dry run as ok with the changes a commit would make', () => { const diff = emptyDiffSummary() - diff.blocks.added.push({ id: 'b-1', key: 'start' }) + diff.blocks.added.push({ id: 'b-1', key: 'start', type: 'start_trigger' }) diff.edges.added = 2 recordSourceDryRun( diff --git a/apps/sim/lib/workflows/application/source-commit.ts b/apps/sim/lib/workflows/application/source-commit.ts index 603d1cd6e24..5f86fc8a561 100644 --- a/apps/sim/lib/workflows/application/source-commit.ts +++ b/apps/sim/lib/workflows/application/source-commit.ts @@ -227,9 +227,11 @@ async function recheckAndWriteSourceMeta( canonicalize(current.state, { catalog, meta: current.meta, resources: plan.resources }), plan.resources ) - if (fingerprint !== args.storedFingerprint) throw new WorkflowSourceStaleError(fingerprint) + if (fingerprint !== args.storedFingerprint) { + throw new WorkflowSourceStaleError(fingerprint, current.meta.name) + } if (options.layoutOnly && !isEqual(withoutBlockLayout(current), withoutBlockLayout(stored))) { - throw new WorkflowSourceStaleError(fingerprint) + throw new WorkflowSourceStaleError(fingerprint, current.meta.name) } } await writeWorkflowSourceMeta( diff --git a/apps/sim/lib/workflows/application/source-diff.ts b/apps/sim/lib/workflows/application/source-diff.ts index 305ad10d919..7ba815b8cb2 100644 --- a/apps/sim/lib/workflows/application/source-diff.ts +++ b/apps/sim/lib/workflows/application/source-diff.ts @@ -1,16 +1,21 @@ -import type { CanonicalDiff, LockIdentity } from '@sim/workflow-code' +import type { CanonicalDiff, CanonicalWorkflow, LockIdentity } from '@sim/workflow-code' interface SourceBlockRef { id: string key: string } +/** An added or removed block, with its type so a reader can spot a probable rename. */ +interface TypedSourceBlockRef extends SourceBlockRef { + type: string +} + /** What the apply changed (or, on a dry run, would change), by stored id and source key. */ export interface SourceDiffSummary { noop: boolean blocks: { - added: SourceBlockRef[] - removed: SourceBlockRef[] + added: TypedSourceBlockRef[] + removed: TypedSourceBlockRef[] renamed: Array<{ id: string; from: string; to: string }> modified: Array } @@ -26,19 +31,27 @@ export interface SourceDiffSummary { */ export function summarizeDiff( diff: CanonicalDiff, - baseIdentity: LockIdentity, - targetIdentity: LockIdentity, + base: { workflow: CanonicalWorkflow; identity: LockIdentity }, + target: { workflow: CanonicalWorkflow; identity: LockIdentity }, noop: boolean, meta: ReadonlyArray<'name' | 'description'> = [] ): SourceDiffSummary { - const targetId = (key: string) => targetIdentity.blocks[key]?.id ?? '' - const baseId = (key: string) => baseIdentity.blocks[key]?.id ?? '' + const targetId = (key: string) => target.identity.blocks[key]?.id ?? '' + const baseId = (key: string) => base.identity.blocks[key]?.id ?? '' return { noop: noop && meta.length === 0, ...(meta.length > 0 ? { meta: [...meta] } : {}), blocks: { - added: diff.blocks.added.map((key) => ({ id: targetId(key), key })), - removed: diff.blocks.removed.map((key) => ({ id: baseId(key), key })), + added: diff.blocks.added.map((key) => ({ + id: targetId(key), + key, + type: target.workflow.blocks[key]?.type ?? '', + })), + removed: diff.blocks.removed.map((key) => ({ + id: baseId(key), + key, + type: base.workflow.blocks[key]?.type ?? '', + })), renamed: diff.blocks.renamed.map(({ from, to }) => ({ id: targetId(to), from, to })), modified: diff.blocks.modified.map((change) => { const fields = change.changes.map((item) => diff --git a/apps/sim/lib/workflows/application/source-error.ts b/apps/sim/lib/workflows/application/source-error.ts index b7b69406c4c..aa1ee7724e4 100644 --- a/apps/sim/lib/workflows/application/source-error.ts +++ b/apps/sim/lib/workflows/application/source-error.ts @@ -49,13 +49,20 @@ export class WorkflowSourceRejectedError extends OrchestrationError { * edited the workflow, toggled a block lock, or a migration changed it. Re-read the source, * rebase the edit onto it and apply again. */ -/** Why a source apply refuses a draft changed since its base, as both its 409 and dry run say. */ -export const WORKFLOW_SOURCE_STALE_MESSAGE = - 'The workflow changed since this source was read. Get the source again, reapply your edit and retry.' +/** + * Why a source apply refuses a draft changed since its base, as both its 409 and dry run say, + * naming the workflow so a workspace apply's line for it says which one. + */ +export function workflowSourceStaleMessage(workflowName: string): string { + return `Workflow ${JSON.stringify(workflowName)} changed since this source was read. Get the source again, reapply your edit and retry.` +} export class WorkflowSourceStaleError extends OrchestrationError { - constructor(readonly currentFingerprint: string) { - super('conflict', WORKFLOW_SOURCE_STALE_MESSAGE) + constructor( + readonly currentFingerprint: string, + workflowName: string + ) { + super('conflict', workflowSourceStaleMessage(workflowName)) this.name = 'WorkflowSourceStaleError' } } diff --git a/apps/sim/lib/workflows/application/source-evaluate.ts b/apps/sim/lib/workflows/application/source-evaluate.ts index 7892e1ecef7..d794cb00cc3 100644 --- a/apps/sim/lib/workflows/application/source-evaluate.ts +++ b/apps/sim/lib/workflows/application/source-evaluate.ts @@ -4,10 +4,13 @@ import { workflowBlocks } from '@sim/db/schema' import { isRecordLike } from '@sim/utils/object' import { type BlockCatalog, + type CanonicalBlock, type CanonicalWorkflow, type CanonicalWorkflowMeta, + type CompileResult, canonicalize, canonicalizeUnderLock, + compileWorkflow, type Diagnostic, diffCanonical, encodeAnchor, @@ -20,6 +23,7 @@ import { renamesBetween, type SourceBase, type SourceHints, + type SourceSpan, type WorkflowLockBlock, type WorkflowLockV1, type WorkflowSourceMetaV1, @@ -116,6 +120,8 @@ export interface SourceEvaluationInput { */ export interface SourceReference extends ResourceKeyRef { member?: string + /** Where the entry file writes the reference, for an MCP tool: the agent's `tools`. */ + span?: SourceSpan } /** A resource an enclosing apply creates or adopts, by key, with the id that stands in for it. */ @@ -229,6 +235,126 @@ function withoutPendingReferences( } } +/** + * `lint` without the block-output references compile already reported in a function's code + * file (`W_UNRESOLVED_REF` on that block, located at the file line): the same dangling `$ref` + * is one warning there, as an offline check reports it, not also a lint error on the + * `codeFile()` call. + */ +function withoutCodeFileReferences( + lint: WorkflowLintReport, + compiled: CompileResult['diagnostics'], + keyOf: (id: string) => string | undefined +): WorkflowLintReport { + const reported = new Set( + compiled.flatMap((diagnostic) => + diagnostic.code === 'W_UNRESOLVED_REF' && + diagnostic.blockKey !== undefined && + diagnostic.span.file !== SOURCE_ENTRY_FILE + ? [diagnostic.blockKey] + : [] + ) + ) + if (reported.size === 0) return lint + return { + ...lint, + unresolvedReferences: lint.unresolvedReferences.filter((reference) => { + const key = keyOf(reference.blockId) + return !( + reference.kind === 'block-output' && + reference.field === 'code' && + key !== undefined && + reported.has(key) + ) + }), + } +} + +/** + * `diagnostics` without the code-file `W_UNRESOLVED_REF` findings a block rename resolves: an + * apply rewrites every `` to the new key, so `$ref.` runs. The compile + * already knew the renames the lock names by ident; a rename materialize inferred (name and + * const both changed) is known only now, so the source is compiled again with every rename and + * a finding stays only where that compile still reports it. + */ +function withoutRenamedCodeFileRefs( + diagnostics: CompileResult['diagnostics'], + renames: ReadonlyMap, + recompile: () => CompileResult +): CompileResult['diagnostics'] { + if (renames.size === 0 || !diagnostics.some((d) => d.code === 'W_UNRESOLVED_REF')) { + return diagnostics + } + const at = (d: CompileResult['diagnostics'][number]) => + `${d.span.file}:${d.span.line}:${d.span.column}` + const still = new Set( + recompile() + .diagnostics.filter((d) => d.code === 'W_UNRESOLVED_REF') + .map(at) + ) + return diagnostics.filter((d) => d.code !== 'W_UNRESOLVED_REF' || still.has(at(d))) +} + +/** `{ span }` when there is one, to spread into a diagnostic. */ +function withSpan(span: SourceSpan | undefined): { span?: SourceSpan } { + return span ? { span } : {} +} + +/** Names in a refusal, the first ten and a count of the rest. */ +function listNames(names: readonly string[]): string { + const shown = names.slice(0, 10).join(', ') + return names.length > 10 ? `${shown}, and ${names.length - 10} more` : shown +} + +/** + * `E_APPLY_BLOCK_DELETE` for the stored blocks an apply removes. When the source adds blocks of + * a removed block's type, they may be renames whose name and const both changed with no identity + * signal tying them to the stored blocks (settings, connections or renamed references): the + * finding names them and says how to keep the ids, or to pass `--allow-delete` to replace them. + */ +/** + * Each removed block with the added block of the same type and parent it most probably became: + * one pair per removed block, in order, each added block used once. + */ +function probableRenames( + removed: readonly CanonicalBlock[], + added: readonly CanonicalBlock[] +): Array<{ removed: CanonicalBlock; added: CanonicalBlock }> { + const free = [...added] + return removed.flatMap((block) => { + const index = free.findIndex( + (candidate) => candidate.type === block.type && candidate.parent === block.parent + ) + if (index === -1) return [] + const [match] = free.splice(index, 1) + return [{ removed: block, added: match }] + }) +} + +function blockDeleteRefusal( + removed: readonly CanonicalBlock[], + added: readonly CanonicalBlock[], + span: SourceSpan | undefined +): Diagnostic { + const removedTypes = new Set(removed.map((block) => block.type)) + const lookalikes = added.filter((block) => removedTypes.has(block.type)) + const deletes = `This apply deletes ${removed.length} block${removed.length === 1 ? '' : 's'} the source no longer declares: ${listNames(removed.map((block) => block.name))}.` + return { + code: 'E_APPLY_BLOCK_DELETE', + severity: 'error', + ...(lookalikes.length > 0 + ? { + message: `${deletes} It adds ${listNames(lookalikes.map((block) => block.name))} of the same type: if those are the same blocks renamed, both their name and const changed, so nothing ties them to the stored blocks and they would get new ids.`, + hint: 'To rename a block and keep its id, keep its const name in this apply (change it in the next), or rename the references to it with it; to replace it with a new block, pass --allow-delete.', + } + : { + message: deletes, + hint: 'Pass --allow-delete (allowDelete=true) to delete them, or declare them again to keep them.', + }), + ...withSpan(span), + } +} + /** Everything a commit (or a dry run) writes and reports, once every check passed. */ export interface SourcePlan { /** @@ -292,9 +418,13 @@ function isResourceKind(kind: string): kind is ResourceKind { * Every `(kind, key)` resource reference in a compiled model, for ident lookup, and the tool each * MCP agent tool (`mcp(key).tool(name)`) names on its server. */ -function collectResourceKeys(value: unknown, refs: SourceReference[]): void { +function collectResourceKeys( + value: unknown, + refs: SourceReference[], + span: SourceSpan | undefined +): void { if (Array.isArray(value)) { - for (const item of value) collectResourceKeys(item, refs) + for (const item of value) collectResourceKeys(item, refs, span) return } if (value === null || typeof value !== 'object') return @@ -308,8 +438,8 @@ function collectResourceKeys(value: unknown, refs: SourceReference[]): void { refs.push({ kind: record.kind, key: record.key }) } const tool = mcpToolReference(record) - if (tool) refs.push(tool) - for (const nested of Object.values(record)) collectResourceKeys(nested, refs) + if (tool) refs.push(span ? { ...tool, span } : tool) + for (const nested of Object.values(record)) collectResourceKeys(nested, refs, span) } /** The server and tool an MCP agent tool entry names, or `null` for anything else. */ @@ -359,14 +489,18 @@ function boundLock(identity: LockIdentity, submitted: WorkflowLockV1): WorkflowL function metaLimitRefusals( meta: CanonicalWorkflowMeta, fields: readonly (keyof CanonicalWorkflowMeta)[] -): Diagnostic[] { - const refusals: Diagnostic[] = [] - const refuse = (message: string, hint: string) => - refusals.push({ code: 'E_WORKFLOW_META_INVALID', severity: 'error', message, hint }) +): Array<{ field: keyof CanonicalWorkflowMeta; refusal: Diagnostic }> { + const refusals: Array<{ field: keyof CanonicalWorkflowMeta; refusal: Diagnostic }> = [] + const refuse = (field: keyof CanonicalWorkflowMeta, message: string, hint: string) => + refusals.push({ + field, + refusal: { code: 'E_WORKFLOW_META_INVALID', severity: 'error', message, hint }, + }) if (fields.includes('name') && meta.name.trim() === '') { - refuse('The workflow name is empty.', "Name the workflow: workflow('Name', ...).") + refuse('name', 'The workflow name is empty.', "Name the workflow: workflow('Name', ...).") } else if (fields.includes('name') && meta.name.length > V2_WORKFLOW_NAME_MAX_LENGTH) { refuse( + 'name', `The workflow name is ${meta.name.length} characters, over the ${V2_WORKFLOW_NAME_MAX_LENGTH}-character limit.`, `Shorten the name to ${V2_WORKFLOW_NAME_MAX_LENGTH} characters or fewer.` ) @@ -374,6 +508,7 @@ function metaLimitRefusals( const description = meta.description ?? '' if (fields.includes('description') && description.length > V2_WORKFLOW_DESCRIPTION_MAX_LENGTH) { refuse( + 'description', `The workflow description is ${description.length} characters, over the ${V2_WORKFLOW_DESCRIPTION_MAX_LENGTH}-character limit.`, `Shorten the description to ${V2_WORKFLOW_DESCRIPTION_MAX_LENGTH} characters or fewer.` ) @@ -411,7 +546,7 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise path !== SOURCE_ENTRY_FILE)), catalog, @@ -420,10 +555,9 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise storedResources.toId(kind, key) === null) if (newKeys.length > 0) { @@ -609,13 +748,21 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise [storedKey, key])) + const compileDiagnostics = withoutRenamedCodeFileRefs(compiled.diagnostics, renamedKeys, () => + compileWorkflow({ ...compileInput, stored: compiled.stored, renamedKeys }) ) const sourceRefusals: Diagnostic[] = [] const headerSpan = compiled.spans[encodeAnchor({ kind: 'header' })] + /** Where a finding about the workflow itself points: the field it names, else the name. */ + const metaSpan = (field: keyof CanonicalWorkflowMeta = 'name') => + compiled.metaSpans[field] ?? compiled.metaSpans.name ?? headerSpan if ( metaChanged.includes('name') && (await workflowNameExistsInFolder({ @@ -630,27 +777,32 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise ({ + ...metaLimitRefusals(meta, isNew ? META_FIELDS : metaChanged).map(({ field, refusal }) => ({ ...refusal, - ...(headerSpan ? { span: headerSpan } : {}), + ...withSpan(metaSpan(field)), })) ) const removedKeys = isNew ? [] : canonicalDiff.blocks.removed + const removedBlocks = removedKeys.map((key) => storedCanonical.workflow.blocks[key]) + const addedBlocks = canonicalDiff.blocks.added.map((key) => materialized.workflow.blocks[key]) if (removedKeys.length > 0 && !input.allowDelete) { - const names = removedKeys.map((key) => storedCanonical.workflow.blocks[key]?.name ?? key) - const shown = names.slice(0, 10).join(', ') - sourceRefusals.push({ - code: 'E_APPLY_BLOCK_DELETE', - severity: 'error', - message: `This apply deletes ${removedKeys.length} block${removedKeys.length === 1 ? '' : 's'} the source no longer declares: ${shown}${names.length > 10 ? `, and ${names.length - 10} more` : ''}.`, - hint: 'Pass --allow-delete (allowDelete=true) to delete them, or declare them again to keep them.', - ...(headerSpan ? { span: headerSpan } : {}), - }) + sourceRefusals.push(blockDeleteRefusal(removedBlocks, addedBlocks, metaSpan())) } + const renameWarnings = input.allowDelete + ? probableRenames(removedBlocks, addedBlocks).map( + ({ removed, added }): Diagnostic => ({ + code: 'W_APPLY_PROBABLE_RENAME', + severity: 'warning', + message: `${removed.name} (${removed.type}) is deleted and ${added.name} (${added.type}) added: probably a rename, but its name and const both changed, so it gets a new id and loses its run history and webhook URL.`, + hint: `To keep the id, apply ${added.name} under the const or name ${removed.name} had first, then change the other.`, + blockKey: added.key, + }) + ) + : [] const storedGraph: SourceGraph = { blocks: state.blocks, edges: state.edges } let blocks = materialized.state.blocks @@ -678,6 +830,7 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise m.id)]) const diagnostics = [ - ...compiled.diagnostics, + ...compileDiagnostics, ...keptDiagnostics, ...materializeDiagnostics, ...refusals, ...normalized.diagnostics, ...lintDiagnostics( - withoutPendingReferences(lint, input.pendingResources ?? []), + withoutCodeFileReferences( + withoutPendingReferences(lint, input.pendingResources ?? []), + compileDiagnostics, + (id) => keyById.get(id) + ), changedIds, (id) => keyById.get(id) ), @@ -761,8 +918,8 @@ export async function evaluateSource(input: SourceEvaluationInput): Promise = { }, } +/** + * Refuses placing a type the acting user cannot see: an unrevealed `preview` block or one the + * `block-visibility` kill switch names, by the catalog's own predicate ({@link isHiddenUnder}), so a + * write cannot add a block the catalog hides. Placed instances keep executing and stay editable, so + * a block keeping its own type is not refused. Reads the visibility in scope: outside + * `withBlockVisibility` preview blocks stay hidden and nothing is kill-switched. + */ +const VISIBLE_TYPE: PlacementGate<'block_not_allowed'> = { + subject: 'placement', + skipTypes: ['block_not_allowed'], + check: ({ blockType, previousType, blockId, operationType, outcome, details }) => { + if (blockType === previousType) return null + const block = getBlock(blockType) + if (!block || !isHiddenUnder(overlayVisibility(), block)) return null + return { + type: 'block_not_allowed', + operationType, + blockId, + reason: `Block type "${blockType}" is not available to you - ${outcome}`, + details: { requestedType: blockType, ...details }, + } + }, +} + /** * Refuses placing a sunset (`legacy` / `deprecated`) type and names its successor. Placed * instances keep executing and stay editable, so a block keeping its own type is not refused. @@ -373,6 +397,7 @@ const PERMITTED_TOOL: ToolEntryGate<'tool_not_allowed'> = { export const BLOCK_GATES: readonly BlockGate[] = [ REGISTERED_TYPE, AVAILABLE_TYPE, + VISIBLE_TYPE, ACTIVE_TYPE, SINGLE_TRIGGER, SINGLE_INSTANCE, diff --git a/apps/sim/lib/workspace-source/api/route-policies.ts b/apps/sim/lib/workspace-source/api/route-policies.ts index 3722b7c8273..ca9dfc24e18 100644 --- a/apps/sim/lib/workspace-source/api/route-policies.ts +++ b/apps/sim/lib/workspace-source/api/route-policies.ts @@ -9,6 +9,7 @@ import { createV2ResourceConcealmentPolicy } from '@/lib/api/server/routes' import type { ApplicationOperation, AuthorizingUseCase } from '@/lib/core/application' import { WorkspaceSourceRejectedError } from '@/lib/workspace-source/application/errors' import { WorkspaceSourceBusyError } from '@/lib/workspace-source/lease/store' +import { SourceStagingFullError } from '@/lib/workspace-source/staging/store' import { v2CaughtOrchestrationError, v2Error, @@ -66,7 +67,8 @@ export function withSourceSlot( * status (`400`, `409` stale, `423` locked, `413` over a limit) with `details.diagnostics`; * `details.code` is `SOURCE_STALE` when a resource changed since the directory was read, else * `SOURCE_REJECTED`. An apply while another one writes the workspace answers `409` with - * `details.code` `SOURCE_BUSY`. + * `details.code` `SOURCE_BUSY`. Staging past the caller's staging caps answers `413` with + * `details.code` `SOURCE_STAGING_FULL`, apart from a body over the request limit. */ export const v2WorkspaceSourceErrorPolicy = createV2ResourceConcealmentPolicy({ notFoundMessage: 'Workspace not found', @@ -85,6 +87,9 @@ export const v2WorkspaceSourceErrorPolicy = createV2ResourceConcealmentPolicy({ if (error instanceof WorkspaceSourceBusyError) { return v2ErrorForOrchestration(error.code, error.message, { code: 'SOURCE_BUSY' }) } + if (error instanceof SourceStagingFullError) { + return v2ErrorForOrchestration(error.code, error.message, { code: 'SOURCE_STAGING_FULL' }) + } return v2CaughtOrchestrationError(error) }, }) diff --git a/apps/sim/lib/workspace-source/application/cross-kind.integration.ts b/apps/sim/lib/workspace-source/application/cross-kind.integration.ts index 78388601d77..d1cc50a3570 100644 --- a/apps/sim/lib/workspace-source/application/cross-kind.integration.ts +++ b/apps/sim/lib/workspace-source/application/cross-kind.integration.ts @@ -84,7 +84,10 @@ import { applyWorkspaceSource, } from '@/lib/workspace-source/application/apply-workspace-source' import { readWorkspaceSource } from '@/lib/workspace-source/application/read-workspace-source' -import { findMissingSourceBlobs } from '@/lib/workspace-source/application/stage-blobs' +import { + findMissingSourceBlobs, + stageSourceBlobs, +} from '@/lib/workspace-source/application/stage-blobs' import { blobManifest, codes, @@ -630,6 +633,41 @@ export default workflow('Answer', {}, () => { } ) + check( + 'renaming a table column an unchanged declared workflow writes warns at the line that writes it', + async () => { + const { workspaceId, dir, base } = await setUp() + const printed = (await get(workspaceId)).files['workflows/ingest/workflow.ts'] + const lines = printed.split('\n') + const dataLine = lines.findIndex((line) => line.includes('email: start.email')) + 1 + const files = { + ...dir.files, + 'tables/leads.ts': dir.files['tables/leads.ts'].replace( + 'column.string({ required: true })', + "column.string({ required: true, name: 'email_address' })" + ), + } + expect(files['tables/leads.ts']).toContain('email_address') + + const planned = await apply(workspaceId, { ...dir, files }, { base, dryRun: true }) + + const renamed = planned.diagnostics.filter( + (finding) => finding.code === 'W_TABLE_COLUMN_RENAMED_IN_USE' + ) + expect(renamed).toEqual([ + expect.objectContaining({ + severity: 'warning', + kind: 'table', + key: 'leads', + message: + 'tables/leads renames column "email" to "email_address", but block "Save Lead" of workflow "Ingest" still writes "email"; the run drops it.', + file: 'workflows/ingest/workflow.ts', + line: dataLine, + }), + ]) + } + ) + check( 'deleting a table a workflow outside the directory uses warns, naming that workflow', async () => { @@ -1009,7 +1047,7 @@ export default workflow('Big', {}, () => { check('workspace files and knowledge documents hash within one per-request budget', async () => { const workspaceId = await newWorkspace() - const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest / 2.5) + const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxHashBytesPerRequest / 2.5) for (const name of ['a.txt', 'b.txt']) { await uploadWorkspaceFile( workspaceId, @@ -1053,10 +1091,17 @@ export default workflow('Big', {}, () => { () => true ) - const refused = await stageBlobsAs( - owner, - workspaceId, - blobs.map((blob) => ({ sha256: blob.sha256, content: blob.content ?? '' })) + const refused = await withPermissionGroupScope(() => + stageSourceBlobs.execute({ + principal: owner, + input: { + workspaceId, + blobs: blobs.map((blob) => ({ + sha256: blob.sha256, + content: Buffer.from(blob.content ?? '', 'base64'), + })), + }, + }) ).then( () => null, (error: unknown) => error @@ -1070,6 +1115,19 @@ export default workflow('Big', {}, () => { }) ) expect(missing.missing).toHaveLength(3) + + const [alone] = blobManifest( + { 'files/alone.bin': Buffer.alloc(WORKSPACE_SOURCE_LIMITS.maxBlobBytes, 'd') }, + () => true + ) + await stageBlobsAs(owner, workspaceId, [{ sha256: alone.sha256, content: alone.content ?? '' }]) + const stillMissing = await withPermissionGroupScope(() => + findMissingSourceBlobs.execute({ + principal: owner, + input: { workspaceId, blobs: [{ sha256: alone.sha256, size: alone.size }] }, + }) + ) + expect(stillMissing.missing).toEqual([]) }) check('a plan already refused authorizes no change', async () => { diff --git a/apps/sim/lib/workspace-source/application/deployments.integration.ts b/apps/sim/lib/workspace-source/application/deployments.integration.ts index 9b58488697c..592d012ead2 100644 --- a/apps/sim/lib/workspace-source/application/deployments.integration.ts +++ b/apps/sim/lib/workspace-source/application/deployments.integration.ts @@ -1055,6 +1055,61 @@ describe('workspace source: deployments', () => { } ) + check( + 'warns when a column rename breaks the live version of a workflow whose draft writes the new name, unless the apply redeploys', + async () => { + const workspaceId = await workspaces.create({ name: 'Deployed column user', ownerId }) + const kinds: WorkspaceKind[] = ['table', 'workflow', 'deployment'] + const created = await apply( + workspaceId, + { + 'tables/leads.ts': LEADS_TABLE, + 'workflows/ingest/workflow.ts': INGEST_LEADS, + 'deployments/ingest.ts': DRAFT, + }, + { kinds } + ) + expect(created.failed).toBeUndefined() + const read = await getAll(workspaceId, kinds) + // The table renames email, and the draft writes the new name; the live version still writes email. + const renamed: Files = { + ...read.files, + 'tables/leads.ts': LEADS_TABLE.replace( + 'email: column.string(),', + "email: column.string({ name: 'email_address' })," + ), + 'workflows/ingest/workflow.ts': INGEST_LEADS.replace( + 'data: { email: start.email }', + 'data: { email_address: start.email }' + ), + } + + const { 'deployments/ingest.ts': _deployment, ...undeclaredDeployment } = renamed + const kept = await apply(workspaceId, undeclaredDeployment, { + kinds: ['table', 'workflow'], + base: read.base, + dryRun: true, + }) + expect(codes(kept.diagnostics)).toEqual([]) + const warning = kept.diagnostics.find( + (finding) => finding.code === 'W_TABLE_COLUMN_RENAMED_IN_USE' + ) + expect(warning).toMatchObject({ kind: 'table', key: 'leads' }) + expect(warning?.message).toContain('live version') + + const redeployed = await apply(workspaceId, renamed, { + kinds, + base: read.base, + dryRun: true, + }) + expect(codes(redeployed.diagnostics)).toEqual([]) + expect(actions(redeployed.plan)).toContain('update deployment:ingest') + expect(codes(redeployed.diagnostics, 'warning')).not.toContain( + 'W_TABLE_COLUMN_RENAMED_IN_USE' + ) + } + ) + check('refuses a deployment of a workflow the workspace does not hold', async () => { const workspaceId = await workspaces.create({ name: 'Deploy nothing', ownerId }) const refused = await refusal(apply(workspaceId, { 'deployments/missing.ts': DRAFT })) diff --git a/apps/sim/lib/workspace-source/application/files.integration.ts b/apps/sim/lib/workspace-source/application/files.integration.ts index 256337c7be5..b049c8cce33 100644 --- a/apps/sim/lib/workspace-source/application/files.integration.ts +++ b/apps/sim/lib/workspace-source/application/files.integration.ts @@ -775,7 +775,7 @@ describe('workspace source files', () => { 'server hashing stops at the per-request budget: a get lists the rest without a hash, and a plan refuses them rather than creating', async () => { const workspaceId = await newWorkspace() - const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest / 2.5) + const size = Math.floor(WORKSPACE_SOURCE_LIMITS.maxHashBytesPerRequest / 2.5) const directory: Record = {} for (const name of ['a.bin', 'b.bin', 'c.bin']) { const bytes = Buffer.alloc(size, name) diff --git a/apps/sim/lib/workspace-source/application/hash-budget.ts b/apps/sim/lib/workspace-source/application/hash-budget.ts index 922d95fedf4..fa4e147af5a 100644 --- a/apps/sim/lib/workspace-source/application/hash-budget.ts +++ b/apps/sim/lib/workspace-source/application/hash-budget.ts @@ -18,7 +18,7 @@ export interface HashBudget { /** A full hashing budget for one request. */ export function newHashBudget(): HashBudget { - return { bytes: WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest, count: MAX_HASHED_PER_REQUEST } + return { bytes: WORKSPACE_SOURCE_LIMITS.maxHashBytesPerRequest, count: MAX_HASHED_PER_REQUEST } } /** Downloads one plan or get runs at once while hashing. */ diff --git a/apps/sim/lib/workspace-source/application/in-use.ts b/apps/sim/lib/workspace-source/application/in-use.ts index 2e6314363c4..98b99bb09bc 100644 --- a/apps/sim/lib/workspace-source/application/in-use.ts +++ b/apps/sim/lib/workspace-source/application/in-use.ts @@ -136,7 +136,7 @@ export function inUseUncheckedFinding(change: Pick { +export function liveVersionsReplaced(changes: readonly PlannedChange[]): Set { const replaced = new Set() for (const change of changes) { if (change.kind !== 'deployment' || change.held) continue diff --git a/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts b/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts index 83a439bc37e..3a82b53167e 100644 --- a/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts +++ b/apps/sim/lib/workspace-source/application/kinds/knowledge/diff.ts @@ -166,7 +166,10 @@ export async function hashDocs( return [...left, ...failed] } -/** The info line a change gets for the documents it indexes, and for chunking it changes. */ +/** + * The info line a change gets for the documents it indexes, and for chunking it changes: + * `storedDocs` counts the stored documents that keep their chunks (neither replaced nor deleted). + */ export function indexingFindings( key: string, uploads: readonly DocUpload[], @@ -355,6 +358,16 @@ export async function diffKnowledgeBase(input: { ) } } - changes.diagnostics.push(...indexingFindings(key, changes.uploads, chunkingChanged, byName.size)) + const reindexed = new Set([ + ...changes.uploads.flatMap((upload) => (upload.replaces ? [upload.replaces.id] : [])), + ...changes.finishes.map((finish) => finish.replaces.id), + ...changes.deletes.map((doc) => doc.id), + ]) + // A finished replacement is already stored and indexed, so it keeps its chunks too. + const keepingChunks = + [...byName.values()].filter((doc) => !reindexed.has(doc.id)).length + changes.finishes.length + changes.diagnostics.push( + ...indexingFindings(key, changes.uploads, chunkingChanged, keepingChunks) + ) return changes } diff --git a/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts index 761c9de0d78..2ae18d1be15 100644 --- a/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/mcp/adapter.ts @@ -23,9 +23,14 @@ import { createLogger } from '@sim/logger' import { mapWithConcurrency } from '@sim/utils/concurrency' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { didYouMean, type Json } from '@sim/workflow-code' +import { didYouMean, type Json, type SourceSpan } from '@sim/workflow-code' import { SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' -import { type BaseResource, fingerprintOf, resourceAddress } from '@sim/workspace-code' +import { + type BaseResource, + type FlatDiagnostic, + fingerprintOf, + resourceAddress, +} from '@sim/workspace-code' import { MCP_DEFAULTS, type McpServerDecl, @@ -50,6 +55,7 @@ import type { McpServerRow } from '@/lib/mcp/queries' import { generateMcpServerId } from '@/lib/mcp/utils' import { admitOnce } from '@/lib/workspace-source/application/context' import { + finding, kindFinding, planUndeclared, serverGoneFinding, @@ -427,8 +433,11 @@ async function listTools( /** Each change that calls tools of an MCP server, with the tools, by server key. */ function toolUsers( changes: readonly PlannedChange[] -): Map }>> { - const byServer = new Map }>>() +): Map }>> { + const byServer = new Map< + string, + Array<{ change: PlannedChange; tools: Map }> + >() for (const change of changes) { if (change.action === 'delete') continue for (const ref of change.refs ?? []) { @@ -436,21 +445,33 @@ function toolUsers( const users = byServer.get(ref.key) ?? [] let user = users.find((entry) => entry.change === change) if (!user) { - user = { change, tools: new Set() } + user = { change, tools: new Map() } users.push(user) } - user.tools.add(ref.member) + if (!user.tools.has(ref.member)) user.tools.set(ref.member, ref.span) byServer.set(ref.key, users) } } return byServer } -/** Where a finding about the tools a change calls points: its workflow file. */ -function fileOf(change: PlannedChange): { file?: string } { - return change.kind === 'workflow' - ? { file: `${workflowFolder(change.key)}/${SOURCE_ENTRY_FILE}` } - : {} +/** + * Where a finding about the tools a change calls points: its workflow file, at the line that + * writes the tool when the workflow's compile located it. + */ +function fileOf(change: PlannedChange, span?: SourceSpan): Partial { + if (change.kind !== 'workflow') return {} + return { + file: `${workflowFolder(change.key)}/${SOURCE_ENTRY_FILE}`, + ...(span + ? { + line: span.line, + column: span.column, + endLine: span.endLine, + endColumn: span.endColumn, + } + : {}), + } } export const mcpAdapter: KindAdapter<'mcp-server', McpServerRow> = { @@ -631,18 +652,18 @@ export const mcpAdapter: KindAdapter<'mcp-server', McpServerRow> = { } const names = new Set(listed.names) for (const { change, tools } of callers) { - for (const tool of tools) { + for (const [tool, span] of tools) { if (names.has(tool)) continue const suggestion = didYouMean(tool, names, (name) => `"${name}"`) change.diagnostics.push( - kindFinding( - change.kind, - change.key, + finding( 'E_MCP_TOOL_UNKNOWN', `${where(change.kind, change.key)} calls tool "${tool}" on MCP server "${key}", which does not list it.${suggestion ? ` ${suggestion}` : ''}`, { hint: `The server lists: ${[...names].slice(0, 20).join(', ') || 'no tools'}.`, - ...fileOf(change), + ...fileOf(change, span), + kind: change.kind, + key: change.key, } ) ) diff --git a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts index 7824cbb7272..8a7944b076b 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts @@ -49,6 +49,7 @@ import { deleteTableUseCase, listTableDefinitionsUseCase, } from '@/lib/table/application/tables' +import { getColumnId } from '@/lib/table/column-keys' import { getTableById } from '@/lib/table/service' import type { RowData } from '@/lib/table/types' import { validateRowSize } from '@/lib/table/validation' @@ -70,6 +71,7 @@ import { isColumnOp, type TableChanges, } from '@/lib/workspace-source/application/kinds/table/diff' +import { checkColumnRenamesInUse } from '@/lib/workspace-source/application/kinds/table/renames' import { baseRows, namedRows, @@ -80,6 +82,7 @@ import { import { printStoredTable, type StoredTableState, + sourceColumns, storedFingerprint, tableAsDecl, } from '@/lib/workspace-source/application/kinds/table/stored' @@ -98,12 +101,18 @@ import type { interface RowsPlan { /** The rows the reshape writes (only those that differ); absent leaves the rows alone. */ write?: ReshapeRows - /** The managed mark to set, by key column name, or `null` to clear it; absent leaves it. */ - mark?: { mode: 'managed'; keyColumn: string } | null + /** + * The rows mark to set (managed rows by key column name, or seed rows), or `null` to clear it; + * absent leaves it. + */ + mark?: RowsMark | null /** The row version the plan read the stored rows at; the apply refuses one that moved. */ version?: number } +/** How a declaration opts its rows in, as the table's mark records it. */ +type RowsMark = { mode: 'managed'; keyColumn: string } | { mode: 'seed' } + /** What the apply carries out for one table, decided at plan time. */ export interface TablePlan { /** The changes an update or adopt makes; absent for a create or delete. */ @@ -111,6 +120,11 @@ export interface TablePlan { /** The base's column bindings the plan's fingerprint was read under. */ bindings?: Record rows?: RowsPlan + /** + * The column keys the write records (`metadata.sourceColumnKeys`), by stored column name: each + * declared column whose key is not its name. Absent leaves the record alone. + */ + keys?: Record } type TableChange = PlannedChange<'table'> @@ -162,8 +176,60 @@ function namesByKey(decl: TableDecl): Map { ) } -/** The managed mark a declaration asks for, by its key column's stored name. */ -function markOf(decl: TableDecl): { mode: 'managed'; keyColumn: string } | null { +/** The keys `decl` gives its columns where a key is not the column's name, by column name. */ +function declaredKeys(decl: TableDecl): Record { + const keys: Record = {} + for (const [key, column] of Object.entries(decl.columns)) { + const name = columnName(key, column) + if (name !== key) setOwn(keys, name, key) + } + return keys +} + +/** + * Whether a read of the stored table without a base (a fresh get, or a promotion's read of the + * target) would print a declared column under another key than `decl` gives it. + */ +function keysUnrecorded(state: StoredTableState, decl: TableDecl): boolean { + const { keyById } = tableAsDecl(state) + const idByName = new Map( + sourceColumns(state.table).map((column) => [column.name, getColumnId(column)]) + ) + return Object.entries(decl.columns).some(([key, column]) => { + const id = idByName.get(columnName(key, column)) + return id !== undefined && keyById.get(id) !== key + }) +} + +/** + * `change` recording `decl`'s column keys: every table write records them, and a no-op becomes a + * write of the record alone when a read without a base would print a column under another key. + */ +function withKeys( + change: TableChange, + state: StoredTableState, + decl: TableDecl, + entry: BaseResource | undefined +): TableChange { + const keys = declaredKeys(decl) + if (change.action === 'update' || change.action === 'adopt') { + return { ...change, detail: { ...change.detail, keys } } + } + if (change.action !== 'noop' || !keysUnrecorded(state, decl)) return change + const bindings = entry?.id === state.table.id ? entry.columns : undefined + return { + ...change, + action: 'update', + storedId: state.table.id, + fields: ['column keys'], + storedFingerprint: storedFingerprint(state, bindings), + detail: { ...(bindings ? { bindings } : {}), keys }, + } +} + +/** The rows mark a declaration asks for: managed rows by their key column's stored name, or seed. */ +function markOf(decl: TableDecl): RowsMark | null { + if (decl.rows?.mode === 'seed') return { mode: 'seed' } const key = decl.rows?.mode === 'managed' ? decl.rows.key : undefined const column = key !== undefined ? decl.columns[key] : undefined return key !== undefined && column @@ -177,7 +243,7 @@ function rowsWrite(decl: TableDecl, allowDelete: boolean): ReshapeRows | undefin if (!rows) return undefined const named = namedRows(rows, namesByKey(decl)) const mark = markOf(decl) - return rows.mode === 'managed' && mark + return rows.mode === 'managed' && mark?.mode === 'managed' ? { mode: 'managed', keyColumn: mark.keyColumn, @@ -285,9 +351,16 @@ async function planRows( const { decl: storedDecl, keyById } = tableAsDecl(state, entry?.columns) const storedKey = storedDecl.rows?.key if (rows?.mode !== 'managed') { - if (storedKey !== undefined) { - outcome.plan = { mark: null } - outcome.fields.push('rows (no longer managed)') + const storedMode = storedDecl.rows?.mode + if (storedMode !== rows?.mode) { + outcome.plan = { mark: markOf(decl) } + outcome.fields.push( + storedMode === 'managed' + ? 'rows (no longer managed)' + : rows + ? 'rows (seed)' + : 'rows (no longer seeded)' + ) } if (rows?.mode === 'seed' && rows.data.length > 0 && !entry) { if (state.table.rowCount === 0) { @@ -456,7 +529,10 @@ async function planTable( // A table whose declaration is unchanged can still hold rows to write. const settled = planned.settled const resource = resolution.action === 'noop' ? resolution.stored : undefined - if (!resource || settled.action !== 'noop' || !decl.rows) return settled + if (!resource || settled.action !== 'noop') return settled + if (!decl.rows && !resource.state.table.metadata?.sourceRows) { + return withKeys(settled, resource.state, decl, entry) + } const rows = await planRows(key, decl, resource.state, entry, ctx) const renamed = rows.plan?.write ? columnsRenamedOnServer(decl, resource.state, entry) : [] if (renamed.length > 0) { @@ -471,7 +547,7 @@ async function planTable( ) return settled } - return withRows(settled, rows, resource.state, entry) + return withKeys(withRows(settled, rows, resource.state, entry), resource.state, decl, entry) } switch (planned.action) { case 'create': { @@ -491,7 +567,7 @@ async function planTable( ...rows.fields, ], pending: pendingOf({ ...change, action: 'create' }, { columns: declaredColumnNames(decl) }), - detail: { rows: rows.plan }, + detail: { rows: rows.plan, keys: declaredKeys(decl) }, } } case 'adopt': @@ -533,45 +609,51 @@ async function planTable( } const action = planned.action === 'adopt' ? 'adopt' : changed ? 'update' : 'noop' const storedId = planned.stored.id - return { - ...change, - action, - storedId, - // A later workflow's dry run checks its row writes against the columns this declares. - ...(action !== 'noop' - ? { - pending: pendingOf( - { ...change, action, storedId }, - { columns: declaredColumnNames(decl) } - ), - } - : {}), - ...(changes.fields.length + rows.fields.length > 0 - ? { fields: [...changes.fields, ...rows.fields] } - : {}), - ...(dropped > 0 || changes.lostCells > 0 || rows.removed > 0 - ? { - destructive: { - what: 'cells', - counts: { - columns: dropped, - ...(changes.removedOptions > 0 ? { options: changes.removedOptions } : {}), - cells: changes.lostCells, - ...(rows.removed > 0 ? { rows: rows.removed } : {}), + const entryOfStored = planned.action === 'update' ? planned.base : undefined + return withKeys( + { + ...change, + action, + storedId, + // A later workflow's dry run checks its row writes against the columns this declares. + ...(action !== 'noop' + ? { + pending: pendingOf( + { ...change, action, storedId }, + { columns: declaredColumnNames(decl) } + ), + } + : {}), + ...(changes.fields.length + rows.fields.length > 0 + ? { fields: [...changes.fields, ...rows.fields] } + : {}), + ...(dropped > 0 || changes.lostCells > 0 || rows.removed > 0 + ? { + destructive: { + what: 'cells', + counts: { + columns: dropped, + ...(changes.removedOptions > 0 ? { options: changes.removedOptions } : {}), + cells: changes.lostCells, + ...(rows.removed > 0 ? { rows: rows.removed } : {}), + }, }, - }, - } - : {}), - storedFingerprint: planFingerprint(storedFingerprint(state, bindings), rows.plan), - ...(action === 'noop' && rows.base - ? { base: { ...printStoredTable(state, key, bindings).base, rows: rows.base } } - : {}), - detail: { - changes, - ...(bindings ? { bindings } : {}), - ...(rows.plan ? { rows: rows.plan } : {}), + } + : {}), + storedFingerprint: planFingerprint(storedFingerprint(state, bindings), rows.plan), + ...(action === 'noop' && rows.base + ? { base: { ...printStoredTable(state, key, bindings).base, rows: rows.base } } + : {}), + detail: { + changes, + ...(bindings ? { bindings } : {}), + ...(rows.plan ? { rows: rows.plan } : {}), + }, }, - } + state, + decl, + entryOfStored + ) } } } @@ -663,6 +745,7 @@ function reshapeInput( ...(rows?.write ? { rows: rows.write } : {}), ...(rows?.mark !== undefined ? { sourceRows: rows.mark } : {}), ...(rows?.version !== undefined ? { expectedRowsVersion: rows.version } : {}), + ...(plan?.keys !== undefined ? { sourceColumnKeys: plan.keys } : {}), } if (!plan?.changes) return { tableId, workspaceId, ...rowsInput } const { table, columns } = plan.changes @@ -700,6 +783,7 @@ async function managedRowsMovedSinceBase( } export const tableAdapter: KindAdapter<'table', StoredTableState> = { + checkRefs: checkColumnRenamesInUse, codec: tableCodec, async load(ctx, { allocate }) { @@ -804,17 +888,21 @@ export const tableAdapter: KindAdapter<'table', StoredTableState> = { }, }) const rows = change.detail?.rows - if (rows?.write || rows?.mark) { + const keys = change.detail?.keys + const recordsKeys = keys !== undefined && Object.keys(keys).length > 0 + if (rows?.write || rows?.mark || recordsKeys) { await reshapeTableUseCase.execute({ principal, - input: reshapeInput(ctx, table.id, { rows }), + input: reshapeInput(ctx, table.id, { rows, ...(recordsKeys ? { keys } : {}) }), }) } return written(ctx, change.key, table.id, decl) } const tableId = change.storedId const plan = change.detail - if (!tableId || !(plan?.changes || plan?.rows)) throw new Error('A table update needs its plan') + if (!tableId || !(plan?.changes || plan?.rows || plan?.keys)) { + throw new Error('A table update needs its plan') + } await reshapeTableUseCase.execute({ principal, input: reshapeInput(ctx, tableId, plan) }) return written(ctx, change.key, tableId, decl) }, diff --git a/apps/sim/lib/workspace-source/application/kinds/table/diff.ts b/apps/sim/lib/workspace-source/application/kinds/table/diff.ts index 88e09ec06c1..6f53119171f 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/diff.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/diff.ts @@ -66,6 +66,10 @@ export interface TableChanges { columns: ColumnChange[] /** Fields that change, for the plan display. */ fields: string[] + /** Stored columns this update renames, old name → new name. */ + renamedColumns: Array<{ from: string; to: string }> + /** Every column name the table holds after the update, in declaration order. */ + columnNames: string[] /** Cells a drop empties, and cells a removed option takes a selection from. */ lostCells: number /** @@ -249,6 +253,10 @@ export async function diffTable( table: {}, columns: [], fields: [], + renamedColumns: [], + columnNames: Object.entries(decl.columns).map(([columnKey, column]) => + columnName(columnKey, column) + ), lostCells: 0, removedOptions: 0, diagnostics, @@ -320,6 +328,7 @@ export async function diffTable( if (target.name !== current.name) { updates.name = target.name fields.push('name') + changes.renamedColumns.push({ from: current.name, to: target.name }) } const retyped = target.type !== current.type const becomesUnique = Boolean(target.unique) && !current.unique diff --git a/apps/sim/lib/workspace-source/application/kinds/table/renames.ts b/apps/sim/lib/workspace-source/application/kinds/table/renames.ts new file mode 100644 index 00000000000..790ee72e3b9 --- /dev/null +++ b/apps/sim/lib/workspace-source/application/kinds/table/renames.ts @@ -0,0 +1,184 @@ +/** + * Workflows that still write a table column by the name a plan renames. A row write, filter or + * sort names a column by its name, so after the rename the run drops the cell (or matches + * nothing); the plan warns on each block, at the line its workflow file writes the old name when + * the directory declares that workflow. A workflow folder the apply evaluates is checked by its own + * lint against the declared columns, so only workflows the apply leaves as stored are scanned here. + */ +import { escapeRegExp } from '@sim/utils/string' +import { splitLines } from '@sim/workflow-code' +import { SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' +import { workflowFolder } from '@sim/workspace-code/kinds/workflow' +import { readWorkflowSource } from '@/lib/workflows/application/read-workflow-source' +import { collectTableBlockFieldIssues } from '@/lib/workflows/editing/lint' +import { + loadDeployedWorkflowState, + loadWorkflowFromNormalizedTables, + NoActiveDeploymentError, +} from '@/lib/workflows/persistence/utils' +import { finding, where } from '@/lib/workspace-source/application/findings' +import { liveVersionsReplaced, workflowsUsing } from '@/lib/workspace-source/application/in-use' +import type { TablePlan } from '@/lib/workspace-source/application/kinds/table/adapter' +import type { PlanContext, PlannedChange } from '@/lib/workspace-source/application/types' + +/** A table this plan renames columns of: its id, name, and each old name → new name. */ +interface RenamingTable { + change: PlannedChange + tableId: string + tableName: string + columnNames: string[] + renamed: Map +} + +function renamingTables(changes: readonly PlannedChange[]): RenamingTable[] { + return changes.flatMap((change) => { + if (change.kind !== 'table' || change.held || change.storedId === undefined) return [] + const plan = change.detail as TablePlan | undefined + const renamed = new Map( + (plan?.changes?.renamedColumns ?? []).map(({ from, to }) => [from, to] as const) + ) + if (renamed.size === 0 || !plan?.changes) return [] + return [ + { + change, + tableId: change.storedId, + tableName: plan.changes.table.name ?? change.key, + columnNames: plan.changes.columnNames, + renamed, + }, + ] + }) +} + +/** + * The 1-based line and column of the first `name` key written after the declaration of block + * `blockName` in `text`, else of the declaration itself; `undefined` when neither is found. + */ +function locateWrite( + text: string, + blockName: string, + name: string +): { line: number; column: number } | undefined { + const lines = splitLines(text) + const quoted = [`'${blockName}'`, `"${blockName}"`] + const declaration = lines.findIndex((line) => quoted.some((name) => line.includes(name))) + if (declaration === -1) return undefined + const keyPattern = new RegExp(`(^|[\\s{,])(['"]?)${escapeRegExp(name)}\\2\\s*:`) + for (let index = declaration; index < lines.length; index++) { + const match = keyPattern.exec(lines[index]) + if (match) return { line: index + 1, column: match.index + match[1].length + 1 } + } + return { line: declaration + 1, column: 1 } +} + +/** The live version of `workflowId`'s deployment, or `null` when it has none. */ +async function liveBlocks(workflowId: string) { + try { + return (await loadDeployedWorkflowState(workflowId)).blocks + } catch (error) { + if (error instanceof NoActiveDeploymentError) return null + throw error + } +} + +/** + * Warns `W_TABLE_COLUMN_RENAMED_IN_USE` on each table change that renames a column some stored + * workflow's Table block still writes, filters or sorts by the old name: one finding per block, + * naming the workflow and the block, pinned to the declared workflow's file and line. A deployed + * workflow's live version runs apart from its draft, so it is checked too (an evaluated draft + * included), unless the apply publishes the draft or undeploys it. + */ +export async function checkColumnRenamesInUse( + changes: readonly PlannedChange[], + ctx: PlanContext +): Promise { + const tables = renamingTables(changes) + if (tables.length === 0) return + const using = await workflowsUsing( + ctx.workspace.workspaceId, + tables.map((table) => table.tableId) + ) + if (using === null) return + /** Declared workflows by stored id: their folder key, and whether the apply evaluates them. */ + const declared = new Map() + for (const change of changes) { + if (change.kind !== 'workflow' || change.storedId === undefined) continue + declared.set(change.storedId, { key: change.key, evaluated: change.refs !== undefined }) + } + const replacedLive = liveVersionsReplaced(changes) + for (const table of tables) { + const columns = new Map([ + [table.tableId, { name: table.tableName, columnNames: table.columnNames }], + ]) + for (const workflow of using.get(table.tableId) ?? []) { + const folder = declared.get(workflow.id) + if ( + changes.some( + (c) => c.kind === 'workflow' && c.storedId === workflow.id && c.action === 'delete' + ) + ) { + continue + } + /** `blockId:field` of each finding already made from the draft. */ + const reported = new Set() + if (workflow.draft && !folder?.evaluated) { + const loaded = await loadWorkflowFromNormalizedTables(workflow.id) + const issues = loaded + ? collectTableBlockFieldIssues(loaded.blocks, columns).filter((issue) => + table.renamed.has(issue.field) + ) + : [] + const text = + folder && issues.length > 0 + ? ( + await readWorkflowSource.execute({ + principal: ctx.principal, + input: { workflowId: workflow.id }, + }) + ).files[SOURCE_ENTRY_FILE] + : undefined + for (const issue of issues) { + reported.add(`${issue.blockId}:${issue.field}`) + const renamedTo = table.renamed.get(issue.field) + const blockName = issue.blockName ?? issue.blockId + const at = text !== undefined ? locateWrite(text, blockName, issue.field) : undefined + table.change.diagnostics.push( + finding( + 'W_TABLE_COLUMN_RENAMED_IN_USE', + `${where('table', table.change.key)} renames column "${issue.field}" to "${renamedTo}", but block "${blockName}" of workflow "${workflow.name}" still ${issue.rowWrite ? 'writes' : 'reads'} "${issue.field}"; the run drops it.`, + { + hint: `Write "${renamedTo}" in that block${folder ? ` (${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE})` : ''}, in this apply or right after it.`, + ...(folder + ? { file: `${workflowFolder(folder.key)}/${SOURCE_ENTRY_FILE}`, ...(at ?? {}) } + : {}), + kind: 'table', + key: table.change.key, + } + ) + ) + } + } + if (!workflow.live || replacedLive.has(workflow.id)) continue + const live = await liveBlocks(workflow.id) + if (!live) continue + for (const issue of collectTableBlockFieldIssues(live, columns)) { + if (!table.renamed.has(issue.field) || reported.has(`${issue.blockId}:${issue.field}`)) { + continue + } + const renamedTo = table.renamed.get(issue.field) + const blockName = issue.blockName ?? issue.blockId + table.change.diagnostics.push( + finding( + 'W_TABLE_COLUMN_RENAMED_IN_USE', + `${where('table', table.change.key)} renames column "${issue.field}" to "${renamedTo}", but block "${blockName}" of workflow "${workflow.name}" (live version) still ${issue.rowWrite ? 'writes' : 'reads'} "${issue.field}"; its runs drop it until the workflow is deployed again.`, + { + hint: `Write "${renamedTo}" in that block and deploy the workflow (\`deployments/.ts\` with live: 'draft'), in this apply or right after it.`, + kind: 'table', + key: table.change.key, + } + ) + ) + } + } + } +} diff --git a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts index d632e0425da..81c109b6608 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts @@ -55,23 +55,34 @@ function columnDecl(column: ColumnDefinition, key: string): TableColumnDecl { /** * A stored table as the declaration it prints as. A column keeps the key `bindings` (the base's - * column key → id) gives it; any other column is keyed by its name, so a fresh directory matches - * columns by name. + * column key → id) gives it, else the key the last source write recorded for it + * (`metadata.sourceColumnKeys`), so a fresh directory and another workspace's read print the key + * the declaration gave it; any other column is keyed by its name. */ export function tableAsDecl( state: StoredTableState, bindings?: Readonly> ): TableAsDecl { const boundKey = new Map(Object.entries(bindings ?? {}).map(([key, column]) => [column.id, key])) + const recordedKey = state.table.metadata?.sourceColumnKeys ?? {} const stored = sourceColumns(state.table) const keyById = new Map() const taken = new Set() - // Bound keys first, so a column keyed by its name never takes a renamed column's key. - for (const column of stored) { - const key = boundKey.get(getColumnId(column)) - if (key === undefined || !isDeclarableColumnKey(key) || taken.has(key)) continue - keyById.set(getColumnId(column), key) - taken.add(key) + // Bound keys first, then recorded ones, so a column keyed by its name never takes a renamed + // column's key. + for (const keyOf of [ + (id: string) => boundKey.get(id), + (id: string) => (Object.hasOwn(recordedKey, id) ? recordedKey[id] : undefined), + ]) { + for (const column of stored) { + const id = getColumnId(column) + const key = keyOf(id) + if (keyById.has(id) || key === undefined || !isDeclarableColumnKey(key) || taken.has(key)) { + continue + } + keyById.set(id, key) + taken.add(key) + } } for (const column of stored) { const id = getColumnId(column) @@ -88,15 +99,20 @@ export function tableAsDecl( const key = keyById.get(getColumnId(column)) ?? column.name setOwn(columns, key, columnDecl(column, key)) } - // Rows source manages print as `rows`, keyed by the column the marker names while it exists. + // Rows source manages print as `rows`, keyed by the column the marker names while it exists; + // seed rows print as their `rows` alone, since the directory keeps their file. const marker = state.table.metadata?.sourceRows - const rowsKey = marker ? keyById.get(marker.keyColumnId) : undefined + const rowsKey = marker?.mode === 'managed' ? keyById.get(marker.keyColumnId) : undefined const decl: TableDecl = { name: state.table.name, ...(state.table.description ? { description: state.table.description } : {}), ...(state.folderPath !== '/' ? { folder: state.folderPath } : {}), columns, - ...(rowsKey !== undefined ? { rows: { mode: 'managed', key: rowsKey, data: [] } } : {}), + ...(rowsKey !== undefined + ? { rows: { mode: 'managed', key: rowsKey, data: [] } } + : marker?.mode === 'seed' + ? { rows: { mode: 'seed', data: [] } } + : {}), } return { decl, keyById, fingerprint: fingerprintOf(tableCodec.canonical(decl)) } } @@ -134,7 +150,7 @@ export function printStoredTable( const { decl, keyById, fingerprint } = tableAsDecl(state, bindings) let rowsEntry: BaseRows | undefined if (rows && 'seed' in rows) { - if (!decl.rows) decl.rows = rows.seed + if (decl.rows?.mode !== 'managed') decl.rows = rows.seed } else if (decl.rows && rows) { decl.rows = { ...decl.rows, data: rows.data } rowsEntry = baseRows(decl.rows, rows.version) diff --git a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts index e83e9c99daf..963ac7223ba 100644 --- a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts @@ -209,13 +209,16 @@ function sourceRefs( evaluated: EvaluatedSource | undefined ): NonNullable { const refs = new Map[number]>() - for (const { kind, key, member } of references) { + for (const { kind, key, member, span } of references) { if (!isWorkspaceKind(kind)) continue if (isResourceKeyed(kind)) { - refs.set(JSON.stringify([kind, key, member ?? null]), { + const id = JSON.stringify([kind, key, member ?? null]) + if (refs.has(id)) continue + refs.set(id, { kind, key, ...(member !== undefined ? { member } : {}), + ...(span !== undefined ? { span } : {}), }) continue } diff --git a/apps/sim/lib/workspace-source/application/knowledge.integration.ts b/apps/sim/lib/workspace-source/application/knowledge.integration.ts index 56f8e4b5cc0..6161edc554c 100644 --- a/apps/sim/lib/workspace-source/application/knowledge.integration.ts +++ b/apps/sim/lib/workspace-source/application/knowledge.integration.ts @@ -517,6 +517,27 @@ describe('knowledge bases as workspace source', () => { } ) + check( + 'a chunking change counts only the stored documents that keep their chunks, not the ones it replaces', + async () => { + const { workspaceId, read } = await setUp() + + const planned = await apply( + workspaceId, + { + files: { + [POLICIES_FILE]: declaration('Policies', '\n chunking: { maxSize: 512 },'), + }, + docs: { ...POLICIES.docs, [REFUNDS]: Buffer.from('# Refunds\nWithin 60 days.\n') }, + }, + { base: read.base, dryRun: true, carry: 'none' } + ) + + const chunking = planned.diagnostics.find((finding) => finding.code === 'I_KB_CHUNKING') + expect(chunking?.message).toContain('1 stored document(s) keep their chunks') + } + ) + check('a get reproduces the apply, and re-applying it changes nothing', async () => { const workspaceId = await newWorkspace() const created = await apply(workspaceId, POLICIES) @@ -1027,6 +1048,38 @@ describe('knowledge bases as workspace source', () => { } ) + check( + 'a chunking change counts the replacement a stopped replace left among the documents keeping their chunks', + async () => { + const { workspaceId, read, dir } = await setUp() + const edited = withDoc(dir, REFUNDS, '# Refunds\nWithin 60 days.\n') + const failing = vi + .spyOn(deleteKnowledgeDocument, 'execute') + .mockRejectedValueOnce(new Error('storage unavailable')) + .mockRejectedValueOnce(new Error('storage unavailable')) + const first = await apply(workspaceId, edited, { base: read.base }).finally(() => + failing.mockRestore() + ) + expect(first.failed).toMatchObject({ key: 'policies' }) + + const planned = await apply( + workspaceId, + { + ...edited, + files: { + ...edited.files, + [POLICIES_FILE]: declaration('Policies', '\n chunking: { maxSize: 512 },'), + }, + }, + { base: first.base, dryRun: true, carry: 'none' } + ) + + // The finished replacement and shipping.txt keep their chunks; the replaced refunds.md goes. + const chunking = planned.diagnostics.find((finding) => finding.code === 'I_KB_CHUNKING') + expect(chunking?.message).toContain('2 stored document(s) keep their chunks') + } + ) + check('a create that fails partway is adopted and finished by the next apply', async () => { const workspaceId = await newWorkspace() const failing = failUpload(2) diff --git a/apps/sim/lib/workspace-source/application/mcp.integration.ts b/apps/sim/lib/workspace-source/application/mcp.integration.ts index 63e55d86bee..cdbd3876195 100644 --- a/apps/sim/lib/workspace-source/application/mcp.integration.ts +++ b/apps/sim/lib/workspace-source/application/mcp.integration.ts @@ -309,6 +309,8 @@ describe('MCP servers as workspace source', () => { kind: 'workflow', key: 'triage', file: 'workflows/triage/workflow.ts', + line: 9, + column: 5, }) expect(unknown?.message).toContain('Did you mean "create_issue"?') } diff --git a/apps/sim/lib/workspace-source/application/promote.integration.ts b/apps/sim/lib/workspace-source/application/promote.integration.ts index aa9a00df6de..2813a116021 100644 --- a/apps/sim/lib/workspace-source/application/promote.integration.ts +++ b/apps/sim/lib/workspace-source/application/promote.integration.ts @@ -11,7 +11,11 @@ import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/featur import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' import { generateId } from '@sim/utils/id' -import { promotionSubmission } from '@sim/workspace-code/promote' +import { + type PromotionRecord, + promotionRecord, + promotionSubmission, +} from '@sim/workspace-code/promote' import { and, eq, inArray, isNull } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, vi } from 'vitest' import { @@ -72,6 +76,14 @@ description: House tone Be brief. ` +const NOTIFY = `// @sim-sdk 1 +import { trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('Notify', {}, () => { + trigger.start('Start', { inputFormat: { email: 'string' } }) +}) +` + const DEV: Record = { 'tables/leads.ts': LEADS, 'skills/tone.md': TONE, @@ -94,6 +106,41 @@ async function promote(dev: Read, target: string, options: { allowDelete?: boole return runner.apply(target, { files: composed.files, base: composed.base, ...options }) } +/** + * Promotes `dev` to `target` as `apply --to` does with the target's promotion record, returning + * the apply's result and the record it leaves. + */ +async function promoteRecorded(dev: Read, target: string, record: PromotionRecord | undefined) { + const prod = await runner.get(target) + const composed = promotionSubmission({ + files: dev.files, + blobPaths: [], + kinds: dev.kinds, + target: prod, + record, + allowDelete: false, + }) + const result = await runner.apply(target, { files: composed.files, base: composed.base }) + return { + result, + record: promotionRecord({ + workspaceId: target, + previous: record, + kinds: dev.kinds, + files: dev.files, + submission: composed, + applied: result, + }), + } +} + +/** `dev` with `edits` applied to it, read back as the directory would hold it. */ +async function editDev(dev: string, edits: Record) { + const read = await runner.get(dev) + await runner.apply(dev, { files: { ...read.files, ...edits }, base: read.base }) + return runner.get(dev) +} + async function blockIds(workspaceId: string) { const rows = await db .select({ id: workflowBlocks.id, name: workflowBlocks.name }) @@ -200,6 +247,142 @@ describe('workspace source promotion', () => { } ) + check( + 'a change made in the target since the last promotion is kept when the directory left it alone, and refused as stale when the directory changed it too', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: DEV }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + expect(first.result.failed).toBeUndefined() + + const prodRead = await runner.get(prod) + await runner.apply(prod, { + files: { + ...prodRead.files, + 'skills/tone.md': TONE.replace('Be brief.', 'Be brief, prod.'), + 'workflows/ingest/workflow.ts': ingest('Prod hotfix.'), + }, + base: prodRead.base, + }) + + const untouched = await promoteRecorded(await runner.get(dev), prod, first.record) + expect(untouched.result.applied).toEqual([]) + const kept = await runner.get(prod) + expect(kept.files['skills/tone.md']).toContain('Be brief, prod.') + expect(kept.files['workflows/ingest/workflow.ts']).toContain('Prod hotfix.') + + const edited = await editDev(dev, { + 'skills/tone.md': TONE.replace('Be brief.', 'Be very brief.'), + 'workflows/ingest/workflow.ts': ingest('Be terse.'), + }) + const stale = await refusal(promoteRecorded(edited, prod, untouched.record)) + expect(stale.stale).toBe(true) + expect(codes(stale.diagnostics)).toContain('E_APPLY_STALE') + const unchanged = await runner.get(prod) + expect(unchanged.files['skills/tone.md']).toContain('Be brief, prod.') + expect(unchanged.files['workflows/ingest/workflow.ts']).toContain('Prod hotfix.') + } + ) + + check( + 'a resource deleted in the target since the last promotion stays deleted when the directory left it alone, and an edit of it since is refused as stale', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: { ...DEV, 'workflows/notify/workflow.ts': NOTIFY } }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + expect(first.result.failed).toBeUndefined() + + const prodRead = await runner.get(prod) + const removed = await runner.apply(prod, { + files: Object.fromEntries( + Object.entries(prodRead.files).filter( + ([path]) => path !== 'skills/tone.md' && !path.startsWith('workflows/notify/') + ) + ), + base: prodRead.base, + allowDelete: true, + }) + expect( + removed.applied.map((change) => `${change.action} ${change.kind}:${change.key}`).sort() + ).toEqual(['delete skill:tone', 'delete workflow:notify']) + + const untouched = await promoteRecorded(await runner.get(dev), prod, first.record) + expect(untouched.result.applied).toEqual([]) + expect(codes(untouched.result.diagnostics, 'info').sort()).toEqual([ + 'I_SERVER_DELETED', + 'I_SERVER_DELETED', + ]) + expect((await liveWorkflows(prod)).map((row) => row.name)).toEqual(['Ingest']) + expect((await runner.get(prod)).files['skills/tone.md']).toBeUndefined() + + const edited = await editDev(dev, { + 'skills/tone.md': TONE.replace('Be brief.', 'Be very brief.'), + 'workflows/notify/workflow.ts': NOTIFY.replace("'Notify'", "'Notify v2'"), + }) + const stale = await refusal(promoteRecorded(edited, prod, untouched.record)) + expect(stale.stale).toBe(true) + expect(codes(stale.diagnostics)).toEqual(['E_APPLY_STALE', 'E_APPLY_STALE']) + expect((await liveWorkflows(prod)).map((row) => row.name)).toEqual(['Ingest']) + } + ) + + check( + 'with the record, a promotion that changed nothing in the target since converges, and the next edit updates it in place', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: DEV }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + const again = await promoteRecorded(await runner.get(dev), prod, first.record) + expect(again.result.applied).toEqual([]) + + const edited = await editDev(dev, { + 'skills/tone.md': TONE.replace('Be brief.', 'Be very brief.'), + 'workflows/ingest/workflow.ts': ingest('Be terse.'), + }) + const updated = await promoteRecorded(edited, prod, again.record) + expect( + updated.result.applied + .map((change) => `${change.action} ${change.kind}:${change.key}`) + .sort() + ).toEqual(['update skill:tone', 'update workflow:ingest']) + const settled = await promoteRecorded(edited, prod, updated.record) + expect(settled.result.applied).toEqual([]) + } + ) + + check('promoting managed rows converges: the second promotion changes nothing', async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + const directory = { + 'tables/stock.ts': `// @sim-sdk 1 +import { column, defineTable } from '@sim/workflow-sdk/workspace' + +export default defineTable({ + name: 'stock', + columns: { + sku: column.string({ required: true, unique: true }), + status: column.select({ options: ['new', 'won'] }), + note_text: column.string({ name: 'memo' }), + }, + rows: { mode: 'managed', key: 'sku' }, +}) +`, + 'tables/stock.rows.jsonl': '{"sku":"A-1","status":"new","note_text":"hi"}\n{"sku":"A-2"}\n', + } + const applied = await runner.apply(dev, { files: directory, kinds: ['table'] }) + const devDir = { ...(await runner.get(dev)), files: applied.files } + + const first = await promote(devDir, prod) + expect(first.applied.map((change) => `${change.action} ${change.kind}:${change.key}`)).toEqual([ + 'create table:stock', + ]) + const second = await promote(devDir, prod) + expect(second.applied).toEqual([]) + }) + check( "the target's own resources are kept, with a warning, unless the promotion allows deletes", async () => { diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index 444d1a986fc..9dce9850bac 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -19,6 +19,7 @@ import { generateId } from '@sim/utils/id' import { compareStrings } from '@sim/utils/string' import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, vi } from 'vitest' +import { updateTableColumnUseCase } from '@/lib/table/application/columns' import { createTableRows } from '@/lib/table/application/rows' import { readCurrentRowsVersion } from '@/lib/table/row-changes' import { getTableById } from '@/lib/table/service' @@ -347,6 +348,112 @@ describe('table rows as workspace source', () => { expect(firedEvents()).toEqual([]) }) + check( + "a column's key survives a fresh get and a rename in Sim, and a table written before keys were recorded records them", + async () => { + const workspaceId = await newWorkspace() + const keyed = `// @sim-sdk 1 +import { column, defineTable } from '@sim/workflow-sdk/workspace' + +export default defineTable({ + name: 'keyed', + columns: { + sku: column.string({ required: true, unique: true }), + note: column.string({ name: 'note_text' }), + }, +}) +` + const created = await apply(workspaceId, { 'tables/keyed.ts': keyed }) + expect(created.failed).toBeUndefined() + expect((await runner.get(workspaceId)).files['tables/keyed.ts']).toContain( + "note: column.string({ name: 'note_text' })" + ) + + const [table] = await db + .select({ id: userTableDefinitions.id, metadata: userTableDefinitions.metadata }) + .from(userTableDefinitions) + .where( + and( + eq(userTableDefinitions.workspaceId, workspaceId), + isNull(userTableDefinitions.archivedAt) + ) + ) + await updateTableColumnUseCase.execute({ + principal: owner, + input: { + tableId: table.id, + workspaceId, + columnName: 'note_text', + updates: { name: 'memo' }, + }, + }) + expect((await runner.get(workspaceId)).files['tables/keyed.ts']).toContain( + "note: column.string({ name: 'memo' })" + ) + + await db + .update(userTableDefinitions) + .set({ metadata: {} }) + .where(eq(userTableDefinitions.id, table.id)) + const legacy = await runner.get(workspaceId) + expect(legacy.files['tables/keyed.ts']).toContain('memo: column.string()') + const recorded = await apply( + workspaceId, + { 'tables/keyed.ts': keyed.replace("name: 'note_text'", "name: 'memo'") }, + { base: legacy.base } + ) + expect(recorded.applied).toMatchObject([ + { kind: 'table', key: 'keyed', action: 'update', fields: ['column keys'] }, + ]) + expect((await runner.get(workspaceId)).files['tables/keyed.ts']).toContain( + "note: column.string({ name: 'memo' })" + ) + } + ) + + check( + 'managed rows resent against a fresh read plan as no change, a column keyed apart from its name included, as a promotion resends them', + async () => { + const workspaceId = await newWorkspace() + const typed = `// @sim-sdk 1 +import { column, defineTable } from '@sim/workflow-sdk/workspace' + +export default defineTable({ + name: 'typed', + columns: { + sku: column.string({ required: true, unique: true }), + shipped: column.date(), + expires: column.ttl(), + price: column.currency({ currencyCode: 'EUR' }), + labels: column.select({ options: ['red', 'green', 'blue'], multiple: true }), + status: column.select({ options: ['new', 'won'] }), + meta: column.json(), + note_text: column.string({ name: 'memo' }), + }, + rows: { mode: 'managed', key: 'sku' }, +}) +` + const directory = { + 'tables/typed.ts': typed, + 'tables/typed.rows.jsonl': [ + '{"sku":"A-1","shipped":"2026-11-02T10:30:00Z","expires":"2027-01-01T00:00:00-00:00","price":19.99,"labels":["red","blue"],"status":"new","meta":{"a":[1,{"b":null}]}}', + '{"sku":"A-2","shipped":"2026-11-01","labels":[],"status":"won","meta":[1,"x",true]}', + '{"sku":"A-9","note_text":"line1\\nline2"}', + ].join('\n'), + } + const created = await apply(workspaceId, directory) + expect(created.failed).toBeUndefined() + const versionBefore = await version(workspaceId) + const read = await runner.get(workspaceId) + expect(read.files['tables/typed.rows.jsonl']).not.toBe(directory['tables/typed.rows.jsonl']) + + const resent = await apply(workspaceId, directory, { base: read.base }) + + expect(resent.applied).toEqual([]) + expect(await version(workspaceId)).toBe(versionBefore) + } + ) + check('a key rename is a delete and an insert, gated by allow-delete', async () => { const workspaceId = await newWorkspace() const read = await managedPlans(workspaceId) @@ -638,8 +745,10 @@ export default defineTable({ expect(again.applied).toEqual([]) expect((await storedRows(workspaceId))[0]).toMatchObject({ seats: 5 }) - // A get leaves seed rows out: they were never the table's identity. + // A get keeps the declaration of seed rows but leaves the rows out: the directory holds them, + // and they were never the table's identity. const read = await runner.get(workspaceId) + expect(read.files['tables/plans.ts']).toContain("rows: { mode: 'seed' }") expect(read.files['tables/plans.rows.jsonl']).toBeUndefined() expect((await apply(workspaceId, read.files, { base: read.base })).applied).toEqual([]) @@ -649,6 +758,43 @@ export default defineTable({ expect(await storedRows(workspaceId)).toHaveLength(2) }) + check( + 'a declaration that stops seeding rows clears it, and one that starts records it', + async () => { + const workspaceId = await newWorkspace() + const seeded = await apply(workspaceId, { + 'tables/plans.ts': SEED, + 'tables/plans.rows.jsonl': ROWS, + }) + const unseeded = PLANS('undefined').replace(' rows: undefined,\n', '') + + const dropped = await apply( + workspaceId, + { 'tables/plans.ts': unseeded }, + { base: seeded.base } + ) + expect(dropped.applied).toMatchObject([ + { key: 'plans', action: 'update', fields: ['rows (no longer seeded)'] }, + ]) + expect((await runner.get(workspaceId)).files['tables/plans.ts']).not.toContain('rows:') + + const read = await runner.get(workspaceId) + const again = await apply( + workspaceId, + { 'tables/plans.ts': SEED, 'tables/plans.rows.jsonl': ROWS }, + { base: read.base } + ) + expect(again.applied).toMatchObject([ + { key: 'plans', action: 'update', fields: ['rows (seed)'] }, + ]) + expect(codes(again.diagnostics, 'info')).not.toContain('I_TABLE_SEED_SKIPPED') + expect((await runner.get(workspaceId)).files['tables/plans.ts']).toContain( + "rows: { mode: 'seed' }" + ) + expect(await storedRows(workspaceId)).toHaveLength(2) + } + ) + check('makes an existing column required in the apply whose managed rows fill it', async () => { const workspaceId = await newWorkspace() const gappy = '{"code":"solo","seats":1,"tier":"basic"}\n{"code":"team","tier":"pro"}\n' diff --git a/apps/sim/lib/workspace-source/application/stage-blobs.ts b/apps/sim/lib/workspace-source/application/stage-blobs.ts index 736d41cbe8f..f3a50d06bb2 100644 --- a/apps/sim/lib/workspace-source/application/stage-blobs.ts +++ b/apps/sim/lib/workspace-source/application/stage-blobs.ts @@ -71,7 +71,8 @@ export interface StageSourceBlobsResult { } /** - * Stages a batch of blobs: refuses one over `maxBlobBytes`, a batch over the request caps, and any + * Stages a batch of blobs: refuses one over `maxBlobBytes`, a batch over the request caps (a single + * blob may fill its request alone), and any * blob whose bytes do not hash to the SHA-256 it was named by (nothing of the batch is staged), * then stores the rest within the scope's caps. Staging a blob already staged only renews it. */ @@ -108,10 +109,10 @@ export const stageSourceBlobs = defineAuthorizedWorkspaceUseCase({ ) } } - if (total > maxBlobBytesPerRequest) { + if (input.blobs.length > 1 && total > maxBlobBytesPerRequest) { throw new OrchestrationError( 'payload_too_large', - `A staging request carries at most ${maxBlobBytesPerRequest} bytes; this one carries ${total}.` + `A staging request of more than one file carries at most ${maxBlobBytesPerRequest} bytes; this one carries ${total}. Send a larger file in a request of its own.` ) } const staged = await stageBlobs( diff --git a/apps/sim/lib/workspace-source/application/types.ts b/apps/sim/lib/workspace-source/application/types.ts index a033afdea00..4f8422e34a1 100644 --- a/apps/sim/lib/workspace-source/application/types.ts +++ b/apps/sim/lib/workspace-source/application/types.ts @@ -1,4 +1,5 @@ import type { Principal } from '@sim/auth/principal' +import type { SourceSpan } from '@sim/workflow-code' import type { BaseResource, BlobEntry, @@ -143,6 +144,8 @@ interface ChangeFields { key: string /** The part of the resource the reference names: an MCP server's tool. */ member?: string + /** Where the declaration's entry file writes the reference, when the pipeline located it. */ + span?: SourceSpan /** * The resource's id, for a kind keyed otherwise than a workflow names it (a workspace file, * keyed by its path): the plan finds that kind's change by id and names it by its key. diff --git a/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts b/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts index 05a4713fb6c..62a331e7e94 100644 --- a/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts +++ b/apps/sim/lib/workspace-source/application/workspace-source.test-helpers.ts @@ -13,6 +13,7 @@ import { sha256Hex } from '@sim/security/hash' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import type { WorkspaceKind } from '@sim/workspace-code' +import { stageBatches } from '@sim/workspace-code/limits' import { inArray, sql } from 'drizzle-orm' import { expect, it } from 'vitest' import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' @@ -124,25 +125,30 @@ export type SourceSubmission = Omit< principal?: Principal } -/** Stages `blobs` (base64 bytes by SHA-256) in `workspaceId` as `principal`, as the CLI does. */ +/** + * Stages `blobs` (base64 bytes by SHA-256) in `workspaceId` as `principal`, as the CLI does: in + * the requests `stageBatches` splits them into. + */ export async function stageBlobsAs( principal: Principal, workspaceId: string, blobs: ReadonlyArray<{ sha256: string; content: string }> ) { - if (blobs.length === 0) return - await withPermissionGroupScope(() => - stageSourceBlobs.execute({ - principal, - input: { - workspaceId, - blobs: blobs.map((blob) => ({ - sha256: blob.sha256, - content: Buffer.from(blob.content, 'base64'), - })), - }, - }) - ) + const decoded = blobs.map((blob) => { + const content = Buffer.from(blob.content, 'base64') + return { sha256: blob.sha256, content, size: content.byteLength } + }) + for (const batch of stageBatches(decoded)) { + await withPermissionGroupScope(() => + stageSourceBlobs.execute({ + principal, + input: { + workspaceId, + blobs: batch.map(({ sha256, content }) => ({ sha256, content })), + }, + }) + ) + } } /** diff --git a/apps/sim/lib/workspace-source/staging/store.ts b/apps/sim/lib/workspace-source/staging/store.ts index da607b45774..d2bd7a7548d 100644 --- a/apps/sim/lib/workspace-source/staging/store.ts +++ b/apps/sim/lib/workspace-source/staging/store.ts @@ -100,6 +100,18 @@ export async function stagedUsage( return { count: Number(usage?.count ?? 0), bytes: Number(usage?.bytes ?? 0) } } +/** + * A caller's staging area has no room for what it stages: `413` like a body over the limit, told + * apart by `details.code` `SOURCE_STAGING_FULL`, since freeing staged files (or waiting for them + * to expire) fixes it and sending smaller requests does not. + */ +export class SourceStagingFullError extends OrchestrationError { + constructor(message: string) { + super('payload_too_large', message) + this.name = 'SourceStagingFullError' + } +} + /** Refuses staging that would take `target` past its blob or byte cap. */ export function assertStagingFits( usage: { count: number; bytes: number }, @@ -107,14 +119,12 @@ export function assertStagingFits( ): void { const { maxStagedBlobs, maxStagedBytes } = WORKSPACE_SOURCE_LIMITS if (usage.count + adding.count > maxStagedBlobs) { - throw new OrchestrationError( - 'payload_too_large', + throw new SourceStagingFullError( `Staging ${adding.count} more binary files would hold ${usage.count + adding.count}, over the ${maxStagedBlobs}-file limit. Staged files expire a day after their last use.` ) } if (usage.bytes + adding.bytes > maxStagedBytes) { - throw new OrchestrationError( - 'payload_too_large', + throw new SourceStagingFullError( `Staging ${adding.bytes} more bytes would hold ${usage.bytes + adding.bytes}, over the ${maxStagedBytes}-byte limit. Staged files expire a day after their last use.` ) } diff --git a/apps/sim/providers/tool-identity.test.ts b/apps/sim/providers/tool-identity.test.ts index a8b12b4e54a..e425df15cd9 100644 --- a/apps/sim/providers/tool-identity.test.ts +++ b/apps/sim/providers/tool-identity.test.ts @@ -78,9 +78,10 @@ describe('provider tool identities', () => { const longId = `tool_${'a'.repeat(80)}` const tools = [providerTool(longId, 'a'), providerTool(longId, 'b')] - assignProviderToolIdentities(tools) + const identities = assignProviderToolIdentities(tools) - expect(tools[0].id).toBe(longId) + expect(tools[0].id).toBe(longId.slice(0, 64)) + expect(identities.toolIdByWireId.get(tools[0].id)).toBe(longId) expect(tools[1].id).toHaveLength(64) expect(tools[1].id).toMatch(/__sim_2$/) }) @@ -171,4 +172,69 @@ describe('provider tool identities', () => { expect(output.toolCalls?.list[0].name).toBe('gmail_send') expect(output.providerTiming?.timeSegments?.[0].name).toBe('gmail_send') }) + + it('gives a custom tool titled with spaces and unicode a provider-safe name that maps back', () => { + const PROVIDER_NAME = /^[a-zA-Z0-9_-]{1,64}$/ + const spaced = providerTool('custom_Get Weather', 'a') + const unicode = providerTool('custom_Météo du jour ☀', 'b') + + const identities = assignProviderToolIdentities([spaced, unicode]) + + expect(spaced.id).toMatch(PROVIDER_NAME) + expect(unicode.id).toMatch(PROVIDER_NAME) + expect(identities.toolIdByWireId.get(spaced.id)).toBe('custom_Get Weather') + expect(identities.toolIdByWireId.get(unicode.id)).toBe('custom_Météo du jour ☀') + + const response: ProviderResponse = { + content: 'done', + model: 'test-model', + toolCalls: [{ name: spaced.id, arguments: {} }], + } + projectProviderResponseToolIdentities(response, identities) + expect(response.toolCalls?.[0].name).toBe('custom_Get Weather') + }) + + it('keeps custom tools whose safe names collide distinct, and a safe id unchanged', () => { + const tools = [ + providerTool('custom_Get Weather', 'a'), + providerTool('custom_Get/Weather', 'b'), + providerTool('custom_Get_Weather', 'c'), + ] + + const identities = assignProviderToolIdentities(tools) + const wireIds = tools.map((tool) => tool.id) + + expect(new Set(wireIds).size).toBe(3) + expect(tools[2].id).toBe('custom_Get_Weather') + expect(identities.toolIdByWireId.get(wireIds[0])).toBe('custom_Get Weather') + expect(identities.toolIdByWireId.get(wireIds[1])).toBe('custom_Get/Weather') + expect(identities.toolIdByWireId.has('custom_Get_Weather')).toBe(false) + + assignProviderToolIdentities(tools) + expect(tools.map((tool) => tool.id)).toEqual(wireIds) + }) + + it('bounds a long custom tool title to the provider name limit, keeping tools that share a prefix distinct', () => { + const PROVIDER_NAME = /^[a-zA-Z0-9_-]{1,64}$/ + const prefix = `custom_${'Look up the customer account '.repeat(3)}` + const tools = [ + providerTool(`${prefix}by email`, 'a'), + providerTool(`${prefix}by phone`, 'b'), + providerTool(`custom_${'x'.repeat(80)}`, 'c'), + ] + + const identities = assignProviderToolIdentities(tools) + const wireIds = tools.map((tool) => tool.id) + + for (const wireId of wireIds) expect(wireId).toMatch(PROVIDER_NAME) + expect(new Set(wireIds).size).toBe(3) + expect(wireIds.map((wireId) => identities.toolIdByWireId.get(wireId))).toEqual([ + `${prefix}by email`, + `${prefix}by phone`, + `custom_${'x'.repeat(80)}`, + ]) + + assignProviderToolIdentities(tools) + expect(tools.map((tool) => tool.id)).toEqual(wireIds) + }) }) diff --git a/apps/sim/providers/tool-identity.ts b/apps/sim/providers/tool-identity.ts index 28b53e98fdc..1be25e13209 100644 --- a/apps/sim/providers/tool-identity.ts +++ b/apps/sim/providers/tool-identity.ts @@ -4,6 +4,17 @@ import type { ProviderResponse, ProviderToolConfig } from '@/providers/types' const MAX_PROVIDER_TOOL_ID_LENGTH = 64 const PROVIDER_ALIAS_MARKER = '__sim_' +/** Characters outside every provider's function-name alphabet (Anthropic: `^[a-zA-Z0-9_-]{1,128}$`). */ +const PROVIDER_UNSAFE_TOOL_ID_CHARS = /[^a-zA-Z0-9_-]/g + +/** + * The id with every character no provider accepts in a function name replaced by `_`. A custom + * tool's canonical id carries its display title (`custom_Get Weather`), which execution resolves by + * title, so only the wire id is made safe and the canonical id keeps the title. + */ +function toProviderSafeToolId(toolId: string): string { + return toolId.replace(PROVIDER_UNSAFE_TOOL_ID_CHARS, '_') +} export interface ProviderToolIdentityMap { /** Provider-only tool id to the canonical registry id used for execution and observability. */ @@ -18,10 +29,12 @@ function buildProviderAlias(toolId: string, occurrence: number, attempt: number) } /** - * Gives duplicate configured tools deterministic provider-safe wire ids. + * Gives configured tools deterministic provider-safe wire ids. * - * The first occurrence and every already-unique tool keep their existing id for backwards - * compatibility. Later occurrences receive opaque ordinal aliases; resource and credential ids + * The first occurrence and every already-unique tool with a provider-safe id keep it for backwards + * compatibility. An id with characters no provider accepts (a custom tool titled `Get Weather`) is + * sent with them replaced by `_`, and one longer than the strictest provider limit is cut to it; + * when either collides with another tool's id it is aliased like a later occurrence. Later occurrences receive opaque ordinal aliases; resource and credential ids * never enter the provider-visible name. Tool objects are updated in place so their instance-bound * params and secret provenance remain attached to the exact object selected by provider adapters. */ @@ -46,11 +59,12 @@ export function assignProviderToolIdentities< const occurrence = (occurrences.get(canonicalId) ?? 0) + 1 occurrences.set(canonicalId, occurrence) - let wireId = canonicalId - if (usedWireIds.has(wireId)) { + const safeId = toProviderSafeToolId(canonicalId) + let wireId = safeId.slice(0, MAX_PROVIDER_TOOL_ID_LENGTH) + if (usedWireIds.has(wireId) || (wireId !== canonicalId && reservedIds.has(wireId))) { let attempt = 0 do { - wireId = buildProviderAlias(canonicalId, occurrence, attempt) + wireId = buildProviderAlias(safeId, occurrence, attempt) attempt += 1 } while (reservedIds.has(wireId) || usedWireIds.has(wireId)) } diff --git a/packages/sim-cli/README.md b/packages/sim-cli/README.md index 016e5b84b0f..be28a0b24b9 100644 --- a/packages/sim-cli/README.md +++ b/packages/sim-cli/README.md @@ -322,8 +322,11 @@ Success exits 0 and most failures exit 1. Two failures any command can hit have `401` (run `sim login`), and 9 when the API answered `429`. Every `GET`, and the source applies, creates and dry runs, are resent up to 6 times first, each wait the longer of `Retry-After` (up to 60 seconds) and a backoff of about 1 to 30 seconds; any other write exits 9 at once. Before this -release both exited 1. Command-specific codes: -[Exit codes](https://docs.sim.ai/cli/scripting#exit-codes). +release both exited 1. Ctrl-C exits 130; a source apply or create already sent is finished and +saved into its directory first. The `workflows source` and `workspaces source` commands also use +5 (stale), 6 (a locked workflow or folder; a locked block exits 1), 7 (too large), 8 (a workspace +apply stopped part-way), 10 (busy), 11 (status: local changes) and 12 (status: the server +changed). Command-specific codes: [Exit codes](https://docs.sim.ai/cli/scripting#exit-codes). ## Workflows and workspaces as code @@ -342,8 +345,8 @@ sim workflows source apply --dir ./support-triage --yes # save the draft `sim workspaces source get|check|status|diff|apply --dir ` does the same for tables, knowledge bases, files, custom tools, skills, MCP servers, workflows and -deployments in one directory. `sim workflows source create --dir ` starts a -new workflow. +deployments in one directory. `sim workflows source create --dir --workspace ` +starts a new workflow in that workspace. Coding agents (Claude Code, Cursor, Codex) start at `.sim/AGENTS.md`, which `get` writes into the directory. The loop is get, edit `workflow.ts`, check, status, diff --git a/packages/sim-cli/src/commands/auth.test.ts b/packages/sim-cli/src/commands/auth.test.ts index 700c1e39ad8..c19a122dcff 100644 --- a/packages/sim-cli/src/commands/auth.test.ts +++ b/packages/sim-cli/src/commands/auth.test.ts @@ -218,6 +218,7 @@ describe('login command', () => { mocks.createInterface.mockReturnValue({ question: vi.fn(async () => 'yes'), close: vi.fn(), + once: vi.fn(), }) vi.spyOn(console, 'log').mockImplementation(() => {}) }) @@ -269,6 +270,7 @@ describe('login command', () => { mocks.createInterface.mockReturnValue({ question: vi.fn(async () => 'no'), close: vi.fn(), + once: vi.fn(), }) await login() @@ -463,6 +465,7 @@ describe('profiles command', () => { mocks.createInterface.mockReturnValue({ question: vi.fn(async () => '1'), close: vi.fn(), + once: vi.fn(), }) vi.spyOn(console, 'log').mockImplementation(() => {}) }) diff --git a/packages/sim-cli/src/commands/auth.ts b/packages/sim-cli/src/commands/auth.ts index fb10fa017b1..0641916a2bd 100644 --- a/packages/sim-cli/src/commands/auth.ts +++ b/packages/sim-cli/src/commands/auth.ts @@ -1,11 +1,11 @@ import { spawn } from 'node:child_process' import { randomBytes } from 'node:crypto' -import { createInterface } from 'node:readline/promises' import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' import { Command, Option } from 'commander' import { printLine } from '#sim-cli/output/io' import { styles } from '#sim-cli/output/presentation' +import { askLine } from '#sim-cli/terminal/line-input' import { buildApprovalUrl, createAuthRequest, @@ -120,15 +120,14 @@ async function confirmProfileOverwrite(profileName: string): Promise { ) } - const prompt = createInterface({ input: process.stdin, output: process.stderr }) - try { - const answer = await prompt.question( + const answer = ( + await askLine( `Profile "${redact(profileName)}" already exists. Replace its login and defaults? (y/N) ` ) - return answer.trim().toLowerCase() === 'y' || answer.trim().toLowerCase() === 'yes' - } finally { - prompt.close() - } + ) + ?.trim() + .toLowerCase() + return answer === 'y' || answer === 'yes' } function selectedProfileName(command: Command): string { @@ -309,20 +308,16 @@ async function chooseWorkspace( printLine(` ${index + 1}) ${safeOneLine(workspace.name)} (${workspace.id})`) } - const prompt = createInterface({ input: process.stdin, output: process.stderr }) - try { - const answer = await prompt.question(`Choose a workspace [1-${workspaces.length}]: `) - const selected = Number(answer.trim()) - if (!Number.isInteger(selected) || selected < 1 || selected > workspaces.length) { - throw new SimApiError( - `Invalid workspace selection "${safeOneLine(answer)}". Choose a number from 1 to ${workspaces.length}.`, - 0 - ) - } - return workspaces[selected - 1] - } finally { - prompt.close() + const answer = await askLine(`Choose a workspace [1-${workspaces.length}]: `) + if (answer === null) throw new SimApiError('No workspace was chosen.', 0, 'CANCELLED') + const selected = Number(answer.trim()) + if (!Number.isInteger(selected) || selected < 1 || selected > workspaces.length) { + throw new SimApiError( + `Invalid workspace selection "${safeOneLine(answer)}". Choose a number from 1 to ${workspaces.length}.`, + 0 + ) } + return workspaces[selected - 1] } function addProfileCommand(): Command { diff --git a/packages/sim-cli/src/commands/protocol/files-get.ts b/packages/sim-cli/src/commands/protocol/files-get.ts index c6b35a34ae9..c587ea7b245 100644 --- a/packages/sim-cli/src/commands/protocol/files-get.ts +++ b/packages/sim-cli/src/commands/protocol/files-get.ts @@ -6,6 +6,7 @@ import { Readable, type Writable } from 'node:stream' import { pipeline } from 'node:stream/promises' import type { Command } from 'commander' import { writeStdout } from '#sim-cli/output/io' +import { interruptsDeferred } from '#sim-cli/terminal/interrupts' import { embedStore } from '../../embed-context' import { V2_OPERATIONS } from '../../generated/v2-api' import { @@ -125,6 +126,9 @@ function reRaise(signal: NodeJS.Signals): void { * The removal is synchronous because the termination that follows gives an * async `rm` no turn to run. * + * An interrupt held by {@link interruptsDeferred} is left alone: the deferral + * raises it again when its write is done, and the cleanup runs then. + * * Exported for its own test: driving it through a real interrupt would take the * test runner down with it. */ @@ -134,6 +138,7 @@ export function removeStagingOnSignal( ): () => void { const installed = STAGE_SIGNALS.map((signal) => { const onSignal = () => { + if (interruptsDeferred()) return process.off(signal, onSignal) const directory = stagingDirectory() if (directory) { diff --git a/packages/sim-cli/src/commands/protocol/index.ts b/packages/sim-cli/src/commands/protocol/index.ts index 31782f51591..d9b37e51f24 100644 --- a/packages/sim-cli/src/commands/protocol/index.ts +++ b/packages/sim-cli/src/commands/protocol/index.ts @@ -21,6 +21,7 @@ import { attachWorkflowRunWait } from './workflow-run-wait' import { attachWorkflowSourceApply, attachWorkflowSourceCreate } from './workflow-source-apply' import { attachWorkflowSourceCheck } from './workflow-source-check' import { attachWorkflowSourceGet } from './workflow-source-get' +import { attachWorkflowSourceParams } from './workflow-source-params' import { attachWorkflowSourceDiff, attachWorkflowSourceStatus } from './workflow-source-status' import { attachWorkspaceOperationWait } from './workspace-operation-wait' import { attachWorkspaceSourceApply } from './workspace-source-apply' @@ -98,6 +99,7 @@ export function attachProtocolCommands(program: Command): void { attachWorkflowSourceCheck(source) attachWorkflowSourceStatus(source) attachWorkflowSourceDiff(source) + attachWorkflowSourceParams(source) const workspaces = group(program, 'workspaces') attachWorkspaceOperationWait(group(workspaces, 'operations')) diff --git a/packages/sim-cli/src/commands/protocol/result.test.ts b/packages/sim-cli/src/commands/protocol/result.test.ts new file mode 100644 index 00000000000..9e8dd331e65 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/result.test.ts @@ -0,0 +1,32 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { printProtocolResult } from './result' + +const ESC = String.fromCharCode(27) +const BEL = String.fromCharCode(7) + +let logged: string[] + +beforeEach(() => { + logged = [] + vi.spyOn(console, 'log').mockImplementation((line: string) => { + logged.push(line) + }) +}) + +describe('printProtocolResult', () => { + it.each(['text', 'table'] as const)( + 'sanitizes a display override built from server text in %s output', + (format) => { + const key = `leads${ESC}]0;pwned${BEL}${ESC}[2A` + printProtocolResult( + format, + { created: [{ kind: 'table', key }] }, + { created: `table ${key}` } + ) + + expect(logged.join('\n')).not.toContain(ESC) + expect(logged.join('\n')).not.toContain(BEL) + expect(logged.join('\n')).toContain('table leads') + } + ) +}) diff --git a/packages/sim-cli/src/commands/protocol/result.ts b/packages/sim-cli/src/commands/protocol/result.ts index 8034aa06e22..d7d39852fc8 100644 --- a/packages/sim-cli/src/commands/protocol/result.ts +++ b/packages/sim-cli/src/commands/protocol/result.ts @@ -1,6 +1,6 @@ import { isRecordLike } from '@sim/utils/object' import type { OutputFormat } from '../../config/index' -import { printRecord, text } from '../../output/render' +import { printRecord, sanitize, text } from '../../output/render' /** One result value as a field: a record, or a list holding one, as compact JSON. */ function fieldText(value: unknown): string { @@ -8,10 +8,20 @@ function fieldText(value: unknown): string { return text(nested ? JSON.stringify(value) : value) } -export function printProtocolResult(format: OutputFormat, result: Record): void { +/** + * Prints a protocol command's result: the record itself in a machine format, else one line per + * field, with `display` replacing the text of the fields it names (a list a person reads as a + * sentence rather than as JSON). An override is built from server text (resource keys, ids), so it + * is sanitized like every other field. + */ +export function printProtocolResult( + format: OutputFormat, + result: Record, + display: Readonly> = {} +): void { const fields = Object.entries(result).map<[string, string]>(([key, value]) => [ key, - fieldText(value), + display[key] === undefined ? fieldText(value) : sanitize(display[key]), ]) printRecord(format, fields, result) } diff --git a/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts new file mode 100644 index 00000000000..b3a2fd47a1e --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts @@ -0,0 +1,158 @@ +import { spawn } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import type { AddressInfo } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +const WORKFLOW_ID = '00000000-0000-4000-8000-00000000000a' +const WORKSPACE_ID = 'ws_local' +const BASE = { + workflowId: WORKFLOW_ID, + workspaceId: WORKSPACE_ID, + fingerprint: 'sf1:a', + catalogHash: 'h', +} +const LOCK = { version: 1, blocks: {}, variables: {} } +const ENTRY = "// @sim-sdk 1\nexport default workflow('W', () => {})\n" +/** How long the server holds the committing request before it answers. */ +const COMMIT_DELAY_MS = 1_500 + +function applied() { + return { + data: { + workflowId: WORKFLOW_ID, + workspaceId: WORKSPACE_ID, + dryRun: false, + applied: true, + diff: { + noop: false, + blocks: { added: [{ id: 'b-1', key: 'start' }], removed: [], renamed: [], modified: [] }, + edges: { added: 1, removed: 0 }, + variables: { added: [], removed: [], modified: [] }, + }, + diagnostics: [], + files: { 'workflow.ts': `${ENTRY}// stored\n` }, + lock: LOCK, + base: { ...BASE, fingerprint: 'sf1:b' }, + mintedIds: {}, + notes: [], + needsRedeployment: true, + }, + } +} + +interface InterruptedRun { + code: number | null + signal: NodeJS.Signals | null + stderr: string +} + +let dir: string +let server: Server +let endpoint: string +/** Resolves the moment the server receives the committing request. */ +let onCommit: () => void + +beforeEach(async () => { + dir = mkdtempSync(join(tmpdir(), 'sim-source-interrupt-')) + mkdirSync(join(dir, 'src')) + mkdirSync(join(dir, 'src/.sim')) + writeFileSync(join(dir, 'src/workflow.ts'), ENTRY) + writeFileSync(join(dir, 'src/.sim/lock.json'), JSON.stringify(LOCK)) + writeFileSync(join(dir, 'src/.sim/base.json'), JSON.stringify(BASE)) + server = createServer((request: IncomingMessage, response: ServerResponse) => { + request.resume() + request.on('end', () => { + onCommit() + setTimeout(() => { + response.setHeader('content-type', 'application/json') + response.end(JSON.stringify(applied())) + }, COMMIT_DELAY_MS) + }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}` +}) + +afterEach(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(() => resolve())) + rmSync(dir, { recursive: true, force: true }) +}) + +/** + * Runs `sim workflows source apply --yes` as a real process and sends it SIGINT `interrupts` + * times once the server holds its commit, 100 ms apart. + */ +function applyInterrupted(interrupts: number): Promise { + const child = spawn( + 'bun', + [ + join(import.meta.dirname, '../../index.ts'), + '--output', + 'json', + 'workflows', + 'source', + 'apply', + '--yes', + '--dir', + join(dir, 'src'), + ], + { + env: { + ...process.env, + SIM_CONFIG_DIR: join(dir, 'config'), + SIM_ENDPOINT: endpoint, + SIM_API_KEY: 'sk-test', + SIM_WORKSPACE: WORKSPACE_ID, + SIM_NO_UPDATE_CHECK: '1', + SIM_CLI_TELEMETRY_DISABLED: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + } + ) + let stderr = '' + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk + }) + child.stdout.resume() + onCommit = () => { + for (let index = 0; index < interrupts; index += 1) { + setTimeout(() => child.kill('SIGINT'), 100 * (index + 1)) + } + } + return new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (code, signal) => resolve({ code, signal, stderr })) + }) +} + +function savedFingerprint(): string { + return JSON.parse(readFileSync(join(dir, 'src/.sim/base.json'), 'utf8')).fingerprint +} + +describe('Ctrl-C while a workflow apply is committing', () => { + it('waits for the answer, saves it into the directory, then ends by the interrupt', async () => { + const run = await applyInterrupted(1) + + expect(run.signal ?? run.code).toSatisfy( + (status: unknown) => status === 'SIGINT' || status === 130 + ) + expect(run.stderr).toContain('the apply was already sent') + expect(savedFingerprint()).toBe('sf1:b') + expect(readFileSync(join(dir, 'src/workflow.ts'), 'utf8')).toContain('// stored') + }, 30_000) + + it('stops at a second Ctrl-C and says the apply may have been saved and how to recover', async () => { + const run = await applyInterrupted(2) + + expect(run.signal ?? run.code).toSatisfy( + (status: unknown) => status === 'SIGINT' || status === 130 + ) + expect(run.stderr).toContain('Stopped before the apply answered. It may have been saved.') + expect(run.stderr).toContain(`sim workflows source get ${WORKFLOW_ID}`) + expect(savedFingerprint()).toBe('sf1:a') + }, 30_000) +}) diff --git a/packages/sim-cli/src/commands/protocol/source-dir.test.ts b/packages/sim-cli/src/commands/protocol/source-dir.test.ts index f53b24a849e..0f01c8f1ade 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.test.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.test.ts @@ -150,27 +150,55 @@ describe('withSourceDirLock', () => { expect(existsSync(join(dir, SOURCE_DIR_LOCK))).toBe(false) }) - it('prints what the interrupted command may have done, then ends with the signal', () => { - const script = ` + /** + * The shape of `workspaces source apply`: the commit and its write-back run deferred inside the + * directory lock, whose own interrupt handler must wait for them too. + */ + function deferredWriteScript(interrupts: number): string { + return ` + import { writeFileSync } from 'node:fs' + import { join } from 'node:path' + import { setTimeout as sleep } from 'node:timers/promises' import { withSourceDirLock } from ${JSON.stringify(join(import.meta.dirname, 'source-dir.ts'))} - let sending = false - await withSourceDirLock( - process.env.SOURCE_DIR, - async () => { - sending = true - process.kill(process.pid, 'SIGINT') - await new Promise(() => {}) - }, - { interruptNote: () => (sending ? 'It may have been written.' : null) } - ) + import { deferInterrupts } from ${JSON.stringify(join(import.meta.dirname, '../../terminal/interrupts.ts'))} + await withSourceDirLock(process.env.SOURCE_DIR, async () => { + await deferInterrupts( + async () => { + for (let sent = 0; sent < ${interrupts}; sent += 1) { + process.kill(process.pid, 'SIGINT') + await sleep(50) + } + await sleep(300) + writeFileSync(join(process.env.SOURCE_DIR, 'saved'), 'yes') + }, + { waiting: 'Waiting for the write.', abandoned: 'Stopped before the write answered.' } + ) + await new Promise(() => {}) + }) ` - const interrupted = spawnSync('bun', ['-e', script], { + } + + it('finishes a deferred write at an interrupt, then releases the lock and ends with the signal', () => { + const interrupted = spawnSync('bun', ['-e', deferredWriteScript(1)], { + env: { ...process.env, SOURCE_DIR: dir }, + timeout: 20_000, + }) + + expect(interrupted.signal).toBe('SIGINT') + expect(interrupted.stderr.toString()).toContain('Waiting for the write.') + expect(existsSync(join(dir, 'saved'))).toBe(true) + expect(existsSync(join(dir, SOURCE_DIR_LOCK))).toBe(false) + }) + + it('abandons a deferred write at a second interrupt, saying so, and releases the lock', () => { + const interrupted = spawnSync('bun', ['-e', deferredWriteScript(2)], { env: { ...process.env, SOURCE_DIR: dir }, timeout: 20_000, }) expect(interrupted.signal).toBe('SIGINT') - expect(interrupted.stderr.toString()).toContain('It may have been written.') + expect(interrupted.stderr.toString()).toContain('Stopped before the write answered.') + expect(existsSync(join(dir, 'saved'))).toBe(false) expect(existsSync(join(dir, SOURCE_DIR_LOCK))).toBe(false) }) diff --git a/packages/sim-cli/src/commands/protocol/source-dir.ts b/packages/sim-cli/src/commands/protocol/source-dir.ts index cd9c8561523..e3d4d6400ea 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir.ts @@ -37,14 +37,15 @@ import { SOURCE_LOCK_FILE, } from '@sim/workflow-code/limits' import { fingerprintOf, KIND_DIRS } from '@sim/workspace-code' -import { printError } from '#sim-cli/output/io' +import type { Command } from 'commander' +import type { ResolvedProfile } from '../../config/index' import { embedStore } from '../../embed-context' import { SOURCE_BUSY_EXIT_CODE } from '../../exit-codes' import { SimApiError } from '../../http/client' import { embeddedFileContent } from '../../transfer/local-file' import { removeStagingOnSignal, saveToFile } from './files-get' -/** Generated SDK declarations: every block in the workspace catalog, plus `workspace.d.ts`. */ +/** Generated SDK declarations: the blocks the source uses (every block's are cached), plus `workspace.d.ts`. */ export const SOURCE_SDK_DIR = '.sim/sdk' /** A short guide for an agent working in the directory; never a top-level `AGENTS.md`. */ export const SOURCE_AGENTS_FILE = '.sim/AGENTS.md' @@ -63,6 +64,18 @@ export const SOURCE_GITIGNORE_FILE = '.gitignore' * edit from a file it may replace. The CLI's own record: never sent to the server. */ export const SOURCE_WRITTEN_FILE = '.sim/written.json' +/** + * The workspace a starter written by `workflows source create` was typed for, so the create that + * follows targets that workspace. The CLI's own record: never sent, and removed once the workflow + * exists. + */ +export const SOURCE_STARTER_FILE = '.sim/starter.json' +/** + * The edges of the workflow a get or apply printed, by lock key, with the base fingerprint they + * belong to, so an offline check tells an edge the stored workflow holds from one the source + * adds. The CLI's own record: never sent to the server. + */ +export const SOURCE_STORED_EDGES_FILE = '.sim/stored-edges.json' /** * Where a workspace get keeps the blobs it verified until they move into place. A get that fails * leaves them there, so the next one downloads only the rest; the one that completes removes the @@ -99,7 +112,7 @@ const GITATTRIBUTES_LINES = [ `${SOURCE_ENTRY_FILE} text eol=lf`, `${SOURCE_FUNCTIONS_DIR}/** text eol=lf`, ] -/** Files read or written at once on a local disk; the declarations run to thousands of small files. */ +/** Files read or written at once on a local disk; a catalog's declarations run to hundreds of small files. */ export const FILE_CONCURRENCY = 32 /** @@ -170,6 +183,51 @@ export async function readSourceJson(dir: string, path: string): Promise { + const dir: unknown = action.getOptionValue('dir') + if (typeof dir === 'string' && !embedStore.getStore()) { + action.setOptionValue('dir', resolve(dir)) + } + }) +} + +/** + * The workspace a source command acts on for a directory read from `readFrom`: that workspace, + * whatever the profile's default, so a directory stays bound to where it came from. Only a + * `--workspace` naming another one is refused, with `next` saying what to run instead (given the + * workspace the directory was read from and the one named). A directory that records no workspace + * takes the profile's (`null` when none is selected). + */ +export function directoryWorkspace( + dir: string, + readFrom: string | null, + profile: Pick, + next: (readFrom: string, named: string) => string +): string | null { + const named = profile.workspaceId + if (readFrom === null) return named + if (profile.sources.workspaceId !== 'flag' || named === null || named === readFrom) { + return readFrom + } + throw new SimApiError( + `${dir} was read from workspace ${readFrom}, but --workspace names ${named}. ${next(readFrom, named)}`, + 0, + 'SOURCE_OTHER_WORKSPACE' + ) +} + +/** The workspace {@link SOURCE_STARTER_FILE} names, or `null` when the directory holds no starter. */ +export async function readStarterWorkspace(dir: string): Promise { + const workspaceId = toRecord(await readSourceJson(dir, SOURCE_STARTER_FILE)).workspaceId + return typeof workspaceId === 'string' ? workspaceId : null +} + /** * A line file (`.gitattributes`, `.gitignore`) with `lines` appended where missing, or `null` * when it has them all. @@ -283,12 +341,17 @@ export interface SourceDirChange { } /** - * The files that record what the directory was read from: each workflow's lock, base and - * `.sim/written.json`, and the workspace base. They move into place only after every other file, + * The files that record what the directory was read from: each workflow's lock, base, + * `.sim/written.json` and stored edges, and the workspace base. They move into place only after every other file, * so a write cut short leaves the old records beside whatever text did land: the next apply is * refused as stale, never sent as if the old text were current. */ -const RECORD_FILES = [SOURCE_BASE_FILE, SOURCE_LOCK_FILE, SOURCE_WRITTEN_FILE] +const RECORD_FILES = [ + SOURCE_BASE_FILE, + SOURCE_LOCK_FILE, + SOURCE_WRITTEN_FILE, + SOURCE_STORED_EDGES_FILE, +] function isRecordFile(path: string): boolean { return RECORD_FILES.some((file) => path === file || path.endsWith(`/${file}`)) @@ -824,14 +887,7 @@ export async function takeOverStaleLock(lock: string, seen: LockSighting): Promi export async function withSourceDirLock( dir: string, run: () => Promise, - options: { - readOnly?: boolean - /** - * What to print when an interrupt (Ctrl-C) ends the command: read when it arrives, so a - * command can name what a write in flight may have done; `null` prints nothing. - */ - interruptNote?: () => string | null - } = {} + options: { readOnly?: boolean } = {} ): Promise { if (embedStore.getStore()) return run() const absent = async (path: string) => @@ -888,14 +944,7 @@ export async function withSourceDirLock( const owner: LockOwner = { pid: process.pid, host: hostname() } // An interrupt (Ctrl-C) releases the lock too: a lock left behind is taken over only by a // command on the same host, so one left on a folder shared with a container would stay. - const disposeSignalRelease = removeStagingOnSignal( - () => lock, - (signal) => { - const note = options.interruptNote?.() - if (note) printError(note) - process.kill(process.pid, signal) - } - ) + const disposeSignalRelease = removeStagingOnSignal(() => lock) try { await writeFile(join(lock, LOCK_OWNER_FILE), JSON.stringify(owner)) return await run() diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.test.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.test.ts new file mode 100644 index 00000000000..e2d3979b4e8 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.test.ts @@ -0,0 +1,35 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import * as exitCodes from '../../exit-codes' +import { WORKFLOW_TREE } from './workflow-source-get' +import { WORKSPACE_TREE } from './workspace-source-dir' + +/** + * Every code a source command can exit with: each `*_EXIT_CODE` the CLI defines except the + * run-wait timeout (4), which no source command returns, plus 0, 1 and Ctrl-C's 130. + */ +const SOURCE_EXIT_CODES = [ + 0, + 1, + ...Object.entries(exitCodes) + .filter(([name]) => name.endsWith('_EXIT_CODE') && name !== 'STILL_RUNNING_EXIT_CODE') + .map(([, code]) => code as number), + 130, +].sort((a, b) => a - b) + +const WORKFLOW_SDK_README = readFileSync( + join(import.meta.dirname, '../../../../workflow-sdk/README.md'), + 'utf8' +) + +describe('the exit-code tables an agent or a reader relies on', () => { + it.each([ + ['the workflow directory guide', WORKFLOW_TREE.agents], + ['the workspace directory guide', WORKSPACE_TREE.agents], + ['the workflow SDK README', WORKFLOW_SDK_README], + ])('%s lists every code a source command exits with', (_name, text) => { + const listed = [...text.matchAll(/^\| (\d+) \|/gm)].map((match) => Number(match[1])) + expect(listed).toEqual(SOURCE_EXIT_CODES) + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts new file mode 100644 index 00000000000..9a4fed61347 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts @@ -0,0 +1,105 @@ +import { DEFERRED_INTERRUPT_BOUND_MS } from '#sim-cli/terminal/interrupts' +import { + AUTH_REQUIRED_EXIT_CODE, + RATE_LIMITED_EXIT_CODE, + SOURCE_BUSY_EXIT_CODE, + SOURCE_DRIFT_EXIT_CODE, + SOURCE_LOCAL_CHANGES_EXIT_CODE, + SOURCE_LOCKED_EXIT_CODE, + SOURCE_PARTIAL_EXIT_CODE, + SOURCE_PLANNED_EXIT_CODE, + SOURCE_STALE_EXIT_CODE, + SOURCE_TOO_LARGE_EXIT_CODE, +} from '../../exit-codes' + +/** The status a shell reports for a command Ctrl-C ended (128 + SIGINT). */ +const INTERRUPTED_EXIT_CODE = 130 + +/** How long an interrupted write waits for its answer before it stops without it. */ +const DEFERRED_SECONDS = DEFERRED_INTERRUPT_BOUND_MS / 1000 + +/** + * Every exit code of the `workflows source` and `workspaces source` commands, as the Markdown + * table both directory guides (`.sim/AGENTS.md`) print, built from the codes the CLI exits with so + * the guides cannot drift from them. `prefix` is the command group the guide is for. + */ +export function sourceExitCodesTable( + prefix: 'sim workflows source' | 'sim workspaces source' +): string { + const rows: Array<[number, string, string]> = [ + [0, 'Applied or created, a dry run or check with no errors, or a clean status', 'Go on'], + [ + 1, + 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed', + 'Fix what it printed and run it again', + ], + [ + SOURCE_STALE_EXIT_CODE, + 'Stale (`SOURCE_STALE`, or `E_APPLY_STALE` on a dry run): the server changed what this directory was read from, and nothing was written', + `\`${prefix} get --dir \`, reapply your edit there, dry-run, apply`, + ], + [ + SOURCE_LOCKED_EXIT_CODE, + 'A workflow or its folder is locked (`E_APPLY_LOCKED`); a locked block exits 1 instead', + 'Ask someone to unlock it in the editor', + ], + [ + SOURCE_TOO_LARGE_EXIT_CODE, + 'The source is over the size limits (on a get, status or diff too)', + 'Split function code, or remove blocks or resources', + ], + [ + SOURCE_PARTIAL_EXIT_CODE, + '`sim workspaces source apply` stopped part-way: what was written is saved into the directory', + 'Apply again until it exits 0', + ], + [ + RATE_LIMITED_EXIT_CODE, + "Still rate limited (HTTP 429) after the CLI's retries (up to 6, at most a minute apart); nothing more was written", + 'Wait a minute and run the same command again', + ], + [ + SOURCE_BUSY_EXIT_CODE, + 'Busy: another apply is writing the workspace (`SOURCE_BUSY`), or another sim command is using this directory (`SOURCE_DIR_BUSY`); nothing was written', + 'Run it again once the other finished', + ], + [ + SOURCE_LOCAL_CHANGES_EXIT_CODE, + '`status`: only your edits, which applying writes', + 'Dry-run, then apply', + ], + [ + SOURCE_DRIFT_EXIT_CODE, + '`status`: the server changed since the get (drift, or a conflict with your edits)', + 'Follow the `Next:` line it prints', + ], + [ + AUTH_REQUIRED_EXIT_CODE, + 'Not logged in, the login expired, or the server refused the credential (`AUTH_REQUIRED`); nothing was sent', + '`sim login`, or fix or unset `SIM_API_KEY` when the key comes from it; then the same command again', + ], + [ + SOURCE_PLANNED_EXIT_CODE, + '`apply` or `create` without `--yes` and no terminal to ask: it printed the plan, found no errors, and wrote nothing', + 'Run the same command with `--yes`', + ], + [ + INTERRUPTED_EXIT_CODE, + `Interrupted (Ctrl-C). An apply or create already sent is finished and saved into the directory first, waiting up to ${DEFERRED_SECONDS}s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written`, + `\`${prefix} status --dir .\``, + ], + ] + return [ + '| Code | Meaning | Do |', + '|---|---|---|', + ...rows.map(([code, meaning, next]) => `| ${code} | ${meaning} | ${next} |`), + ].join('\n') +} + +/** The exit codes every source write (apply, create) shares, appended to its help. */ +export const SOURCE_WRITE_EXIT_CODES_HELP = ` +Exit ${AUTH_REQUIRED_EXIT_CODE}: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. +Run \`sim login\`, or fix or unset SIM_API_KEY when the key comes from it. Exit ${INTERRUPTED_EXIT_CODE}: interrupted +(Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to +${DEFERRED_SECONDS}s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may +have been written and how to check.` diff --git a/packages/sim-cli/src/commands/protocol/source-primer.ts b/packages/sim-cli/src/commands/protocol/source-primer.ts new file mode 100644 index 00000000000..3d77d91b74a --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-primer.ts @@ -0,0 +1,111 @@ +/** + * The primer a source get prints and writes atop `.sim/AGENTS.md`: the few rules an agent needs to + * edit a source directory without reading the guide, plus a snippet for each builder the source + * already calls, so the blocks in front of it need no lookup. The guide below it is the reference. + */ +import type { BlockCatalog } from '@sim/workflow-code' +import { SOURCE_BASE_FILE, SOURCE_ENTRY_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' +import { SDK_SPECIAL_BUILDERS } from '@sim/workflow-code/sdk' +import { + builderSnippet, + describeBuilder, + referencedBuilders, + typecheckCommand, +} from '@sim/workflow-sdk/codegen' +import { + RATE_LIMITED_EXIT_CODE, + SOURCE_BUSY_EXIT_CODE, + SOURCE_DRIFT_EXIT_CODE, + SOURCE_LOCAL_CHANGES_EXIT_CODE, + SOURCE_STALE_EXIT_CODE, +} from '../../exit-codes' +import { SOURCE_AGENTS_FILE } from './source-dir' + +/** Which directory a primer is for. */ +export type PrimerKind = 'workflow' | 'workspace' + +/** The primer without its builder snippets stays within this many bytes. */ +export const PRIMER_MAX_BYTES = 3072 + +/** At most this many snippets, and this many bytes of them; the rest are named for `params`. */ +const MAX_SNIPPETS = 12 +const MAX_SNIPPET_BYTES = 4096 + +/** The command that prints any builder's params (`workflow-source-params.ts`). */ +const PARAMS_COMMAND = 'sim workflows source params' + +/** The rules every source directory shares, after the kind's own edit loop. */ +function sharedRules(group: string): string { + return `- **Identity**: a block is its entry in \`${SOURCE_LOCK_FILE}\`, which binds it by its const name or its display name. Change the display name (first argument) freely: the block keeps its id. Renaming the const too keeps the id when its settings, neighbours or references still tie it to the old block; otherwise the apply refuses it as a delete (\`E_APPLY_BLOCK_DELETE\`), and the dry run names it. Safest: keep the const in the same apply. +- **References**: in \`${SOURCE_ENTRY_FILE}\` read an output as \`x.content\`, a loop or parallel item as \`p.currentItem\` (\`p.index\`), text as \`\` t\`…\` \`\`, env as \`env.NAME\`. In a function code file (\`codeFile('./functions/x.js')\`) read an output as \`$ref..\`, the key being the display name lowercased with whitespace and dots removed ("Triage Agent" → \`$ref.triageagent.content\`); in a loop body \`$ref.loop.currentItem\` / \`$ref.loop.index\`, in a parallel \`$ref.parallel.currentItem\` / \`$ref.parallel.index\`; env as \`$env.NAME\`. +- **Look up a builder**: \`${PARAMS_COMMAND} \` (\`table.upsertRow\`, \`agent\`, \`trigger.schedule\`; a namespace such as \`gmail\` lists its builders), offline. +- **Exit codes**: 0 ok, 1 errors (fix what it printed), ${SOURCE_STALE_EXIT_CODE} stale (\`${group} get --dir \`, redo the edit there), ${RATE_LIMITED_EXIT_CODE} rate limited, ${SOURCE_BUSY_EXIT_CODE} busy, ${SOURCE_LOCAL_CHANGES_EXIT_CODE}/${SOURCE_DRIFT_EXIT_CODE} \`status\`: your edits / the server changed. Every code: \`${SOURCE_AGENTS_FILE}\`.` +} + +const WORKFLOW_PRIMER = `# Sim workflow source: primer + +\`${SOURCE_ENTRY_FILE}\` is a Sim workflow as TypeScript; the server compiles it statically and nothing in it runs. Commands read the workflow id from \`${SOURCE_BASE_FILE}\`; run them from this directory. + +**Edit loop**: edit \`${SOURCE_ENTRY_FILE}\` (and \`functions/*\`) → \`sim workflows source check --dir .\` (offline; it also adds new blocks to \`tsconfig.json\`) → \`${typecheckCommand()}\` → \`sim workflows source apply --dir . --dry-run\` → \`--yes\`. Rerun each step until it exits 0. Apply saves the draft in canonical form: read \`${SOURCE_ENTRY_FILE}\` again before the next edit. + +- **Deletes**: removing a block's declaration deletes the block only with \`apply --allow-delete\`; without it the apply is refused (\`E_APPLY_BLOCK_DELETE\`). +- **Deploy**: apply saves a draft; \`apply --deploy\` also deploys it, or \`sim workflows deploy \` publishes the saved draft later. +${sharedRules('sim workflows source')} +` + +const WORKSPACE_PRIMER = `# Sim workspace source: primer + +This directory is a Sim workspace as files (\`tables/\`, \`workflows//workflow.ts\`, …); the server reads them statically and nothing runs. Commands read the workspace id from \`${SOURCE_BASE_FILE}\`; run them from this directory. + +**Edit loop**: edit the files → \`sim workspaces source check --dir .\` (offline; regenerates \`.sim/sdk/workspace.d.ts\`, adds new blocks to \`tsconfig.json\`) → \`${typecheckCommand()}\` → \`sim workspaces source apply --dir . --dry-run\` (the plan) → \`--yes\`. Rerun each step until it exits 0. Apply rewrites the directory from what it stored: read a file again before the next edit. + +- **Deletes**: a removed file (a resource this directory managed), table column or workflow block is deleted only with \`apply --allow-delete\`; without it the apply is refused. A kind is managed only when its folder exists. Deletes cannot be undone. +- **Deploy**: \`deployments/.ts\` (\`defineDeployment({ live: 'draft' })\`) deploys that workflow on apply; without one, apply never deploys. +${sharedRules('sim workspaces source')} +` + +/** The primer for a `kind` directory, without builder snippets. */ +export function sourcePrimer(kind: PrimerKind): string { + return kind === 'workflow' ? WORKFLOW_PRIMER : WORKSPACE_PRIMER +} + +/** + * A snippet for each builder `sources` call ({@link builderSnippet}), integration builders before + * the core ones (`agent`, `fn`), whose shape the source shows. Over the cap, the rest are named + * for `params`. Empty when the source calls none. + */ +function builderSnippets(catalog: BlockCatalog, sources: Iterable): string { + const core = (key: string) => Object.hasOwn(SDK_SPECIAL_BUILDERS, key) || key === 'agent' + const keys = referencedBuilders(catalog, sources) + const ordered = [...keys.filter((key) => !core(key)), ...keys.filter(core)] + const snippets: string[] = [] + const skipped: string[] = [] + let bytes = 0 + for (const key of ordered) { + const reference = describeBuilder(catalog, key) + if (!reference) continue + const snippet = builderSnippet(reference) + if (snippets.length >= MAX_SNIPPETS || bytes + snippet.length > MAX_SNIPPET_BYTES) { + skipped.push(key) + continue + } + snippets.push(snippet) + bytes += snippet.length + } + if (snippets.length === 0) return '' + const rest = + skipped.length > 0 + ? `\nAlso used (\`${PARAMS_COMMAND} \`): ${skipped.map((key) => `\`${key}\``).join(', ')}.\n` + : '' + return `## Builders this source uses\n\n${snippets.join('\n')}\n${rest}` +} + +/** The primer with the snippets of the builders `sources` call. */ +export function primerFor( + kind: PrimerKind, + catalog: BlockCatalog, + sources: Iterable +): string { + const snippets = builderSnippets(catalog, sources) + return snippets ? `${sourcePrimer(kind)}\n${snippets}` : sourcePrimer(kind) +} diff --git a/packages/sim-cli/src/commands/protocol/source-status.ts b/packages/sim-cli/src/commands/protocol/source-status.ts index 96e0d092ea6..820dbcc78e1 100644 --- a/packages/sim-cli/src/commands/protocol/source-status.ts +++ b/packages/sim-cli/src/commands/protocol/source-status.ts @@ -12,6 +12,7 @@ import { createTwoFilesPatch, FILE_HEADERS_ONLY } from 'diff' import type { OutputFormat } from '../../config/index' import { AUTH_REQUIRED_CODE, + AUTH_REQUIRED_EXIT_CODE, SOURCE_DRIFT_EXIT_CODE, SOURCE_LOCAL_CHANGES_EXIT_CODE, SOURCE_TOO_LARGE_EXIT_CODE, @@ -47,8 +48,9 @@ read); ${SOURCE_LOCAL_CHANGES_EXIT_CODE} local changes only (applying writes the directory was read: drift (a resource edited, added or deleted there that the directory leaves alone) or a conflict (also changed here, so applying it is refused as stale), with or without local changes; ${SOURCE_TOO_LARGE_EXIT_CODE} the source is too large for the server to read in one dry run; 1 the command -failed. A server that cannot be reached is not a failure: the status shows the local changes only -and says so (exit 0 or ${SOURCE_LOCAL_CHANGES_EXIT_CODE}).` +failed; ${AUTH_REQUIRED_EXIT_CODE} not logged in (AUTH_REQUIRED) or the credential was refused: the status shows the +local changes only, then run \`sim login\`. A server that cannot be reached is not a failure: the +status shows the local changes only and says so (exit 0 or ${SOURCE_LOCAL_CHANGES_EXIT_CODE}).` /** What `diff` compares, and its exit codes. */ export const DIFF_HELP = ` @@ -60,8 +62,12 @@ file or folder (and in a workspace directory by kind:key, such as table:leads). differences were shown, whether or not there were any; ${SOURCE_TOO_LARGE_EXIT_CODE} the source is too large for the server to read; 1 the command failed.` -/** The exit status of a status report: drift or conflict, else local changes, else clean. */ -export function statusExitCode(state: SourceStatus['state']): number { +/** + * The exit status of a status report: no login (`login` holds the command to run), else drift or + * conflict, else local changes, else clean. + */ +export function statusExitCode(state: SourceStatus['state'], login: string | null): number { + if (login) return AUTH_REQUIRED_EXIT_CODE if (state === 'conflict' || state === 'drift') return SOURCE_DRIFT_EXIT_CODE return state === 'local' ? SOURCE_LOCAL_CHANGES_EXIT_CODE : 0 } @@ -69,28 +75,34 @@ export function statusExitCode(state: SourceStatus['state']): number { /** Gateway statuses that mean the server was not reached, rather than that it refused. */ const UNREACHABLE_STATUSES = new Set([502, 503, 504]) +/** + * Whether `error` says this profile cannot authenticate: the CLI's own refusal of a profile with + * no login, or a credential the server refused (`401`, which exits 13 like it). + */ +export function credentialRefused(error: unknown): boolean { + return ( + error instanceof SimApiError && + ((error.status === 0 && error.code === AUTH_REQUIRED_CODE) || error.status === 401) + ) +} + /** * Why the server could not be reached (no connection, a timeout, a gateway that gave up, no login - * on the profile), as a sentence, or `null` for any other failure (a cancelled request included), - * which a status reports rather than hides. + * on the profile or a credential it refused), as a sentence, or `null` for any other failure (a + * cancelled request included), which a status reports rather than hides. */ export function unreachableReason(error: unknown): string | null { if (!(error instanceof SimApiError)) return null - const offline = - noAnswer(error, UNREACHABLE_STATUSES) || - (error.status === 0 && error.code === AUTH_REQUIRED_CODE) - if (!offline) return null + if (!noAnswer(error, UNREACHABLE_STATUSES) && !credentialRefused(error)) return null return /[.!?]$/.test(error.message) ? error.message : `${error.message}.` } /** - * The login a status recommends when `error` is the CLI's own refusal of a profile with no login - * (which a status treats as an unreachable server), or `null` for any other failure. + * The login a status recommends when `error` says the profile cannot authenticate (which a status + * treats as an unreachable server), or `null` for any other failure. */ export function loginCommand(error: unknown, authProfile: string): string | null { - return error instanceof SimApiError && error.status === 0 && error.code === AUTH_REQUIRED_CODE - ? `sim login --profile ${authProfile}` - : null + return credentialRefused(error) ? `sim login --profile ${authProfile}` : null } const SERVER_WORDS = { diff --git a/packages/sim-cli/src/commands/protocol/source-tree.ts b/packages/sim-cli/src/commands/protocol/source-tree.ts index bbc729c84df..f5ba410c59f 100644 --- a/packages/sim-cli/src/commands/protocol/source-tree.ts +++ b/packages/sim-cli/src/commands/protocol/source-tree.ts @@ -5,14 +5,19 @@ * the agent guide and the LF lines. */ import { createHash } from 'node:crypto' -import { join } from 'node:path' -import { isRecordLike } from '@sim/utils/object' +import { rmSync } from 'node:fs' +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { mapWithConcurrency } from '@sim/utils/concurrency' +import { isRecordLike, toRecord } from '@sim/utils/object' import { escapeRegExp } from '@sim/utils/string' import type { BlockCatalog } from '@sim/workflow-code' import { catalogBlock } from '@sim/workflow-code/catalog/view' +import { SOURCE_BASE_FILE } from '@sim/workflow-code/limits' import { COOKBOOK_EXAMPLES, CORE_DTS, + checkedBlockTypes, defaultBlockFiles, fullBlockFiles, generateSdkTypes, @@ -24,7 +29,8 @@ import { sdkIndexHeader, type WorkspaceManifest, } from '@sim/workflow-sdk/codegen' -import { printError } from '#sim-cli/output/io' +import { printError, printLine } from '#sim-cli/output/io' +import type { OutputFormat } from '../../config/index' import { readJsonFile, writeJsonFile } from '../../config/json-file' import { configDir } from '../../config/paths' import { embedStore } from '../../embed-context' @@ -33,7 +39,10 @@ import type { GetSourceResourcesResponse, GetWorkflowSourceResponse } from '../. import { SimApiError } from '../../http/client' import type { OperationClient } from '../../runtime/called-operations' import { cliVersion } from '../../version' +import { printProtocolResult } from './result' import { + FILE_CONCURRENCY, + readSourceJson, readSourceText, SOURCE_SDK_DIR as SDK_DIR, SOURCE_AGENTS_FILE, @@ -44,6 +53,7 @@ import { updateSourceDir, withMissingLines, } from './source-dir' +import { type PrimerKind, primerFor } from './source-primer' const TSCONFIG_FILE = 'tsconfig.json' /** Type-checks the source against every current block, for a block the default one leaves out. */ @@ -75,26 +85,61 @@ function isBlockCatalog(value: unknown): value is BlockCatalog { ) } -/** Workspace ids are uuids; anything else is never used to name a cache file. */ +/** Workspace ids are uuids and catalog hashes hex; anything else never names a cache file. */ const CACHE_KEY = /^[\w-]+$/ +/** The CLI's catalog cache: every catalog by its hash, and each workspace's current hash. */ +function catalogCacheDir(): string { + return join(configDir(), 'cache', 'source-catalog') +} + /** - * Where the workspace's catalog is cached: one file per workspace in the CLI's config directory, - * so every `source get` in the workspace (whatever the workflow or output directory) revalidates - * the same copy. Callers gate the id with `CACHE_KEY` first. + * Where a catalog is cached: by its content hash, so the workspace catalog and every per-workflow + * catalog share one file per distinct catalog, however many workflows and workspaces name it. + * Callers gate the hash with `CACHE_KEY` first. */ -function catalogCachePath(workspaceId: string): string { - return join(configDir(), 'cache', 'source-catalog', `${workspaceId}.json`) +function catalogCachePath(hash: string): string { + return join(catalogCacheDir(), `${hash}.json`) } /** - * The cached catalog, or `null`. Standalone runs only: an embedded run has no local disk of its - * own to cache on. The file is read without following a link and under a size cap. + * Where the hash of a workspace's catalog is recorded, so a get revalidates it by ETag and an + * offline command finds it. Callers gate the id with `CACHE_KEY` first. */ +function workspaceCatalogPointerPath(workspaceId: string): string { + return join(catalogCacheDir(), 'workspaces', `${workspaceId}.json`) +} + +/** A pointer is one hash; a larger file was not written by this CLI. */ +const MAX_POINTER_BYTES = 4096 + +/** + * The cached catalog of `hash`, or `null`. Standalone runs only: an embedded run has no local disk + * of its own to cache on. The file is read without following a link and under a size cap, and + * kept only when the catalog in it names the hash its file does. + */ +export function cachedCatalogByHash(hash: unknown): BlockCatalog | null { + if (embedStore.getStore() || typeof hash !== 'string' || !CACHE_KEY.test(hash)) return null + const cached = readJsonFile(catalogCachePath(hash), MAX_CATALOG_CACHE_BYTES) + return isBlockCatalog(cached) && cached.hash === hash ? cached : null +} + +/** The cached catalog of the workspace, by the hash its last catalog read recorded, or `null`. */ export function cachedCatalog(workspaceId: string): BlockCatalog | null { if (embedStore.getStore() || !CACHE_KEY.test(workspaceId)) return null - const cached = readJsonFile(catalogCachePath(workspaceId), MAX_CATALOG_CACHE_BYTES) - return isBlockCatalog(cached) ? cached : null + const pointer = readJsonFile(workspaceCatalogPointerPath(workspaceId), MAX_POINTER_BYTES) + return isRecordLike(pointer) ? cachedCatalogByHash(pointer.hash) : null +} + +/** + * The cached catalog a workflow source directory was printed against, by the hash its + * `.sim/base.json` records (the workspace's, or the workflow's own for a workflow holding a type + * the workspace catalog lacks), or `null` when the directory has no base or that catalog is not + * cached. An offline check compiles against it, so it agrees with the dry run, which compiles + * against the same catalog. + */ +export async function cachedSourceCatalog(dir: string): Promise { + return cachedCatalogByHash(toRecord(await readSourceJson(dir, SOURCE_BASE_FILE)).catalogHash) } /** The catalog a `getSourceCatalog` answer carries. */ @@ -105,22 +150,71 @@ function catalogFrom(response: { data: unknown }): BlockCatalog { return response.data } -/** Replaces the cached catalog atomically; a failed write only costs the next get a download. */ -function writeCachedCatalog(workspaceId: string, catalog: BlockCatalog): void { +/** + * Caches `catalog` by its hash, replacing the file atomically; a failed write only costs the next + * get a download. An unchanged catalog is not rewritten. + */ +function writeCachedCatalog(catalog: BlockCatalog): void { + if (embedStore.getStore() || !CACHE_KEY.test(catalog.hash)) return + if (cachedCatalogByHash(catalog.hash)) return + writeJsonFile(catalogCachePath(catalog.hash), catalog) +} + +/** + * Records `hash` as the workspace's catalog, dropping the per-workspace copy and the + * per-workflow copies an older CLI cached under the workspace id. + */ +function writeWorkspaceCatalogPointer(workspaceId: string, hash: string): void { if (embedStore.getStore() || !CACHE_KEY.test(workspaceId)) return - writeJsonFile(catalogCachePath(workspaceId), catalog) + writeJsonFile(workspaceCatalogPointerPath(workspaceId), { hash }) + for (const legacy of [`${workspaceId}.json`, workspaceId]) { + if (legacy === `${hash}.json`) continue + try { + rmSync(join(catalogCacheDir(), legacy), { recursive: true, force: true }) + } catch {} + } } /** - * The workspace's block catalog, revalidated by ETag against the per-workspace cache: a `304` - * reuses it, anything else replaces it. A workflow can hold a type the workspace no longer offers - * (a legacy version, a block an allowlist now withholds); only then is that workflow's compile - * catalog fetched instead, uncached, so its types still resolve. + * The catalog of the workflow `workflowId` (the workspace catalog plus the types it stores that the + * workspace catalog lacks), fetched and cached by hash; `knownHash`, a hash the workflow's catalog + * may have, is sent as its ETag, so a catalog already cached answers `304` with no body. + */ +export async function loadWorkflowCatalog( + client: OperationClient<'getSourceCatalog'>, + workspaceId: string, + workflowId: string, + knownHash?: string +): Promise { + const known = cachedCatalogByHash(knownHash) + try { + const catalog = catalogFrom( + await client.request<{ data: unknown }>('getSourceCatalog', { + params: { workspaceId }, + query: { workflowId }, + ...(known ? { headers: { 'if-none-match': `"${known.hash}"` } } : {}), + }) + ) + writeCachedCatalog(catalog) + return catalog + } catch (error) { + if (!known || !(error instanceof SimApiError) || error.status !== 304) throw error + return known + } +} + +/** + * The workspace's block catalog, revalidated by ETag against the cached one: a `304` reuses it, + * anything else replaces it. A workflow can hold a type the workspace no longer offers (a legacy + * version, a block an allowlist now withholds); only then is that workflow's own catalog used + * instead, so its types still resolve: the one cached under the hash its base records + * (`catalogHash`), else fetched ({@link loadWorkflowCatalog}). */ export async function loadCatalog( client: OperationClient<'getSourceCatalog'>, source: Pick & { workflowId?: string + catalogHash?: string } ): Promise { const cached = cachedCatalog(source.workspaceId) @@ -132,23 +226,18 @@ export async function loadCatalog( ...(cached ? { headers: { 'if-none-match': `"${cached.hash}"` } } : {}), }) ) - writeCachedCatalog(source.workspaceId, catalog) + writeCachedCatalog(catalog) } catch (error) { if (!cached || !(error instanceof SimApiError) || error.status !== 304) throw error catalog = cached } - if ( - source.workflowId === undefined || - source.blockTypes.every((type) => catalogBlock(catalog, type) !== undefined) - ) { - return catalog - } - return catalogFrom( - await client.request<{ data: unknown }>('getSourceCatalog', { - params: { workspaceId: source.workspaceId }, - query: { workflowId: source.workflowId }, - }) - ) + writeWorkspaceCatalogPointer(source.workspaceId, catalog.hash) + const declaresAll = (candidate: BlockCatalog) => + source.blockTypes.every((type) => catalogBlock(candidate, type) !== undefined) + if (source.workflowId === undefined || declaresAll(catalog)) return catalog + const printedWith = cachedCatalogByHash(source.catalogHash) + if (printedWith && declaresAll(printedWith)) return printedWith + return loadWorkflowCatalog(client, source.workspaceId, source.workflowId) } /** @@ -334,16 +423,72 @@ function sdkGeneratorLine(): string { return `// generated by sim ${cliVersion()} · core.d.ts ${core}` } -/** The generated declarations, with the generator line under the index's catalog header. */ -function sdkTypeFiles(catalog: BlockCatalog): Array { +/** Under the index's catalog header, the generator line; `blockTypes` as `generateSdkTypes` takes it. */ +function sdkTypeFiles( + catalog: BlockCatalog, + blockTypes?: ReadonlySet +): Array { const generator = sdkGeneratorLine() - return generateSdkTypes(catalog).map((file) => { - if (file.path !== SDK_INDEX_FILE) return [`${SDK_DIR}/${file.path}`, file.text] as const + return generateSdkTypes(catalog, blockTypes ? { blockTypes } : {}).map((file) => { + if (file.path !== SDK_INDEX_FILE) return [file.path, file.text] as const const [header, ...rest] = file.text.split('\n') - return [`${SDK_DIR}/${file.path}`, [header, generator, ...rest].join('\n')] as const + return [file.path, [header, generator, ...rest].join('\n')] as const }) } +/** The head a current `index.d.ts` starts with: the catalog header and the generator line. */ +function sdkIndexHead(catalog: BlockCatalog): string { + return `${sdkIndexHeader(catalog)}\n${sdkGeneratorLine()}` +} + +/** + * Where the declarations of every block of `catalog` are cached, once per catalog and generator + * (the generator line's digest), beside the catalog itself. `tsconfig.full.json` includes the + * `index.d.ts` there, so a directory holds only the blocks it uses. Callers gate the hash with + * `CACHE_KEY` first. + */ +function sdkCacheDir(catalog: BlockCatalog): string { + const generator = createHash('sha256').update(sdkGeneratorLine(), 'utf8').digest('hex') + return join(catalogCacheDir(), catalog.hash, `sdk-${generator.slice(0, 16)}`) +} + +/** + * The cached full declarations' `index.d.ts` for `catalog` (an absolute path), generating them + * into a staging folder moved into place on the first call; later calls only check the index. + * `null` in an embedded run, which has no disk of its own, or when the cache cannot be written: + * the directory then holds every block itself, as before the cache. + */ +async function sdkCacheIndex(catalog: BlockCatalog): Promise { + if (embedStore.getStore() || !CACHE_KEY.test(catalog.hash)) return null + const dir = sdkCacheDir(catalog) + const index = join(dir, SDK_INDEX_FILE) + const head = sdkIndexHead(catalog) + const current = await readFile(index, 'utf8').catch(() => null) + if (current?.startsWith(`${head}\n`)) return index + try { + await mkdir(dirname(dir), { recursive: true }) + const staging = await mkdtemp(`${dir}.staging-`) + try { + const files = sdkTypeFiles(catalog) + const parents = new Set(files.map(([path]) => dirname(join(staging, path)))) + for (const parent of parents) await mkdir(parent, { recursive: true }) + await mapWithConcurrency(files, FILE_CONCURRENCY, ([path, text]) => + writeFile(join(staging, path), text) + ) + await rm(dir, { recursive: true, force: true }) + // Another command that generated the same files first already moved them into place. + await rename(staging, dir).catch(async (error: unknown) => { + if (!(await readFile(index, 'utf8').catch(() => null))?.startsWith(`${head}\n`)) throw error + }) + } finally { + await rm(staging, { recursive: true, force: true }) + } + return index + } catch { + return null + } +} + /** What a tree write reports: the same fields after a get and a create scaffold. */ export interface SourceTree { /** How many files were written. */ @@ -352,6 +497,8 @@ export interface SourceTree { workspaceTypes: boolean /** The agent guide to read before editing. */ guide: typeof SOURCE_AGENTS_FILE + /** The primer atop the guide (`source-primer.ts`), printed so an agent needs no other read. */ + primer: string } /** Reports a `tsconfig.json` the CLI left alone and summarizes a tree write. */ @@ -359,20 +506,46 @@ export function finishTree( dir: string, written: number, workspaceTypes: WorkspaceTypes | null, - ownsTsconfig: boolean + tree: { ownsTsconfig: boolean; primer: string } ): SourceTree { - if (!ownsTsconfig) { + if (!tree.ownsTsconfig) { printError( `note: ${TSCONFIG_FILE} in ${dir} is not the CLI's, so it was left as it is. To type-check the source, map "@sim/workflow-sdk" to ["./${SDK_DIR}/core.d.ts"] in its compilerOptions.paths.` ) } - return { files: written, workspaceTypes: workspaceTypes !== null, guide: SOURCE_AGENTS_FILE } + return { + files: written, + workspaceTypes: workspaceTypes !== null, + guide: SOURCE_AGENTS_FILE, + primer: tree.primer, + } +} + +/** + * Prints a tree write's result: in JSON or YAML one document with the primer as its `primer` + * field, otherwise the fields and then the primer as text, so it reads as written. + */ +export function printTreeResult( + format: OutputFormat, + result: Record & { primer: string } +): void { + if (format === 'json' || format === 'yaml') { + printProtocolResult(format, result) + return + } + const { primer, ...fields } = result + printProtocolResult(format, fields) + printLine(`\n${primer}`) } /** What a directory's editable tree holds beside the printed source. */ export interface SourceTreeProfile { - /** `.sim/AGENTS.md`. */ + /** Which primer opens `.sim/AGENTS.md`. */ + kind: PrimerKind + /** The reference in `.sim/AGENTS.md`, below the primer. */ agents: string + /** More guides beside `.sim/AGENTS.md` (path → text), rewritten every time like it. */ + guides?: ReadonlyArray /** * A `tsconfig.json` the CLI writes, starting with {@link TSCONFIG_MARKER}, that type-checks the * source with the SDK declarations `sdkFiles` (directory-relative paths). @@ -391,13 +564,15 @@ export interface SourceTreeProfile { /** * The SDK files each written tsconfig includes for source that uses the `used` block types: * `tsconfig.json` the models, `workspace.d.ts` and only those blocks (plus the core builders'), - * so `tsc` reads a few dozen files instead of every block; `tsconfig.full.json` every current - * block for adding one the source does not use yet. + * which the directory holds, so `tsc` reads a few dozen files instead of every block; + * `tsconfig.full.json` every current block for adding one the source does not use yet, through + * `fullIndex` (the cached index, else the directory's own). */ function sdkIncludes( catalog: BlockCatalog, used: readonly string[], - hasWorkspaceTypes: boolean + hasWorkspaceTypes: boolean, + fullIndex: string ): { tsconfig: string[]; full: string[] } { const workspace = hasWorkspaceTypes ? [WORKSPACE_TYPES_FILE] : [] const inSdk = (files: readonly string[]) => files.map((file) => `${SDK_DIR}/${file}`) @@ -407,25 +582,33 @@ function sdkIncludes( ...workspace, ...inSdk(defaultBlockFiles(catalog, used)), ], - full: [...inSdk(fullBlockFiles(catalog, used)), ...workspace], + full: [ + ...fullBlockFiles(catalog, used).map((file) => + file === SDK_INDEX_FILE ? fullIndex : `${SDK_DIR}/${file}` + ), + ...workspace, + ], } } +/** The block types source that uses `use` names: the server's, and the builders it calls. */ +function usedBlockTypes(catalog: BlockCatalog, use: SourceTreeUse): string[] { + return [...new Set([...(use.blockTypes ?? []), ...referencedBlockTypes(catalog, use.sources)])] +} + /** - * The CLI's tsconfig files for source that uses `use`: each written when it is absent or still + * The CLI's tsconfig files for the `used` block types: each written when it is absent or still * starts with {@link TSCONFIG_MARKER}, so a person's own file is never replaced. */ function tsconfigWrites( catalog: BlockCatalog, hasWorkspaceTypes: boolean, profile: Pick, - use: SourceTreeUse, + used: readonly string[], + fullIndex: string, current: { tsconfig: string | null; full: string | null } ): { write: Array; ownsTsconfig: boolean } { - const used = [ - ...new Set([...(use.blockTypes ?? []), ...referencedBlockTypes(catalog, use.sources)]), - ] - const includes = sdkIncludes(catalog, used, hasWorkspaceTypes) + const includes = sdkIncludes(catalog, used, hasWorkspaceTypes, fullIndex) const owned = (text: string | null) => text === null || text.startsWith(TSCONFIG_MARKER) const write: Array = [] const ownsTsconfig = owned(current.tsconfig) @@ -435,13 +618,36 @@ function tsconfigWrites( } /** - * Rewrites the CLI's tsconfig files for the source `use` names: after an apply, which keeps the - * declarations but may have added a block `tsconfig.json` should now include, and on a check, so - * a block the edit added type-checks before anything is applied. `incremental` (a check) keeps - * the blocks the files include now, since a check scans only the source text and not the - * server's block types, and rewrites only files that exist. It reads the workspace's cached - * catalog, so it costs no request; without one nothing changes until the next get. A file whose - * text would not change is not rewritten. + * The generated declarations a directory holds for the `used` block types, under the SDK + * directory: the models, an index and `INDEX.md` of its blocks, and the file of each block + * `tsconfig.json` includes. Without a cache (`cached` false) every block, as the full config then + * reads the directory's own index. + */ +function localSdkFiles( + catalog: BlockCatalog, + used: readonly string[], + cached: boolean +): Array { + const blockTypes = cached ? new Set(checkedBlockTypes(catalog, used)) : undefined + return sdkTypeFiles(catalog, blockTypes).map( + ([path, text]) => [`${SDK_DIR}/${path}`, text] as const + ) +} + +/** `tsconfig.full.json`'s index: the cached one, else the directory's own. */ +function fullIndexPath(cacheIndex: string | null): string { + return cacheIndex ?? SDK_INDEX_PATH +} + +/** + * Brings the CLI's tsconfig files and the block declarations the directory holds up to date with + * the source `use` names: after an apply, which may have added a block, and on a check, so a block + * the edit added type-checks before anything is applied. `incremental` (a check) keeps the blocks + * `tsconfig.json` includes now, since a check scans only the source text and not the server's + * block types, and rewrites `tsconfig.json` only when it exists (`tsconfig.full.json`, which names + * this machine's cache, is written again when missing, as a fresh clone lacks it). It reads the + * cached catalog, so it costs no request; without one nothing changes until the next get. A file + * whose text would not change is not rewritten. */ export async function refreshTsconfigs( dir: string, @@ -450,27 +656,45 @@ export async function refreshTsconfigs( use: SourceTreeUse, options: { incremental?: boolean } = {} ): Promise { - const catalog = cachedCatalog(workspaceId) + const catalog = (await cachedSourceCatalog(dir)) ?? cachedCatalog(workspaceId) if (!catalog) return - const [tsconfig, full, workspaceTypes] = await Promise.all([ + const [tsconfig, full, workspaceTypes, index, cacheIndex] = await Promise.all([ readSourceText(dir, TSCONFIG_FILE), readSourceText(dir, TSCONFIG_FULL_FILE), readSourceText(dir, WORKSPACE_TYPES_FILE), + readSourceText(dir, SDK_INDEX_PATH), + sdkCacheIndex(catalog), ]) const kept = options.incremental ? includedBlockTypes(tsconfig) : [] + const used = usedBlockTypes(catalog, { + ...use, + blockTypes: [...(use.blockTypes ?? []), ...kept], + }) const { write } = tsconfigWrites( catalog, workspaceTypes !== null, profile, - { ...use, blockTypes: [...(use.blockTypes ?? []), ...kept] }, + used, + fullIndexPath(cacheIndex), { tsconfig, full } ) - const current: Record = { - [TSCONFIG_FILE]: tsconfig, - [TSCONFIG_FULL_FILE]: full, - } - const changed = write.filter( - ([path, text]) => current[path] !== text && (!options.incremental || current[path] !== null) + const current = new Map([ + [TSCONFIG_FILE, tsconfig], + [TSCONFIG_FULL_FILE, full], + ]) + // Declarations of another catalog or generator are left for the next get, which replaces them. + const declarations = + cacheIndex !== null && index?.startsWith(`${sdkIndexHead(catalog)}\n`) + ? localSdkFiles(catalog, used, true).filter(([path]) => path.startsWith(`${SDK_DIR}/blocks/`)) + : [] + const texts = await mapWithConcurrency(declarations, FILE_CONCURRENCY, ([path]) => + readSourceText(dir, path) + ) + declarations.forEach(([path], position) => current.set(path, texts[position])) + const changed = [...write, ...declarations].filter( + ([path, text]) => + current.get(path) !== text && + (!options.incremental || path !== TSCONFIG_FILE || current.get(path) !== null) ) if (changed.length > 0) await updateSourceDir(dir, { write: changed }) } @@ -502,12 +726,15 @@ export interface SourceTreeUse { } /** - * Everything beside the printed source that makes a directory editable: the agent guide and the - * example workflows (both rewritten every time and never in `.sim/written.json`), the - * declarations (regenerated only when the catalog or generator changed), `workspace.d.ts`, the - * CLI's `tsconfig.json` and `tsconfig.full.json`, and the LF lines. `incremental` (a write that - * leaves workflows on disk unread, such as a get of only some kinds) keeps the blocks - * `tsconfig.json` includes now, as {@link refreshTsconfigs} does, adding the ones `use` needs. + * Everything beside the printed source that makes a directory editable: the agent guide (the + * primer for the builders `use` calls, then the profile's reference) and the example workflows + * (both rewritten every time and never in `.sim/written.json`), the declarations of the blocks + * the source uses (every block's are cached once per catalog, for `tsconfig.full.json`), + * `workspace.d.ts`, the CLI's `tsconfig.json` and `tsconfig.full.json`, and the LF lines. + * Declarations of another catalog or generator are removed whole; otherwise only the block files, + * so a block the source no longer uses loses its file. `incremental` (a write that leaves + * workflows on disk unread, such as a get of only some kinds) keeps the blocks `tsconfig.json` + * includes now, as {@link refreshTsconfigs} does, adding the ones `use` needs. */ export async function sourceTreeChange( dir: string, @@ -516,9 +743,12 @@ export async function sourceTreeChange( profile: SourceTreeProfile, use: SourceTreeUse, options: { incremental?: boolean } = {} -): Promise<{ change: SourceDirChange; ownsTsconfig: boolean }> { +): Promise<{ change: SourceDirChange; ownsTsconfig: boolean; primer: string }> { + const sources = [...use.sources] + const primer = primerFor(profile.kind, catalog, sources) const write: Array = [ - [SOURCE_AGENTS_FILE, profile.agents], + [SOURCE_AGENTS_FILE, `${primer}\n${profile.agents}`], + ...(profile.guides ?? []), ...Object.entries(COOKBOOK_EXAMPLES).map( ([file, text]) => [`${SOURCE_EXAMPLES_DIR}/${file}`, text] as const ), @@ -527,19 +757,25 @@ export async function sourceTreeChange( ...(profile.sdk ?? []), ] const remove: string[] = [] - const [gitattributes, gitignore, index, tsconfig, tsconfigFull] = await Promise.all([ + const [gitattributes, gitignore, index, tsconfig, tsconfigFull, cacheIndex] = await Promise.all([ readSourceText(dir, SOURCE_GITATTRIBUTES_FILE), profile.gitignore ? readSourceText(dir, SOURCE_GITIGNORE_FILE) : null, readSourceText(dir, SDK_INDEX_PATH), readSourceText(dir, TSCONFIG_FILE), readSourceText(dir, TSCONFIG_FULL_FILE), + sdkCacheIndex(catalog), ]) const kept = options.incremental ? includedBlockTypes(tsconfig) : [] + const used = usedBlockTypes(catalog, { + sources, + blockTypes: [...(use.blockTypes ?? []), ...kept], + }) const tsconfigs = tsconfigWrites( catalog, workspaceTypes !== null, profile, - { ...use, blockTypes: [...(use.blockTypes ?? []), ...kept] }, + used, + fullIndexPath(cacheIndex), { tsconfig, full: tsconfigFull } ) write.push(...tsconfigs.write) @@ -548,10 +784,13 @@ export async function sourceTreeChange( if (mergedAttributes !== null) write.push([SOURCE_GITATTRIBUTES_FILE, mergedAttributes]) const mergedIgnore = profile.gitignore ? withMissingLines(gitignore, profile.gitignore) : null if (mergedIgnore !== null) write.push([SOURCE_GITIGNORE_FILE, mergedIgnore]) - const indexHead = index?.split('\n', 2).join('\n') - if (indexHead !== `${sdkIndexHeader(catalog)}\n${sdkGeneratorLine()}`) { - remove.push(SDK_DIR) - write.push(...sdkTypeFiles(catalog)) + const current = index?.startsWith(`${sdkIndexHead(catalog)}\n`) === true + if (!current) remove.push(SDK_DIR) + if (cacheIndex !== null) { + if (current) remove.push(`${SDK_DIR}/blocks`) + write.push(...localSdkFiles(catalog, used, true)) + } else if (!current) { + write.push(...localSdkFiles(catalog, used, false)) } if (workspaceTypes === null) remove.push(WORKSPACE_TYPES_FILE) else write.push([WORKSPACE_TYPES_FILE, workspaceTypes.text]) @@ -560,7 +799,7 @@ export async function sourceTreeChange( } else { remove.push(WORKSPACE_MANIFEST_FILE) } - return { change: { write, remove }, ownsTsconfig } + return { change: { write, remove }, ownsTsconfig, primer } } /** A read the server refused as over the source size limits exits 7, as a refused apply does. */ diff --git a/packages/sim-cli/src/commands/protocol/source-ui.ts b/packages/sim-cli/src/commands/protocol/source-ui.ts index a9ec4068f05..c39e7eb14f2 100644 --- a/packages/sim-cli/src/commands/protocol/source-ui.ts +++ b/packages/sim-cli/src/commands/protocol/source-ui.ts @@ -3,16 +3,18 @@ * findings (each with its location, frame and hint), the confirmation prompt, the refusal of a * write nobody confirmed, and an API refusal re-thrown with the exit status a script branches on. */ -import { createInterface } from 'node:readline/promises' import { isRecordLike, toRecord } from '@sim/utils/object' import { SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' import { type FlatDiagnostic, type RefusalClass, refusalClass } from '@sim/workspace-code' +import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' +import { askLine } from '#sim-cli/terminal/line-input' import type { OutputFormat } from '../../config/index' -import { embedStore } from '../../embed-context' +import { embedStore, setSoftExitCode } from '../../embed-context' import { SOURCE_BUSY_EXIT_CODE, SOURCE_LOCKED_EXIT_CODE, + SOURCE_PLANNED_EXIT_CODE, SOURCE_STALE_EXIT_CODE, SOURCE_TOO_LARGE_EXIT_CODE, } from '../../exit-codes' @@ -26,7 +28,7 @@ import { noAnswer, REQUEST_CANCELLED, SimApiError } from '../../http/client' type WorkflowDiagnostic = ApplyWorkflowSourceResponse['data']['diagnostics'][number] /** A workflow finding, or a workspace one: the same fields, over both code vocabularies. */ -type Diagnostic = Omit & { +export type Diagnostic = Omit & { code: | WorkflowDiagnostic['code'] | ApplyWorkspaceSourceResponse['data']['diagnostics'][number]['code'] @@ -74,28 +76,58 @@ function formatDiagnostic(diagnostic: Diagnostic): string { return lines.join('\n') } +/** How many other locations a collapsed line lists before counting the rest. */ +const COLLAPSED_LOCATIONS = 8 + +/** Where a finding sits, as its printed line starts with: `file:line:column`, or empty. */ +function locationOf(diagnostic: Diagnostic): string { + return formatDiagnostic({ ...diagnostic, frame: undefined, hint: undefined }) + .split(` ${diagnostic.severity} `)[0] + .trim() +} + /** - * Prints a write's findings: every error and warning with its location, frame and hint, and each - * info code folded into one line (a new workflow places every block by autolayout, which is no - * reason to print a frame per block and bury the warnings). + * Prints a write's findings on stderr for a person: every error with its location, frame and + * hint; the first warning of each code the same way, and the rest of that code on one line + * listing where they are, each with its own message when it differs from the first (one fix + * usually serves them all, and twenty frames of the same warning buried the errors); each info + * code folded into one line. */ -function printDiagnostics(diagnostics: Diagnostic[]): void { - const infos = new Map() +function printDiagnostics(diagnostics: readonly Diagnostic[]): void { + const groups = new Map() for (const diagnostic of diagnostics) { - if (diagnostic.severity !== 'info') { + if (diagnostic.severity === 'error') { printError(formatDiagnostic(diagnostic)) continue } - const group = infos.get(diagnostic.code) + const key = `${diagnostic.severity}:${diagnostic.code}` + const group = groups.get(key) if (group) group.push(diagnostic) - else infos.set(diagnostic.code, [diagnostic]) + else groups.set(key, [diagnostic]) } - for (const [code, group] of infos) { - const [first] = group + for (const [first, ...rest] of groups.values()) { + if (first.severity === 'info') { + printError( + rest.length === 0 + ? formatDiagnostic({ ...first, frame: undefined }) + : `info ${first.code} on ${rest.length + 1} blocks: ${first.message} (and ${rest.length} more like it)` + ) + continue + } + printError(formatDiagnostic(first)) + if (rest.length === 0) continue + // A collapsed finding saying something else than the first keeps its words beside its place. + const listed = rest + .slice(0, COLLAPSED_LOCATIONS) + .map((diagnostic) => { + const location = locationOf(diagnostic) + if (diagnostic.message === first.message) return location + return location ? `${location} (${diagnostic.message})` : diagnostic.message + }) + .filter(Boolean) + const unlisted = rest.length - listed.length printError( - group.length === 1 - ? formatDiagnostic({ ...first, frame: undefined }) - : `info ${code} on ${group.length} blocks: ${first.message} (and ${group.length - 1} more like it)` + ` ${rest.length} more ${first.code} like it${listed.length > 0 ? `: ${listed.join(', ')}` : ''}${unlisted > 0 && listed.length > 0 ? `, and ${unlisted} more` : ''}` ) } } @@ -103,16 +135,31 @@ function printDiagnostics(diagnostics: Diagnostic[]): void { /** A finding in a source command's `--output json` (or `yaml`) result: the frame stays human-only. */ type ResultDiagnostic = Omit & { code: Diagnostic['code'] } +/** Whether `format` prints one machine document on stdout rather than lines for a person. */ +function isMachine(format: OutputFormat): boolean { + return format === 'json' || format === 'yaml' +} + /** * The `diagnostics` a source command's result carries in a machine format, so the one document on - * stdout holds every finding; a display format prints them on stderr only. + * stdout holds every finding: errors and warnings, and info only with `verbose` (a new workflow + * places every block by autolayout, which is no finding to act on), except `I_FINDINGS_TRUNCATED`, + * which says the list is incomplete. A display format prints them on stderr only. */ -export function resultDiagnostics( +function resultDiagnostics( format: OutputFormat, - diagnostics: readonly Diagnostic[] + diagnostics: readonly Diagnostic[], + verbose = false ): { diagnostics?: ResultDiagnostic[] } { - if (format !== 'json' && format !== 'yaml') return {} - return { diagnostics: diagnostics.map(({ frame: _frame, ...finding }) => finding) } + if (!isMachine(format)) return {} + return { + diagnostics: diagnostics + .filter( + (finding) => + verbose || finding.severity !== 'info' || finding.code === 'I_FINDINGS_TRUNCATED' + ) + .map(({ frame: _frame, ...finding }) => finding), + } } function findingKey(diagnostic: Diagnostic): string { @@ -128,11 +175,64 @@ export function fresh(printed: Set, key: string): boolean { } /** - * Prints the findings `printed` does not hold yet and records them, so a finding the preview - * showed is not shown again by the write that follows it. + * Where a source command's findings go, chosen once by its output format, so each is shown once: + * a display format prints them on stderr as frames (and never in the result), a machine format + * only in the one document on stdout (and never as frames, which an agent reading both streams + * read twice). */ -export function printNewDiagnostics(diagnostics: Diagnostic[], printed: Set): void { - printDiagnostics(diagnostics.filter((diagnostic) => fresh(printed, findingKey(diagnostic)))) +export interface FindingsOutput { + /** Whether the result is one machine document on stdout. */ + machine: boolean + /** + * Prints the findings not printed yet on stderr in a display format, so a finding the preview + * showed is not shown again by the write that follows it; nothing in a machine format. + */ + print(diagnostics: readonly Diagnostic[]): void + /** The result's `diagnostics` field: see {@link resultDiagnostics}. */ + result(diagnostics: readonly Diagnostic[]): { diagnostics?: ResultDiagnostic[] } +} + +/** The {@link FindingsOutput} of `format`; `verbose` keeps info findings in a machine result. */ +export function findingsOutput(format: OutputFormat, verbose = false): FindingsOutput { + const printed = new Set() + const machine = isMachine(format) + return { + machine, + print(diagnostics) { + const unseen = diagnostics.filter((diagnostic) => fresh(printed, findingKey(diagnostic))) + if (!machine) printDiagnostics(unseen) + }, + result: (diagnostics) => resultDiagnostics(format, diagnostics, verbose), + } +} + +/** A resource a source write created, as its result lists it; `id` once it is written. */ +export interface CreatedResource { + kind: string + key?: string + id?: string +} + +/** How many created resources the display line names before counting the rest. */ +const NAMED_CREATED = 5 + +/** The created resources as one line for a person: `workflow (abc), block start (b-1), …`. */ +export function createdText(created: readonly CreatedResource[]): string { + if (created.length === 0) return 'none' + const named = created.slice(0, NAMED_CREATED).map(({ kind, key, id }) => { + const label = [kind, key].filter((part) => part !== undefined).join(' ') + return id === undefined ? label : `${label} (${id})` + }) + const more = created.length - named.length + return `${named.join(', ')}${more > 0 ? `, and ${more} more` : ''}` +} + +/** Adds `--verbose` to a source command whose result can carry info findings. */ +export function verboseFlag(command: Command): Command { + return command.option( + '--verbose', + 'With --output json, also list info findings (such as blocks placed by autolayout)' + ) } function isDiagnosticList(value: unknown): value is Diagnostic[] { @@ -146,13 +246,8 @@ function canPrompt(): boolean { /** Asks `question` on the terminal; anything but `y` or `yes` is no. */ export async function confirm(question: string): Promise { - const prompt = createInterface({ input: process.stdin, output: process.stderr }) - try { - const answer = (await prompt.question(`${question} (y/N) `)).trim().toLowerCase() - return answer === 'y' || answer === 'yes' - } finally { - prompt.close() - } + const answer = (await askLine(`${question} (y/N) `))?.trim().toLowerCase() + return answer === 'y' || answer === 'yes' } /** The exit status of each `409` the server tells apart by `details.code`; any other exits 1. */ @@ -185,19 +280,35 @@ function answerLost(error: SimApiError): boolean { return noAnswer(error, LOST_ANSWER_STATUSES) || error.code === REQUEST_CANCELLED } +/** + * The server's pointer to the findings in the error document (`See details.diagnostics.`): a + * display format prints them instead, so the sentence names a JSON field the reader never sees. + */ +const DETAILS_REFERENCE = / See details\.diagnostics\.$/ + /** * Re-throws an API refusal after printing its diagnostics, so each shows its source line, with * the exit status a script branches on. In a machine `format` the error document keeps the * refusal's details with each finding as a success result carries it ({@link resultDiagnostics}); - * a display format prints the findings once, above the error. `ifAnswerLost` is the next step for - * a write whose answer never came back, appended to its message. + * a display format prints the findings once, above the error, and `verbose` keeps info findings in + * the machine document as it does in a result. `ifAnswerLost` is the next step for a write whose + * answer never came back, appended to its message. */ -export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost?: string): never { +export function printRefusal( + error: unknown, + format: OutputFormat, + options: { ifAnswerLost?: string; verbose?: boolean } = {} +): never { + const { ifAnswerLost, verbose = false } = options if (!(error instanceof SimApiError)) throw error const details = toRecord(error.details) const diagnostics = details.diagnostics const printed = isDiagnosticList(diagnostics) - if (printed) printDiagnostics(diagnostics) + const machine = isMachine(format) + if (printed && !machine) printDiagnostics(diagnostics) + const message = machine + ? error.message + : error.message.replace(DETAILS_REFERENCE, printed ? ' Each finding is printed above.' : '') const nameTaken = error.status === 409 && details.code === 'SOURCE_NAME_TAKEN' const hint = nameTaken ? 'Pass --name to create it under another name, or apply to that workflow.' @@ -205,10 +316,10 @@ export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost? ? ifAnswerLost : null throw new SimApiError( - hint ? `${error.message} ${hint}` : error.message, + hint ? `${message} ${hint}` : message, error.status, error.code, - printed ? refusalDetails(format, details, diagnostics) : error.details, + printed ? refusalDetails(format, details, diagnostics, verbose) : error.details, refusalExitCode(error, details.code), error.requestId ) @@ -218,26 +329,42 @@ export function printRefusal(error: unknown, format: OutputFormat, ifAnswerLost? function refusalDetails( format: OutputFormat, details: Record, - diagnostics: Diagnostic[] + diagnostics: Diagnostic[], + verbose: boolean ): Record | undefined { - const flattened = resultDiagnostics(format, diagnostics) + const flattened = resultDiagnostics(format, diagnostics, verbose) return flattened.diagnostics ? { ...details, ...flattened } : undefined } /** - * A write without `--yes` or `--dry-run` where no terminal can ask: the generic refusal, naming - * what the write does (`writes the workflow draft`) and its verb (`apply`). + * How a source write proceeds: `dry-run` checks only; `write` writes at once (`--yes`); `ask` + * previews and asks on a terminal; `plan` (no `--yes` and no terminal to ask) previews, prints the + * plan and stops with {@link SOURCE_PLANNED_EXIT_CODE}, so an agent's edit loop sees the plan + * without a separate dry run and applies it with `--yes`. */ -export function requireConsent( - options: { dryRun?: boolean; yes?: boolean }, - action: string, - verb: string -): void { - if (!options.dryRun && !options.yes && !canPrompt()) { - throw new SimApiError( - `This ${action}. Re-run with --yes to ${verb} it, or with --dry-run to check it without writing.`, - 0, - 'CONFIRMATION_REQUIRED' - ) +export type WriteMode = 'dry-run' | 'write' | 'ask' | 'plan' + +/** The {@link WriteMode} of a write's `--dry-run` and `--yes` flags, here. */ +export function writeMode(options: { dryRun?: boolean; yes?: boolean }): WriteMode { + if (options.dryRun) return 'dry-run' + if (options.yes) return 'write' + return canPrompt() ? 'ask' : 'plan' +} + +/** `value` as one shell word: as it is when it needs no quoting, else single-quoted. */ +export function shellWord(value: string): string { + return /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'` +} + +/** The next step a `plan` run names: the same write with `--yes`. */ +export function planNext(command: string): { command: string; reason: string } { + return { + command: `${command} --yes`, + reason: 'Nothing was written: this is the plan. Run it with --yes to apply it.', } } + +/** Marks a `plan` run that found no errors: nothing was written, applying needs `--yes`. */ +export function stopAtPlan(): void { + setSoftExitCode(SOURCE_PLANNED_EXIT_CODE) +} diff --git a/packages/sim-cli/src/commands/protocol/stored-edges.ts b/packages/sim-cli/src/commands/protocol/stored-edges.ts new file mode 100644 index 00000000000..04306d11abb --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/stored-edges.ts @@ -0,0 +1,180 @@ +import { isRecordLike, toRecord } from '@sim/utils/object' +import { + type BlockCatalog, + compileWorkflow, + parseWorkflowLock, + type StoredEdgeRef, + setOwn, +} from '@sim/workflow-code' +import { SOURCE_BASE_FILE, SOURCE_ENTRY_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' +import { KIND_DIRS } from '@sim/workspace-code' +import type { OperationClient } from '../../runtime/called-operations' +import { readSourceJson, SOURCE_STORED_EDGES_FILE } from './source-dir' +import { cachedCatalogByHash, loadWorkflowCatalog } from './source-tree' + +/** A printed source as a get, apply or create answers it. */ +interface PrintedAnswer { + files: Record + lock: unknown + base: unknown +} + +/** + * The `.sim/stored-edges.json` write for a printed source: its edges, compiled against `catalog` + * (the one it was printed with), tagged with its base fingerprint. `null` when the answer has no + * fingerprint or entry, or nothing to compile it with; the record left on disk then names an + * older fingerprint, which a check ignores. + */ +export function storedEdgesWrite( + answer: PrintedAnswer, + catalog: BlockCatalog | null, + folder = '' +): readonly [string, string] | null { + const { fingerprint } = toRecord(answer.base) + const entry = answer.files[SOURCE_ENTRY_FILE] + const { lock } = parseWorkflowLock(answer.lock) + if (!catalog || typeof fingerprint !== 'string' || entry === undefined || !lock) return null + const { edges } = compileWorkflow({ + entry: { path: SOURCE_ENTRY_FILE, text: entry }, + files: new Map(Object.entries(answer.files).filter(([path]) => path !== SOURCE_ENTRY_FILE)), + catalog, + sdkMajor: catalog.sdkMajor, + knownKeys: new Set(Object.keys(lock.blocks)), + storedUnread: lock, + }) + return [ + `${folder}${SOURCE_STORED_EDGES_FILE}`, + `${JSON.stringify({ version: 1, fingerprint, edges }, null, 2)}\n`, + ] +} + +/** The workflow folders a workspace answer's files hold, by key, each as a printed source. */ +function printedWorkflowFolders(files: Record): Map { + const folders = new Map() + const prefix = `${KIND_DIRS.workflow}/` + for (const [path, text] of Object.entries(files)) { + if (!path.startsWith(prefix)) continue + const slash = path.indexOf('/', prefix.length) + if (slash < 0) continue + const key = path.slice(prefix.length, slash) + const relative = path.slice(slash + 1) + const folder = folders.get(key) ?? { files: {}, lock: undefined, base: undefined } + if (relative === SOURCE_LOCK_FILE) folder.lock = parseJson(text) + else if (relative === SOURCE_BASE_FILE) folder.base = parseJson(text) + else setOwn(folder.files, relative, text) + folders.set(key, folder) + } + return folders +} + +function parseJson(text: string): unknown { + try { + return JSON.parse(text) + } catch { + return undefined + } +} + +/** The stored-edges records of a workspace answer, and the folders left without one. */ +export interface WorkflowFolderRecords { + records: Array + /** Folder keys whose catalog could not be fetched: their record is skipped, not failed. */ + unrecorded: string[] +} + +/** + * The `.sim/stored-edges.json` write of each workflow folder a workspace get or apply answered + * with, each compiled against the catalog its base names. A workflow holding a type the workspace + * catalog lacks was printed with its own catalog: one not cached yet is fetched, once per distinct + * hash, and cached, so an offline check compiles that folder against it too. The records are a + * cache fill, never part of the write: a catalog fetch that fails skips the folders printed with + * that catalog (named in `unrecorded`) rather than failing a write the server already committed. + */ +export async function workflowFolderRecords( + client: OperationClient<'getSourceCatalog'>, + workspaceId: string, + workspaceCatalog: BlockCatalog, + files: Record +): Promise { + const folders = printedWorkflowFolders(files) + const uncached = new Map() + for (const { base } of folders.values()) { + const { workflowId, catalogHash } = toRecord(base) + if (typeof workflowId !== 'string' || typeof catalogHash !== 'string') continue + if (catalogHash === workspaceCatalog.hash || cachedCatalogByHash(catalogHash)) continue + if (!uncached.has(catalogHash)) uncached.set(catalogHash, workflowId) + } + const fetched = new Map() + for (const [catalogHash, workflowId] of uncached) { + const catalog = await loadWorkflowCatalog(client, workspaceId, workflowId).catch(() => null) + if (catalog) fetched.set(catalogHash, catalog) + } + const records: Array = [] + const unrecorded: string[] = [] + for (const [key, folder] of folders) { + const { catalogHash } = toRecord(folder.base) + const catalog = + catalogHash === workspaceCatalog.hash + ? workspaceCatalog + : typeof catalogHash === 'string' + ? (fetched.get(catalogHash) ?? cachedCatalogByHash(catalogHash)) + : null + if (typeof catalogHash === 'string' && uncached.has(catalogHash) && !catalog) { + unrecorded.push(key) + continue + } + const write = storedEdgesWrite(folder, catalog, `${KIND_DIRS.workflow}/${key}/`) + if (write) records.push(write) + } + return { records, unrecorded } +} + +/** + * The warning for workflow folders {@link workflowFolderRecords} could not record, or `null`. + * Their edges are unrecorded, so an offline check of each one compiles without its stored edges + * until the next get records them. + */ +export function unrecordedFoldersWarning(unrecorded: readonly string[]): string | null { + if (unrecorded.length === 0) return null + return `Warning: could not fetch the block catalog of ${unrecorded.map((key) => `${KIND_DIRS.workflow}/${key}`).join(', ')}, so its stored edges are not recorded; everything else was saved. The next \`sim workspaces source get\` records them.` +} + +function isStoredEdge(value: unknown): value is StoredEdgeRef { + return ( + isRecordLike(value) && + typeof value.from === 'string' && + typeof value.port === 'string' && + typeof value.to === 'string' + ) +} + +/** + * The stored edges recorded for the base `dir` holds now, or `undefined` when none are: no + * record, one an older CLI or another base wrote, or a malformed one. + */ +export async function readStoredEdges(dir: string): Promise { + const [record, base] = await Promise.all([ + readSourceJson(dir, SOURCE_STORED_EDGES_FILE), + readSourceJson(dir, SOURCE_BASE_FILE), + ]) + const { version, fingerprint, edges } = toRecord(record) + if ( + version !== 1 || + typeof fingerprint !== 'string' || + fingerprint !== toRecord(base).fingerprint || + !Array.isArray(edges) || + !edges.every(isStoredEdge) + ) { + return undefined + } + return edges +} + +/** The stored-edges records of the workflow folders whose base a write removes (`paths`). */ +export function removedWorkflowRecords(paths: Iterable): string[] { + return [...paths].flatMap((path) => + path.startsWith(`${KIND_DIRS.workflow}/`) && path.endsWith(`/${SOURCE_BASE_FILE}`) + ? [`${path.slice(0, -SOURCE_BASE_FILE.length)}${SOURCE_STORED_EDGES_FILE}`] + : [] + ) +} diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index ec73df948bb..5a34c301403 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -1,23 +1,39 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { hostname, tmpdir } from 'node:os' import { join } from 'node:path' +import { toRecord } from '@sim/utils/object' import { compileWorkflow } from '@sim/workflow-code' import { fingerprintOf } from '@sim/workspace-code' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' import { buildGeneratedCommands } from '../../runtime/build' +import { cacheCatalogText } from '../../test/catalog-cache' import { contextMockFns, contextMockState } from '../../test/context-mock' import { digest } from './source-dir' import { TSCONFIG_MARKER } from './source-tree' import { attachWorkflowSourceApply, attachWorkflowSourceCreate } from './workflow-source-apply' -const prompt = vi.hoisted(() => ({ question: vi.fn() })) +const prompt = vi.hoisted(() => ({ + question: vi.fn(), + /** The open readline interface, to end its input as Ctrl-D does. */ + open: null as import('node:events').EventEmitter | null, +})) vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) -vi.mock('node:readline/promises', () => ({ - createInterface: () => ({ question: prompt.question, close: () => {} }), -})) +vi.mock('node:readline/promises', async () => { + const { EventEmitter } = await import('node:events') + return { + createInterface: () => { + const readline = Object.assign(new EventEmitter(), { + question: prompt.question, + close: () => readline.emit('close'), + }) + prompt.open = readline + return readline + }, + } +}) const { mockRequest: request } = contextMockFns @@ -47,13 +63,17 @@ function outcome(overrides: { dryRun?: boolean; diagnostics?: unknown[]; notes?: diff: { noop: false, blocks: { - added: [{ id: 'b-1', key: 'start' }], - removed: [], + added: [{ id: 'b-1', key: 'start', type: 'start_trigger' }] as Array<{ + id: string + key: string + type: string + }>, + removed: [] as Array<{ id: string; key: string; type: string }>, renamed: [], modified: [] as Array<{ id: string; key: string; fields: string[] }>, }, edges: { added: 1, removed: 0 }, - variables: { added: [], removed: [], modified: [] }, + variables: { added: [] as string[], removed: [] as string[], modified: [] as string[] }, }, diagnostics: overrides.diagnostics ?? [], files: { 'workflow.ts': `${ENTRY}// stored\n` }, @@ -143,28 +163,62 @@ function dryRunFlags() { } describe('sim workflows source apply', () => { - it('refuses without --yes when nobody is at a terminal, sending nothing', async () => { + it('without --yes and nobody at a terminal, prints the plan, writes nothing and exits 14', async () => { setTerminal(false) + request.mockResolvedValue(outcome({ dryRun: true })) - const error = await apply(WORKFLOW_ID, '--dir', dir).catch((caught: unknown) => caught) + await apply(WORKFLOW_ID, '--dir', dir, '--allow-delete') - expect(error).toBeInstanceOf(SimApiError) - expect((error as SimApiError).code).toBe('CONFIRMATION_REQUIRED') - expect((error as SimApiError).message).toContain('--yes') - expect(request).not.toHaveBeenCalled() + expect(dryRunFlags()).toEqual([true]) + expect(process.exitCode).toBe(14) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(ENTRY) + expect(stdoutJson()).toMatchObject({ + applied: false, + dryRun: true, + next: { command: `sim workflows source apply --dir ${dir} --allow-delete --yes` }, + }) + expect(stderrLines().join('\n')).toContain('Blocks: 1 added') }) - it('refuses a create without --yes and sends nothing', async () => { + it('without --yes, exits with the refusal and names no next step when the plan has errors', async () => { setTerminal(false) + request.mockResolvedValue( + outcome({ + dryRun: true, + diagnostics: [{ code: 'E_UNKNOWN_PARAM', severity: 'error', message: 'No param x.' }], + }) + ) - const error = await program() - .parseAsync(['node', 'sim', 'workflows', 'source', 'create', '--dir', dir]) - .catch((caught: unknown) => caught) + await apply(WORKFLOW_ID, '--dir', dir) - expect(error).toBeInstanceOf(SimApiError) - expect((error as SimApiError).code).toBe('CONFIRMATION_REQUIRED') - expect((error as SimApiError).message).toContain('--yes to create') - expect(request).not.toHaveBeenCalled() + expect(process.exitCode).toBe(1) + expect(stdoutJson()).not.toHaveProperty('next') + }) + + it('plans a create without --yes and creates nothing', async () => { + setTerminal(false) + rmSync(join(dir, '.sim/base.json')) + request.mockResolvedValue(outcome({ dryRun: true })) + + await program().parseAsync(['node', 'sim', 'workflows', 'source', 'create', '--dir', dir]) + + expect(dryRunFlags()).toEqual([true]) + expect(process.exitCode).toBe(14) + expect(stdoutJson()).toMatchObject({ + next: { + command: `sim workflows source create --dir ${dir} --workspace ${WORKSPACE_ID} --yes`, + }, + }) + }) + + it('with --yes, prints the summary of what it applied', async () => { + setTerminal(false) + request.mockResolvedValue(outcome()) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes') + + expect(dryRunFlags()).toEqual([false]) + expect(stderrLines().join('\n')).toContain('Blocks: 1 added, 0 removed') }) it('applies at once with --yes and saves what the server stored', async () => { @@ -203,11 +257,7 @@ describe('sim workflows source apply', () => { it('points tsconfig.json at a block the applied source added, from the cached catalog', async () => { setTerminal(false) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) - mkdirSync(join(dir, 'config/cache/source-catalog'), { recursive: true }) - writeFileSync( - join(dir, `config/cache/source-catalog/${WORKSPACE_ID}.json`), - readFileSync(FIXTURE_CATALOG, 'utf8') - ) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, readFileSync(FIXTURE_CATALOG, 'utf8')) writeFileSync(join(dir, 'tsconfig.json'), `${TSCONFIG_MARKER}\n{}\n`) const stored = outcome() stored.data.files = { 'workflow.ts': `${ENTRY}// slack.sendMessage('Post', {})\n` } @@ -257,6 +307,71 @@ describe('sim workflows source apply', () => { expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(ENTRY) }) + it.each([ + [ + 'Node closes the prompt', + () => { + queueMicrotask(() => prompt.open?.emit('close')) + return new Promise(() => {}) + }, + ], + [ + 'Bun aborts the question', + async () => { + throw Object.assign(new Error('The operation was aborted.'), { name: 'AbortError' }) + }, + ], + ])( + 'at a terminal, applies nothing when input ends at the prompt (%s)', + async (_runtime, answer) => { + setTerminal(true) + request.mockResolvedValue(outcome({ dryRun: true })) + prompt.question.mockImplementation(answer) + + await expect(apply(WORKFLOW_ID, '--dir', dir)).rejects.toMatchObject({ + code: 'CANCELLED', + exitCode: 1, + message: 'Nothing was applied.', + }) + + expect(dryRunFlags()).toEqual([true]) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(ENTRY) + } + ) + + it.each([ + [ + 'table', + 'The workflow source has 1 error(s); nothing was written. Each finding is printed above.', + ], + ['json', 'The workflow source has 1 error(s); nothing was written. See details.diagnostics.'], + ] as const)('names where the findings of a refusal are in %s output', async (output, message) => { + contextMockState.output = output + setTerminal(false) + request.mockRejectedValue( + new SimApiError( + 'The workflow source has 1 error(s); nothing was written. See details.diagnostics.', + 400, + 'BAD_REQUEST', + { + code: 'SOURCE_REJECTED', + diagnostics: [ + { + code: 'E_PARSE', + severity: 'error', + message: 'Unexpected token.', + file: 'workflow.ts', + line: 2, + column: 1, + }, + ], + } + ) + ) + + await expect(apply(WORKFLOW_ID, '--dir', dir, '--yes')).rejects.toMatchObject({ message }) + }) + it('at a terminal, does not ask when the dry run already found errors', async () => { setTerminal(true) request.mockResolvedValue( @@ -328,6 +443,45 @@ describe('sim workflows source apply', () => { }) }) + it('names each added and removed block with its type in the result and the preview', async () => { + setTerminal(true) + const changed = outcome({ dryRun: true }) + changed.data.diff.blocks.added = [{ id: 'b-3', key: 'bang', type: 'function' }] + changed.data.diff.blocks.removed = [{ id: 'b-2', key: 'boom', type: 'function' }] + request.mockResolvedValue(changed) + prompt.question.mockResolvedValue('n') + + await apply(WORKFLOW_ID, '--dir', dir).catch(() => {}) + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stderrLines().some((line) => line.includes('Added: bang (function)'))).toBe(true) + expect(stderrLines().some((line) => line.includes('Removed: boom (function)'))).toBe(true) + const printed = vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '' + expect(JSON.parse(printed)).toMatchObject({ + added: [{ block: 'bang', type: 'function' }], + removed: [{ block: 'boom', type: 'function' }], + }) + }) + + it('names each changed variable in the result and the preview', async () => { + setTerminal(true) + const changed = outcome({ dryRun: true }) + changed.data.diff.variables = { added: ['retries'], removed: [], modified: ['threshold'] } + request.mockResolvedValue(changed) + prompt.question.mockResolvedValue('n') + + await apply(WORKFLOW_ID, '--dir', dir).catch(() => {}) + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect( + stderrLines().some((line) => line.includes('Variables: added retries; modified threshold')) + ).toBe(true) + const printed = vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '' + expect(JSON.parse(printed)).toMatchObject({ + variables: { added: ['retries'], removed: [], modified: ['threshold'] }, + }) + }) + it('prints the modified blocks readably in text output', async () => { contextMockState.output = 'text' const modified = outcome({ dryRun: true }) @@ -359,12 +513,17 @@ describe('sim workflows source apply', () => { ]) const printed = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') - expect(printed).toMatchObject({ created: false, dryRun: true }) + expect(printed).toMatchObject({ + dryRun: true, + created: [{ kind: 'workflow' }, { kind: 'block', key: 'start' }], + }) expect(printed).not.toHaveProperty('workflowId') + expect(JSON.stringify(printed.created)).not.toContain('"id"') }) it('folds info findings into one line per code and keeps frames on warnings', async () => { setTerminal(false) + contextMockState.output = 'text' const layout = (key: string, line: number) => ({ code: 'I_MAT_NEEDS_LAYOUT', severity: 'info', @@ -404,6 +563,7 @@ describe('sim workflows source apply', () => { it('prints every note on a dry run, so a kept name or a dropped tool is never silent', async () => { setTerminal(false) + contextMockState.output = 'text' const note = "The workflow name 'New' differs from the stored one and was not changed." request.mockResolvedValue(outcome({ dryRun: true, notes: [note] })) @@ -412,18 +572,19 @@ describe('sim workflows source apply', () => { expect(stderrLines()).toContain(`note: ${note}`) }) - it('counts notes apart from warnings in the result', async () => { + it('lists notes apart from warnings in the JSON result, and prints none on stderr', async () => { setTerminal(false) request.mockResolvedValue(outcome({ dryRun: true, notes: ['a kept name'] })) await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') - const printed = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') - expect(printed).toMatchObject({ warnings: 0, notes: 1 }) + expect(stdoutJson()).toMatchObject({ warnings: 0, notes: ['a kept name'] }) + expect(stderrLines()).toEqual([]) }) it('at a terminal, prints the notes of the preview and of the applied write', async () => { setTerminal(true) + contextMockState.output = 'text' request.mockImplementation(async (_path: string, options: { query?: { dryRun?: boolean } }) => outcome({ dryRun: options.query?.dryRun === true, @@ -441,6 +602,7 @@ describe('sim workflows source apply', () => { it('at a terminal, prints a warning and a note the preview showed once, not again on the write', async () => { setTerminal(true) + contextMockState.output = 'text' const warning = { code: 'W_REQUIRED_MISSING', severity: 'warning', @@ -483,6 +645,236 @@ describe('sim workflows source apply', () => { }) }) + describe('findings and results an agent reads', () => { + const untyped = (line: number) => ({ + code: 'W_JSON_REF_UNTYPED', + severity: 'warning', + message: `Nothing declares the type of .`, + file: 'workflow.ts', + line, + column: 7, + frame: `> ${line} | n: score.result.n${line},`, + hint: 'Leave it bare when it holds a number.', + }) + const layout = { + code: 'I_MAT_NEEDS_LAYOUT', + severity: 'info', + message: 'Block "start" is placed by autolayout', + file: 'workflow.ts', + line: 3, + column: 3, + } + + it('with --output json, prints findings only in the JSON on stdout, never as frames on stderr', async () => { + setTerminal(false) + request.mockResolvedValue(outcome({ dryRun: true, diagnostics: [untyped(4)] })) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stdoutJson()).toMatchObject({ diagnostics: [{ code: 'W_JSON_REF_UNTYPED' }] }) + expect(stderrLines().join('\n')).not.toContain('W_JSON_REF_UNTYPED') + }) + + it('leaves info findings out of the JSON unless --verbose', async () => { + setTerminal(false) + request.mockResolvedValue(outcome({ dryRun: true, diagnostics: [layout, untyped(4)] })) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + expect(stdoutJson().diagnostics).toEqual([expect.objectContaining({ severity: 'warning' })]) + + vi.mocked(console.log).mockClear() + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run', '--verbose') + expect(stdoutJson().diagnostics).toEqual([ + expect.objectContaining({ code: 'I_MAT_NEEDS_LAYOUT' }), + expect.objectContaining({ code: 'W_JSON_REF_UNTYPED' }), + ]) + }) + + it('in a display format, prints the first warning of a code whole and the rest on one line', async () => { + setTerminal(false) + contextMockState.output = 'text' + request.mockResolvedValue( + outcome({ dryRun: true, diagnostics: [4, 5, 6, 7].map((line) => untyped(line)) }) + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + const lines = stderrLines() + expect(lines.filter((line) => line.includes('> '))).toHaveLength(1) + expect(lines).toContain( + ' 3 more W_JSON_REF_UNTYPED like it: workflow.ts:5:7 (Nothing declares the type of .), workflow.ts:6:7 (Nothing declares the type of .), workflow.ts:7:7 (Nothing declares the type of .)' + ) + }) + + it('in a display format, names each collapsed warning by location alone when they say the same', async () => { + setTerminal(false) + contextMockState.output = 'text' + request.mockResolvedValue( + outcome({ + dryRun: true, + diagnostics: [4, 5].map((line) => ({ ...untyped(line), message: 'Same words.' })), + }) + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stderrLines()).toContain(' 1 more W_JSON_REF_UNTYPED like it: workflow.ts:5:7') + }) + + it('with --output json and --verbose, keeps info findings in a refusal too', async () => { + setTerminal(false) + request.mockRejectedValue( + new SimApiError('Refused. See details.diagnostics.', 422, 'SOURCE_REJECTED', { + diagnostics: [layout, STALE], + }) + ) + + const refused = await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--verbose').then( + () => null, + (error: unknown) => error as SimApiError + ) + + expect(toRecord(refused?.details).diagnostics).toEqual([ + expect.objectContaining({ code: 'I_MAT_NEEDS_LAYOUT' }), + expect.objectContaining({ code: 'E_APPLY_STALE' }), + ]) + }) + + it('lists the workflow and blocks a create made, with their ids', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + rmSync(join(dir, '.sim/base.json')) + const catalog = JSON.parse(readFileSync(FIXTURE_CATALOG, 'utf8')) + request.mockImplementation(async (path: string) => { + if (path.endsWith('/workflows/source/catalog')) return { data: catalog } + if (path.endsWith('/workflows/source/resources')) { + return { + data: { + workspaceId: WORKSPACE_ID, + tables: [], + knowledgeBases: [], + credentials: [], + workflows: [], + env: [], + truncated: false, + }, + } + } + return outcome() + }) + + await program().parseAsync([ + 'node', + 'sim', + 'workflows', + 'source', + 'create', + '--dir', + dir, + '--yes', + ]) + + expect(stdoutJson()).toMatchObject({ + workflowId: WORKFLOW_ID, + created: [ + { kind: 'workflow', id: WORKFLOW_ID }, + { kind: 'block', key: 'start', id: 'b-1' }, + ], + }) + }) + + it('names what an apply created on one line in a display format', async () => { + setTerminal(false) + contextMockState.output = 'text' + request.mockResolvedValue(outcome()) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes') + + expect(vi.mocked(console.log).mock.calls.map((call) => call.join(' '))).toContain( + 'created\tblock start (b-1)' + ) + }) + }) + + describe('--deploy', () => { + const DEPLOY_PATH = `/api/v2/workflows/${WORKFLOW_ID}/deploy` + const deployed = { + data: { + id: WORKFLOW_ID, + isDeployed: true, + deployedAt: null, + warnings: [], + activeDeployment: null, + latestDeploymentAttempt: null, + version: 4, + lint: null, + }, + } + + it('deploys through the deploy operation once the apply is saved', async () => { + setTerminal(false) + request.mockImplementation(async (path: string) => + path === DEPLOY_PATH ? deployed : outcome() + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH, DEPLOY_PATH]) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(`${ENTRY}// stored\n`) + expect(stdoutJson()).toMatchObject({ + applied: true, + deployed: { version: 4, isDeployed: true }, + needsRedeployment: false, + }) + }) + + it('without --deploy, names the deploy command when the deployed workflow now differs', async () => { + setTerminal(false) + request.mockResolvedValue(outcome()) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes') + + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH]) + expect(stdoutJson()).toMatchObject({ + needsRedeployment: true, + next: { command: `sim workflows deploy ${WORKFLOW_ID}` }, + }) + }) + + it('deploys nothing after a refused apply, and refuses beside --dry-run', async () => { + setTerminal(false) + request.mockResolvedValue(outcome({ diagnostics: [STALE] })) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH]) + + request.mockClear() + await expect(apply(WORKFLOW_ID, '--dir', dir, '--dry-run', '--deploy')).rejects.toThrow( + /--dry-run saves nothing/ + ) + expect(request).not.toHaveBeenCalled() + }) + + it('says the apply was saved when the deploy after it fails', async () => { + setTerminal(false) + request.mockImplementation(async (path: string) => { + if (path === DEPLOY_PATH) throw new SimApiError('Forbidden', 403, 'FORBIDDEN') + return outcome() + }) + + const error = await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy').catch( + (caught: unknown) => caught + ) + + expect(error).toMatchObject({ status: 403, code: 'FORBIDDEN' }) + expect((error as Error).message).toContain(`The apply was saved into ${dir}`) + expect((error as Error).message).toContain(`sim workflows deploy ${WORKFLOW_ID}`) + expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(`${ENTRY}// stored\n`) + // The saved write's result still prints, before the deploy's failure. + expect(stdoutJson()).toMatchObject({ workflowId: WORKFLOW_ID, applied: true }) + }) + }) + it('exits 5 when a dry run finds the directory stale', async () => { setTerminal(false) request.mockResolvedValue(outcome({ dryRun: true, diagnostics: [STALE] })) @@ -755,6 +1147,75 @@ describe('sim workflows source create on a directory with no workflow.ts', () => expect(printed.files).toBeGreaterThan(0) }) + /** Answers the starter's catalog and manifest reads, and a create with a stored workflow. */ + function answerStarterAndCreate() { + request.mockImplementation(async (path: string, init: { query?: { dryRun?: boolean } }) => { + if (path.endsWith('/workflows/source/catalog')) return { data: CATALOG } + if (path.endsWith('/workflows/source/resources')) return MANIFEST + if (path.endsWith('/workflows/source')) return outcome({ dryRun: init.query?.dryRun }) + throw new Error(`unexpected request ${path}`) + }) + } + + function createCalls(): string[] { + return request.mock.calls + .map((call) => call[0] as string) + .filter((path) => path.endsWith('/workflows/source')) + } + + it('records the workspace the starter was typed for and names it in the next step', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + const empty = join(dir, 'new') + answerStarterAndCreate() + + await create('--dir', empty) + + expect(JSON.parse(readFileSync(join(empty, '.sim/starter.json'), 'utf8'))).toEqual({ + workspaceId: WORKSPACE_ID, + }) + expect(stderrLines().join('\n')).toContain( + `sim workflows source create --dir ${empty} --workspace ${WORKSPACE_ID} --yes` + ) + }) + + it('creates a starter in the workspace it was typed for, whatever the profile default', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + const starter = join(dir, 'new') + answerStarterAndCreate() + await create('--dir', starter) + contextMockState.workspaceId = 'ws_other' + contextMockState.workspaceSource = 'config' + + try { + await create('--dir', starter, '--yes') + expect(createCalls()).toEqual([`/api/v2/workspaces/${WORKSPACE_ID}/workflows/source`]) + } finally { + contextMockState.workspaceId = WORKSPACE_ID + } + }) + + it('creates a starter where --workspace names, and drops the starter record once created', async () => { + setTerminal(false) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + const starter = join(dir, 'new') + answerStarterAndCreate() + await create('--dir', starter) + contextMockState.workspaceId = 'ws_other' + contextMockState.workspaceSource = 'flag' + + try { + await create('--dir', starter, '--yes') + expect(createCalls()).toEqual(['/api/v2/workspaces/ws_other/workflows/source']) + expect(existsSync(join(starter, '.sim/starter.json'))).toBe(false) + expect(existsSync(join(starter, '.sim/base.json'))).toBe(true) + } finally { + contextMockState.workspaceId = WORKSPACE_ID + contextMockState.workspaceSource = 'config' + } + }) + it.each(['--dry-run', '--yes'])('refuses %s instead of scaffolding', async (flag) => { setTerminal(false) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 69b7f704ce1..ee3233cefd7 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -1,4 +1,5 @@ import { mapWithConcurrency } from '@sim/utils/concurrency' +import { toRecord } from '@sim/utils/object' import { parseSourceBase, parseWorkflowLock, type WorkflowLockV1 } from '@sim/workflow-code' import { codeFileReferences } from '@sim/workflow-code/code-files/references' import { @@ -9,34 +10,49 @@ import { } from '@sim/workflow-code/limits' import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' +import { DEFERRED_INTERRUPT_BOUND_MS, deferInterrupts } from '#sim-cli/terminal/interrupts' import { setSoftExitCode } from '../../embed-context' import type { ApplyWorkflowSourceBody, ApplyWorkflowSourceResponse, CreateWorkflowFromSourceBody, + DeployWorkflowResponse, } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' import { printProtocolResult } from './result' import { + absoluteDirOption, FILE_CONCURRENCY, lockCodeFiles, readSourceText, + readStarterWorkspace, SOURCE_AGENTS_FILE, + SOURCE_STARTER_FILE, sourceRelativePath, + updateSourceDir, WORKSPACE_FOLDER_BUSY_HELP, withWorkspaceFolderLock, writeBack, } from './source-dir' +import { SOURCE_WRITE_EXIT_CODES_HELP } from './source-exit-codes' +import { cachedCatalogByHash, printTreeResult } from './source-tree' import { + type CreatedResource, confirm, + createdText, + type FindingsOutput, + findingsOutput, fresh, - printNewDiagnostics, + planNext, printRefusal, refusedExitCode, - requireConsent, - resultDiagnostics, + shellWord, + stopAtPlan, + verboseFlag, + writeMode, } from './source-ui' +import { storedEdgesWrite } from './stored-edges' import { refreshWorkflowTsconfigs, writeSourceDirectory, @@ -50,15 +66,20 @@ const APPLY_EXIT_CODES_HELP = ` Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); -6 the workflow or its folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after -the CLI's retries, so nothing was written: wait a minute and run it again.` +6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, +E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's +retries, so nothing was written: wait a minute and run it again; 14 without --yes and with no +terminal to ask: the plan was printed with no errors and nothing was written (run the same +command with --yes to apply it).` const CREATE_EXIT_CODES_HELP = ` Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken -(SOURCE_NAME_TAKEN: pass --name), or the command failed; 6 the folder is locked; 7 the source -is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a -minute and run it again.` +(SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the +workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the +source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: +wait a minute and run it again; 14 without --yes and with no terminal to ask: the plan was +printed with no errors and nothing was created (run the same command with --yes).` type SourceApplyData = ApplyWorkflowSourceResponse['data'] @@ -69,13 +90,26 @@ interface SourceWriteOptions { dryRun?: boolean yes?: boolean allowDelete?: boolean + verbose?: boolean + deploy?: boolean } -/** The text of a file the directory must hold, or a refusal that says how to write it. */ +/** + * The text of a file the directory must hold, or a refusal that says how to write it: `hint`, or + * for a starter no workflow was created from yet, the create that makes it one. + */ async function requireText(directory: string, path: string, hint: string): Promise { const text = await readSourceText(directory, path) - if (text === null) throw new SimApiError(`${directory} has no ${path}. ${hint}`, 0) - return text + if (text !== null) return text + const starterWorkspace = await readStarterWorkspace(directory) + throw new SimApiError( + `${directory} has no ${path}. ${ + starterWorkspace === null + ? hint + : `It holds a starter no workflow was created from yet: run \`${createCommand(directory, starterWorkspace)} --dry-run\`, then the same with --yes.` + }`, + 0 + ) } async function readJson(directory: string, path: string, hint: string): Promise { @@ -105,14 +139,15 @@ function codeFilePaths(entry: string, lock: WorkflowLockV1 | null): string[] { } /** - * Prints a write's diagnostics and notes (such as a custom tool the save could not store), - * skipping any already in `printed` and adding the rest: after a terminal preview, the + * Prints a write's diagnostics and, in a display format, its notes (such as a custom tool the + * save could not store; a machine result lists them), each once: after a terminal preview, the * write it confirmed prints only what the preview did not. */ -function printFindings(data: SourceApplyData, printed: Set): void { - printNewDiagnostics(data.diagnostics, printed) +function printFindings(data: SourceApplyData, findings: FindingsOutput, notes: Set): void { + findings.print(data.diagnostics) + if (findings.machine) return for (const note of data.notes) { - if (fresh(printed, `note:${note}`)) printError(`note: ${note}`) + if (fresh(notes, note)) printError(`note: ${note}`) } } @@ -126,12 +161,17 @@ function diffSummary(data: SourceApplyData): string { `${blocks.renamed.length} renamed`, `${blocks.modified.length} modified`, ] - const variableCount = - variables.added.length + variables.removed.length + variables.modified.length + const variableChanges = (['added', 'removed', 'modified'] as const).flatMap((change) => + variables[change].length > 0 ? [`${change} ${variables[change].join(', ')}`] : [] + ) const modified = blocks.modified.map(({ key, fields }) => `${key} (${fields.join(', ')})`) + const typed = (list: ReadonlyArray<{ key: string; type: string }>) => + list.map(({ key, type }) => `${key} (${type})`).join('; ') return `Blocks: ${parts.join(', ')}. Connections: +${edges.added} -${edges.removed}.${ - variableCount > 0 ? ` Variables changed: ${variableCount}.` : '' - }${modified.length > 0 ? `\nModified: ${modified.join('; ')}` : ''}` + blocks.added.length > 0 ? `\nAdded: ${typed(blocks.added)}` : '' + }${blocks.removed.length > 0 ? `\nRemoved: ${typed(blocks.removed)}` : ''}${ + modified.length > 0 ? `\nModified: ${modified.join('; ')}` : '' + }${variableChanges.length > 0 ? `\nVariables: ${variableChanges.join('; ')}` : ''}` } const GET_HINT = 'Write the directory with `sim workflows source get` first.' @@ -176,6 +216,11 @@ export async function readLock(directory: string): Promise { ) } +/** The directory's `workflow.ts`, or a refusal naming the command that writes one. */ +export function requireEntry(directory: string): Promise { + return requireText(directory, ENTRY_FILE, GET_HINT) +} + /** * The files a write sends: the entry file and the code files it or the lock references. A * referenced file that is missing is left out, and the server reports it with its source line. @@ -184,7 +229,7 @@ export async function readSourceFiles( directory: string, lock: WorkflowLockV1 | null ): Promise> { - const entry = await requireText(directory, ENTRY_FILE, GET_HINT) + const entry = await requireEntry(directory) const files: Record = { [ENTRY_FILE]: entry } const paths = codeFilePaths(entry, lock) const texts = await mapWithConcurrency(paths, FILE_CONCURRENCY, (path) => @@ -202,13 +247,15 @@ type SourceWriteOperation = 'applyWorkflowSource' | 'createWorkflowFromSource' /** * The shared write flow of `apply` and `create`: refuse a write without consent when no terminal * can ask, preview and ask on a terminal, send, print the diagnostics, and after a successful - * write save it into the directory with the caller's `save`. + * write save it into the directory with the caller's `save`. From the send through the save an + * interrupt waits for them ({@link deferInterrupts}), so Ctrl-C never strands a stored write the + * directory did not record. */ async function runSourceWrite(args: { options: SourceWriteOptions creating: boolean question: string - client: OperationClient + client: OperationClient operation: Operation params: Record body: ApplyWorkflowSourceBody | CreateWorkflowFromSourceBody @@ -217,6 +264,8 @@ async function runSourceWrite(args: { save: (data: SourceApplyData) => Promise /** The next step when the write's answer never came back, so it may have been stored. */ ifAnswerLost: string + /** This write as a command, without `--yes`: what a `plan` run names next with it. */ + command: string }): Promise { const { options, creating } = args /** Sends the write (or its dry run); a refusal prints its diagnostics and rethrows. */ @@ -234,38 +283,161 @@ async function runSourceWrite(args: { }) return response.data } catch (error) { - return printRefusal(error, args.output, dryRun ? undefined : args.ifAnswerLost) + return printRefusal(error, args.output, { + ...(dryRun ? {} : { ifAnswerLost: args.ifAnswerLost }), + verbose: options.verbose === true, + }) } } - const printed = new Set() - if (!options.dryRun && !options.yes) { + const findings = findingsOutput(args.output, options.verbose) + const notes = new Set() + const report = (data: SourceApplyData, deployment: DeployOutcome = {}) => { + const result = { ...summarize(data, creating, findings.machine), ...deployment } + printProtocolResult( + args.output, + { ...result, ...findings.result(data.diagnostics) }, + { + ...(result.created ? { created: createdText(result.created) } : {}), + ...(deployment.next ? { next: deployment.next.command } : {}), + ...(deployment.deployed + ? { deployed: `version ${deployment.deployed.version ?? '?'}` } + : {}), + } + ) + } + const mode = writeMode(options) + if (mode === 'ask' || mode === 'plan') { const preview = await send(true) - printFindings(preview, printed) + printFindings(preview, findings, notes) if (preview.diagnostics.some((diagnostic) => diagnostic.severity === 'error')) { setSoftExitCode(refusedExitCode(preview.diagnostics)) - printProtocolResult(args.output, { - ...summarize(preview, creating), - ...resultDiagnostics(args.output, preview.diagnostics), - }) + report(preview) return } printError(diffSummary(preview)) + if (mode === 'plan') { + stopAtPlan() + report(preview, { next: planNext(args.command) }) + return + } if (!(await confirm(args.question))) { throw new SimApiError('Nothing was applied.', 0, 'CANCELLED') } } - const data = await send(options.dryRun === true) - printFindings(data, printed) - const refused = data.diagnostics.some((diagnostic) => diagnostic.severity === 'error') - if (refused) setSoftExitCode(refusedExitCode(data.diagnostics)) + const verb = creating ? 'create' : 'apply' + let refused = false + const sendAndSave = async (): Promise => { + const data = await send(options.dryRun === true) + printFindings(data, findings, notes) + refused = data.diagnostics.some((diagnostic) => diagnostic.severity === 'error') + if (refused) setSoftExitCode(refusedExitCode(data.diagnostics)) + if (!data.dryRun && !refused && (data.applied || !creating)) await args.save(data) + return data + } + const data = options.dryRun + ? await sendAndSave() + : await deferInterrupts(sendAndSave, { + waiting: `Interrupted: the ${verb} was already sent. Waiting up to ${DEFERRED_INTERRUPT_BOUND_MS / 1000}s for its answer and saving it into ${options.dir}, so the directory matches the workflow; press Ctrl-C again to stop now.`, + abandoned: `Stopped before the ${verb} answered. ${args.ifAnswerLost}`, + }) - if (!data.dryRun && !refused && (data.applied || !creating)) await args.save(data) + const stored = !data.dryRun && !refused && (data.applied || !creating) + if (mode === 'write' && stored) printError(diffSummary(data)) + let deployment: DeployOutcome = {} + if (stored) { + try { + deployment = await deployStep(args.client, data, { + ...options, + verb, + machine: findings.machine, + }) + } catch (error) { + // The write stands: its result prints before the deploy's failure. + report(data) + throw error + } + } + report(data, deployment) +} - printProtocolResult(args.output, { - ...summarize(data, creating), - ...resultDiagnostics(args.output, data.diagnostics), - }) +/** What a write adds to its result about deploying: the deploy it ran, or the one to run. */ +interface DeployOutcome { + deployed?: { version?: number; isDeployed: boolean; warnings?: string[] } + needsRedeployment?: boolean + next?: { command: string; reason: string } +} + +/** + * After a stored write: with `--deploy`, deploys the workflow through the same operation as + * `sim workflows deploy` (so the same permission check, on the server); without it, names that + * command as the next step when the workflow is deployed and its draft now differs. A failed + * deploy says the write itself was saved. + */ +async function deployStep( + client: OperationClient<'deployWorkflow'>, + data: SourceApplyData, + options: { deploy?: boolean; dir: string; verb: string; machine: boolean } +): Promise { + const deployCommand = `sim workflows deploy ${data.workflowId}` + if (!options.deploy) { + if (!data.needsRedeployment) return {} + return { + next: { + command: deployCommand, + reason: `The workflow is deployed and its draft now differs from the live version: deploy to make the change live (or ${options.verb} with --deploy).`, + }, + } + } + const deployed = await client + .request('deployWorkflow', { + params: { workflowId: data.workflowId }, + body: {}, + }) + .then( + (response) => response.data, + (error: unknown) => { + throw deployFailure(error, options, deployCommand) + } + ) + if (!options.machine) { + for (const warning of deployed.warnings) printError(`deploy warning: ${warning}`) + } + return { + deployed: { + ...(deployed.version !== undefined ? { version: deployed.version } : {}), + isDeployed: deployed.isDeployed, + ...(options.machine && deployed.warnings.length > 0 ? { warnings: deployed.warnings } : {}), + }, + needsRedeployment: false, + } +} + +/** A failed deploy after a saved write, saying the write itself stands and how to deploy it. */ +function deployFailure( + error: unknown, + options: { dir: string; verb: string }, + deployCommand: string +): unknown { + if (!(error instanceof SimApiError)) return error + return new SimApiError( + `The ${options.verb} was saved into ${options.dir}, but the deploy failed: ${error.message} Fix that, then run \`${deployCommand}\`.`, + error.status, + error.code, + error.details, + error.exitCode, + error.requestId + ) +} + +/** Refuses `--deploy` beside `--dry-run`: a dry run stores nothing to deploy. */ +function assertDeployable(options: SourceWriteOptions): void { + if (options.deploy && options.dryRun) { + throw new SimApiError( + '--deploy deploys what the write saves, and --dry-run saves nothing: drop one of them.', + 0 + ) + } } const DIR_DESCRIPTION = @@ -282,7 +454,12 @@ const DIR_DESCRIPTION = * needs `--yes`. `--dry-run` writes nothing, so it never asks. */ export function attachWorkflowSourceApply(source: Command): void { - const [command, connectApply] = apiCommand(source, 'apply', ['applyWorkflowSource']) + const [command, connectApply] = apiCommand(source, 'apply', [ + 'applyWorkflowSource', + 'deployWorkflow', + ]) + absoluteDirOption(command) + verboseFlag(command) command .argument( '[workflowId]', @@ -291,16 +468,23 @@ export function attachWorkflowSourceApply(source: Command): void { .requiredOption('--dir ', DIR_DESCRIPTION) .option('--dry-run', 'Check the source and show the outcome without saving or writing files') .option('-y, --yes', 'Apply without asking for confirmation') + .option( + '--deploy', + 'After the write is saved, deploy the workflow (as `sim workflows deploy ` does)' + ) .option( '--allow-delete', 'Delete blocks the source no longer declares; without it, an apply that deletes a block is refused' ) .allowExcessArguments(false) .description('Apply a workflow source directory to a workflow draft') - .addHelpText('after', `${APPLY_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}`) + .addHelpText( + 'after', + `${APPLY_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}${SOURCE_WRITE_EXIT_CODES_HELP}` + ) .action(async (named: string | undefined, options: SourceWriteOptions) => withWorkspaceFolderLock(options.dir, async () => { - requireConsent(options, 'writes the workflow draft', 'apply') + assertDeployable(options) const { client, profile } = connectApply() const lock = await readLock(options.dir) const base = await readBase(options.dir, named) @@ -310,13 +494,18 @@ export function attachWorkflowSourceApply(source: Command): void { options, creating: false, question: `Apply these changes to workflow ${workflowId}?`, + command: `sim workflows source apply --dir ${shellWord(options.dir)}${options.allowDelete ? ' --allow-delete' : ''}${options.deploy ? ' --deploy' : ''}`, output: profile.output, client, operation: 'applyWorkflowSource', params: { workflowId }, body: { files, lock, base, provenance: { client: 'cli' } }, save: async (data) => { - await writeBack(options.dir, data, lock) + const edges = storedEdgesWrite( + data, + cachedCatalogByHash(toRecord(data.base).catalogHash) + ) + await writeBack(options.dir, data, lock, { write: edges ? [edges] : [] }) await refreshWorkflowTsconfigs(options.dir, data) }, ifAnswerLost: `It may have been saved. Run \`sim workflows source apply ${workflowId} --dir ${options.dir} --dry-run\`: exit 5 (stale) means the draft changed since this directory was read, by this apply or by someone else. Run \`sim workflows source get ${workflowId} --dir \` and compare it with ${options.dir}: if it holds your edit, \`sim workflows source get ${workflowId} --dir ${options.dir} --force\` brings this directory up to date; if not, reapply your edit in the fresh directory. Any other outcome means it was not saved: apply again.`, @@ -336,7 +525,10 @@ export function attachWorkflowSourceCreate(source: Command): void { 'createWorkflowFromSource', 'getSourceCatalog', 'getSourceResources', + 'deployWorkflow', ]) + absoluteDirOption(command) + verboseFlag(command) command .requiredOption( '--dir ', @@ -346,9 +538,16 @@ export function attachWorkflowSourceCreate(source: Command): void { .option('--name ', "The workflow name (default: the source's)") .option('--folder ', 'The folder to create it in (default: the root)') .option('-y, --yes', 'Create without asking for confirmation') + .option( + '--deploy', + 'After the write is saved, deploy the workflow (as `sim workflows deploy ` does)' + ) .allowExcessArguments(false) .description('Create a workflow from a workflow source directory') - .addHelpText('after', `${CREATE_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}`) + .addHelpText( + 'after', + `${CREATE_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}${SOURCE_WRITE_EXIT_CODES_HELP}` + ) .action(async (options: SourceWriteOptions & { name?: string; folder?: string }) => withWorkspaceFolderLock(options.dir, async () => { if ((await readSourceText(options.dir, ENTRY_FILE)) === null) { @@ -359,21 +558,27 @@ export function attachWorkflowSourceCreate(source: Command): void { ) } const { client, profile } = connectCreate() - const tree = await writeSourceScaffold(client, options.dir, client.requireWorkspace()) + const workspaceId = client.requireWorkspace() + const tree = await writeSourceScaffold(client, options.dir, workspaceId) printError( - `${options.dir} had no ${ENTRY_FILE}, so a starter, the SDK types, tsconfig.json, .sim/AGENTS.md and the examples in .sim/examples/ were written and nothing was created. Edit ${ENTRY_FILE}, then run \`sim workflows source create --dir ${options.dir} --yes\`.` + `${options.dir} had no ${ENTRY_FILE}, so a starter, the SDK types for workspace ${workspaceId}, tsconfig.json, .sim/AGENTS.md and the examples in .sim/examples/ were written and nothing was created. Edit ${ENTRY_FILE}, then run \`${createCommand(options.dir, workspaceId)} --yes\`.` ) - printProtocolResult(profile.output, { created: false, scaffolded: true, ...tree }) + printTreeResult(profile.output, { created: false, scaffolded: true, ...tree }) return } - requireConsent(options, 'creates a workflow', 'create') + assertDeployable(options) const { client, profile } = connectCreate() - const workspaceId = client.requireWorkspace() + // A starter is bound to the workspace its types were written for, as a get directory + // is to the one it was read from; --workspace still chooses another. + const starterWorkspace = + profile.sources.workspaceId === 'flag' ? null : await readStarterWorkspace(options.dir) + const workspaceId = client.requireWorkspace(starterWorkspace ?? undefined) const files = await readSourceFiles(options.dir, null) await runSourceWrite<'createWorkflowFromSource'>({ options, creating: true, question: 'Create this workflow?', + command: `${createCommand(shellWord(options.dir), workspaceId)}${options.name !== undefined ? ` --name ${shellWord(options.name)}` : ''}${options.folder !== undefined ? ` --folder ${shellWord(options.folder)}` : ''}${options.deploy ? ' --deploy' : ''}`, output: profile.output, client, operation: 'createWorkflowFromSource', @@ -381,6 +586,7 @@ export function attachWorkflowSourceCreate(source: Command): void { ifAnswerLost: `It may have been created. Run this command again: a SOURCE_NAME_TAKEN refusal means it was, so find its id with \`sim workflows list\` and run \`sim workflows source get --dir ${options.dir} --force\`.`, save: async (data) => { await writeSourceDirectory(client, options.dir, data, []) + await updateSourceDir(options.dir, { write: [], remove: [SOURCE_STARTER_FILE] }) }, body: { files, @@ -393,27 +599,65 @@ export function attachWorkflowSourceCreate(source: Command): void { ) } -/** A create that did not apply stored nothing, so its provisional workflow id names no workflow. */ -function summarize(data: SourceApplyData, created: boolean) { +/** Whether a diff adds, removes or changes any workflow variable. */ +function variablesChanged(variables: SourceApplyData['diff']['variables']): boolean { + return variables.added.length > 0 || variables.removed.length > 0 || variables.modified.length > 0 +} + +/** The create command for `dir` in `workspaceId`, as the next step a starter prints. */ +function createCommand(dir: string, workspaceId: string): string { + return `sim workflows source create --dir ${dir} --workspace ${workspaceId}` +} + +/** + * What a write created (on a dry run, would create; ids are only known once it is written): the + * workflow of a create, then each block added. + */ +function createdResources(data: SourceApplyData, creating: boolean): CreatedResource[] { + const written = !data.dryRun && data.applied + return [ + ...(creating ? [{ kind: 'workflow', ...(written ? { id: data.workflowId } : {}) }] : []), + ...data.diff.blocks.added.map(({ key, id }) => ({ + kind: 'block', + key, + ...(written ? { id } : {}), + })), + ] +} + +/** + * The result of a write. A create that did not apply stored nothing, so its provisional workflow + * id names no workflow. Notes are listed in a `machine` result, and counted in a display one, + * which prints them on stderr. + */ +function summarize(data: SourceApplyData, creating: boolean, machine: boolean) { + const created = data.dryRun || data.applied ? createdResources(data, creating) : [] return { - ...(created && !data.applied ? {} : { workflowId: data.workflowId }), - ...(created ? { created: data.applied } : {}), + ...(creating && !data.applied ? {} : { workflowId: data.workflowId }), applied: data.applied, dryRun: data.dryRun, noop: data.diff.noop, + ...(created.length > 0 ? { created } : {}), blocksAdded: data.diff.blocks.added.length, + ...(data.diff.blocks.added.length > 0 && { + added: data.diff.blocks.added.map(({ key, type }) => ({ block: key, type })), + }), blocksRemoved: data.diff.blocks.removed.length, + ...(data.diff.blocks.removed.length > 0 && { + removed: data.diff.blocks.removed.map(({ key, type }) => ({ block: key, type })), + }), blocksModified: data.diff.blocks.modified.length, ...(data.diff.blocks.modified.length > 0 && { modified: data.diff.blocks.modified.map(({ key, fields }) => ({ block: key, fields })), }), edgesAdded: data.diff.edges.added, edgesRemoved: data.diff.edges.removed, + ...(variablesChanged(data.diff.variables) && { variables: data.diff.variables }), errors: data.diagnostics.filter((diagnostic) => diagnostic.severity === 'error').length, warnings: data.diagnostics.filter((diagnostic) => diagnostic.severity === 'warning').length, - notes: data.notes.length, + notes: machine ? data.notes : data.notes.length, fingerprint: data.base.fingerprint, needsRedeployment: data.needsRedeployment, - ...(created && data.applied && !data.dryRun ? { guide: SOURCE_AGENTS_FILE } : {}), + ...(creating && data.applied && !data.dryRun ? { guide: SOURCE_AGENTS_FILE } : {}), } } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index c597f47ff97..e2989388885 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -4,7 +4,8 @@ import { join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { SimApiError } from '../../http/client' -import { contextMockFns } from '../../test/context-mock' +import { cacheCatalogText } from '../../test/catalog-cache' +import { contextMockFns, mockedProfile } from '../../test/context-mock' import { attachWorkflowSourceCheck } from './workflow-source-check' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -33,11 +34,7 @@ async function check(): Promise { } function cacheCatalog(): void { - mkdirSync(join(dir, 'config/cache/source-catalog'), { recursive: true }) - writeFileSync( - join(dir, `config/cache/source-catalog/${WORKSPACE_ID}.json`), - readFileSync(FIXTURE_CATALOG, 'utf8') - ) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, readFileSync(FIXTURE_CATALOG, 'utf8')) } function result(): Record { @@ -51,10 +48,18 @@ function stdoutJson(): Record { return JSON.parse(calls[0].join(' ')) } -function stderr(): string { - return vi - .mocked(console.error) - .mock.calls.map((call) => call.join(' ')) +/** + * The findings of the JSON result on stdout as a person reads them on stderr in a display format + * (`file:line:column severity CODE: message`, then its hint): with `--output json` they are + * printed only there. + */ +function findingLines(): string { + const diagnostics = (result().diagnostics ?? []) as Array> + return diagnostics + .map( + ({ file, line, column, severity, code, message, hint }) => + `${file === undefined ? '' : `${file}:${line}:${column} `}${severity} ${code}: ${message}${hint === undefined ? '' : `\n hint: ${hint}`}` + ) .join('\n') } @@ -208,7 +213,11 @@ beforeEach(() => { JSON.stringify({ workflowId: 'wf', workspaceId: WORKSPACE_ID, fingerprint: 'sf1:a' }) ) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) - contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_other', output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_other', + output: 'json', + }) contextMockFns.mockRequest.mockReset() vi.spyOn(console, 'log').mockImplementation(() => {}) vi.spyOn(console, 'error').mockImplementation(() => {}) @@ -242,7 +251,7 @@ describe('sim workflows source check', () => { await check() expect(result()).toMatchObject({ errors: 1 }) - expect(stderr()).toMatch(/workflow\.ts:6:\d+ error E_UNKNOWN_PARAM/) + expect(findingLines()).toMatch(/workflow\.ts:6:\d+ error E_UNKNOWN_PARAM/) expect(process.exitCode).toBe(1) }) @@ -269,6 +278,19 @@ describe('sim workflows source check', () => { }) }) + it('with --output json, prints each finding in the JSON only, not again as a frame on stderr', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace('{ body: { received: start.message } }', '{ bdy: { received: start.message } }') + ) + + await check() + + expect(stdoutJson()).toMatchObject({ diagnostics: [{ code: 'E_UNKNOWN_PARAM' }] }) + expect(vi.mocked(console.error)).not.toHaveBeenCalled() + }) + it('leaves stored() to the server, which holds the stored values', async () => { cacheCatalog() writeFileSync( @@ -287,7 +309,7 @@ describe('sim workflows source check', () => { await check() - expect(stderr()).not.toContain('E_STORED_VALUE_MISSING') + expect(findingLines()).not.toContain('E_STORED_VALUE_MISSING') }) it('refuses without a cached catalog, naming the get that caches one', async () => { @@ -299,6 +321,29 @@ describe('sim workflows source check', () => { expect((error as SimApiError).message).toContain('sim workflows source get') }) + it('refuses a directory without workflow.ts before looking for a catalog, by its absolute path', async () => { + const error = await check().catch((caught: unknown) => caught) + + expect((error as SimApiError).code).not.toBe('CATALOG_NOT_CACHED') + expect((error as SimApiError).message).toContain(`${dir} has no workflow.ts`) + }) + + it('refuses a --workspace naming another workspace than .sim/base.json, and only that', async () => { + cacheCatalog() + writeFileSync(join(dir, 'workflow.ts'), CLEAN) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_other', + output: 'json', + sources: { workspaceId: 'flag' }, + }) + + const error = await check().catch((caught: unknown) => caught) + + expect((error as SimApiError).code).toBe('SOURCE_OTHER_WORKSPACE') + expect((error as SimApiError).message).toContain(`was read from workspace ${WORKSPACE_ID}`) + }) + it('points a workspace root at the workspace check', async () => { cacheCatalog() writeWorkspace() @@ -335,8 +380,10 @@ describe('sim workflows source check', () => { await checkIn(folder) expect(result()).toMatchObject({ errors: 2 }) - expect(stderr()).toMatch(/workflow\.ts:6:\d+ error E_MAT_UNKNOWN_RESOURCE: No table "contact"/) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( + /workflow\.ts:6:\d+ error E_MAT_UNKNOWN_RESOURCE: No table "contact"/ + ) + expect(findingLines()).toMatch( /error E_MAT_UNKNOWN_RESOURCE: No credential "ops-slak"[\s\S]*Keys: ops-slack/ ) expect(process.exitCode).toBe(1) @@ -353,7 +400,7 @@ describe('sim workflows source check', () => { await checkIn(folder) expect(result()).toMatchObject({ errors: 0, warnings: 1 }) - expect(stderr()).toContain('W_ENV_UNKNOWN') + expect(findingLines()).toContain('W_ENV_UNKNOWN') }) it('fails a row write naming a column or select option the table does not declare', async () => { @@ -370,10 +417,10 @@ describe('sim workflows source check', () => { await checkIn(folder) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( /error W_TABLE_UNKNOWN_COLUMN: data writes emial[\s\S]*Columns: email, status/ ) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( /error W_UNKNOWN_OPTION: status of table "contacts" has no option done/ ) expect(process.exitCode).toBe(1) @@ -384,7 +431,7 @@ describe('sim workflows source check', () => { writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) await check() - expect(stderr()).toMatch(/error W_UNKNOWN_OPTION/) + expect(findingLines()).toMatch(/error W_UNKNOWN_OPTION/) expect(process.exitCode).toBe(1) process.exitCode = undefined @@ -394,6 +441,71 @@ describe('sim workflows source check', () => { expect(process.exitCode).toBeUndefined() }) + it('reports an untyped JSON reference only on a block the lock does not bind, and on all with --strict', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace( + "const done = response('Done', { body: { received: start.message } })", + "const call = api('Call', { url: 'https://x', method: 'POST', body: { n: start.missing.deep } })\n const done = response('Done', { body: { received: start.message } })" + ).replace('import { response,', 'import { api, response,') + ) + const untyped = () => + ((result().diagnostics ?? []) as Array<{ code: string }>).filter( + (finding) => finding.code === 'W_JSON_REF_UNTYPED' + ) + + await check() + expect(untyped()).toHaveLength(1) + + writeLock(['start', 'done', 'call']) + await check() + expect(untyped()).toEqual([]) + + await checkIn(dir, '--strict') + expect(untyped()).toHaveLength(1) + }) + + it('fails on a dangling $ref in a new function file by default, and on any with --strict', async () => { + cacheCatalog() + mkdirSync(join(dir, 'functions'), { recursive: true }) + writeFileSync(join(dir, 'functions/total.js'), 'return $ref.ordertotals.sum\n') + writeFileSync( + join(dir, 'workflow.ts'), + `// @sim-sdk 1 +import { codeFile, fn, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('W', {}, () => { + const start = trigger.start('Start', {}) + const total = fn('Total', { code: codeFile('./functions/total.js') }) + start.to(total) +}) +` + ) + + await check() + expect(result().diagnostics).toContainEqual( + expect.objectContaining({ + code: 'W_UNRESOLVED_REF', + severity: 'error', + file: 'functions/total.js', + line: 1, + column: 8, + }) + ) + expect(process.exitCode).toBe(1) + + process.exitCode = undefined + writeLock(['start', 'total']) + await check() + expect(result()).toMatchObject({ errors: 0, warnings: 1 }) + expect(process.exitCode).toBeUndefined() + + await checkIn(dir, '--strict') + expect(result()).toMatchObject({ errors: 1 }) + expect(process.exitCode).toBe(1) + }) + it('fails on a bound block with --strict and on nothing with --no-strict', async () => { cacheCatalog() writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) @@ -416,10 +528,12 @@ describe('sim workflows source check', () => { writeLock(['start', 'l', 'inner', 'fallback', 'a', 'route']) await check() - expect(stderr()).toMatch(/warning W_ERROR_PORT_STORED/) - expect(stderr()).toMatch(/warning W_EDGE_CYCLE/) - expect(stderr()).toMatch(/warning W_BRANCH_UNWIRED/) - expect(stderr()).toMatch(/keeps this error edge only if the stored workflow already holds it/) + expect(findingLines()).toMatch(/warning W_ERROR_PORT_STORED/) + expect(findingLines()).toMatch(/warning W_EDGE_CYCLE/) + expect(findingLines()).toMatch(/warning W_BRANCH_UNWIRED/) + expect(findingLines()).toMatch( + /keeps this error edge only if the stored workflow already holds it/ + ) expect(result()).toMatchObject({ errors: 0 }) expect(process.exitCode).toBeUndefined() @@ -434,8 +548,8 @@ describe('sim workflows source check', () => { writeLock(['start', 'l', 'inner', 'fallback', 'a', 'route']) await check() - expect(stderr()).toMatch(/warning W_ERROR_PORT_STORED/) - expect(stderr()).toMatch(/warning W_EDGE_CYCLE/) + expect(findingLines()).toMatch(/warning W_ERROR_PORT_STORED/) + expect(findingLines()).toMatch(/warning W_EDGE_CYCLE/) expect(result()).toMatchObject({ errors: 0 }) expect(process.exitCode).toBeUndefined() }) @@ -449,8 +563,8 @@ describe('sim workflows source check', () => { writeLock(['start', 'l', 'inner', 'fallback', 'a', 'route']) await check() - expect(stderr()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) - expect(stderr()).toMatch(/error E_EDGE_CYCLE/) + expect(findingLines()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) + expect(findingLines()).toMatch(/error E_EDGE_CYCLE/) expect(process.exitCode).toBe(1) }) @@ -460,8 +574,8 @@ describe('sim workflows source check', () => { writeLock(['start', 'inner', 'fallback', 'route']) await check() - expect(stderr()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) - expect(stderr()).toMatch(/error E_EDGE_CYCLE/) + expect(findingLines()).toMatch(/error E_ERROR_PORT_UNSUPPORTED/) + expect(findingLines()).toMatch(/error E_EDGE_CYCLE/) expect(process.exitCode).toBe(1) }) @@ -484,6 +598,5 @@ describe('sim workflows source check', () => { expect(includes(join(dir, 'tsconfig.json'))).toEqual( expect.arrayContaining(['.sim/sdk/blocks/slack_v2.d.ts', '.sim/sdk/blocks/gmail_v2.d.ts']) ) - expect(() => readFileSync(join(dir, 'tsconfig.full.json'))).toThrow() }) }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index 743b8593d81..c8bb5177c82 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -23,15 +23,24 @@ import { profileFrom } from '../../context' import { setSoftExitCode } from '../../embed-context' import { SimApiError } from '../../http/client' import { printProtocolResult } from './result' -import { readSourceJson, readSourceText, workspaceFolderOf } from './source-dir' +import { + absoluteDirOption, + directoryWorkspace, + readSourceJson, + readSourceText, + readStarterWorkspace, + workspaceFolderOf, +} from './source-dir' import { cachedCatalog, + cachedSourceCatalog, type LocalDeclarations, refreshTsconfigs, refreshWorkspaceTypes, } from './source-tree' -import { printNewDiagnostics, resultDiagnostics } from './source-ui' -import { readSourceFiles } from './workflow-source-apply' +import { type Diagnostic, findingsOutput, verboseFlag } from './source-ui' +import { readStoredEdges } from './stored-edges' +import { readSourceFiles, requireEntry } from './workflow-source-apply' import { WORKFLOW_TREE } from './workflow-source-get' import { kindFolders, @@ -60,6 +69,7 @@ const STRICT_CODES: ReadonlySet = new Set([ 'W_TABLE_UNKNOWN_COLUMN', 'W_CHILD_INPUT_UNKNOWN', 'W_BRANCH_UNWIRED', + 'W_UNRESOLVED_REF', ]) /** @@ -70,6 +80,14 @@ const STRICT_CODES: ReadonlySet = new Set([ */ const STORED_EXEMPT_CODES: ReadonlySet = new Set(['W_BRANCH_UNWIRED']) +/** + * Warnings a check reports only on a block the lock does not bind (one added since the get), and + * on every block with `--strict`: printed source holds them on blocks nobody edited, and the + * server, which compares with the stored workflow, already reports them only for an edited field. + * `W_JSON_REF_UNTYPED`: a bare reference nothing types, in a JSON field. + */ +const EDIT_SCOPED_CODES: ReadonlySet = new Set(['W_JSON_REF_UNTYPED']) + /** * Which {@link STRICT_CODES} warnings fail a check: `new` those on a block the lock does not bind * (one added since the get, or every block of a workflow that has no lock yet), `all` every one, @@ -85,7 +103,7 @@ type StrictMode = 'new' | 'all' | 'off' const KEPT_AS_WRITTEN = /[;,] (?:it|the value|the text) is kept as written\.$/ /** One finding of a check, as the write commands print theirs. */ -export type CheckFinding = Parameters[0][number] +export type CheckFinding = Diagnostic /** The `--strict` / `--no-strict` pair: absent selects the default, {@link StrictMode} `new`. */ export interface StrictOptions { @@ -108,7 +126,8 @@ does not bind (a block added since the get, or every block of a new workflow): ${[...STRICT_CODES].map((code) => ` ${code}`).join(',\n')}. --strict fails on them in every block, except ${[...STORED_EXEMPT_CODES].join(', ')} on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict never -fails on a warning.` +fails on a warning. ${[...EDIT_SCOPED_CODES].join(', ')} is reported only on blocks the lock does not +bind, and with --strict on every block.` /** Adds the `--strict` / `--no-strict` pair to a check command. */ export function strictFlags(command: Command): Command { @@ -346,16 +365,23 @@ function declarationFindings( } /** - * Compiles the workflow source directory `dir` against `catalog`, checking env names against the - * names `secrets.ts` declares when `declarations` hold them. + * Compiles the workflow source directory `dir` against the catalog its base names when that one is + * cached (a workflow holding a type the workspace catalog lacks was printed with its own), else + * `catalog`, checking env names against the names `secrets.ts` declares when `declarations` hold + * them. */ async function compileWorkflowDir( dir: string, - catalog: BlockCatalog, + workspaceCatalog: BlockCatalog, declarations?: WorkspaceDeclarations ) { const { lock } = parseWorkflowLock(await readSourceJson(dir, SOURCE_LOCK_FILE)) - const files = await readSourceFiles(dir, lock ?? null) + const [files, storedEdges, printedWith] = await Promise.all([ + readSourceFiles(dir, lock ?? null), + readStoredEdges(dir), + cachedSourceCatalog(dir), + ]) + const catalog = printedWith ?? workspaceCatalog const envNames = declarations?.envNames const compiled = compileWorkflow({ entry: { path: SOURCE_ENTRY_FILE, text: files[SOURCE_ENTRY_FILE] }, @@ -364,6 +390,7 @@ async function compileWorkflowDir( sdkMajor: catalog.sdkMajor, knownKeys: new Set(Object.keys(lock?.blocks ?? {})), storedUnread: lock ?? undefined, + ...(storedEdges ? { storedUnreadEdges: storedEdges } : {}), ...(envNames ? { envNames, envDeclaredIn: KIND_DIRS.secret } : {}), }) return { lock, files, compiled } @@ -457,8 +484,11 @@ export async function checkWorkflowDir( !(STORED_EXEMPT_CODES.has(code) && bound(blockKey)) && (options.strict === 'all' || (options.strict === 'new' && !bound(blockKey))) const prefix = options.prefix ?? '' + const reported = (code: string, blockKey: string | undefined) => + !SERVER_ONLY_CODES.has(code) && + !(EDIT_SCOPED_CODES.has(code) && options.strict !== 'all' && bound(blockKey)) const findings = [ - ...compiled.diagnostics.filter((diagnostic) => !SERVER_ONLY_CODES.has(diagnostic.code)), + ...compiled.diagnostics.filter((diagnostic) => reported(diagnostic.code, diagnostic.blockKey)), ...declared, ].map(({ code, severity, message, hint, frame, span, blockKey }): CheckFinding => { const raised = severity === 'warning' && fails(code, blockKey) @@ -474,13 +504,18 @@ export async function checkWorkflowDir( return { findings, entry: entryText } } -/** Prints a check's findings and result, and exits 1 on any error. */ +/** + * Prints a check's findings and result, and exits 1 on any error. `verbose` keeps info findings in + * a machine result. + */ export function reportCheck( output: Parameters[0], findings: CheckFinding[], - result: Record + result: Record, + verbose = false ): void { - printNewDiagnostics(findings, new Set()) + const shown = findingsOutput(output, verbose) + shown.print(findings) const errors = findings.filter((finding) => finding.severity === 'error').length if (errors > 0) setSoftExitCode(1) printProtocolResult(output, { @@ -488,13 +523,16 @@ export function reportCheck( errors, warnings: findings.filter((finding) => finding.severity === 'warning').length, ...result, - ...resultDiagnostics(output, findings), + ...shown.result(findings), }) } -/** The cached catalog of `workspaceId`, or a refusal naming `get`, the command that caches one. */ -export function requireCachedCatalog(workspaceId: string, get: string): BlockCatalog { - const catalog = cachedCatalog(workspaceId) +/** The catalog a cache read found, or a refusal naming `get`, the command that caches one. */ +function cachedOrRefuse( + catalog: BlockCatalog | null, + workspaceId: string, + get: string +): BlockCatalog { if (catalog) return catalog throw new SimApiError( `No block catalog is cached for workspace ${workspaceId}. Run ${get} once, then check again.`, @@ -503,75 +541,97 @@ export function requireCachedCatalog(workspaceId: string, get: string): BlockCat ) } +/** The cached catalog of `workspaceId`, or a refusal naming `get`, the command that caches one. */ +export function requireCachedCatalog(workspaceId: string, get: string): BlockCatalog { + return cachedOrRefuse(cachedCatalog(workspaceId), workspaceId, get) +} + /** * `sim workflows source check [--dir ]`: compiles a source directory locally with the bundled * compiler and the cached catalog, so an agent's edit loop gets syntax and builder findings in * milliseconds and without the network. The dry run stays the authority. */ export function attachWorkflowSourceCheck(source: Command): void { - strictFlags( - source - .command('check') - .option('--dir ', 'Source directory: workflow.ts, functions/* and .sim/', '.') + verboseFlag( + strictFlags( + absoluteDirOption(source.command('check')).option( + '--dir ', + 'Source directory: workflow.ts, functions/* and .sim/', + '.' + ) + ) ) .allowExcessArguments(false) .description('Compile a workflow source directory locally, without the server') .addHelpText('after', CHECK_HELP) - .action(async (options: { dir: string } & StrictOptions, command: Command) => { - const profile = profileFrom(command) - const base = toRecord(await readSourceJson(options.dir, SOURCE_BASE_FILE)) - if ( - typeof base.workspaceId === 'string' && - base.workflowId === undefined && - (await readSourceText(options.dir, SOURCE_ENTRY_FILE)) === null - ) { - throw new SimApiError( - `${options.dir} is a workspace source directory: check it with \`sim workspaces source check --dir ${options.dir}\`.`, - 0 - ) - } - const workspace = await workspaceFolderOf(options.dir) - const workspaceId = - typeof base.workspaceId === 'string' - ? base.workspaceId - : (workspace?.workspaceId ?? profile.workspaceId) - if (!workspaceId) { - throw new SimApiError( - `${options.dir} has no ${SOURCE_BASE_FILE} and no workspace is selected: pass --workspace .`, - 0 - ) - } - const catalog = requireCachedCatalog( - workspaceId, - workspace - ? `\`sim workspaces source get --dir ${workspace.root}\`` - : `\`sim workflows source get --dir ${options.dir}\` (or \`create\`)` - ) - const declarations = workspace - ? { - ...(await readWorkspaceDeclarations(workspace.root)), - workflowInputs: await workspaceWorkflowInputs(workspace.root, catalog), - } - : undefined - const { findings, entry } = await checkWorkflowDir(options.dir, catalog, { - strict: strictMode(options), - ...(declarations ? { declarations } : {}), - }) - if (workspace && declarations) { - await refreshWorkspaceTypes( - workspace.root, - withWorkflowInputs(declarations.local, declarations.workflowInputs) - ) - await refreshWorkspaceTsconfigs(workspace.root, workspaceId) - } else { - await refreshTsconfigs( + .action( + async (options: { dir: string; verbose?: boolean } & StrictOptions, command: Command) => { + const profile = profileFrom(command) + const base = toRecord(await readSourceJson(options.dir, SOURCE_BASE_FILE)) + if ( + typeof base.workspaceId === 'string' && + base.workflowId === undefined && + (await readSourceText(options.dir, SOURCE_ENTRY_FILE)) === null + ) { + throw new SimApiError( + `${options.dir} is a workspace source directory: check it with \`sim workspaces source check --dir ${options.dir}\`.`, + 0 + ) + } + await requireEntry(options.dir) + const workspace = await workspaceFolderOf(options.dir) + const recorded = + typeof base.workspaceId === 'string' + ? base.workspaceId + : (workspace?.workspaceId ?? + (profile.sources.workspaceId === 'flag' + ? null + : await readStarterWorkspace(options.dir))) + const workspaceId = directoryWorkspace( options.dir, + recorded, + profile, + (from) => `Run it without --workspace to check it against the catalog of ${from}.` + ) + if (!workspaceId) { + throw new SimApiError( + `${options.dir} has no ${SOURCE_BASE_FILE} and no workspace is selected: pass --workspace .`, + 0 + ) + } + const catalog = cachedOrRefuse( + (await cachedSourceCatalog(options.dir)) ?? cachedCatalog(workspaceId), workspaceId, - WORKFLOW_TREE, - { sources: [entry] }, - { incremental: true } + workspace + ? `\`sim workspaces source get --dir ${workspace.root}\`` + : `\`sim workflows source get --dir ${options.dir}\` (or \`create\`)` ) + const declarations = workspace + ? { + ...(await readWorkspaceDeclarations(workspace.root)), + workflowInputs: await workspaceWorkflowInputs(workspace.root, catalog), + } + : undefined + const { findings, entry } = await checkWorkflowDir(options.dir, catalog, { + strict: strictMode(options), + ...(declarations ? { declarations } : {}), + }) + if (workspace && declarations) { + await refreshWorkspaceTypes( + workspace.root, + withWorkflowInputs(declarations.local, declarations.workflowInputs) + ) + await refreshWorkspaceTsconfigs(workspace.root, workspaceId) + } else { + await refreshTsconfigs( + options.dir, + workspaceId, + WORKFLOW_TREE, + { sources: [entry] }, + { incremental: true } + ) + } + reportCheck(profile.output, findings, { catalogHash: catalog.hash }, options.verbose) } - reportCheck(profile.output, findings, { catalogHash: catalog.hash }) - }) + ) } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts index 1ff6b340033..55ab1fb1b47 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.test.ts @@ -1,13 +1,16 @@ +import { execFileSync } from 'node:child_process' import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' import { COOKBOOK_EXAMPLES } from '@sim/workflow-sdk/codegen' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' import { buildGeneratedCommands } from '../../runtime/build' -import { contextMockFns, contextMockState } from '../../test/context-mock' +import { contextMockFns, contextMockState, mockedProfile } from '../../test/context-mock' import { digest } from './source-dir' +import { PRIMER_MAX_BYTES, sourcePrimer } from './source-primer' +import { attachWorkflowSourceCheck } from './workflow-source-check' import { attachWorkflowSourceGet } from './workflow-source-get' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -64,6 +67,21 @@ function program(): Command { return root } +/** Type-checks the output directory with one of its tsconfig files; returns tsc's output, or ''. */ +function typecheck(project: string): string { + try { + execFileSync( + join(__dirname, '../../../../../node_modules/.bin/tsc'), + ['--noEmit', '-p', join(outDir, project)], + { encoding: 'utf8', stdio: 'pipe' } + ) + return '' + } catch (error) { + const failure = error as { stdout?: string; stderr?: string } + return `${failure.stdout ?? ''}${failure.stderr ?? ''}` + } +} + /** The `include` list of a tsconfig the CLI wrote into the output directory. */ function tsconfigInclude(project: string): string[] { const [, ...json] = readFileSync(join(outDir, project), 'utf8').split('\n') @@ -121,7 +139,7 @@ afterEach(() => { }) describe('sim workflows source get --dir', () => { - it('writes the source, lock, base, an agent guide and declarations for every block in the workspace catalog', async () => { + it('writes the source, lock, base, an agent guide and the declarations of the blocks it uses', async () => { serve({ [SOURCE_PATH]: () => source({ 'workflow.ts': '// @sim-sdk 1\n', 'functions/score.js': 'return 1\n' }), @@ -136,7 +154,7 @@ describe('sim workflows source get --dir', () => { ) expect(existsSync(join(outDir, '.sim/sdk/core.d.ts'))).toBe(true) expect(existsSync(join(outDir, '.sim/sdk/blocks/agent.d.ts'))).toBe(true) - expect(existsSync(join(outDir, '.sim/sdk/blocks/slack_v2.d.ts'))).toBe(true) + expect(existsSync(join(outDir, '.sim/sdk/blocks/slack_v2.d.ts'))).toBe(false) const workspaceTypes = readFileSync(join(outDir, '.sim/sdk/workspace.d.ts'), 'utf8') expect(workspaceTypes).toContain("'docs': true") expect(workspaceTypes).toContain("interface WorkspaceSkills {\n 'triage': true") @@ -145,6 +163,67 @@ describe('sim workflows source get --dir', () => { expect(callsTo(CATALOG_PATH)[0][1].query).toBeUndefined() }) + it('prints the primer, which opens the guide, with a snippet for each builder the source calls and no other', async () => { + serve({ + [SOURCE_PATH]: () => + source({ 'workflow.ts': "// @sim-sdk 1\nconst mail = gmail.sendEmail('Mail', {})\n" }), + }) + + await get('--dir', outDir) + + const { primer } = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') + expect(readFileSync(join(outDir, '.sim/AGENTS.md'), 'utf8').startsWith(primer)).toBe(true) + expect(primer).toContain('- `gmail.sendEmail` (Gmail: Send Email)') + expect(primer).not.toContain('`slack.sendMessage`') + }) + + it('keeps the primer without snippets within its byte budget for both directory kinds', () => { + for (const kind of ['workflow', 'workspace'] as const) { + expect(Buffer.byteLength(sourcePrimer(kind))).toBeLessThanOrEqual(PRIMER_MAX_BYTES) + } + }) + + it('type-checks a builder of a block the directory does not hold: through the cache at once, and locally after a check', async () => { + const workflow = [ + '// @sim-sdk 1', + "import { trigger, workflow } from '@sim/workflow-sdk'", + '', + "export default workflow('W', {}, () => {", + " const start = trigger.start('Start', {})", + '})', + '', + ].join('\n') + serve({ [SOURCE_PATH]: () => source({ 'workflow.ts': workflow }, []) }) + await get('--dir', outDir) + expect(existsSync(join(outDir, '.sim/sdk/blocks/gmail_v2.d.ts'))).toBe(false) + + writeFileSync( + join(outDir, 'workflow.ts'), + workflow + .replace('{ trigger, workflow }', '{ gmail, trigger, workflow }') + .replace( + " const start = trigger.start('Start', {})\n", + " const start = trigger.start('Start', {})\n const mail = gmail.sendEmail('Mail', { to: 'a@b.c', subject: 's', body: 'b', bogusParam: 1 })\n start.to(mail)\n" + ) + ) + const unknownParam = /'bogusParam' does not exist in type 'GmailSendEmailParams'/ + expect(typecheck('tsconfig.full.json')).toMatch(unknownParam) + + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: WORKSPACE_ID, + output: 'json', + }) + const check = new Command('source') + attachWorkflowSourceCheck(check) + check.commands.forEach((command) => command.exitOverride()) + await check.parseAsync(['node', 'source', 'check', '--dir', outDir]) + process.exitCode = undefined + + expect(existsSync(join(outDir, '.sim/sdk/blocks/gmail_v2.d.ts'))).toBe(true) + expect(typecheck('tsconfig.json')).toMatch(unknownParam) + }) + it('writes the example workflows beside the guide, outside what the tsconfig files type-check', async () => { serve() @@ -160,7 +239,7 @@ describe('sim workflows source get --dir', () => { } }) - it('type-checks only the blocks the source uses by default, and every current block in tsconfig.full.json', async () => { + it('type-checks only the blocks the source uses by default, and every current block through the cache in tsconfig.full.json', async () => { serve({ [SOURCE_PATH]: () => source( @@ -193,20 +272,26 @@ describe('sim workflows source get --dir', () => { ) expect(include).not.toContain('.sim/sdk/blocks/google_sheets_v2.d.ts') expect(include.every((path) => existsSync(join(outDir, path)))).toBe(true) - expect(tsconfigInclude('tsconfig.full.json')).toEqual([ - 'workflow.ts', - '.sim/sdk/index.d.ts', - '.sim/sdk/workspace.d.ts', - ]) + const [entry, fullIndex, workspace] = tsconfigInclude('tsconfig.full.json') + expect([entry, workspace]).toEqual(['workflow.ts', '.sim/sdk/workspace.d.ts']) + expect(fullIndex.startsWith(join(rootDir, 'config/cache/source-catalog', CATALOG.hash))).toBe( + true + ) + expect(readFileSync(fullIndex, 'utf8')).toContain("import './blocks/google_sheets_v2'") + expect(existsSync(join(dirname(fullIndex), 'blocks/google_sheets_v2.d.ts'))).toBe(true) }) - it('writes one declaration file per block, and the builder index', async () => { + it('lists the builders of the blocks it holds in the builder index, and every builder in the cached one', async () => { serve() await get('--dir', outDir) expect(existsSync(join(outDir, '.sim/sdk/operations'))).toBe(false) - expect(readFileSync(join(outDir, '.sim/sdk/INDEX.md'), 'utf8')).toContain( + const local = readFileSync(join(outDir, '.sim/sdk/INDEX.md'), 'utf8') + expect(local).toContain('- `agent` — Agent → `blocks/agent.d.ts` (`AgentParams`)') + expect(local).not.toContain('`gmail.sendEmail`') + const [, fullIndex] = tsconfigInclude('tsconfig.full.json') + expect(readFileSync(join(dirname(fullIndex), 'INDEX.md'), 'utf8')).toContain( '- `gmail.sendEmail` — Gmail: Send Email → `blocks/gmail_v2.d.ts` (`GmailSendEmailParams`)' ) }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts index 72cb9ad3022..b4d719da980 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-get.ts @@ -11,24 +11,27 @@ import type { GetWorkflowSourceResponse } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { callsOperations, type OperationClient } from '../../runtime/called-operations' import { buildRequest } from '../../runtime/request' -import { printProtocolResult } from './result' import { + absoluteDirOption, assertPrintedSourcePaths, locallyEditedFiles, readPreviousLock, readSourceJson, SOURCE_SDK_DIR as SDK_DIR, SOURCE_EXAMPLES_DIR, + SOURCE_STARTER_FILE, updateSourceDir, WORKSPACE_FOLDER_BUSY_HELP, withWorkspaceFolderLock, writeBack, } from './source-dir' +import { sourceExitCodesTable } from './source-exit-codes' import { assertSdkMajor, finishTree, loadCatalog, loadWorkspaceTypes, + printTreeResult, refreshTsconfigs, type SourceTree, type SourceTreeProfile, @@ -39,6 +42,7 @@ import { type WorkspaceTypes, withSourceExitCode, } from './source-tree' +import { storedEdgesWrite } from './stored-edges' const DIR_FLAG = 'dir' const FORCE_FLAG = 'force' @@ -66,31 +70,29 @@ function workflowTsconfig(sdkFiles: readonly string[]): string { )}\n` } -/** The guide an agent working in the directory reads first; the docs page has the rest. */ -const AGENTS_GUIDE = `# Sim workflow source - -This directory is a Sim workflow written as TypeScript. The server compiles \`${SOURCE_ENTRY_FILE}\` statically; nothing in it runs. The workflow id is \`workflowId\` in \`${SOURCE_BASE_FILE}\`; every command below reads it from there. - -## Start here +/** + * The reference below the primer in `.sim/AGENTS.md`, by task: the edit loop in detail, a new or + * forked workflow, the file shape, lookups, the rules and every exit code. The docs page has the + * rest. + */ +const AGENTS_GUIDE = `# Reference -1. Read the examples in \`${SOURCE_EXAMPLES_DIR}/\` before writing: \`cookbook.ts\` (agent with structured output, condition, function, table query and insert, knowledge search, Slack, human in the loop, loop, response), \`integrations.ts\` (webhook trigger, weekday cron schedule, HTTP request with headers, table update by row id, Gmail, an error path) and \`agents.ts\` (router, agent tools: custom tool, MCP tool, integration operation, child workflow; run a child workflow; parallel). Their resource keys are examples; this workspace's are in \`${WORKSPACE_TYPES_FILE}\`. -2. For any other block, find its builder in \`${SDK_DIR}/${SDK_GUIDE_FILE}\` (one line per builder: \`grep -i "gmail" ${SDK_DIR}/${SDK_GUIDE_FILE}\`). It names the builder, the file and the params type; read just that type: \`grep -n -A40 "export type GmailSendEmailParams" ${SDK_DIR}/blocks/gmail_v2.d.ts\`. Builders are named for the service, not the block type: \`gmail_v2\` is \`gmail.*\`, \`workflow_input\` is \`runWorkflow\`, the HTTP request block is \`api\`, and \`legacy.*\` builders are old versions not to write. -3. Do not read \`${SDK_DIR}/blocks/*.d.ts\` or \`${SDK_DIR}/models.d.ts\` whole: they are large. \`grep\` them for a type or a model id. +The primer above is enough to edit; read a section here when a step needs it. -## Edit loop (run from this directory) +## Edit loop in detail -1. Edit \`${SOURCE_ENTRY_FILE}\` (and \`functions/*\`). -2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. Fix and rerun until it exits 0. -3. \`${typecheckCommand()}\` type-checks with the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. -4. \`sim workflows source status --dir .\` before applying: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed it since (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\`: fix every error it prints, rerun until it exits 0. -5. \`sim workflows source apply --dir . --yes\`: saves the draft (never deploys). It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out), so read the file again before the next edit rather than editing from memory. -6. Exit 5 (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get --dir \`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare it with this one. Only when it holds your edit, run \`sim workflows source get --dir . --force\`; otherwise reapply your edit in the fresh directory. +1. Edit \`${SOURCE_ENTRY_FILE}\` (and \`functions/*\`). \`${SOURCE_EXAMPLES_DIR}/\` holds correct examples to copy from: \`cookbook.ts\` (agent with structured output, condition, function, table query and insert, knowledge search, Slack, human in the loop, loop, response), \`integrations.ts\` (webhook trigger, weekday cron schedule, HTTP request with headers, table update by row id, Gmail, an error path) and \`agents.ts\` (router, agent tools: custom tool, MCP tool, integration operation, child workflow; run a child workflow; parallel). Their resource keys are examples; this workspace's are in \`${WORKSPACE_TYPES_FILE}\`. +2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. +3. \`${typecheckCommand()}\`: about a second; catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It runs the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply: a builder from another block fails with \`has no exported member\` until you run step 2 (or use \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine. +4. \`sim workflows source status --dir .\`: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\` until it exits 0. +5. \`sim workflows source apply --dir . --yes\`: saves the draft, refusing errors, block deletions without \`--allow-delete\` and a stale directory, so it needs no separate dry run (without \`--yes\` and a terminal it prints the plan and exits 14). \`--deploy\` also deploys it; without it, a deployed workflow's next \`sim workflows deploy \` is printed. It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out). +6. Exit 5 (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get --dir \`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare. Only when it holds your edit, run \`sim workflows source get --dir . --force\`; otherwise reapply your edit in the fresh directory. -An apply that deletes blocks (a declaration removed from \`${SOURCE_ENTRY_FILE}\`) is refused with \`E_APPLY_BLOCK_DELETE\` naming them; pass \`--allow-delete\` when the deletion is intended. Editing the \`workflow('…')\` name or description renames or redescribes the workflow; a name another workflow in its folder holds is refused with \`E_NAME_TAKEN\`. +Editing the \`workflow('…')\` name or description renames or redescribes the workflow; a name another workflow in its folder holds is refused with \`E_NAME_TAKEN\`. Lines starting \`note:\` explain something the write did differently than the source says; read them. -Lines starting \`note:\` explain something the write did differently than the source says; read them. +## A new workflow, or a fork -A new workflow: \`sim workflows source create --dir \` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, these types, \`tsconfig.json\` and this guide, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, run step 2 and 3, then \`sim workflows source create --dir --dry-run\` and \`sim workflows source create --dir --yes\`: it prints the new workflow id and writes the lock files. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir --name --yes\`. +\`sim workflows source create --dir --workspace \` (without \`--dry-run\` or \`--yes\`) on a directory without \`${SOURCE_ENTRY_FILE}\` writes a starter \`${SOURCE_ENTRY_FILE}\`, the types of that workspace, \`tsconfig.json\` and this guide, records the workspace in \`${SOURCE_STARTER_FILE}\`, and creates nothing. Until the workflow exists there is no \`${SOURCE_BASE_FILE}\`, so \`status\`, \`diff\` and \`apply\` refuse the directory: edit the starter, check and type-check it, then \`sim workflows source create --dir --workspace --dry-run\` and the same with \`--yes\` (the starter prints both with its workspace filled in): it prints the new workflow id and writes the lock files. A create goes to the starter's workspace whatever the profile's default, unless \`--workspace\` names another. Exit 1 with \`SOURCE_NAME_TAKEN\` means another workflow has that name: pass \`--name\`. To fork a workflow, copy its directory, delete the copy's \`.sim/\` and run \`sim workflows source create --dir --workspace --name --yes\`. ## File shape @@ -108,26 +110,28 @@ export default workflow('Workflow name', {}, () => { The \`// @sim-sdk 1\` header must be the first line, and the file has exactly one \`export default workflow(name, options, () => { ... })\`; \`options\` holds \`description\` and \`variables\` and may be \`{}\`. -## Types +## Finding a builder and its types -- \`${SDK_DIR}/core.d.ts\`: the SDK (\`workflow\`, \`t\`, \`env\`, \`credential\`, \`table\`, \`BlockMeta\`). -- \`${SDK_DIR}/${SDK_GUIDE_FILE}\`: every builder, its block and operation, file and params type. -- \`${SDK_DIR}/blocks/.d.ts\`: one block's builders with their params and outputs. +- \`sim workflows source params \`: one builder's params (required first, enums, defaults) and outputs; a namespace (\`gmail\`) lists its builders; a miss names the closest. Builders are named for the service, not the block type: \`gmail_v2\` is \`gmail.*\`, \`workflow_input\` is \`runWorkflow\`, the HTTP request block is \`api\`, and \`legacy.*\` builders are old versions not to write. +- \`${SDK_DIR}/core.d.ts\`: the SDK (\`workflow\`, \`t\`, \`env\`, \`credential\`, \`table\`, \`loop\`, \`parallel\`, \`BlockMeta\`). +- \`${SDK_DIR}/blocks/.d.ts\`: the builders of a block the source uses, with their params and outputs (check adds the file of a block an edit starts using). Do not read them, or \`${SDK_DIR}/models.d.ts\`, whole: \`params\` prints the part you need, and \`grep\` finds a model id. - \`${WORKSPACE_TYPES_FILE}\`: this workspace's tables, knowledge bases, credentials, workflows and env names. -## Diagnostics and exit codes - -Findings print as \`file:line:column severity CODE: message\`, then the code frame and a hint. \`E_*\` refuses the whole apply; \`W_*\` is saved as written and names a real problem. \`check\` prints the \`W_*\` codes step 2 lists as \`error\` on a block you added, since they fail it there; the apply still saves them as warnings. Exit 0 applied, 1 source errors (or a name another workflow holds), 5 stale, 6 workflow locked, 7 source too large (on a get too), 9 rate limited (HTTP 429) after the CLI's retries (up to 6, at most a minute apart): nothing was written, so wait a minute and run the same command again; 13 not logged in (\`AUTH_REQUIRED\`): run \`sim login\`, then the command again. - ## Rules - Beside the header and the imports from \`@sim/workflow-sdk\`, the file holds only the default-exported workflow. Its callback holds block declarations (\`const x = builder('Name', params)\`) and wiring (\`x.to(y)\`). Helpers, loops, conditionals and other imports have no stored form, so the compiler refuses them: use \`loop.forEach\`, \`condition\` and \`runWorkflow\` blocks instead. -- Logic that does not fit a block goes in a function block with \`code: codeFile('./functions/name.js')\`; in JavaScript read outputs as \`$ref.blockKey.path\` (the block's key: its name lowercased with every whitespace run and every \`.\` removed, so "Triage Agent" is \`triageagent\`; not its variable) and env vars as \`$env.NAME\`. +- Logic that does not fit a block goes in a function block with \`code: codeFile('./functions/name.js')\` (references as the primer says). - Hosted Sim supplies an agent's model key. On self-hosted Sim the agent needs \`apiKey: env.NAME\` (a secret the workspace holds); without it \`check\` reports \`W_REQUIRED_MISSING: apiKey is required\`. - A child workflow's output is \`c.result\`, untyped: its \`response\` block's \`{ data, status, headers }\`, so a field of the child's response body is \`c.result.data.field\` (\`${SDK_DIR}/${SDK_GUIDE_FILE}\` "How it runs" has the rest). - A table block's row is \`{ id, data }\`: read a cell as \`row.data.column\` (in a loop over \`query.rows\`, \`p.currentItem.data.column\`). - Keep \`${SOURCE_LOCK_FILE}\` and \`${SOURCE_BASE_FILE}\` as written: they bind each block to its stored id and make the next apply safe. +## Diagnostics and exit codes + +Findings print as \`file:line:column severity CODE: message\`, then the code frame and a hint. \`E_*\` refuses the whole apply; \`W_*\` is saved as written and names a real problem. \`check\` prints the \`W_*\` codes it fails on as \`error\` on a block you added; the apply still saves them as warnings. Exit codes of every \`workflows source\` command: + +${sourceExitCodesTable('sim workflows source')} + The full guide: ${SOURCE_DOCS_URL} ` @@ -155,12 +159,19 @@ async function writeSourceTree( blockTypes, }), ]) - const written = await writeBack(dir, source, previousLock, tree.change) - return finishTree(dir, written, workspaceTypes, tree.ownsTsconfig) + const edges = storedEdgesWrite(source, catalog) + const written = await writeBack( + dir, + source, + previousLock, + edges ? { ...tree.change, write: [...tree.change.write, edges] } : tree.change + ) + return finishTree(dir, written, workspaceTypes, tree) } /** The workflow directory's tree beside the printed source. */ export const WORKFLOW_TREE: SourceTreeProfile = { + kind: 'workflow', agents: AGENTS_GUIDE, tsconfig: workflowTsconfig, } @@ -180,7 +191,8 @@ export default workflow('New workflow', { description: 'Describe what this workf /** * Starts a new source directory: a starter `workflow.ts` plus the tree a get writes (the * declarations for the workspace catalog and its resources, `tsconfig.json`, the agent guide), - * with no lock or base, since no workflow exists yet. Only for a directory with no `workflow.ts`. + * with no lock or base, since no workflow exists yet, and {@link SOURCE_STARTER_FILE} naming the + * workspace those declarations are for. Only for a directory with no `workflow.ts`. */ export async function writeSourceScaffold( client: OperationClient<'getSourceCatalog' | 'getSourceResources'>, @@ -195,9 +207,13 @@ export async function writeSourceScaffold( const tree = await sourceTreeChange(dir, catalog, workspaceTypes, WORKFLOW_TREE, { sources: [STARTER_SOURCE], }) - const write = [[SOURCE_ENTRY_FILE, STARTER_SOURCE] as const, ...tree.change.write] + const write = [ + [SOURCE_ENTRY_FILE, STARTER_SOURCE] as const, + [SOURCE_STARTER_FILE, `${JSON.stringify({ workspaceId }, null, 2)}\n`] as const, + ...tree.change.write, + ] await updateSourceDir(dir, { ...tree.change, write }) - return finishTree(dir, write.length, workspaceTypes, tree.ownsTsconfig) + return finishTree(dir, write.length, workspaceTypes, tree) } /** @@ -216,6 +232,7 @@ export async function writeSourceDirectory( loadCatalog(client, { workspaceId: source.workspaceId, workflowId: source.workflowId, + catalogHash: source.base.catalogHash, blockTypes: [...blockTypes], }), loadWorkspaceTypes(client, source.workspaceId), @@ -280,11 +297,11 @@ get into another directory, or pass --force); 7 the workflow is too large to pri /** * Teaches the generated `workflows source get` leaf `--dir `: instead of printing the * response, it writes `workflow.ts`, `functions/*`, `.sim/lock.json`, `.sim/base.json`, SDK - * declarations for every block in the workspace's catalog (`.sim/sdk/**`, so a block an agent - * adds resolves too; the catalog is ETag-cached per workspace in the CLI's config directory), - * `.sim/sdk/workspace.d.ts` from the resource manifest, a `tsconfig.json` that resolves them, a - * short `.sim/AGENTS.md`, the example `.sim/examples/cookbook.ts` and the `.gitattributes` lines - * that keep the source LF. Without `--dir` the generated handler runs unchanged. Commander offers + * declarations for the blocks the source uses (`.sim/sdk/**`; every block's are generated once + * per catalog into the CLI's cache, which `tsconfig.full.json` includes, and the catalog itself is + * ETag-cached), `.sim/sdk/workspace.d.ts` from the resource manifest, the tsconfig files, the + * primer it prints atop `.sim/AGENTS.md`, the examples in `.sim/examples/` and the + * `.gitattributes` lines that keep the source LF. Without `--dir` the generated handler runs unchanged. Commander offers * no way to read the action it holds, so it is captured and delegated to, as `runs get` does. */ export function attachWorkflowSourceGet(source: Command): void { @@ -308,6 +325,7 @@ export function attachWorkflowSourceGet(source: Command): void { `--${FORCE_FLAG}`, "With --dir: replace another workflow's source or edits no apply has saved" ) + absoluteDirOption(get) get.addHelpText('after', `${GET_EXIT_CODES_HELP}${WORKSPACE_FOLDER_BUSY_HELP}`) get.action(async (workflowId: string, _options: unknown, command: Command): Promise => { const flags = command.optsWithGlobals() as Record @@ -349,7 +367,7 @@ export function attachWorkflowSourceGet(source: Command): void { workflowSource, workflowSource.blockTypes ) - printProtocolResult(profile.output, { + printTreeResult(profile.output, { workflowId: workflowSource.workflowId, dir, ...tree, diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-params.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-params.test.ts new file mode 100644 index 00000000000..b5cf39a7a8c --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/workflow-source-params.test.ts @@ -0,0 +1,127 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Command } from 'commander' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { SimApiError } from '../../http/client' +import { cacheCatalogText } from '../../test/catalog-cache' +import { contextMockFns, mockedProfile } from '../../test/context-mock' +import { attachWorkflowSourceParams } from './workflow-source-params' + +vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) + +const WORKSPACE_ID = 'ws_params' +/** A real catalog projected from registry blocks, shared with the SDK's own type checks. */ +const FIXTURE_TEXT = readFileSync( + join(__dirname, '../../../../workflow-sdk/test/fixtures/catalog.json'), + 'utf8' +) + +let dir: string + +async function params(...argv: string[]): Promise { + const source = new Command('source') + attachWorkflowSourceParams(source) + source.commands.forEach((command) => command.exitOverride()) + await source.parseAsync(['node', 'source', 'params', ...argv, '--dir', dir]) +} + +function stdoutJson(): Record { + const calls = vi.mocked(console.log).mock.calls + expect(calls).toHaveLength(1) + return JSON.parse(calls[0].join(' ')) +} + +async function refusal(run: Promise): Promise { + return (await run.then( + () => { + throw new Error('expected the command to refuse') + }, + (error: unknown) => error + )) as SimApiError +} + +/** The fixture catalog under another hash, without the `table.upsertRow` builder. */ +function catalogWithoutUpsert(): string { + const catalog = JSON.parse(FIXTURE_TEXT) + const { 'table.upsertRow': _removed, ...builders } = catalog.builders + return JSON.stringify({ ...catalog, builders, hash: 'f'.repeat(64) }) +} + +function writeBase(catalogHash?: string): void { + writeFileSync( + join(dir, '.sim/base.json'), + JSON.stringify({ + workflowId: 'wf', + workspaceId: WORKSPACE_ID, + fingerprint: 'sf1:a', + catalogHash, + }) + ) +} + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'sim-source-params-')) + mkdirSync(join(dir, '.sim')) + vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_other', + output: 'json', + }) + contextMockFns.mockRequest.mockReset() + vi.spyOn(console, 'log').mockImplementation(() => {}) +}) + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }) +}) + +describe('sim workflows source params', () => { + it("reads the catalog the directory was printed with, not the workspace's newer one", async () => { + const printedWith = JSON.parse(FIXTURE_TEXT).hash as string + cacheCatalogText(join(dir, 'config'), null, FIXTURE_TEXT) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, catalogWithoutUpsert()) + writeBase(printedWith) + + await params('table.upsertRow') + + const { builder } = stdoutJson() as { + builder: { key: string; params: { common: Array<{ key: string; required: boolean }> } } + } + expect(builder.key).toBe('table.upsertRow') + expect(builder.params.common.filter((param) => param.required).map((p) => p.key)).toEqual([ + 'tableId', + 'data', + ]) + expect(contextMockFns.mockRequest).not.toHaveBeenCalled() + }) + + it('reads an agent tool form as the builder it names', async () => { + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, FIXTURE_TEXT) + writeBase() + + await params('slack.tool.sendMessage') + + expect(stdoutJson()).toMatchObject({ builder: { key: 'slack.sendMessage' } }) + }) + + it('refuses a builder the catalog lacks, naming the closest one', async () => { + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, FIXTURE_TEXT) + writeBase() + + const error = await refusal(params('slack.postMessage')) + + expect(error.code).toBe('BUILDER_UNKNOWN') + expect(error.message).toContain('Did you mean slack.sendMessage?') + }) + + it('refuses without a cached catalog, making no request', async () => { + writeBase() + + const error = await refusal(params('agent')) + + expect(error.code).toBe('CATALOG_NOT_CACHED') + expect(contextMockFns.mockRequest).not.toHaveBeenCalled() + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-params.ts b/packages/sim-cli/src/commands/protocol/workflow-source-params.ts new file mode 100644 index 00000000000..632867ed1c2 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/workflow-source-params.ts @@ -0,0 +1,90 @@ +import { toRecord } from '@sim/utils/object' +import type { BlockCatalog } from '@sim/workflow-code' +import { SOURCE_BASE_FILE } from '@sim/workflow-code/limits' +import { lookupBuilder, renderBuilderReference } from '@sim/workflow-sdk/codegen' +import type { Command } from 'commander' +import { printLine } from '#sim-cli/output/io' +import { profileFrom } from '../../context' +import { SimApiError } from '../../http/client' +import { printProtocolResult } from './result' +import { readSourceJson, workspaceFolderOf } from './source-dir' +import { cachedCatalog, cachedSourceCatalog } from './source-tree' + +const PARAMS_HELP = ` +Prints a builder's params (required first, with enums and defaults), the gate field that picks +which others apply, and its outputs, from the block catalog the last get, create or check cached. +No request is made. is the dotted builder path as source writes it: table.upsertRow, +agent, trigger.schedule, slack.sendMessage (slack.tool.sendMessage names the same builder). A +namespace (gmail, trigger.gmail) lists its builders; a construct core.d.ts declares (loop, +parallel, t, env) prints its declaration. Run it from a workflow or workspace source directory, +or pass --dir. + +Exit codes: 0 printed; 1 no builder or namespace of that name (the closest one is named), or no +cached catalog (run a get once).` + +/** + * The catalog the directory was printed with (its `.sim/base.json` hash), else the one cached for + * its workspace, the workspace a workflow folder belongs to, or the profile's. + */ +async function lookupCatalog(dir: string, profileWorkspace: string | null): Promise { + const printedWith = await cachedSourceCatalog(dir) + if (printedWith) return printedWith + const base = toRecord(await readSourceJson(dir, SOURCE_BASE_FILE)) + const workspaceId = + typeof base.workspaceId === 'string' + ? base.workspaceId + : ((await workspaceFolderOf(dir))?.workspaceId ?? profileWorkspace) + const cached = workspaceId ? cachedCatalog(workspaceId) : null + if (cached) return cached + throw new SimApiError( + workspaceId + ? `No block catalog is cached for workspace ${workspaceId}. Run \`sim workflows source get --dir \` (or \`sim workspaces source get --dir \`) once, then look it up again.` + : `${dir} is not a source directory and no workspace is selected: pass --dir or --workspace .`, + 0, + 'CATALOG_NOT_CACHED' + ) +} + +/** + * `sim workflows source params [--dir ]`: one builder's params and outputs from the + * cached catalog, so an agent reads the shape of a block it is about to write in one command + * instead of searching the declarations. + */ +export function attachWorkflowSourceParams(source: Command): void { + source + .command('params') + .argument('', 'Builder path (table.upsertRow, agent, trigger.schedule) or namespace') + .option('--dir ', 'Source directory whose catalog to read', '.') + .allowExcessArguments(false) + .description("Print a builder's params and outputs from the cached block catalog (offline)") + .addHelpText('after', PARAMS_HELP) + .action(async (query: string, options: { dir: string }, command: Command) => { + const profile = profileFrom(command) + const catalog = await lookupCatalog(options.dir, profile.workspaceId) + const found = lookupBuilder(catalog, query) + const machine = profile.output === 'json' || profile.output === 'yaml' + switch (found.kind) { + case 'builder': + if (machine) printProtocolResult(profile.output, { builder: found.reference }) + else printLine(renderBuilderReference(found.reference)) + return + case 'namespace': + if (machine) printProtocolResult(profile.output, found) + else { + printLine(`${found.namespace} has ${found.builders.length} builders:`) + for (const builder of found.builders) printLine(` ${builder.key} — ${builder.summary}`) + } + return + case 'core': + if (machine) printProtocolResult(profile.output, found) + else printLine(`${found.name} is declared in .sim/sdk/core.d.ts:\n\n${found.declaration}`) + return + case 'missing': + throw new SimApiError( + `No builder or namespace is named ${query}.${found.hint ? ` ${found.hint}` : ''}`, + 0, + 'BUILDER_UNKNOWN' + ) + } + }) +} diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 8a48a6bdb02..358f555de29 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -18,23 +18,25 @@ import { SimApiError } from '../../http/client' import { buildGeneratedCommands } from '../../runtime/build' import { attachProtocolCommands } from './index' +/** The profile every command resolves; a test that reads display output sets `output` to text. */ +const profile = vi.hoisted(() => ({ + name: 'default', + endpoint: 'https://sim.example', + authProfile: 'default', + apiKey: 'key', + oauth: null, + workspaceId: '00000000-0000-4000-8000-00000000000b', + output: 'json' as 'json' | 'text', + sources: { + endpoint: 'default' as const, + credential: 'env' as const, + workspaceId: 'env' as const, + output: 'default' as const, + }, +})) + vi.mock('../../context', async () => { const { SimClient } = await import('../../http/client') - const profile = { - name: 'default', - endpoint: 'https://sim.example', - authProfile: 'default', - apiKey: 'key', - oauth: null, - workspaceId: '00000000-0000-4000-8000-00000000000b', - output: 'json' as const, - sources: { - endpoint: 'default' as const, - credential: 'env' as const, - workspaceId: 'env' as const, - output: 'default' as const, - }, - } return { globalsOf: () => ({}), profileFrom: () => profile, @@ -186,7 +188,7 @@ const MANIFEST = { /** What the mocked server answers; each test replaces the routes it exercises. */ interface Server { source: () => Response - catalog: (headers: Headers) => Response + catalog: (headers: Headers, url: URL) => Response manifest: () => Response apply: () => Response create: () => Response @@ -201,6 +203,7 @@ let catalogRequests: Headers[] let stderr: string[] beforeEach(() => { + profile.output = 'json' root = mkdtempSync(join(tmpdir(), 'sim-source-roundtrip-')) dir = join(root, 'source') vi.stubEnv('SIM_CONFIG_DIR', join(root, 'config')) @@ -233,7 +236,7 @@ beforeEach(() => { if (url.pathname === `/api/v2/workspaces/${WORKSPACE_ID}/workflows/source/catalog`) { const headers = new Headers(init?.headers) catalogRequests.push(headers) - return server.catalog(headers) + return server.catalog(headers, url) } if (url.pathname === `/api/v2/workspaces/${WORKSPACE_ID}/workflows/source/resources`) { return server.manifest() @@ -493,13 +496,14 @@ describe('catalog and declarations', () => { expect(existsSync(join(dir, 'workflow.ts'))).toBe(false) }) - it('leaves the declarations alone while the index on disk names the same catalog', async () => { + it('drops the file of a block the source no longer uses while the index names the same catalog', async () => { await get() - writeFileSync(join(dir, '.sim/sdk/blocks/kept_block.d.ts'), 'declare const x: 1\n') + writeFileSync(join(dir, '.sim/sdk/blocks/unused_block.d.ts'), 'declare const x: 1\n') await get() - expect(existsSync(join(dir, '.sim/sdk/blocks/kept_block.d.ts'))).toBe(true) + expect(existsSync(join(dir, '.sim/sdk/blocks/unused_block.d.ts'))).toBe(false) + expect(existsSync(join(dir, '.sim/sdk/blocks/agent.d.ts'))).toBe(true) }) it('regenerates the declarations for the same catalog when another CLI release generated them', async () => { @@ -527,8 +531,12 @@ describe('catalog and declarations', () => { }) it('heals a torn catalog cache by fetching the catalog whole and rewriting it', async () => { - const cache = join(root, 'config/cache/source-catalog', `${WORKSPACE_ID}.json`) - mkdirSync(join(cache, '..'), { recursive: true }) + const cache = join(root, 'config/cache/source-catalog', `${CATALOG.hash}.json`) + mkdirSync(join(cache, '../workspaces'), { recursive: true }) + writeFileSync( + join(cache, '../workspaces', `${WORKSPACE_ID}.json`), + JSON.stringify({ hash: CATALOG.hash }) + ) writeFileSync(cache, '{"hash":"torn","catalog":{"blo') await get() @@ -782,6 +790,7 @@ describe('apply refusals', () => { it('prints each compile error as file:line:column with its frame and hint, then exits 1', async () => { await get() + profile.output = 'text' server.apply = () => refusal(400, 'SOURCE_INVALID', { diagnostics: [ @@ -853,3 +862,277 @@ describe('reading the directory', () => { expect(applyBodies[0].files).toEqual({ 'workflow.ts': WORKFLOW_TS }) }) }) + +/** + * The block types the workspace catalog withholds from the caller (an unrevealed preview, an + * allowlist) while a stored workflow still holds them: only that workflow's own source catalog, + * which the get fetches with `workflowId`, declares them, and the dry run compiles against it. + */ +const WITHHELD_TYPES = new Set(['table_v2', 'knowledge']) + +/** {@link CATALOG} as the workspace catalog of a caller it withholds {@link WITHHELD_TYPES} from. */ +const WORKSPACE_CATALOG = { + ...CATALOG, + hash: 'workspace-catalog', + blocks: Object.fromEntries( + Object.entries(CATALOG.blocks).filter(([type]) => !WITHHELD_TYPES.has(type)) + ), + builders: Object.fromEntries( + Object.entries(CATALOG.builders as Record).filter( + ([, builder]) => !WITHHELD_TYPES.has(builder.type) + ) + ), +} + +/** Every builder path of a withheld type, in catalog order. */ +const WITHHELD_BUILDERS = Object.entries(CATALOG.builders as Record) + .filter(([, builder]) => WITHHELD_TYPES.has(builder.type)) + .map(([path]) => path) + +/** A ticket router as a get prints it: a condition whose else branch writes a table row. */ +const TICKET_ROUTER = `// @sim-sdk 1 +import { + condition, + js, + knowledge, + knowledgeBase, + response, + t, + table, + trigger, + workflow, +} from '@sim/workflow-sdk' + +export default workflow('Ticket Router', {}, () => { + const start = trigger.start('Start', { inputFormat: { subject: 'string', priority: 'string' } }) + const isUrgent = condition('Is Urgent', { if: js\`\${start.priority} === 'high'\`, else: true }) + const policy = knowledge.search('Policy', { + knowledgeBaseId: knowledgeBase('policies'), + query: start.subject, + }) + const logTicket = table.insertRow('Log Ticket', { + tableId: table('tickets'), + data: { subject: start.subject, status: 'open' }, + }) + const upsertTicket = table.upsertRow('Upsert Ticket', { + tableId: table('tickets'), + data: { subject: start.subject, status: 'open' }, + }) + const recent = table.queryRows('Recent Tickets', { tableId: table('tickets'), limit: 10 }) + const done = response('Done', { body: { rowId: t\`\${logTicket.row.id}\`, rows: recent.rows } }) + + start.to(isUrgent) + isUrgent.if.to(policy) + isUrgent.else.to(logTicket) + logTicket.to(upsertTicket) + upsertTicket.to(recent) + recent.to(done) +}) +` + +/** One block of every withheld builder, each with no params, as an edit could add them. */ +const EVERY_WITHHELD_BUILDER = [ + '// @sim-sdk 1', + "import { knowledge, table, trigger, workflow } from '@sim/workflow-sdk'", + '', + "export default workflow('Every Builder', {}, () => {", + ...WITHHELD_BUILDERS.map((path, index) => ` const b${index} = ${path}('Block ${index}', {})`), + '})', + '', +].join('\n') + +const check = (...flags: string[]) => sim('workflows', 'source', 'check', '--dir', dir, ...flags) + +function checked(): { errors: number; warnings: number; diagnostics: unknown[] } { + return JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') +} + +describe('checking a workflow holding a type the workspace catalog withholds', () => { + afterEach(() => { + process.exitCode = undefined + }) + + beforeEach(() => { + server.catalog = (_, url) => + json({ data: url.searchParams.has('workflowId') ? CATALOG : WORKSPACE_CATALOG }) + }) + + function serve(source: string, catalogHash: string = CATALOG.hash): void { + const { data } = sourceResponse( + { 'workflow.ts': source }, + { version: 1, blocks: {}, variables: {} } + ) + server.source = () => + json({ + data: { + ...data, + base: { ...data.base, catalogHash }, + blockTypes: ['start_trigger', 'condition', 'response', ...WITHHELD_TYPES], + }, + }) + } + + it('passes the unedited source of a get, with the else branch wired to the table write', async () => { + serve(TICKET_ROUTER) + await get() + + await check() + + expect(checked()).toMatchObject({ errors: 0, warnings: 0, diagnostics: [] }) + expect(process.exitCode).toBeUndefined() + }) + + it('finds what the dry run finds for every builder only the workflow catalog declares', async () => { + serve(EVERY_WITHHELD_BUILDER) + await get() + const dryRun = compileWorkflow({ + entry: { path: 'workflow.ts', text: EVERY_WITHHELD_BUILDER }, + files: new Map(), + catalog: CATALOG, + sdkMajor: CATALOG.sdkMajor, + }).diagnostics.map(({ code, message, span }) => ({ code, message, line: span?.line })) + + await check('--no-strict') + + expect(WITHHELD_BUILDERS).toEqual( + expect.arrayContaining(['table.insertRow', 'knowledge.search']) + ) + expect(dryRun.map(({ code }) => code)).not.toContain('E_UNKNOWN_BUILDER') + expect( + (checked().diagnostics as Array<{ code: string; message: string; line?: number }>).map( + ({ code, message, line }) => ({ code, message, line }) + ) + ).toEqual(dryRun) + }) + + it('caches each catalog once by its hash, fetching the workflow catalog only while uncached', async () => { + const cacheDir = join(root, 'config/cache/source-catalog') + mkdirSync(join(cacheDir, WORKSPACE_ID), { recursive: true }) + writeFileSync(join(cacheDir, `${WORKSPACE_ID}.json`), '{}') + serve(TICKET_ROUTER) + const queries: string[] = [] + const answer = server.catalog + server.catalog = (headers, url) => { + queries.push(url.searchParams.get('workflowId') ?? 'workspace') + return answer(headers, url) + } + + await get() + await get() + + expect(readdirSync(cacheDir).sort()).toEqual( + [CATALOG.hash, `${CATALOG.hash}.json`, `${WORKSPACE_CATALOG.hash}.json`, 'workspaces'].sort() + ) + expect(queries.filter((query) => query === WORKFLOW_ID)).toHaveLength(1) + }) + + it('checks against the workspace catalog again once a later get no longer needs its own', async () => { + serve(TICKET_ROUTER) + await get() + const revealed = { ...CATALOG, hash: 'revealed' } + server.catalog = () => json({ data: revealed }) + serve(TICKET_ROUTER, revealed.hash) + await get() + + await check() + + expect(checked()).toMatchObject({ errors: 0, catalogHash: revealed.hash }) + }) +}) + +/** A get's source holding a stored cycle between A and B, every block bound by the lock. */ +const STORED_CYCLE = `// @sim-sdk 1 +import { api, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('W', {}, () => { + const start = trigger.start('Start', {}) + const a = api('A', { url: 'https://example.com', method: 'GET' }) + const b = api('B', { url: 'https://example.com', method: 'GET' }) + const c = api('C', { url: 'https://example.com', method: 'GET' }) + + start.to(a) + a.to(b) + b.to(a) + b.to(c) +}) +` + +const STORED_CYCLE_LOCK = { + version: 1, + blocks: Object.fromEntries( + ['start', 'a', 'b', 'c'].map((key, order) => [ + key, + { id: `id-${key}`, parent: null, ident: key, order }, + ]) + ), + variables: {}, +} + +describe('checking wiring between blocks the lock binds', () => { + beforeEach(() => { + server.source = () => + json({ + data: { + ...sourceResponse({ 'workflow.ts': STORED_CYCLE }, STORED_CYCLE_LOCK).data, + blockTypes: ['start_trigger', 'api'], + }, + }) + }) + + afterEach(() => { + process.exitCode = undefined + }) + + const check = (...flags: string[]) => sim('workflows', 'source', 'check', '--dir', dir, ...flags) + + function codes(): string[] { + const printed = JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') + return (printed.diagnostics as Array<{ code: string }>).map(({ code }) => code) + } + + function edit(change: (source: string) => string): void { + writeFileSync(join(dir, 'workflow.ts'), change(read('workflow.ts'))) + } + + it('refuses a cycle the edit closes between bound blocks, as the dry run does', async () => { + await get() + edit((source) => source.replace(' b.to(c)\n', ' b.to(c)\n c.to(b)\n')) + + await check('--strict') + + expect(codes()).toEqual(['E_EDGE_CYCLE']) + expect(process.exitCode).toBe(1) + }) + + it('keeps the cycle the get printed a warning', async () => { + await get() + + await check('--strict') + + expect(codes()).toEqual(['W_EDGE_CYCLE']) + expect(process.exitCode).toBeUndefined() + }) + + it('keeps the printed cycle a warning through a block renamed in source', async () => { + await get() + edit((source) => source.replace("api('B',", "api('Bee',")) + + await check('--strict') + + expect(codes()).toEqual(['W_EDGE_CYCLE']) + expect(process.exitCode).toBeUndefined() + }) + + it('records the edges an apply printed, for the base it saves', async () => { + await get() + server.apply = () => + json(appliedResponse({ 'workflow.ts': STORED_CYCLE }, { lock: STORED_CYCLE_LOCK })) + await apply() + edit((source) => source.replace(' b.to(c)\n', ' b.to(c)\n c.to(b)\n')) + + await check() + + expect(JSON.parse(read('.sim/stored-edges.json')).fingerprint).toBe(SECOND) + expect(codes()).toEqual(['E_EDGE_CYCLE']) + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts index a59cb309b46..1f210b95193 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts @@ -4,7 +4,7 @@ import { basename, dirname, join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { REQUEST_CANCELLED, SimApiError } from '../../http/client' -import { contextMockFns, contextMockState } from '../../test/context-mock' +import { contextMockFns, contextMockState, mockedProfile } from '../../test/context-mock' import { digest } from './source-dir' import { attachWorkflowSourceDiff, attachWorkflowSourceStatus } from './workflow-source-status' @@ -102,7 +102,7 @@ const STALE = { code: 'E_APPLY_STALE', severity: 'error', message: 'The workflow beforeEach(() => { contextMockState.output = 'json' mockRequest.mockReset() - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'json' }) + mockProfileFrom.mockReturnValue({ ...mockedProfile(), workspaceId: WORKSPACE_ID, output: 'json' }) root = mkdtempSync(join(tmpdir(), 'sim-workflow-status-')) writeWorkflowDir(root, null) vi.spyOn(console, 'log').mockImplementation(() => {}) @@ -115,6 +115,20 @@ afterEach(() => { }) describe('workflows source status', () => { + it.each(['status', 'diff'] as const)( + 'on a starter no workflow was created from, %s names the create to run, not a get', + async (command) => { + const starter = join(root, 'starter') + write(starter, 'workflow.ts', GOT['workflow.ts']) + write(starter, '.sim/starter.json', JSON.stringify({ workspaceId: WORKSPACE_ID })) + + await expect(run(starter, command)).rejects.toThrow( + `It holds a starter no workflow was created from yet: run \`sim workflows source create --dir ${starter} --workspace ${WORKSPACE_ID} --dry-run\`` + ) + expect(mockRequest).not.toHaveBeenCalled() + } + ) + it('is clean, exit 0, with nothing changed on either side', async () => { mockRequest.mockResolvedValue(dryRun()) @@ -241,7 +255,40 @@ describe('workflows source status', () => { // A dry run would only exit 13: the login is the next step. next: { command: 'sim login --profile default' }, }) - expect(process.exitCode).toBe(11) + expect(process.exitCode).toBe(13) + }) + + it('exits 13 without a login even when nothing changed here, like every other command', async () => { + mockRequest.mockRejectedValue( + new SimApiError('Not logged in on profile "default".', 0, 'AUTH_REQUIRED', undefined, 13) + ) + + await run(root, 'status') + + expect(printed()).toMatchObject({ server: 'unreachable', state: 'clean' }) + expect(process.exitCode).toBe(13) + }) + + it('degrades to the local view when the server refuses the credential, naming the login', async () => { + write(root, 'workflow.ts', `${ENTRY}// edited\n`) + mockRequest.mockRejectedValue( + new SimApiError( + 'Invalid API key — run sim login --profile default', + 401, + 'UNAUTHORIZED', + undefined, + 13 + ) + ) + + await run(root, 'status') + + expect(printed()).toMatchObject({ + server: 'unreachable', + state: 'local', + next: { command: 'sim login --profile default' }, + }) + expect(process.exitCode).toBe(13) }) it('fails on a cancelled request instead of reporting the server unreachable', async () => { @@ -384,7 +431,11 @@ describe('workflows source diff', () => { it('prints a colorless unified diff for a person when stdout is not a terminal', async () => { write(root, 'functions/score.js', 'return 2\n') mockRequest.mockResolvedValue({ data: { ...dryRun().data, files: GOT } }) - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'text' }) + mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: WORKSPACE_ID, + output: 'text', + }) await run(root, 'diff') diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts index 100d8160830..4bb903d4e7f 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts @@ -22,6 +22,7 @@ import type { ApplyWorkflowSourceResponse, GetWorkflowSourceResponse } from '../ import { SimApiError } from '../../http/client' import { apiCommand } from '../../runtime/called-operations' import { + absoluteDirOption, digest, listSourceFiles, readFolderWritten, @@ -134,6 +135,7 @@ interface StatusOptions { */ export function attachWorkflowSourceStatus(source: Command): void { const [command, connect] = apiCommand(source, 'status', ['applyWorkflowSource']) + absoluteDirOption(command) command .option('--dir ', 'Source directory: workflow.ts, functions/* and .sim/', '.') .option('--offline', 'Report local changes only, without asking the server') @@ -213,7 +215,7 @@ export function attachWorkflowSourceStatus(source: Command): void { }), } printStatusReport(output, `Workflow ${workflowId} in ${options.dir}`, report, planLine) - setSoftExitCode(statusExitCode(status.state)) + setSoftExitCode(statusExitCode(status.state, login)) }) } @@ -231,6 +233,7 @@ interface DiffOptions { */ export function attachWorkflowSourceDiff(source: Command): void { const [command, connect] = apiCommand(source, 'diff', ['getWorkflowSource']) + absoluteDirOption(command) command .argument('[paths...]', 'Files or folders to compare (default: every file)') .option('--dir ', 'Source directory: workflow.ts, functions/* and .sim/', '.') diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 6fb1071a98d..1cbc484917f 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -6,6 +6,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + realpathSync, rmSync, truncateSync, unlinkSync, @@ -13,10 +14,11 @@ import { writeFileSync, } from 'node:fs' import { hostname, tmpdir } from 'node:os' -import { dirname, join } from 'node:path' +import { basename, dirname, join } from 'node:path' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' +import { cacheCatalogText } from '../../test/catalog-cache' import { contextMockFns, contextMockState } from '../../test/context-mock' import { runWorkspaceSource } from '../../test/workspace-source-command' import { digest } from './source-dir' @@ -129,6 +131,38 @@ afterEach(() => { }) describe('workspaces source apply', () => { + it('without --yes and nobody at a terminal, prints the plan, writes nothing and exits 14', async () => { + request.mockResolvedValue( + answer({ dryRun: true, plan: [{ kind: 'table', key: 'leads', action: 'create' }] }) + ) + + await apply('--allow-delete') + + expect(request.mock.calls.map((call) => call[1].query?.dryRun === true)).toEqual([true]) + expect(process.exitCode).toBe(14) + const printed = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) + expect(printed).toMatchObject({ + dryRun: true, + created: [{ kind: 'table', key: 'leads' }], + next: { command: `sim workspaces source apply --dir ${dir} --allow-delete --yes` }, + }) + expect(vi.mocked(console.error).mock.calls.flat().join('\n')).toContain('+ leads') + }) + + it('with --yes, prints the plan it applied', async () => { + request.mockResolvedValue( + answer({ + plan: [{ kind: 'table', key: 'leads', action: 'create' }], + applied: [{ kind: 'table', key: 'leads', action: 'create', id: 'tbl_1' }], + }) + ) + + await apply('--yes') + + expect(request).toHaveBeenCalledTimes(1) + expect(vi.mocked(console.error).mock.calls.flat().join('\n')).toContain('+ leads') + }) + it('puts each finding of a dry run in the JSON on stdout', async () => { const finding = { code: 'E_DECL_SHAPE', @@ -196,9 +230,8 @@ describe('workspaces source apply', () => { expect(Object.keys(bodies()[0].files)).toContain('workflows/ingest/workflow.ts') }) - it('sends an unchanged workflow folder whole when applying a copy to another workspace', async () => { + it('sends an unchanged workflow folder copied from another workspace whole', async () => { const copied = folderBase('ws_other') - write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) write('workflows/ingest/.sim/base.json', copied) write( '.sim/written.json', @@ -209,7 +242,6 @@ describe('workspaces source apply', () => { }, }) ) - contextMockState.workspaceSource = 'flag' request.mockResolvedValue(answer()) await apply('--yes') @@ -289,6 +321,46 @@ describe('workspaces source apply', () => { expect(readFileSync(join(dir, 'tables/leads.ts'), 'utf8')).toBe(LEADS) }) + it('records what a promotion left in the target, and compares the next promotion with it rather than the target as read then', async () => { + const leads = (fingerprint: string) => ({ + id: 'tbl_prod', + fingerprint: `rf1:${fingerprint.repeat(64)}`, + }) + let live = { version: 1, workspaceId: 'ws_prod', resources: { 'table:leads': leads('a') } } + request.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path === '/api/v2/workspaces/ws_prod/source' && options.body === undefined) { + return { + data: { + workspaceId: 'ws_prod', + kinds: ['table'], + files: { 'tables/leads.ts': LEADS }, + blobs: [], + base: live, + skipped: [], + diagnostics: [], + }, + } + } + return answer({ workspaceId: 'ws_prod', base: live }) + }) + + await apply('--to', 'ws_prod', '--only', 'tables', '--dry-run') + expect(existsSync(join(dir, '.sim/targets/ws_prod.json'))).toBe(false) + await apply('--to', 'ws_prod', '--only', 'tables', '--yes') + expect(JSON.parse(readFileSync(join(dir, '.sim/targets/ws_prod.json'), 'utf8'))).toMatchObject({ + version: 1, + workspaceId: 'ws_prod', + resources: { 'table:leads': leads('a') }, + }) + + live = { ...live, resources: { 'table:leads': leads('b') } } + request.mockClear() + await apply('--to', 'ws_prod', '--only', 'tables', '--yes') + + const post = request.mock.calls.find(([, options]) => options.body !== undefined) + expect(post?.[1].body.base.resources['table:leads']).toEqual(leads('a')) + }) + it('counts the changes that delete data in a dry run, and says re-applying an older directory does not restore it', async () => { request.mockResolvedValue( answer({ @@ -359,7 +431,7 @@ describe('workspaces source apply', () => { expect(summary).toMatchObject({ workspaceId: 'ws_prod', dryRun: true, - created: 1, + created: [{ kind: 'table', key: 'leads' }], diagnostics: [expect.objectContaining({ code: 'W_UNDECLARED', key: 'audit' })], }) }) @@ -526,7 +598,11 @@ describe('workspaces source apply', () => { .mocked(console.log) .mock.calls.map(([line]) => String(line)) .join('\n') - expect(JSON.parse(printed)).toMatchObject({ applied: 1, created: 1, failed: 'workflow:ingest' }) + expect(JSON.parse(printed)).toMatchObject({ + applied: 1, + created: [{ kind: 'table', key: 'leads', id: 'tbl_1' }], + failed: 'workflow:ingest', + }) expect(readFileSync(join(dir, 'tables/leads.ts'), 'utf8')).toBe(`${LEADS}// stored\n`) expect(JSON.parse(readFileSync(join(dir, '.sim/base.json'), 'utf8')).resources).toEqual({ 'table:leads': { id: 'tbl_1' }, @@ -647,6 +723,56 @@ describe('workspaces source apply', () => { expect(request).not.toHaveBeenCalled() }) + it('a staging request the server refuses as too large applies nothing, names the upload and the fix, and exits 7', async () => { + write('files/a.bin', Buffer.alloc(3_000, 1)) + mockRequest.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path.endsWith('/source/blobs')) { + throw new SimApiError('Request body is too large', 413, 'PAYLOAD_TOO_LARGE') + } + return path.includes('/source/blobs') ? stagingAnswer(path, options) : request(path, options) + }) + + const refused = await apply('--yes').then( + () => null, + (error: unknown) => error + ) + + expect(refused).toMatchObject({ status: 413, code: 'PAYLOAD_TOO_LARGE', exitCode: 7 }) + expect((refused as SimApiError).message).toContain('files/a.bin') + expect((refused as SimApiError).message).toContain('sim update') + expect(request).not.toHaveBeenCalled() + }) + + it('keeps the server’s refusal of a staging area with no room, which no CLI update or proxy fixes', async () => { + write('files/a.bin', Buffer.alloc(3_000, 1)) + const full = + 'Staging 3000 more bytes would hold 2000000001, over the 2000000000-byte limit. Staged files expire a day after their last use.' + mockRequest.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path.endsWith('/source/blobs')) { + throw new SimApiError(full, 413, 'PAYLOAD_TOO_LARGE', { code: 'SOURCE_STAGING_FULL' }) + } + return path.includes('/source/blobs') ? stagingAnswer(path, options) : request(path, options) + }) + + const refused = await apply('--yes').then( + () => null, + (error: unknown) => error + ) + + expect(refused).toMatchObject({ status: 413, message: full }) + expect(request).not.toHaveBeenCalled() + }) + + it('uploads two files that would overflow one request body in two requests', async () => { + write('files/a.bin', Buffer.alloc(6_000_000, 1)) + write('files/b.bin', Buffer.alloc(6_000_000, 2)) + request.mockResolvedValueOnce(answer()) + + await apply('--yes') + + expect(staging.uploaded.map((batch) => batch.length)).toEqual([1, 1]) + }) + describe('rate limiting', () => { const limited = () => new SimApiError('API rate limit exceeded', 429, 'RATE_LIMITED', undefined, 9) @@ -767,17 +893,36 @@ describe('workspaces source apply', () => { await expect(apply('--yes')).rejects.toThrow(/unsafe file path/) }) - it('refuses a directory read from another workspace unless --workspace names the target', async () => { + it('applies to the workspace the directory was read from, refusing only a --workspace naming another', async () => { write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) request.mockResolvedValue(answer()) - await expect(apply('--yes')).rejects.toMatchObject({ code: 'SOURCE_OTHER_WORKSPACE' }) - expect(request).not.toHaveBeenCalled() - - contextMockState.workspaceSource = 'flag' await apply('--yes') expect(request).toHaveBeenCalledTimes(1) + expect(request.mock.calls[0][0]).toBe('/api/v2/workspaces/ws_other/source') + + write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) + contextMockState.workspaceSource = 'flag' + await expect(apply('--yes')).rejects.toMatchObject({ + code: 'SOURCE_OTHER_WORKSPACE', + message: expect.stringContaining('--to ws_local'), + }) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('names the directory by its absolute path when --dir is relative', async () => { + const cwd = process.cwd() + process.chdir(dirname(dir)) + try { + contextMockState.workspaceSource = 'flag' + write('.sim/base.json', JSON.stringify({ ...BASE, workspaceId: 'ws_other' })) + await expect(runWorkspaceSource('apply', basename(dir), '--yes')).rejects.toMatchObject({ + message: expect.stringContaining(`${realpathSync(dir)} was read from workspace ws_other`), + }) + } finally { + process.chdir(cwd) + } }) it('refuses a base that records blobs no get downloaded, but sends one the user removed', async () => { @@ -993,11 +1138,12 @@ describe('workspaces source apply', () => { ).resolves.toMatchObject({ blobs: [] }) }) - it('warns that a hidden file under a kind folder is not sent', async () => { + it('warns on stderr that a hidden file under a kind folder is not sent', async () => { write('files/.gitkeep', '') write('files/.env', 'SECRET=1') write('files/notes.md', 'hi') request.mockResolvedValue(answer()) + contextMockState.output = 'text' await apply('--yes') @@ -1010,6 +1156,29 @@ describe('workspaces source apply', () => { expect(warned).not.toContain('files/.gitkeep') }) + it('names each file it does not send in its own warning, in the JSON result and its count', async () => { + write('files/a:b.txt', 'x') + write('files/q?.txt', 'x') + write('workflows/.hidden-wf/workflow.ts', INGEST) + request.mockResolvedValue(answer()) + + await apply('--yes') + + const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) + const notSent = summary.diagnostics.filter( + (finding: { code: string }) => finding.code === 'W_PATH_NOT_SENT' + ) + expect(notSent.map((finding: { file: string }) => finding.file).sort()).toEqual([ + 'files/a:b.txt', + 'files/q?.txt', + 'workflows/.hidden-wf/workflow.ts', + ]) + expect(notSent.every((finding: { severity: string }) => finding.severity === 'warning')).toBe( + true + ) + expect(summary.warnings).toBe(3) + }) + it('sends secrets.ts and credentials.ts as their kinds, and no other root file', async () => { const secrets = "// @sim-sdk 1\nexport default defineSecrets({ OPENAI_API_KEY: { description: 'Models' } })\n" @@ -1065,4 +1234,59 @@ describe('workspaces source apply', () => { .join('\n') expect(warned).toContain('Held back, not written: workflow ingest') }) + + it('saves the applied answer when the catalog of a workflow it reprinted cannot be fetched, and says its edges are unrecorded', async () => { + const configDir = mkdtempSync(join(tmpdir(), 'sim-workspace-apply-config-')) + vi.stubEnv('SIM_CONFIG_DIR', configDir) + cacheCatalogText( + configDir, + 'ws_local', + readFileSync(join(__dirname, '../../../../workflow-sdk/test/fixtures/catalog.json'), 'utf8') + ) + const storedBase = JSON.stringify({ + version: 1, + workflowId: 'wf_1', + workspaceId: 'ws_local', + fingerprint: 'sf1:b', + catalogHash: 'workflow-own-catalog', + }) + request.mockImplementation( + async (path: string, options: { query?: { workflowId?: string } }) => { + if (path.endsWith('/source/catalog') && options.query?.workflowId) { + throw new SimApiError('Service unavailable', 503, 'UNAVAILABLE') + } + return answer({ + plan: [{ kind: 'workflow', key: 'ingest', action: 'update' }], + applied: [{ kind: 'workflow', key: 'ingest', action: 'update', id: 'wf_1' }], + files: { + 'workflows/ingest/workflow.ts': `${INGEST}// stored\n`, + 'workflows/ingest/.sim/base.json': storedBase, + }, + base: { ...BASE, resources: { 'workflow:ingest': { id: 'wf_1' } } }, + }) + } + ) + + try { + await apply('--yes') + } finally { + vi.unstubAllEnvs() + rmSync(configDir, { recursive: true, force: true }) + } + + expect(process.exitCode).toBeUndefined() + expect(readFileSync(join(dir, 'workflows/ingest/workflow.ts'), 'utf8')).toBe( + `${INGEST}// stored\n` + ) + expect(readFileSync(join(dir, 'workflows/ingest/.sim/base.json'), 'utf8')).toBe(storedBase) + expect(JSON.parse(readFileSync(join(dir, '.sim/base.json'), 'utf8')).resources).toEqual({ + 'workflow:ingest': { id: 'wf_1' }, + }) + expect(existsSync(join(dir, 'workflows/ingest/.sim/stored-edges.json'))).toBe(false) + const warned = vi + .mocked(console.error) + .mock.calls.map(([line]) => String(line)) + .join('\n') + expect(warned).toContain('could not fetch the block catalog of workflows/ingest') + }) }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 46da2dec2a5..77619db4da0 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -12,9 +12,16 @@ import { } from '@sim/workspace-code' import { isBlobPath, KIND_LAYOUTS } from '@sim/workspace-code/kinds/registry' import { kindOfPath, WORKSPACE_BASE_FILE } from '@sim/workspace-code/layout' -import { promotionSubmission } from '@sim/workspace-code/promote' +import { + type PromotionRecord, + parsePromotionRecord, + promotionRecord, + promotionRecordPath, + promotionSubmission, +} from '@sim/workspace-code/promote' import type { Command } from 'commander' import { printError } from '#sim-cli/output/io' +import { DEFERRED_INTERRUPT_BOUND_MS, deferInterrupts } from '#sim-cli/terminal/interrupts' import { setSoftExitCode } from '../../embed-context' import { SOURCE_PARTIAL_EXIT_CODE } from '../../exit-codes' import type { @@ -26,6 +33,8 @@ import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' import { printProtocolResult } from './result' import { + absoluteDirOption, + directoryWorkspace, isMissing, orphanedCodeFiles, readSourceJson, @@ -38,14 +47,25 @@ import { writtenFile, writtenRecord, } from './source-dir' +import { SOURCE_WRITE_EXIT_CODES_HELP } from './source-exit-codes' +import { cachedCatalog } from './source-tree' import { confirm, - printNewDiagnostics, + createdText, + findingsOutput, + planNext, printRefusal, refusedExitCode, - requireConsent, - resultDiagnostics, + shellWord, + stopAtPlan, + verboseFlag, + writeMode, } from './source-ui' +import { + removedWorkflowRecords, + unrecordedFoldersWarning, + workflowFolderRecords, +} from './stored-edges' import { appliedBlobDigests, assertBlobsFetched, @@ -66,6 +86,7 @@ type Change = ApplyData['plan'][number] interface ApplyOptions { dir: string + verbose?: boolean dryRun?: boolean yes?: boolean allowDelete?: boolean @@ -78,18 +99,21 @@ Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printe file:line:column with its frame and hint), or changes were held back because they use a declared secret the workspace has no value for or a declared credential key nothing binds to (the rest was applied: set the secret or connect the credential, then apply again), the directory was read from -another workspace than the profile's (SOURCE_OTHER_WORKSPACE: pass --workspace to choose, or --to -to promote it), its +another workspace than the one --workspace names (SOURCE_OTHER_WORKSPACE: drop --workspace, or +promote it with --to; without --workspace an apply targets the workspace the directory was read +from), its base records binary files no get downloaded (SOURCE_BLOBS_NOT_FETCHED: get again first), or the command failed; 5 a resource changed or was deleted on the server since the directory was read -(SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource is -locked; 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was +(SOURCE_STALE: get it again into a fresh directory, reapply the edit, retry); 6 a resource (a +workflow or its folder, a table) is locked (a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the directory is too large; 8 the apply stopped part-way on a server failure: what was written is saved into the directory, and applying again continues. An apply a resource refused part-way (stale, locked, forbidden) saves what was written too, and exits as that refusal: 5, 6 or 1; 9 rate limited (HTTP 429) after the CLI's retries, before anything was written: wait a minute and apply again (binary files already uploaded are not sent again); 10 another apply is writing the workspace (SOURCE_BUSY), or another command is using the directory -(SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished.` +(SOURCE_DIR_BUSY): nothing was written, run the command again once the other finished; 14 +without --yes and with no terminal to ask: the plan was printed with no errors and nothing was +written (run the same command with --yes to apply it).` const SYMBOLS: Record = { create: '+', @@ -298,13 +322,15 @@ function droppedResourceFiles( * their digests in `.sim/written.json` (`written` is what it held), beside the digests of the * blobs the server now holds. The function files a written workflow no longer uses, and the files * a reprinted resource no longer has, are removed from disk and the record; the record also drops - * the files of resources the apply deleted ({@link deletedRecordedPaths}). + * the files of resources the apply deleted ({@link deletedRecordedPaths}). `records` are the + * stored-edges records of the workflows it reprinted. */ async function saveApplied( dir: string, data: ApplyData, submission: WorkspaceSubmission, - written: Written + written: Written, + records: ReadonlyArray ): Promise { assertWorkspacePaths(Object.keys(data.files)) const orphaned = [ @@ -324,31 +350,12 @@ async function saveApplied( ...Object.entries(data.files), [WORKSPACE_BASE_FILE, `${JSON.stringify(data.base, null, 2)}\n`], [SOURCE_WRITTEN_FILE, writtenFile(record)], + ...records, ], - remove: orphaned, + remove: [...orphaned, ...removedWorkflowRecords(deleted)], }) } -/** - * Refuses a directory read from another workspace than the one targeted, unless `--workspace` - * names the target: applying it would create every resource anew there, so the choice is the - * caller's to make explicitly. - */ -function assertBaseWorkspace( - dir: string, - base: unknown, - workspaceId: string, - explicit: boolean -): void { - const readFrom = baseWorkspaceOf(base) - if (readFrom === null || readFrom === workspaceId || explicit) return - throw new SimApiError( - `${dir} was read from workspace ${readFrom}, but this profile targets ${workspaceId}. Pass --workspace ${readFrom} to apply it there, or --to ${workspaceId} to promote it: its resources bind by key to that workspace's own, and the directory stays bound to ${readFrom}.`, - 0, - 'SOURCE_OTHER_WORKSPACE' - ) -} - /** The workflow a directory's base belongs to (a workflow source directory), or `null`. */ function baseWorkflowOf(base: unknown): string | null { return isRecordLike(base) && typeof base.workflowId === 'string' ? base.workflowId : null @@ -359,11 +366,21 @@ function baseWorkspaceOf(base: unknown): string | null { return isRecordLike(base) && typeof base.workspaceId === 'string' ? base.workspaceId : null } +/** A promotion's submission, with what it needs to record the target once applied. */ +interface Promotion { + submission: WorkspaceSubmission + /** The directory's own text files, before the target's records replaced its workflows'. */ + files: Record + /** The record the last promotion to this target left, if any. */ + record: PromotionRecord | undefined +} + /** * The submission that promotes `submission` (read for the target, so every workflow folder is * whole) to `target`: the target's get supplies its base and each workflow's lock and base, so - * its resources are bound by key and updated in place. Refuses a target the directory was read - * from, which a plain apply serves. + * its resources are bound by key and updated in place, and the target's promotion record (what + * the last promotion from this directory left there) is the base each resource it holds compares + * with. Refuses a target the directory was read from, which a plain apply serves. */ async function promoted( client: OperationClient<'getWorkspaceSource'>, @@ -371,7 +388,7 @@ async function promoted( submission: WorkspaceSubmission, target: string, allowDelete: boolean -): Promise { +): Promise { if (baseWorkspaceOf(submission.base) === target) { throw new SimApiError( `${dir} was read from workspace ${target}; apply it without --to.`, @@ -379,6 +396,10 @@ async function promoted( 'SOURCE_PROMOTE_SAME_WORKSPACE' ) } + const recordPath = promotionRecordPath(target) + const record = recordPath + ? parsePromotionRecord(await readSourceJson(dir, recordPath), target) + : undefined const { data } = await client.request('getWorkspaceSource', { params: { workspaceId: target }, query: { kinds: submission.kinds.join(',') }, @@ -392,9 +413,41 @@ async function promoted( blobPaths: submission.blobs.map((blob) => blob.path), kinds: submission.kinds, target: { files: data.files, base: targetBase }, + record, allowDelete, }) - return { ...submission, files: composed.files, base: composed.base } + return { + submission: { ...submission, files: composed.files, base: composed.base }, + files: submission.files, + record, + } +} + +/** + * Saves what a promotion applied to `target` as its promotion record in the directory, the base + * the next promotion there compares with. Nothing else in the directory changes. + */ +async function savePromotion( + dir: string, + target: string, + promotion: Promotion, + data: ApplyData +): Promise { + const path = promotionRecordPath(target) + const base = parseWorkspaceBase(data.base).base + if (!path || !base) return + const record = promotionRecord({ + workspaceId: target, + previous: promotion.record, + kinds: promotion.submission.kinds, + files: promotion.files, + submission: { + files: promotion.submission.files, + base: parseWorkspaceBase(promotion.submission.base).base ?? base, + }, + applied: { base, files: data.files, plan: data.plan }, + }) + await updateSourceDir(dir, { write: [[path, `${JSON.stringify(record, null, 2)}\n`]] }) } /** @@ -410,7 +463,10 @@ export function attachWorkspaceSourceApply(source: Command): void { 'findMissingWorkspaceSourceBlobs', 'stageWorkspaceSourceBlobs', 'getWorkspaceSource', + 'getSourceCatalog', ]) + absoluteDirOption(command) + verboseFlag(command) command .requiredOption('--dir ', 'Workspace source directory written by `workspaces source get`') .option('--dry-run', 'Plan and check without writing anything') @@ -419,16 +475,27 @@ export function attachWorkspaceSourceApply(source: Command): void { .option('--only ', `Apply only these kinds (${ONLY_KINDS_HELP})`) .option( '--to ', - "Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place; the directory is not changed" + "Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too" ) .allowExcessArguments(false) .description('Apply a workspace source directory') - .addHelpText('after', EXIT_CODES_HELP) + .addHelpText('after', `${EXIT_CODES_HELP}${SOURCE_WRITE_EXIT_CODES_HELP}`) .action(async (options: ApplyOptions) => { - requireConsent(options, 'writes the workspace', 'apply') + const mode = writeMode(options) const only = parseKinds(options.only) const { client, profile } = connect() - const workspaceId = options.to ?? client.requireWorkspace() + const readFrom = baseWorkspaceOf(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) + const workspaceId = + options.to ?? + client.requireWorkspace( + directoryWorkspace( + options.dir, + readFrom, + profile, + (from, named) => + `Run it without --workspace to apply it to ${from}, or promote it with --to ${named}: its resources bind by key to that workspace's own, and the directory stays bound to ${from}.` + ) ?? undefined + ) const post = async (submission: WorkspaceSubmission, dryRun: boolean) => { const body: ApplyWorkspaceSourceBody = { files: submission.files, @@ -458,7 +525,7 @@ export function attachWorkspaceSourceApply(source: Command): void { try { staged = await stageBlobs(client, workspaceId, options.dir, submission.blobs, paths) } catch (error) { - return printRefusal(error, profile.output) + return printRefusal(error, profile.output, { verbose: options.verbose === true }) } if (staged.count > 0) { printError(`Uploaded ${staged.count} binary file(s), ${staged.bytes} bytes.`) @@ -484,15 +551,12 @@ export function attachWorkspaceSourceApply(source: Command): void { refusal = error } } - return printRefusal( - refusal, - profile.output, - dryRun ? undefined : answerLostHint(options.dir) - ) + return printRefusal(refusal, profile.output, { + ...(dryRun ? {} : { ifAnswerLost: answerLostHint(options.dir) }), + verbose: options.verbose === true, + }) } - // Set while the apply write is in flight, so an interrupt names what it may have done. - let writing = false await withSourceDirLock( options.dir, async () => { @@ -503,55 +567,73 @@ export function attachWorkspaceSourceApply(source: Command): void { if (read.kinds.length === 0 || baseWorkflowOf(read.base) !== null) { await assertWorkspaceDir(options.dir) } - if (!options.to) { - assertBaseWorkspace( - options.dir, - read.base, - workspaceId, - profile.sources.workspaceId === 'flag' - ) - } assertBlobsFetched(options.dir, read, written.digests) - const submission = options.to + const promotion = options.to ? await promoted(client, options.dir, read, options.to, options.allowDelete === true) - : read - if (submission.ignored.length > 0) { - printError( - `Not sent (a hidden name, or a character no workspace path holds): ${submission.ignored.join(', ')}` + : null + const submission = promotion?.submission ?? read + // The apply repeats findings the preview already showed: each is printed once. + const findings = findingsOutput(profile.output, options.verbose) + const notSent = notSentFindings(submission.ignored) + findings.print(notSent) + /** Prints `data` as the result, with the findings about files the CLI did not send. */ + const report = (data: ApplyData, next?: { command: string; reason: string }) => { + const diagnostics = [...notSent, ...data.diagnostics] + const result = { ...summarize({ ...data, diagnostics }), ...(next ? { next } : {}) } + printProtocolResult( + profile.output, + { ...result, ...findings.result(diagnostics) }, + { created: createdText(result.created), ...(next ? { next: next.command } : {}) } ) } - // The apply repeats findings the preview already showed. - const printed = new Set() let preview: ApplyData | null = null - if (options.dryRun || !options.yes) { + if (mode !== 'write') { preview = await send(submission, true) printPlan(preview.plan) - printNewDiagnostics(preview.diagnostics, printed) + findings.print(preview.diagnostics) const errors = preview.diagnostics.some((finding) => finding.severity === 'error') if (errors) setSoftExitCode(refusedExitCode(preview.diagnostics)) - if (options.dryRun || refusesApply(preview)) { - printProtocolResult(profile.output, { - ...summarize(preview), - ...resultDiagnostics(profile.output, preview.diagnostics), - }) + if (mode === 'dry-run' || refusesApply(preview)) { + report(preview) return } - if ( - !options.yes && - !(await confirm(`Apply these changes to workspace ${workspaceId}?`)) - ) { + if (mode === 'plan') { + stopAtPlan() + report(preview, planNext(applyCommand(options))) + return + } + if (!(await confirm(`Apply these changes to workspace ${workspaceId}?`))) { throw new SimApiError('Nothing was applied.', 0, 'CANCELLED') } } // Without a preview, stage the blobs the base does not hold as they are. await stage(submission, preview ? new Set(preview.needBlobs) : blobsToStage(submission)) - writing = true - const data = await send(submission, false) - writing = false - printNewDiagnostics(data.diagnostics, printed) - // A promotion leaves the directory bound to the workspace it was read from. - if (!options.to) await saveApplied(options.dir, data, submission, written) + const data = await deferInterrupts( + async () => { + const answer = await send(submission, false) + findings.print(answer.diagnostics) + // A promotion leaves the directory bound to the workspace it was read from, and + // records what it left in the target for the next promotion to compare with. + if (promotion && options.to) { + await savePromotion(options.dir, options.to, promotion, answer) + } else { + const workspaceCatalog = cachedCatalog(workspaceId) + const { records, unrecorded } = workspaceCatalog + ? await workflowFolderRecords(client, workspaceId, workspaceCatalog, answer.files) + : { records: [], unrecorded: [] } + await saveApplied(options.dir, answer, submission, written, records) + const unrecordedWarning = unrecordedFoldersWarning(unrecorded) + if (unrecordedWarning) printError(unrecordedWarning) + } + return answer + }, + { + waiting: `Interrupted: the apply was already sent. Waiting up to ${DEFERRED_INTERRUPT_BOUND_MS / 1000}s for its answer and saving it into ${options.dir}, so the directory matches the workspace; press Ctrl-C again to stop now.`, + abandoned: `Stopped before the apply answered. ${answerLostHint(options.dir)}`, + } + ) + if (mode === 'write') printPlan(data.plan) const held = data.plan.filter((change) => change.held) if (held.length > 0) { printError( @@ -563,22 +645,37 @@ export function attachWorkspaceSourceApply(source: Command): void { printError(`Stopped at ${data.failed.kind} ${data.failed.key}: ${data.failed.message}`) setSoftExitCode(failedExitCode(data.failed)) } - printProtocolResult(profile.output, { - ...summarize(data), - ...resultDiagnostics(profile.output, data.diagnostics), - }) + report(data) }, - { - readOnly: options.dryRun === true, - interruptNote: () => - writing - ? `Interrupted while the apply was being sent. ${answerLostHint(options.dir)}` - : null, - } + { readOnly: mode === 'dry-run' || mode === 'plan' } ) }) } +/** This apply as a command, without `--yes` or `--dry-run`: what a plan run names next. */ +function applyCommand(options: ApplyOptions): string { + return [ + `sim workspaces source apply --dir ${shellWord(options.dir)}`, + ...(options.only ? [`--only ${shellWord(options.only)}`] : []), + ...(options.to ? [`--to ${shellWord(options.to)}`] : []), + ...(options.allowDelete ? ['--allow-delete'] : []), + ].join(' ') +} + +/** + * A `W_PATH_NOT_SENT` warning for each file under a kind folder the CLI does not send: a hidden + * name, or a character no workspace path holds. + */ +function notSentFindings(paths: readonly string[]): ApplyData['diagnostics'] { + return paths.map((path) => ({ + code: 'W_PATH_NOT_SENT' as const, + severity: 'warning' as const, + message: `${path} is not sent: a hidden name, or a character no workspace path holds. The apply writes and deletes nothing for it.`, + hint: 'Rename it to send it, or move it out of the kind folder.', + file: path, + })) +} + /** * The next step for an apply write whose answer never came back (a timeout, a dropped connection, * an interrupt): it may or may not have been written. @@ -587,7 +684,10 @@ function answerLostHint(dir: string): string { return `It may have been written. Run \`sim workspaces source apply --dir ${dir} --dry-run\`: no changes, or only the workflows you edited refused as stale (exit 5), means it may have been, or someone else changed those resources. Run \`sim workspaces source get --dir \` and compare it with ${dir}: if it holds your edits, \`sim workspaces source get --dir ${dir} --force\` brings this directory up to date; if not, reapply your edits in the fresh directory. Any other outcome means it was not written: apply again.` } -/** The result line: what a dry run would write, or what an apply wrote (not what it planned). */ +/** + * The result line: what a dry run would write, or what an apply wrote (not what it planned). + * `created` lists each resource created, with its id once written. + */ function summarize(data: ApplyData) { const changes = data.dryRun ? data.plan : data.applied const count = (action: Change['action']) => @@ -596,7 +696,9 @@ function summarize(data: ApplyData) { workspaceId: data.workspaceId, dryRun: data.dryRun, applied: data.applied.length, - created: count('create'), + created: changes + .filter((change) => change.action === 'create' && !change.held) + .map(({ kind, key, id }) => ({ kind, key, ...(id !== undefined ? { id } : {}) })), updated: count('update') + count('adopt'), deleted: count('delete'), destructive: destructiveChanges(changes), diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts b/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts index b309a864756..ed1a1d94e12 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-blobs.ts @@ -12,6 +12,7 @@ import { Readable, Transform } from 'node:stream' import { pipeline } from 'node:stream/promises' import { mapWithConcurrency } from '@sim/utils/concurrency' import { generateShortId } from '@sim/utils/id' +import { toRecord } from '@sim/utils/object' import { APPLY_ORDER, type BlobEntry, @@ -29,7 +30,11 @@ import { KIND_LAYOUTS, } from '@sim/workspace-code/kinds/registry' import { isWorkspaceSourcePath, kindOfPath } from '@sim/workspace-code/layout' -import { stageBatches, WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' +import { + stageBatches, + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' import type { FindMissingWorkspaceSourceBlobsResponse, GetWorkspaceSourceResponse, @@ -470,7 +475,7 @@ type StagingClient = OperationClient< /** * Stages the bytes of the blobs at `paths` for an apply to `workspaceId`: asks which of their * hashes the server does not hold staged for this caller, then uploads only those, in requests of - * at most `maxBlobBytesPerRequest`. A blob staged by an earlier, interrupted run is not sent + * at most `maxBlobBytesPerRequest` (a larger blob alone). A blob staged by an earlier, interrupted run is not sent * again. A file whose bytes no longer hash to what the directory read is refused before it is * sent. Returns how many blobs and bytes were uploaded. */ @@ -516,11 +521,37 @@ export async function stageBlobs( form.append(blob.sha256, new Blob([new Uint8Array(content)]), blob.sha256) bytes += content.byteLength } - await client.request('stageWorkspaceSourceBlobs', { - params: { workspaceId }, - form, - retryRateLimited: true, - }) + try { + await client.request('stageWorkspaceSourceBlobs', { + params: { workspaceId }, + form, + retryRateLimited: true, + }) + } catch (error) { + throw stagingTooLarge(error, batch) + } } return { count: sending.length, bytes } } + +/** + * A staging request the server refused as too large (`413`), re-raised naming what it carried and + * what to do: nothing was applied, and since this CLI batches within the limits it shares with the + * server, a refusal means the server runs other limits (another version, or a proxy in front of + * it with a smaller body limit). A staging area with no room (`SOURCE_STAGING_FULL`), which only + * freeing staged files fixes, and any other error pass through. + */ +function stagingTooLarge(error: unknown, batch: readonly BlobEntry[]): unknown { + if (!(error instanceof SimApiError) || error.status !== 413) return error + if (toRecord(error.details).code === 'SOURCE_STAGING_FULL') return error + const bytes = batch.reduce((sum, blob) => sum + blob.size, 0) + const largest = batch.reduce((top, blob) => (blob.size > top.size ? blob : top), batch[0]) + return new SimApiError( + `The server refused an upload of ${batch.length} binary file(s), ${bytes} bytes (largest ${largest.path}), as too large: ${error.message} Nothing was applied. Update the CLI (\`sim update\`) so it splits uploads the way this server accepts; if a proxy in front of the server limits request bodies, raise its limit above ${WORKSPACE_SOURCE_STAGE_MAX_BYTES} bytes.`, + error.status, + error.code, + error.details, + error.exitCode, + error.requestId + ) +} diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts index 969ade98a8f..d966ca640d5 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.test.ts @@ -4,7 +4,8 @@ import { join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' -import { contextMockFns } from '../../test/context-mock' +import { cacheCatalogText } from '../../test/catalog-cache' +import { contextMockFns, mockedProfile } from '../../test/context-mock' import { attachWorkspaceSourceCheck } from './workspace-source-check' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -52,10 +53,18 @@ function stdoutJson(): Record { return JSON.parse(calls[0].join(' ')) } -function stderr(): string { - return vi - .mocked(console.error) - .mock.calls.map((call) => call.join(' ')) +/** + * The findings of the JSON result on stdout as a person reads them on stderr in a display format + * (`file:line:column severity CODE: message`, then its hint): with `--output json` they are + * printed only there. + */ +function findingLines(): string { + const diagnostics = (result().diagnostics ?? []) as Array> + return diagnostics + .map( + ({ file, line, column, severity, code, message, hint }) => + `${file === undefined ? '' : `${file}:${line}:${column} `}${severity} ${code}: ${message}${hint === undefined ? '' : `\n hint: ${hint}`}` + ) .join('\n') } @@ -74,12 +83,13 @@ beforeEach(() => { '', ].join('\n') ) - writeFile( - `config/cache/source-catalog/${WORKSPACE_ID}.json`, - readFileSync(FIXTURE_CATALOG, 'utf8') - ) + cacheCatalogText(join(dir, 'config'), WORKSPACE_ID, readFileSync(FIXTURE_CATALOG, 'utf8')) vi.stubEnv('SIM_CONFIG_DIR', join(dir, 'config')) - contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_other', output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_other', + output: 'json', + }) contextMockFns.mockRequest.mockReset() vi.spyOn(console, 'log').mockImplementation(() => {}) vi.spyOn(console, 'error').mockImplementation(() => {}) @@ -111,10 +121,12 @@ describe('sim workspaces source check', () => { await check() expect(result()).toMatchObject({ errors: 2, workflows: 3 }) - expect(stderr()).toMatch( + expect(findingLines()).toMatch( /workflows\/second\/workflow\.ts:5:\d+ error E_MAT_UNKNOWN_RESOURCE: No table "contact"/ ) - expect(stderr()).toMatch(/workflows\/third\/workflow\.ts:5:\d+ error W_TABLE_UNKNOWN_COLUMN/) + expect(findingLines()).toMatch( + /workflows\/third\/workflow\.ts:5:\d+ error W_TABLE_UNKNOWN_COLUMN/ + ) expect(process.exitCode).toBe(1) }) @@ -132,8 +144,8 @@ describe('sim workspaces source check', () => { await check() - expect(stderr()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes emial/) - expect(stderr()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes seen/) + expect(findingLines()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes emial/) + expect(findingLines()).toMatch(/W_TABLE_UNKNOWN_COLUMN: data writes seen/) expect(process.exitCode).toBe(1) }) @@ -155,10 +167,7 @@ describe('sim workspaces source check', () => { }) it('in a fresh clone (no cached catalog), downloads the catalog once and checks', async () => { - const catalog = readFileSync( - join(dir, `config/cache/source-catalog/${WORKSPACE_ID}.json`), - 'utf8' - ) + const catalog = readFileSync(FIXTURE_CATALOG, 'utf8') rmSync(join(dir, 'config'), { recursive: true }) writeFile('workflows/first/workflow.ts', insertWorkflow()) contextMockFns.mockRequest.mockResolvedValue({ data: JSON.parse(catalog) }) @@ -308,6 +317,30 @@ describe('sim workspaces source check', () => { expect(process.exitCode).toBe(1) }) + it('a rows file with more bad lines than one file lists exits 1, as source errors, not 7 as a size limit', async () => { + writeFile( + 'tables/plans.ts', + [ + '// @sim-sdk 1', + "import { column, defineTable } from '@sim/workflow-sdk/workspace'", + 'export default defineTable({', + " name: 'plans',", + ' columns: { code: column.string({ required: true, unique: true }) },', + " rows: { mode: 'managed', key: 'code' },", + '})', + '', + ].join('\n') + ) + writeFile('tables/plans.rows.jsonl', `${Array.from({ length: 150 }, () => '{').join('\n')}\n`) + + await check() + + const found = stdoutJson().diagnostics as Array<{ code: string; severity: string }> + expect(found.at(-1)).toMatchObject({ code: 'I_FINDINGS_TRUNCATED', severity: 'info' }) + expect(found.filter((finding) => finding.severity === 'error')).toHaveLength(100) + expect(process.exitCode).toBe(1) + }) + it('accepts seed rows the columns hold, coercing a numeric string as the write does', async () => { writeFile( 'tables/seeds.ts', diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts index cf7c90ffdfc..c16c3caa9ca 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts @@ -4,12 +4,18 @@ import { KIND_DIRS } from '@sim/workspace-code' import { WORKSPACE_BASE_FILE } from '@sim/workspace-code/layout' import type { Command } from 'commander' import { profileFrom } from '../../context' -import { AUTH_REQUIRED_CODE } from '../../exit-codes' import { SimApiError } from '../../http/client' import { apiCommand, type Connection } from '../../runtime/called-operations' -import { readSourceJson, SOURCE_DIR_BUSY_HELP, withSourceDirLock } from './source-dir' -import { unreachableReason } from './source-status' +import { + absoluteDirOption, + directoryWorkspace, + readSourceJson, + SOURCE_DIR_BUSY_HELP, + withSourceDirLock, +} from './source-dir' +import { credentialRefused, unreachableReason } from './source-status' import { cachedCatalog, loadCatalog, refreshWorkspaceTypes } from './source-tree' +import { verboseFlag } from './source-ui' import { type CheckFinding, checkWorkflowDir, @@ -67,12 +73,9 @@ async function fetchedCatalog( try { return await loadCatalog(connection.client, { workspaceId, blockTypes: [] }) } catch (error) { - // No login keeps its own refusal (exit 13, naming `sim login`); a server that did not answer + // No login, or a refused credential, keeps its own refusal (exit 13, naming `sim login`); a server that did not answer // leaves the catalog uncached, which the get that caches it fixes once the server answers. - const reason = - error instanceof SimApiError && error.code !== AUTH_REQUIRED_CODE - ? unreachableReason(error) - : null + const reason = credentialRefused(error) ? null : unreachableReason(error) if (reason === null) throw error throw new SimApiError( `No block catalog is cached for workspace ${workspaceId}, and downloading it failed: ${reason} Run \`sim workspaces source get --dir ${dir}\` once the server can be reached, then check again.`, @@ -89,11 +92,14 @@ async function fetchedCatalog( */ export function attachWorkspaceSourceCheck(source: Command): void { const [command, connect] = apiCommand(source, 'check', ['getSourceCatalog']) - strictFlags( - command.option( - '--dir ', - `Workspace source directory: the kind folders and ${WORKSPACE_BASE_FILE}`, - '.' + absoluteDirOption(command) + verboseFlag( + strictFlags( + command.option( + '--dir ', + `Workspace source directory: the kind folders and ${WORKSPACE_BASE_FILE}`, + '.' + ) ) ) .allowExcessArguments(false) @@ -101,57 +107,76 @@ export function attachWorkspaceSourceCheck(source: Command): void { 'Compile every workflow of a workspace source directory locally, without the server' ) .addHelpText('after', `${CHECK_HELP}\n${SOURCE_DIR_BUSY_HELP}`) - .action(async (options: { dir: string } & StrictOptions, command: Command) => { - const profile = profileFrom(command) - const base = toRecord(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) - await assertWorkspaceDir(options.dir) - const workspaceId = - typeof base.workspaceId === 'string' ? base.workspaceId : profile.workspaceId - if (!workspaceId) { - throw new SimApiError( - `${options.dir} has no ${WORKSPACE_BASE_FILE} and no workspace is selected: pass --workspace .`, - 0 + .action( + async (options: { dir: string; verbose?: boolean } & StrictOptions, command: Command) => { + const profile = profileFrom(command) + const base = toRecord(await readSourceJson(options.dir, WORKSPACE_BASE_FILE)) + await assertWorkspaceDir(options.dir) + const workspaceId = directoryWorkspace( + options.dir, + typeof base.workspaceId === 'string' ? base.workspaceId : null, + profile, + (from) => `Run it without --workspace to check it against the catalog of ${from}.` ) - } - await withSourceDirLock(options.dir, async () => { - const catalog = - cachedCatalog(workspaceId) ?? (await fetchedCatalog(connect, workspaceId, options.dir)) - const workflowInputs = await workspaceWorkflowInputs(options.dir, catalog) - const declarations = { - ...(await readWorkspaceDeclarations(options.dir)), - workflowInputs, - } - const keys = await workspaceWorkflowKeys(options.dir) - const findings: CheckFinding[] = [] - for (const key of keys) { - const checked = await checkWorkflowDir( - join(options.dir, KIND_DIRS.workflow, key), - catalog, - { - declarations, - strict: strictMode(options), - prefix: `${KIND_DIRS.workflow}/${key}/`, - } + if (!workspaceId) { + throw new SimApiError( + `${options.dir} has no ${WORKSPACE_BASE_FILE} and no workspace is selected: pass --workspace .`, + 0 ) - findings.push(...checked.findings) } - for (const { code, message, hint, file, line, column } of declarations.rowFindings) { - findings.push({ + await withSourceDirLock(options.dir, async () => { + const catalog = + cachedCatalog(workspaceId) ?? (await fetchedCatalog(connect, workspaceId, options.dir)) + const workflowInputs = await workspaceWorkflowInputs(options.dir, catalog) + const declarations = { + ...(await readWorkspaceDeclarations(options.dir)), + workflowInputs, + } + const keys = await workspaceWorkflowKeys(options.dir) + const findings: CheckFinding[] = [] + for (const key of keys) { + const checked = await checkWorkflowDir( + join(options.dir, KIND_DIRS.workflow, key), + catalog, + { + declarations, + strict: strictMode(options), + prefix: `${KIND_DIRS.workflow}/${key}/`, + } + ) + findings.push(...checked.findings) + } + for (const { code, - severity: 'error', + severity, message, - ...(hint !== undefined ? { hint } : {}), + hint, file, line, column, - }) - } - await refreshWorkspaceTypes( - options.dir, - withWorkflowInputs(declarations.local, workflowInputs) - ) - await refreshWorkspaceTsconfigs(options.dir, workspaceId) - reportCheck(profile.output, findings, { workflows: keys.length, catalogHash: catalog.hash }) - }) - }) + } of declarations.rowFindings) { + findings.push({ + code, + severity, + message, + ...(hint !== undefined ? { hint } : {}), + file, + line, + column, + }) + } + await refreshWorkspaceTypes( + options.dir, + withWorkflowInputs(declarations.local, workflowInputs) + ) + await refreshWorkspaceTsconfigs(options.dir, workspaceId) + reportCheck( + profile.output, + findings, + { workflows: keys.length, catalogHash: catalog.hash }, + options.verbose + ) + }) + } + ) } diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index 73f5263129e..9608a974d94 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -9,7 +9,6 @@ import { mapWithConcurrency } from '@sim/utils/concurrency' import { toRecord } from '@sim/utils/object' import type { WorkflowInput } from '@sim/workflow-code' import { - SDK_GUIDE_FILE, SOURCE_COMPILER_OPTIONS, typecheckCommand, WORKSPACE_COOKBOOK, @@ -18,6 +17,7 @@ import { import { APPLY_ORDER, type BlobEntry, + type FlatDiagnostic, isFileKind, KIND_DIRS, WORKSPACE_KIND_SPECS, @@ -57,9 +57,11 @@ import { SOURCE_DOWNLOADS_DIR, SOURCE_SDK_DIR, SOURCE_STAGING_PREFIX, + SOURCE_STORED_EDGES_FILE, type StatDigest, type Written, } from './source-dir' +import { sourceExitCodesTable } from './source-exit-codes' import { type LocalDeclarations, type SourceTreeProfile, @@ -102,9 +104,18 @@ function workspaceTsconfig(sdkFiles: readonly string[]): string { /** Where the workspace example is written: `.sim/examples/workspace/`. */ const WORKSPACE_EXAMPLES_DIR = '.sim/examples/workspace' -const AGENTS_GUIDE = `# Sim workspace source +/** Where the workspace guide's deep dives are written, beside `.sim/AGENTS.md`. */ +const WORKSPACE_ADVANCED_FILE = '.sim/GUIDE-advanced.md' -This directory is a Sim workspace written as files. The server reads every file statically; nothing in it runs. The workspace id is \`workspaceId\` in \`${WORKSPACE_BASE_FILE}\`. \`${WORKSPACE_EXAMPLES_DIR}/\` holds one correct example of each kind and a workflow using them, and \`.sim/examples/*.ts\` one workflow example of each common block (agent with structured output, condition, HTTP request, child workflow, Slack, loop, …): copy from them. Find any other builder in \`${SOURCE_SDK_DIR}/${SDK_GUIDE_FILE}\`. +/** + * The reference below the primer in `.sim/AGENTS.md`: the layout, the edit loop in detail, what + * to know and every exit code. + */ +const AGENTS_GUIDE = `# Reference + +The primer above is enough to edit; read a section here when a step needs it. Deletes in depth, Git and CI, and promoting to another workspace: \`${WORKSPACE_ADVANCED_FILE}\`. + +\`${WORKSPACE_EXAMPLES_DIR}/\` holds one correct example of each kind and a workflow using them, and \`.sim/examples/*.ts\` one workflow example of each common block (agent with structured output, condition, HTTP request, child workflow, Slack, loop, …): copy from them. Look up any other builder with \`sim workflows source params \`. ## Layout @@ -116,13 +127,34 @@ One apply creates resources before the workflows that use them, so a workflow ma Secrets never appear in source: a workflow reads an environment variable as \`env.NAME\` and tool code as \`$env.NAME\`, both by name, and a saved credential is \`credential('')\`. Declare the ones workflows use in \`${KIND_DIRS.secret}\` and \`${KIND_DIRS.credential}\`: a workflow using a declared secret the workspace has no value for, or a declared credential key nothing binds to, is held back (exit 1, with the command that fixes it) while the rest of the apply is written. The same directory applied to another workspace binds each credential key to that workspace's own credential. -## Edit loop (run from this directory) +## Edit loop in detail 1. Edit the files. 2. \`sim workspaces source check --dir .\`: compiles every workflow offline with the server's compiler, against what this directory declares: an unknown resource key (skills included) or table column, a child workflow's inputs, and an env name \`${KIND_DIRS.secret}\` lacks (a warning); and checks each \`${KIND_DIRS.table}/.rows.jsonl\` line against its table as the apply would. It fails (exit 1) on errors and on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block), regenerates \`.sim/sdk/workspace.d.ts\` and adds the blocks the workflows now use to \`tsconfig.json\`. In a workflow folder, \`sim workflows source check --dir .\` checks that one workflow the same way. Fix and rerun until it exits 0. -3. \`${typecheckCommand()}\` type-checks the files with the TypeScript release the declarations are built for. \`tsconfig.json\` includes only the blocks the workflows used at the last check or get, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). \`${SOURCE_SDK_DIR}/${SDK_GUIDE_FILE}\` lists every builder with its file and params type; \`grep\` it rather than reading the declarations. The dry run is the authority: a printed file you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. +3. Run \`${typecheckCommand()}\` after every edit, before any dry run or apply, and fix what it reports: it takes about a second and catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It type-checks the files with the TypeScript release the declarations are built for. \`tsconfig.json\` includes only the blocks the workflows used at the last check or get, which keeps it fast: a builder from another block fails with \`has no exported member\` until you run step 2 (or check with \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). \`sim workflows source params \` prints one builder's params and outputs; do not read the declarations whole. The dry run is the authority: a printed file you did not edit that \`tsc\` flags and the dry run accepts is fine, so leave it as it is. 4. \`sim workspaces source status --dir .\` before applying: your changes by resource (\`sim workspaces source diff --dir .\` shows the edits), what changed on the server since the get (a workflow edited in the editor included), the conflicts, and the command to run next. Exit 0 nothing changed, 11 only your edits, 12 the server changed since (\`--offline\`: skip the server). Then \`sim workspaces source apply --dir . --dry-run\`: shows the plan (\`+\` create, \`~\` update, \`=\` adopt an existing resource of the same name, \`-\` delete) and every error; fix and rerun until it exits 0. -5. \`sim workspaces source apply --dir . --yes\`: writes it and rewrites this directory from what was stored. +5. \`sim workspaces source apply --dir . --yes\`: writes it and rewrites this directory from what was stored; it prints the plan it applied (without \`--yes\` and a terminal it prints the plan and exits 14, writing nothing). + +## Good to know + +- A column's object key stays bound to its stored column: to rename one keep the key and set \`name\` (\`email: column.string({ name: 'email_address' })\`). Changing the key drops the column and its cells. +- Renaming a skill (its \`name\`) does not rewrite the workflows that use it: each agent keeps the name it stored until its \`skills\` list is next edited. The key is what binds them. +- A workflow folder you did not edit is sent with only its base, and the server does not compare it with the stored workflow: unlike other kinds, an apply does not report that a workflow changed on the server since the get (\`status\`, which sends every folder whole, does, and the next get shows it). An edit of the folder is checked against its own \`.sim/base.json\` and refused as stale (exit 5) when the workflow changed since. +- A knowledge base's \`embeddingModel\` is fixed when it is created: an apply that changes it is refused. +- A document or workspace file over 10 MB, or whose name no directory can hold, is left out of a get and listed in \`.sim/skipped.json\`; an apply never writes or deletes it. +- A hidden file (a name starting with a dot) under a kind folder is never sent: an apply names it in a warning. + +## Exit codes + +${sourceExitCodesTable('sim workspaces source')} + +Exit 5 right after an apply that ended without an answer (a timeout, a dropped connection) means that apply or someone else changed the workspace: get into a fresh directory and compare, and only when it holds your edits run \`sim workspaces source get --dir . --force\`. A get that exits 9 keeps the binary files it downloaded and fetches only the rest. + +Keep \`${WORKSPACE_BASE_FILE}\` and each workflow's \`.sim/\` files as written. +` + +/** The workspace guide's deep dives, which `.sim/AGENTS.md` points to in one line. */ +const ADVANCED_GUIDE = `# Sim workspace source: advanced ## Deletes @@ -137,7 +169,7 @@ Secrets never appear in source: a workflow reads an environment variable as \`en ## Git and CI -- Commit the whole directory, \`.sim/\` included: \`${WORKSPACE_BASE_FILE}\`, each workflow's \`.sim/\` and \`.sim/written.json\` are what a later apply compares with, from any clone. The generated \`.gitignore\` leaves out only what a command keeps while it runs (\`.sim/downloads/\`, \`.sim/command.lock/\`, \`.sim-staging-*\`), and \`.gitattributes\` keeps workspace files and documents byte for byte. +- Commit the whole directory, \`.sim/\` included: \`${WORKSPACE_BASE_FILE}\`, each workflow's \`.sim/\` and \`.sim/written.json\` are what a later apply compares with, from any clone. The generated \`.gitignore\` leaves out only what a command keeps while it runs (\`.sim/downloads/\`, \`.sim/command.lock/\`, \`.sim-staging-*\`) and \`${TSCONFIG_FULL_FILE}\`, which names the CLI's cache on this machine (\`check\` writes it again), and \`.gitattributes\` keeps workspace files and documents byte for byte. - A fresh clone runs \`check\` (it downloads the block catalog once when none is cached), \`apply --dry-run\` and \`apply --yes\` with only the committed files. A workspace API key may apply tables, rows, knowledge bases, files, custom tools, MCP servers and credential bindings; workflows, skills, secrets and deployments need a member's personal API key (deployments an admin's). - Plan on every pull request (\`apply --dry-run\`), apply on merge, and commit the files the apply rewrote. Two branches that each applied: merge them, then run the dry run. No changes means the merged files match the workspace. Exit 5 means a resource changed since the base this branch holds: \`sim workspaces source get --dir \`, carry the edit over, and apply from there. - \`sim workspaces source get --dir .\` refreshes the workspace \`${WORKSPACE_BASE_FILE}\` names, whatever the profile's default; a \`--workspace\` naming another is refused. @@ -145,27 +177,14 @@ Secrets never appear in source: a workflow reads an environment variable as \`en ## Promote to another workspace -\`sim workspaces source apply --dir . --to --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name), and a change made in the target since is refused as stale (exit 5). Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w \`) and connect its credentials, or the workflows using them are held back. This directory is not changed and stays bound to the workspace it was read from. Without \`--to\`, an apply refuses a directory read from another workspace (\`SOURCE_OTHER_WORKSPACE\`). - -## Good to know - -- A column's object key stays bound to its stored column: to rename one keep the key and set \`name\` (\`email: column.string({ name: 'email_address' })\`). Changing the key drops the column and its cells. -- Renaming a skill (its \`name\`) does not rewrite the workflows that use it: each agent keeps the name it stored until its \`skills\` list is next edited. The key is what binds them. -- A workflow folder you did not edit is sent with only its base, and the server does not compare it with the stored workflow: unlike other kinds, an apply does not report that a workflow changed on the server since the get (\`status\`, which sends every folder whole, does, and the next get shows it). An edit of the folder is checked against its own \`.sim/base.json\` and refused as stale (exit 5) when the workflow changed since. -- A knowledge base's \`embeddingModel\` is fixed when it is created: an apply that changes it is refused. -- A document or workspace file over 10 MB, or whose name no directory can hold, is left out of a get and listed in \`.sim/skipped.json\`; an apply never writes or deletes it. -- A hidden file (a name starting with a dot) under a kind folder is never sent: an apply names it in a warning. - -## Exit codes - -0 applied; 1 errors (each printed as file:line:column with a hint); 5 a resource changed on the server since this directory was read (get it again into a fresh directory, reapply the edit; right after an apply that ended without an answer, such as a timeout, that apply or someone else changed it: get into a fresh directory and compare, and only when it holds your edits run \`sim workspaces source get --dir . --force\`); 6 locked; 7 too large; 8 stopped part-way (a resource failed, or a later round stayed rate limited): what was written is saved here, so apply again, as often as it takes, until it exits 0; 9 rate limited (HTTP 429) after the CLI's retries (up to 6, at most a minute apart): nothing more was written, so wait a minute and run the same command again (a get keeps the binary files it downloaded and fetches only the rest); 10 another apply (a CI job, a teammate) is writing this workspace, or another command is using this directory: nothing was written, so wait for it to finish and run the command again. 13 not logged in (\`AUTH_REQUIRED\`): run \`sim login\`, then the command again. - -Keep \`${WORKSPACE_BASE_FILE}\` and each workflow's \`.sim/\` files as written. +\`sim workspaces source apply --dir . --to --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name). Each promote records what it left in the target in \`.sim/targets/.json\` (commit it), and the next promote compares with that: a resource changed in the target since is kept when this directory left it alone, and refused as stale (exit 5) when this directory changed it too. A resource with no record (the first promote, or one created in the target since) is updated to match this directory; delete the record to take the target as it stands again. Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w \`) and connect its credentials, or the workflows using them are held back. This directory stays bound to the workspace it was read from. Without \`--to\`, \`check\`, \`status\`, \`diff\` and \`apply\` act on the workspace this directory was read from, whatever the profile's default; a \`--workspace\` naming another is refused (\`SOURCE_OTHER_WORKSPACE\`). ` /** The workspace directory's tree beside the printed source. */ export const WORKSPACE_TREE: SourceTreeProfile = { + kind: 'workspace', agents: AGENTS_GUIDE, + guides: [[WORKSPACE_ADVANCED_FILE, ADVANCED_GUIDE]], tsconfig: workspaceTsconfig, // Workspace files and knowledge base documents are bytes, kept as they are (`-text`, after the // LF lines it overrides): a commit that rewrote their line endings would change their hashes. @@ -177,7 +196,13 @@ export const WORKSPACE_TREE: SourceTreeProfile = { `${KIND_DIRS.file}/** -text`, `${KIND_DIRS['knowledge-base']}/*/docs/** -text`, ], - gitignore: [`${SOURCE_DOWNLOADS_DIR}/`, `${SOURCE_DIR_LOCK}/`, `${SOURCE_STAGING_PREFIX}*/`], + gitignore: [ + `${SOURCE_DOWNLOADS_DIR}/`, + `${SOURCE_DIR_LOCK}/`, + `${SOURCE_STAGING_PREFIX}*/`, + // It includes the cached declarations by this machine's absolute path; `check` writes it again. + TSCONFIG_FULL_FILE, + ], sdk: Object.entries(WORKSPACE_SDK_DTS).map( ([name, text]) => [`${WORKSPACE_SDK_DIR}/${name}`, text] as const ), @@ -277,9 +302,13 @@ export interface WorkspaceDeclarations { workflowInputs?: ReadonlyMap } -/** A finding about a line of a table's rows file (`E_TABLE_ROWS`, an error). */ +/** + * A finding about a line of a table's rows file: `E_TABLE_ROWS`, an error, or the + * `I_FINDINGS_TRUNCATED` that ends a long list. + */ interface RowFinding { code: WorkspaceDiagnosticCode + severity: FlatDiagnostic['severity'] message: string hint?: string file: string @@ -347,8 +376,9 @@ function tableRowFindings(key: string, decl: TableDecl, text: string | undefined if (!rows || text === undefined) return [] const parsed = parseTableRows(key, text, rows, new Set(Object.keys(decl.columns))) const read = parsed.diagnostics.map( - ({ code, message, hint, span }): RowFinding => ({ + ({ code, severity, message, hint, span }): RowFinding => ({ code, + severity, message, ...(hint ? { hint } : {}), file, @@ -359,6 +389,7 @@ function tableRowFindings(key: string, decl: TableDecl, text: string | undefined const refused = refusedTableRows(decl, parsed.data).map( ({ row, errors }): RowFinding => ({ code: 'E_TABLE_ROWS', + severity: 'error', message: `Line ${parsed.lines[row] ?? 1} does not fit the declared columns: ${errors.join('; ')}.`, hint: `Write a value each column of ${KIND_DIRS.table}/${key}${DECLARATION_EXTENSION} holds (a select cell names one of its options), fill every required column, and keep unique values unique.`, file, @@ -406,6 +437,26 @@ function declaredTable( return codec.parse(key, text, []) } +/** + * The rows files of the tables `files` (a get's answer) declares seed rows for. A server read + * prints a seed table's `rows` but never its rows, which only the directory holds, so a get keeps + * the file it finds there. + */ +export function seedRowsFiles(files: Readonly>): Set { + const kept = new Set() + const paths = Object.keys(files).filter((path) => kindOfPath(path) === 'table') + for (const [key, tablePaths] of KIND_LAYOUTS.table.group(paths).groups) { + const rowsPath = tableRowsPath(key) + const { decl } = declaredTable( + key, + tablePaths.filter((path) => path !== rowsPath), + files + ) + if (decl?.rows?.mode === 'seed') kept.add(rowsPath) + } + return kept +} + /** A table declaration's columns by stored name, as a workflow's row writes name them. */ function declaredColumns(decl: TableDecl): ReadonlyMap { return new Map( @@ -428,6 +479,11 @@ export function assertWorkspacePaths(paths: Iterable): void { } } +/** A workflow folder's CLI-only record (`.sim/stored-edges.json`): never sent, never ignored. */ +function isWorkflowRecord(kind: WorkspaceKind, path: string): boolean { + return kind === 'workflow' && path.endsWith(`/${SOURCE_STORED_EDGES_FILE}`) +} + /** * The folders (and root files) of `kinds`, each walked once: `present`, the kinds whose folder * holds any file (its `.gitkeep` marker included) or whose root file exists; `paths`, their source @@ -452,7 +508,7 @@ export async function kindFolders( if (files.length > 0) present.push(kind) for (const path of files) { if (isWorkspaceSourcePath(path)) paths.push(path) - else if (path !== keepFile(kind)) ignored.push(path) + else if (path !== keepFile(kind) && !isWorkflowRecord(kind, path)) ignored.push(path) } } return { present, paths, ignored } diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts index e25e1a229b0..858451de70c 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts @@ -18,9 +18,10 @@ import { fingerprintOf } from '@sim/workspace-code' import { knowledgeCodec } from '@sim/workspace-code/kinds/knowledge' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' -import { contextMockFns, contextMockState } from '../../test/context-mock' +import { contextMockFns, contextMockState, mockedProfile } from '../../test/context-mock' import { runWorkflowSourceCheck, runWorkspaceSource } from '../../test/workspace-source-command' import { blobBytes, blobDigests } from './workspace-source-blobs' +import { kindFolders } from './workspace-source-dir' vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) @@ -89,7 +90,11 @@ beforeEach(() => { contextMockState.workspaceSource = 'config' request.mockReset() requestRaw.mockReset() - contextMockFns.mockProfileFrom.mockReturnValue({ workspaceId: 'ws_local', output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: 'ws_local', + output: 'json', + }) rootDir = mkdtempSync(join(tmpdir(), 'sim-workspace-get-')) dir = join(rootDir, 'out') vi.stubEnv('SIM_CONFIG_DIR', join(rootDir, 'config')) @@ -124,6 +129,40 @@ describe('workspaces source get', () => { expect(readFileSync(join(dir, 'tables/leads.ts'), 'utf8')).toBe(`${LEADS}// server\n`) }) + it('keeps the seed rows file of a table the answer declares seed rows for, which no read prints', async () => { + const seeded = (name: string) => + `// @sim-sdk 1\nimport { column, defineTable } from '@sim/workflow-sdk/workspace'\n\nexport default defineTable({ name: '${name}', columns: { code: column.string() }, rows: { mode: 'seed' } })\n` + const rows = '{"code":"a"}\n' + serve({ + 'tables/leads.ts': seeded('leads'), + 'tables/leads.rows.jsonl': rows, + 'tables/accounts.ts': seeded('accounts'), + 'tables/accounts.rows.jsonl': rows, + }) + await get() + + serve({ 'tables/leads.ts': seeded('leads'), 'tables/accounts.ts': ACCOUNTS }) + await get() + + expect(readFileSync(join(dir, 'tables/leads.rows.jsonl'), 'utf8')).toBe(rows) + expect(existsSync(join(dir, 'tables/accounts.rows.jsonl'))).toBe(false) + }) + + it('writes the workspace primer atop the guide, and the deep dives to the advanced guide it names', async () => { + serve({ 'tables/leads.ts': LEADS }) + + await get() + + const agents = readFileSync(join(dir, '.sim/AGENTS.md'), 'utf8') + expect(agents.startsWith('# Sim workspace source: primer')).toBe(true) + expect(agents).toContain('`.sim/GUIDE-advanced.md`') + expect(agents).not.toContain('## Git and CI') + const advanced = readFileSync(join(dir, '.sim/GUIDE-advanced.md'), 'utf8') + for (const section of ['## Deletes', '## Git and CI', '## Promote to another workspace']) { + expect(advanced).toContain(section) + } + }) + it('refuses to restore a file deleted since the CLI wrote it, a pending delete', async () => { serve({ 'tables/leads.ts': LEADS, 'tables/accounts.ts': ACCOUNTS }) await get() @@ -649,7 +688,7 @@ describe('workspaces source get', () => { '---\nname: fresh\ndescription: New\n---\nAdded after the get.\n' ) expect(typecheck(uses('fresh'))).toContain('"fresh"') - contextMockFns.mockProfileFrom.mockReturnValue({ output: 'json' }) + contextMockFns.mockProfileFrom.mockReturnValue({ ...mockedProfile(), output: 'json' }) request.mockClear() await runWorkspaceSource('check', dir) expect(request).not.toHaveBeenCalled() @@ -897,3 +936,142 @@ describe('workspaces source get', () => { expect(request).not.toHaveBeenCalled() }) }) + +/** Block types the workspace catalog withholds while a stored workflow holds them. */ +const WITHHELD_TYPES = new Set(['table_v2', 'knowledge']) + +/** {@link CATALOG} as the workspace catalog of a caller it withholds {@link WITHHELD_TYPES} from. */ +const WORKSPACE_CATALOG = { + ...CATALOG, + hash: 'workspace-catalog', + blocks: Object.fromEntries( + Object.entries(CATALOG.blocks).filter(([type]) => !WITHHELD_TYPES.has(type)) + ), + builders: Object.fromEntries( + Object.entries(CATALOG.builders as Record).filter( + ([, builder]) => !WITHHELD_TYPES.has(builder.type) + ) + ), +} + +const ROUTER = `// @sim-sdk 1 +import { condition, js, knowledge, knowledgeBase, table, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('Router', {}, () => { + const start = trigger.start('Start', { inputFormat: { subject: 'string', priority: 'string' } }) + const isUrgent = condition('Is Urgent', { if: js\`\${start.priority} === 'high'\`, else: true }) + const policy = knowledge.search('Policy', { + knowledgeBaseId: knowledgeBase('policies'), + query: start.subject, + }) + const logTicket = table.insertRow('Log Ticket', { + tableId: table('tickets'), + data: { subject: start.subject }, + }) + + start.to(isUrgent) + isUrgent.if.to(policy) + isUrgent.else.to(logTicket) +}) +` + +/** A stored cycle between A and B, every block bound by the lock. */ +const CYCLE = `// @sim-sdk 1 +import { api, trigger, workflow } from '@sim/workflow-sdk' + +export default workflow('Cycle', {}, () => { + const start = trigger.start('Start', {}) + const a = api('A', { url: 'https://example.com', method: 'GET' }) + const b = api('B', { url: 'https://example.com', method: 'GET' }) + const c = api('C', { url: 'https://example.com', method: 'GET' }) + + start.to(a) + a.to(b) + b.to(a) + b.to(c) +}) +` + +function lockOf(keys: readonly string[]): string { + return JSON.stringify({ + version: 1, + blocks: Object.fromEntries( + keys.map((key, order) => [key, { id: `id-${key}`, parent: null, ident: key, order }]) + ), + variables: {}, + }) +} + +/** + * A workspace answer holding workflow `key` (`source`, bound by a lock on `keys`) printed with + * the catalog of hash `catalogHash`; the catalog route answers the full catalog for a workflow and + * the withholding one for the workspace. + */ +function serveWorkflow(key: string, source: string, keys: readonly string[], catalogHash: string) { + const files = { + [`workflows/${key}/workflow.ts`]: source, + [`workflows/${key}/.sim/lock.json`]: lockOf(keys), + [`workflows/${key}/.sim/base.json`]: JSON.stringify({ + version: 1, + workflowId: `wf-${key}`, + workspaceId: 'ws_local', + fingerprint: `sf1:${key}`, + catalogHash, + }), + } + serve(files, { + kinds: ['workflow'], + resources: { [`workflow:${key}`]: { id: `wf-${key}`, fingerprint: `rf1:${key}` } }, + }) + const answer = request.getMockImplementation() + request.mockImplementation( + async (path: string, options?: { query?: { workflowId?: string } }) => { + if (path !== CATALOG_PATH) return answer?.(path, options) + return { data: options?.query?.workflowId ? CATALOG : WORKSPACE_CATALOG } + } + ) +} + +function checked(): { errors: number; diagnostics: Array<{ code: string }> } { + return JSON.parse(vi.mocked(console.log).mock.calls.at(-1)?.join(' ') ?? '{}') +} + +describe('workspaces source check after a get', () => { + afterEach(() => { + process.exitCode = undefined + }) + + it('checks a workflow holding a type the workspace catalog withholds against its own catalog', async () => { + serveWorkflow('router', ROUTER, ['start', 'isurgent', 'policy', 'logticket'], CATALOG.hash) + await get() + + await runWorkspaceSource('check', dir) + + expect(checked()).toMatchObject({ errors: 0, diagnostics: [] }) + expect(existsSync(join(dir, 'workflows/router/.sim/stored-edges.json'))).toBe(true) + expect((await kindFolders(dir, ['workflow'])).ignored).toEqual([]) + expect( + request.mock.calls.filter( + ([path, options]) => path === CATALOG_PATH && options?.query?.workflowId === 'wf-router' + ) + ).toHaveLength(1) + }) + + it('refuses a cycle an edit closes between bound blocks of a workflow folder', async () => { + serveWorkflow('cycle', CYCLE, ['start', 'a', 'b', 'c'], WORKSPACE_CATALOG.hash) + await get() + await runWorkspaceSource('check', dir, '--strict') + expect(checked().diagnostics.map(({ code }) => code)).toEqual(['W_EDGE_CYCLE']) + expect(process.exitCode).toBeUndefined() + const entry = join(dir, 'workflows/cycle/workflow.ts') + writeFileSync( + entry, + readFileSync(entry, 'utf8').replace(' b.to(c)\n', ' b.to(c)\n c.to(b)\n') + ) + + await runWorkspaceSource('check', dir, '--strict') + + expect(checked().diagnostics.map(({ code }) => code)).toEqual(['E_EDGE_CYCLE']) + expect(process.exitCode).toBe(1) + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index 7d869ceb9e8..6cd5c22d924 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -16,13 +16,15 @@ import { WORKSPACE_SKIPPED_FILE, } from '@sim/workspace-code/layout' import type { Command } from 'commander' +import { printError } from '#sim-cli/output/io' import type { GetWorkspaceSourceResponse } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { apiCommand } from '../../runtime/called-operations' import { removeStagingOnSignal } from './files-get' -import { printProtocolResult } from './result' import { + absoluteDirOption, checkedPath, + directoryWorkspace, editedFiles, FILE_CONCURRENCY, readSourceJson, @@ -42,9 +44,15 @@ import { finishTree, loadCatalog, loadWorkspaceTypes, + printTreeResult, sourceTreeChange, withSourceExitCode, } from './source-tree' +import { + removedWorkflowRecords, + unrecordedFoldersWarning, + workflowFolderRecords, +} from './stored-edges' import { assertListedBlobs, blobDigests, @@ -58,6 +66,7 @@ import { keepFile, kindFolders, localDeclarations, + seedRowsFiles, skippedFile, WORKSPACE_TREE, } from './workspace-source-dir' @@ -160,17 +169,19 @@ async function writeWorkspaceSource( dir: string, source: WorkspaceSource, force: boolean -): Promise<{ written: number; removed: number; guide: string }> { +): Promise<{ written: number; removed: number; guide: string; primer: string }> { assertWorkspacePaths(Object.keys(source.files)) assertListedBlobs(source.blobs) const blobs = new Map(source.blobs.map((blob) => [blob.path, blob])) const written = await readWritten(dir) const { paths: onDisk } = await kindFolders(dir, source.kinds) + const seedRows = seedRowsFiles(source.files) const gone = new Set( onDisk.filter( (path) => !Object.hasOwn(source.files, path) && !blobs.has(path) && + !seedRows.has(path) && Object.hasOwn(written.digests, path) ) ) @@ -206,6 +217,12 @@ async function writeWorkspaceSource( loadWorkspaceTypes(client, source.workspaceId, localDeclarations(source.files)), ]) assertSdkMajor(catalog) + const { records: workflowRecords, unrecorded } = await workflowFolderRecords( + client, + source.workspaceId, + catalog, + source.files + ) // A get that leaves the workflows unread keeps the blocks those on disk type-check against. const tree = await sourceTreeChange( dir, @@ -276,6 +293,7 @@ async function writeWorkspaceSource( [WORKSPACE_BASE_FILE, `${JSON.stringify(base, null, 2)}\n`], [SOURCE_WRITTEN_FILE, writtenFile(record)], ...(skipped ? [[WORKSPACE_SKIPPED_FILE, skipped] as const] : []), + ...workflowRecords, ...tree.change.write, ] await updateSourceDir(dir, { @@ -285,13 +303,21 @@ async function writeWorkspaceSource( ), remove: [ ...gone, + ...removedWorkflowRecords(gone), ...(skipped ? [] : [WORKSPACE_SKIPPED_FILE]), ...(tree.change.remove ?? []), ], }) await rm(downloads, { recursive: true, force: true }) - const summary = finishTree(dir, write.length + stale.length, workspaceTypes, tree.ownsTsconfig) - return { written: summary.files, removed: gone.size, guide: summary.guide } + const unrecordedWarning = unrecordedFoldersWarning(unrecorded) + if (unrecordedWarning) printError(unrecordedWarning) + const summary = finishTree(dir, write.length + stale.length, workspaceTypes, tree) + return { + written: summary.files, + removed: gone.size, + guide: summary.guide, + primer: summary.primer, + } } finally { if (partial) await rm(partial, { recursive: true, force: true }) disposeSignalCleanup() @@ -332,29 +358,6 @@ async function keepDownloads( ) } -/** - * The workspace a get writes `dir` from. A directory whose base was read from a workspace stays - * that workspace's: a get refreshes it, whatever the profile's default, so a refresh never - * rewrites it into another workspace. A `--workspace` naming another one is refused (get into - * another directory, or promote with `apply --to`). A directory without a base takes `profile`. - */ -async function directoryWorkspace( - dir: string, - profile: string, - explicit: boolean -): Promise { - // Read as leniently as an apply reads it: a base the parser refuses still names its workspace. - const recorded = toRecord(await readSourceJson(dir, WORKSPACE_BASE_FILE)).workspaceId - const readFrom = typeof recorded === 'string' ? recorded : null - if (readFrom === null || readFrom === profile) return profile - if (!explicit) return readFrom - throw new SimApiError( - `${dir} was read from workspace ${readFrom}, but --workspace names ${profile}. Get ${profile} into another directory, or run the get without --workspace to refresh ${readFrom}; to apply this directory to ${profile}, use \`sim workspaces source apply --to ${profile}\`.`, - 0, - 'SOURCE_OTHER_WORKSPACE' - ) -} - /** * `sim workspaces source get --dir [--only kinds]`: writes the workspace as a source * directory: `tables/`, `tools/`, `skills/`, `knowledge/`, `files/` and `workflows//` (each a @@ -369,6 +372,7 @@ export function attachWorkspaceSourceGet(source: Command): void { 'downloadFile', 'downloadKnowledgeDocument', ]) + absoluteDirOption(command) command .requiredOption('--dir ', 'Directory to write the workspace source into') .option('--only ', `Read only these kinds (${ONLY_KINDS_HELP})`) @@ -380,10 +384,18 @@ export function attachWorkspaceSourceGet(source: Command): void { const kinds = parseKinds(options.only) const { client, profile } = connect() await withSourceDirLock(options.dir, async () => { - const workspaceId = await directoryWorkspace( - options.dir, - client.requireWorkspace(), - profile.sources.workspaceId === 'flag' + // Read as leniently as an apply reads it: a base the parser refuses still names its workspace. + const recorded = toRecord( + await readSourceJson(options.dir, WORKSPACE_BASE_FILE) + ).workspaceId + const workspaceId = client.requireWorkspace( + directoryWorkspace( + options.dir, + typeof recorded === 'string' ? recorded : null, + profile, + (readFrom, named) => + `Get ${named} into another directory, or run the get without --workspace to refresh ${readFrom}; to apply this directory to ${named}, use \`sim workspaces source apply --to ${named}\`.` + ) ?? undefined ) const { data } = await client .request('getWorkspaceSource', { @@ -394,7 +406,7 @@ export function attachWorkspaceSourceGet(source: Command): void { throw withSourceExitCode(error) }) const result = await writeWorkspaceSource(client, options.dir, data, options.force === true) - printProtocolResult(profile.output, { + printTreeResult(profile.output, { workspaceId: data.workspaceId, dir: options.dir, kinds: data.kinds, @@ -403,6 +415,7 @@ export function attachWorkspaceSourceGet(source: Command): void { removed: result.removed, skipped: data.skipped.length, guide: result.guide, + primer: result.primer, }) }) }) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-staging.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-staging.test.ts new file mode 100644 index 00000000000..38360cb006a --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/workspace-source-staging.test.ts @@ -0,0 +1,80 @@ +import { createHash } from 'node:crypto' +import { + SOURCE_REQUEST_BODY_CEILING_BYTES, + STAGE_PART_FRAMING_BYTES, + stageBatches, + WORKSPACE_SOURCE_LIMITS, + WORKSPACE_SOURCE_STAGE_MAX_BYTES, +} from '@sim/workspace-code/limits' +import { describe, expect, it } from 'vitest' + +interface Sized { + size: number + fill: number +} + +/** The bytes of each staging request `blobs` split into, serialized as the CLI sends them. */ +async function requestBodies(blobs: Sized[]): Promise { + const sizes: number[] = [] + for (const batch of stageBatches(blobs)) { + const form = new FormData() + for (const blob of batch) { + const bytes = Buffer.alloc(blob.size, blob.fill) + const sha256 = createHash('sha256').update(bytes).digest('hex') + form.append(sha256, new Blob([new Uint8Array(bytes)]), sha256) + } + sizes.push((await new Response(form).arrayBuffer()).byteLength) + } + return sizes +} + +const of = (count: number, size: number): Sized[] => + Array.from({ length: count }, (_, index) => ({ size, fill: index % 256 })) + +describe('staging requests', () => { + it.each([ + ['two 6 MB files', of(2, 6_000_000)], + ['two files just under the per-request cap together', of(2, 4_900_000)], + ['one file of the largest size a directory holds', of(1, WORKSPACE_SOURCE_LIMITS.maxBlobBytes)], + ['a large file between small ones', [...of(3, 1_000), ...of(1, 9_999_999), ...of(3, 1_000)]], + [ + 'as many tiny files as one request carries', + of(WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest, 1), + ], + ])( + '%s: every request body fits the route cap, which fits the proxy ceiling', + async (_, blobs) => { + const bodies = await requestBodies(blobs) + + for (const bytes of bodies) + expect(bytes).toBeLessThanOrEqual(WORKSPACE_SOURCE_STAGE_MAX_BYTES) + expect(WORKSPACE_SOURCE_STAGE_MAX_BYTES).toBeLessThan(SOURCE_REQUEST_BODY_CEILING_BYTES) + } + ) + + it('sends a file too large to share a request in one of its own, never dropping it', () => { + const blobs = [ + ...of(1, 3_000_000), + ...of(1, WORKSPACE_SOURCE_LIMITS.maxBlobBytes), + ...of(1, 10), + ] + + const batches = stageBatches(blobs) + + expect(batches.map((batch) => batch.map((blob) => blob.size))).toEqual([ + [3_000_000], + [WORKSPACE_SOURCE_LIMITS.maxBlobBytes], + [10], + ]) + }) + + it('budgets the framing of each part, so a full request of tiny files is split by count', () => { + const count = WORKSPACE_SOURCE_LIMITS.maxBlobsPerRequest + 1 + const perPart = Math.floor(WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest / 10) + + expect(stageBatches(of(count, 0)).map((batch) => batch.length)).toEqual([count - 1, 1]) + expect( + stageBatches(of(10, perPart - STAGE_PART_FRAMING_BYTES + 1)).map((batch) => batch.length) + ).toEqual([9, 1]) + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts index a2e9cc02570..68858293f3a 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts @@ -5,7 +5,7 @@ import { dirname, join } from 'node:path' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' -import { contextMockFns, contextMockState } from '../../test/context-mock' +import { contextMockFns, contextMockState, mockedProfile } from '../../test/context-mock' import { digest } from './source-dir' import { attachWorkspaceSourceDiff, attachWorkspaceSourceStatus } from './workspace-source-status' @@ -94,7 +94,7 @@ beforeEach(() => { contextMockState.workspaceSource = 'config' contextMockState.workspaceId = WORKSPACE_ID mockRequest.mockReset() - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'json' }) + mockProfileFrom.mockReturnValue({ ...mockedProfile(), workspaceId: WORKSPACE_ID, output: 'json' }) dir = mkdtempSync(join(tmpdir(), 'sim-workspace-status-')) write('.sim/base.json', JSON.stringify(BASE)) for (const [path, text] of Object.entries(GOT)) write(path, text) @@ -277,19 +277,27 @@ describe('workspaces source status', () => { }) }) - it('recommends the workspace a status was pointed at with --workspace', async () => { - contextMockState.workspaceSource = 'flag' + it('compares with the workspace the directory was read from, whatever the profile default', async () => { contextMockState.workspaceId = 'ws_other' write('tables/leads.ts', LEADS.replace('columns: {}', 'columns: { email: column.string() }')) mockRequest.mockResolvedValue(dryRun([{ kind: 'table', key: 'leads', action: 'update' }])) await run('status') + expect(mockRequest.mock.calls[0][0]).toBe(`/api/v2/workspaces/${WORKSPACE_ID}/source`) expect(printed().next).toMatchObject({ - command: `sim workspaces source apply --dir ${dir} --yes --workspace ws_other`, + command: `sim workspaces source apply --dir ${dir} --yes`, }) }) + it('refuses a --workspace naming another workspace than the directory was read from', async () => { + contextMockState.workspaceSource = 'flag' + contextMockState.workspaceId = 'ws_other' + + await expect(run('status')).rejects.toMatchObject({ code: 'SOURCE_OTHER_WORKSPACE' }) + expect(mockRequest).not.toHaveBeenCalled() + }) + it('recommends --allow-delete for an update that drops data, as applying it needs', async () => { write('tables/leads.ts', LEADS.replace("name: 'leads'", "name: 'leads', description: 'x'")) mockRequest.mockResolvedValue( @@ -541,6 +549,7 @@ describe('workspaces source status', () => { server: 'unreachable', next: { command: 'sim login --profile default' }, }) + expect(process.exitCode).toBe(13) }) it('fails rather than hides a server refusal that is not about reaching it', async () => { @@ -560,7 +569,11 @@ describe('workspaces source status', () => { it('prints one screen ending with the next command for a person', async () => { write('tables/leads.ts', `${LEADS}// edited\n`) mockRequest.mockResolvedValue(dryRun([{ kind: 'table', key: 'leads', action: 'update' }])) - mockProfileFrom.mockReturnValue({ workspaceId: WORKSPACE_ID, output: 'text' }) + mockProfileFrom.mockReturnValue({ + ...mockedProfile(), + workspaceId: WORKSPACE_ID, + output: 'text', + }) await run('status') diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts index 44ee49eb403..2bd45bf9cf5 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts @@ -33,7 +33,9 @@ import type { import { SimApiError } from '../../http/client' import { apiCommand, type Connection } from '../../runtime/called-operations' import { + absoluteDirOption, digest, + directoryWorkspace, readSourceJson, readSourceText, readWritten, @@ -146,17 +148,24 @@ function localPaths(local: LocalWorkspace): string[] { } /** - * The workspace a status or diff asks about: the one `--workspace` names, else the one the - * directory was read from, else the profile's. + * The workspace a status or diff asks about: the one the directory was read from (a `--workspace` + * naming another is refused), else the profile's. */ function targetWorkspace( connection: Connection<'applyWorkspaceSource' | 'getWorkspaceSource'>, + dir: string, readFrom: string | null ): string { const { client, profile } = connection - return profile.sources.workspaceId === 'flag' || readFrom === null - ? client.requireWorkspace() - : readFrom + return client.requireWorkspace( + directoryWorkspace( + dir, + readFrom, + profile, + (from, named) => + `Run it without --workspace to compare with ${from}, or get ${named} into another directory.` + ) ?? undefined + ) } /** The resources `.sim/skipped.json` lists: never written by a get, so never drift. */ @@ -285,6 +294,7 @@ interface StatusOptions { */ export function attachWorkspaceSourceStatus(source: Command): void { const [command, connect] = apiCommand(source, 'status', ['applyWorkspaceSource']) + absoluteDirOption(command) command .option('--dir ', 'Workspace source directory written by `workspaces source get`', '.') .option('--only ', `Report only these kinds (${ONLY_KINDS_HELP})`) @@ -314,10 +324,11 @@ export function attachWorkspaceSourceStatus(source: Command): void { let login: string | null = null if (!options.offline) { const connection = connect() - const target = targetWorkspace(connection, local.readFrom) + const target = targetWorkspace(connection, options.dir, local.readFrom) workspaceId = target const { profile } = connection - if (profile.sources.workspaceId === 'flag' || profile.workspaceId !== target) { + // A directory with a base is bound to its workspace; only one without names it. + if (local.readFrom === null && profile.sources.workspaceId === 'flag') { workspaceScope = ` --workspace ${target}` } let submission = local.submission @@ -392,7 +403,7 @@ export function attachWorkspaceSourceStatus(source: Command): void { report, planLine ) - setSoftExitCode(statusExitCode(status.state)) + setSoftExitCode(statusExitCode(status.state, login)) }, READ_ONLY ) @@ -523,6 +534,7 @@ function serverDiffs( */ export function attachWorkspaceSourceDiff(source: Command): void { const [command, connect] = apiCommand(source, 'diff', ['getWorkspaceSource']) + absoluteDirOption(command) command .argument('[paths...]', 'Files, folders or kind:key resources to compare (default: every file)') .option('--dir ', 'Workspace source directory written by `workspaces source get`', '.') @@ -543,7 +555,7 @@ export function attachWorkspaceSourceDiff(source: Command): void { const selects = filter.selects const read = async (kinds: readonly WorkspaceKind[]) => { const connection = connect() - const workspaceId = targetWorkspace(connection, local.readFrom) + const workspaceId = targetWorkspace(connection, options.dir, local.readFrom) try { const response = await connection.client.request( 'getWorkspaceSource', diff --git a/packages/sim-cli/src/commands/secrets.test.ts b/packages/sim-cli/src/commands/secrets.test.ts index dcfa206109f..4b2b413a835 100644 --- a/packages/sim-cli/src/commands/secrets.test.ts +++ b/packages/sim-cli/src/commands/secrets.test.ts @@ -156,13 +156,10 @@ describe('secrets set cancellation', () => { it('exits 130 when the user aborts the prompt', async () => { mockPromptSecret.mockRejectedValue(new MockCancelled()) - vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { - throw new Error(`exit ${code}`) - }) as never) await expect( program().parseAsync(['node', 'sim', 'secrets', 'set', 'K', '--scope', 'workspace']) - ).rejects.toThrow('exit 130') + ).rejects.toMatchObject({ code: 130 }) expect(mockRequest).not.toHaveBeenCalled() }) }) diff --git a/packages/sim-cli/src/commands/telemetry.ts b/packages/sim-cli/src/commands/telemetry.ts index f30b31418e3..0897968fbdb 100644 --- a/packages/sim-cli/src/commands/telemetry.ts +++ b/packages/sim-cli/src/commands/telemetry.ts @@ -1,4 +1,5 @@ import { Command } from 'commander' +import { printLine } from '#sim-cli/output/io' import { builtInIngestTarget, DO_NOT_TRACK_VARIABLE, @@ -40,7 +41,7 @@ function currentStatus(): TelemetryStatus { */ function setEnabled(enabled: boolean): void { writeTelemetryState({ ...loadTelemetryState(), enabled }) - console.log(describe(currentStatus())) + printLine(describe(currentStatus())) } export function telemetryCommand(): Command { @@ -54,8 +55,8 @@ export function telemetryCommand(): Command { 'Telemetry commands always print text. DO_NOT_TRACK and SIM_TELEMETRY_DISABLED override the saved preference; a build without a reporting destination remains off. See the usage-data guide for the data collected.' ) .action(() => { - console.log(describe(currentStatus())) - console.log(`Learn more: ${USAGE_DATA_DOCS_URL}`) + printLine(describe(currentStatus())) + printLine(`Learn more: ${USAGE_DATA_DOCS_URL}`) }) telemetry diff --git a/packages/sim-cli/src/embed-context.ts b/packages/sim-cli/src/embed-context.ts index 812b8895f76..df7999d046e 100644 --- a/packages/sim-cli/src/embed-context.ts +++ b/packages/sim-cli/src/embed-context.ts @@ -75,8 +75,11 @@ export function setSoftExitCode(code: number): void { export const embedStore = new AsyncLocalStorage() -/** Thrown in place of process.exit inside an embedded run. */ -export class EmbeddedExit extends Error { +/** + * Thrown in place of `process.exit`: an embedded run returns its code to the host, and the + * terminal CLI exits with it once its output is written out. + */ +export class CliExit extends Error { constructor(readonly code: number) { super(`CLI exited with code ${code}`) } @@ -106,8 +109,7 @@ export function embeddedProfile(): ResolvedProfile | null { } } -/** Exits the terminal CLI without terminating an embedding host. */ +/** Ends the command with `code`, without terminating an embedding host. */ export function exitCli(code: number): never { - if (embedStore.getStore()) throw new EmbeddedExit(code) - return process.exit(code) + throw new CliExit(code) } diff --git a/packages/sim-cli/src/embed.ts b/packages/sim-cli/src/embed.ts index 984c44f4b1e..be26c306f55 100644 --- a/packages/sim-cli/src/embed.ts +++ b/packages/sim-cli/src/embed.ts @@ -2,9 +2,9 @@ import { getErrorMessage } from '@sim/utils/errors' import { type Command, CommanderError, type OutputConfiguration } from 'commander' import { ProfileConfigError } from './config/index' import { + CliExit, type EmbedContext, type EmbeddedCliIdentity, - EmbeddedExit, type EmbeddedFileSnapshot, embedStore, } from './embed-context' @@ -173,7 +173,7 @@ export async function runEmbeddedCli( /** Mirrors the installed CLI's top-level error handling (src/index.ts), minus process.exit. */ function renderEmbeddedError(ctx: EmbedContext, error: unknown): number { if (error instanceof EmbeddedOutputLimitError) return 1 - if (error instanceof EmbeddedExit) return error.code + if (error instanceof CliExit) return error.code if (error instanceof CommanderError) { // commander's CommanderError from exitOverride: usage/parse errors already // printed through the (captured) output; help/version exit 0. diff --git a/packages/sim-cli/src/exit-codes.ts b/packages/sim-cli/src/exit-codes.ts index 4b4e9f73ba9..ff1a53caad9 100644 --- a/packages/sim-cli/src/exit-codes.ts +++ b/packages/sim-cli/src/exit-codes.ts @@ -72,5 +72,12 @@ export const SOURCE_DRIFT_EXIT_CODE = 12 */ export const AUTH_REQUIRED_EXIT_CODE = 13 +/** + * A `workflows source apply`/`create` or `workspaces source apply` run without `--yes` where no + * terminal can ask: it printed the plan a dry run found, with no errors, and wrote nothing. Run + * the same command with `--yes` to apply that plan. + */ +export const SOURCE_PLANNED_EXIT_CODE = 14 + /** The error code of an {@link AUTH_REQUIRED_EXIT_CODE} refusal the CLI raises itself. */ export const AUTH_REQUIRED_CODE = 'AUTH_REQUIRED' diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 329cfe3139d..f4c2aac3294 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -780,10 +780,12 @@ export type ApplyWorkflowSourceResponse = { added: Array<{ id: string key: string + type: string }> removed: Array<{ id: string key: string + type: string }> renamed: Array<{ id: string @@ -949,6 +951,7 @@ export type ApplyWorkflowSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -1209,6 +1212,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_DECL_SHAPE' | 'E_DECL_UNKNOWN_FIELD' | 'E_DECL_UNKNOWN_FILE' + | 'W_PATH_NOT_SENT' | 'E_BLOB_MANIFEST' | 'E_BLOB_MISSING' | 'E_LIMIT_EXCEEDED' @@ -1283,6 +1287,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_PLAN_REF_DELETED' | 'E_PLAN_REF_HELD' | 'W_RESOURCE_IN_USE' + | 'W_TABLE_COLUMN_RENAMED_IN_USE' | 'E_NAME_TAKEN' | 'E_WORKFLOW_FILES_MISSING' | 'E_WORKFLOW_FOLDER_COPIED' @@ -1432,6 +1437,7 @@ export type ApplyWorkspaceSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -1487,6 +1493,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_DECL_SHAPE' | 'E_DECL_UNKNOWN_FIELD' | 'E_DECL_UNKNOWN_FILE' + | 'W_PATH_NOT_SENT' | 'E_BLOB_MANIFEST' | 'E_BLOB_MISSING' | 'E_LIMIT_EXCEEDED' @@ -1561,6 +1568,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_PLAN_REF_DELETED' | 'E_PLAN_REF_HELD' | 'W_RESOURCE_IN_USE' + | 'W_TABLE_COLUMN_RENAMED_IN_USE' | 'E_NAME_TAKEN' | 'E_WORKFLOW_FILES_MISSING' | 'E_WORKFLOW_FOLDER_COPIED' @@ -1710,6 +1718,7 @@ export type ApplyWorkspaceSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -5199,10 +5208,12 @@ export type CreateWorkflowFromSourceResponse = { added: Array<{ id: string key: string + type: string }> removed: Array<{ id: string key: string + type: string }> renamed: Array<{ id: string @@ -5368,6 +5379,7 @@ export type CreateWorkflowFromSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -10462,6 +10474,7 @@ export type GetWorkflowSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' @@ -10917,6 +10930,7 @@ export type GetWorkspaceSourceResponse = { | 'E_DECL_SHAPE' | 'E_DECL_UNKNOWN_FIELD' | 'E_DECL_UNKNOWN_FILE' + | 'W_PATH_NOT_SENT' | 'E_BLOB_MANIFEST' | 'E_BLOB_MISSING' | 'E_LIMIT_EXCEEDED' @@ -10991,6 +11005,7 @@ export type GetWorkspaceSourceResponse = { | 'E_PLAN_REF_DELETED' | 'E_PLAN_REF_HELD' | 'W_RESOURCE_IN_USE' + | 'W_TABLE_COLUMN_RENAMED_IN_USE' | 'E_NAME_TAKEN' | 'E_WORKFLOW_FILES_MISSING' | 'E_WORKFLOW_FOLDER_COPIED' @@ -11140,6 +11155,7 @@ export type GetWorkspaceSourceResponse = { | 'W_APPLY_TYPE_CHANGED' | 'W_APPLY_TRIGGER_REMOVED' | 'W_APPLY_CATALOG_MOVED' + | 'W_APPLY_PROBABLE_RENAME' | 'E_APPLY_LINT' | 'W_APPLY_LINT' | 'W_TABLE_UNKNOWN_COLUMN' diff --git a/packages/sim-cli/src/http/client.test.ts b/packages/sim-cli/src/http/client.test.ts index 68eba1e667e..e8ef2101888 100644 --- a/packages/sim-cli/src/http/client.test.ts +++ b/packages/sim-cli/src/http/client.test.ts @@ -264,11 +264,32 @@ describe('workspace-key refusals', () => { await expect(client().request('/api/v2/secrets')).rejects.toMatchObject({ message: - 'Workspace API key cannot perform this operation — this operation does not support workspace API keys; use an OAuth login or personal API key: sim login --profile default', + 'Workspace API key cannot perform this operation — this operation does not support workspace API keys; use an OAuth login or personal API key: the key comes from SIM_API_KEY, which overrides any login: fix it, or unset it and run sim login --profile default', code: 'FORBIDDEN', }) }) + it('names SIM_API_KEY, not a login that it would override, when the server refuses that key', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response(JSON.stringify({ error: { message: 'Invalid API key' } }), { + status: 401, + headers: { 'content-type': 'application/json' }, + }) + ) + ) + + const refused = client().request('/api/v2/meta') + + await expect(refused).rejects.toMatchObject({ + message: + 'Invalid API key — the key comes from SIM_API_KEY, which overrides any login: fix it, or unset it and run sim login --profile default', + exitCode: 13, + }) + }) + it('invents no remedy for other forbidden codes', async () => { vi.stubGlobal( 'fetch', @@ -998,3 +1019,73 @@ describe('OAuth bearer credentials', () => { expect(init.headers).not.toHaveProperty('authorization') }) }) + +describe('encoded answers', () => { + function profile(overrides: Partial[0]>) { + return { + name: 'default', + endpoint: 'https://sim.example', + authProfile: 'default', + apiKey: 'key', + oauth: null, + workspaceId: 'ws_1', + output: 'json' as const, + sources: { + endpoint: 'default' as const, + credential: 'env' as const, + workspaceId: 'env' as const, + output: 'default' as const, + }, + ...overrides, + } + } + + /** A local server that answers brotli when asked, recording each request's Accept-Encoding. */ + async function encodingServer(body: unknown) { + const { createServer } = await import('node:http') + const { brotliCompressSync } = await import('node:zlib') + const asked: Array = [] + const server = createServer((request, response) => { + asked.push(request.headers['accept-encoding']) + const json = Buffer.from(JSON.stringify(body)) + const br = /\bbr\b/.test(request.headers['accept-encoding'] ?? '') + response.writeHead(200, { + 'content-type': 'application/json', + ...(br ? { 'content-encoding': 'br' } : {}), + }) + response.end(br ? brotliCompressSync(json) : json) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + const port = typeof address === 'object' && address ? address.port : 0 + return { endpoint: `http://127.0.0.1:${port}`, asked, close: () => server.close() } + } + + it('asks for brotli on the wire and reads the decoded answer', async () => { + const body = { data: { blocks: 'x'.repeat(50_000) } } + const server = await encodingServer(body) + try { + const wire = new SimClient(profile({ endpoint: server.endpoint })) + + await expect(wire.request('/api/v2/catalog')).resolves.toEqual(body) + expect(server.asked).toEqual(['br, gzip']) + } finally { + server.close() + } + }) + + it('never asks an in-process transport for an encoding it would hand back undecoded', async () => { + const transport = vi.fn( + async (_url: string | URL | Request, _init?: RequestInit) => + new Response(JSON.stringify({ data: 1 }), { + headers: { 'content-type': 'application/json' }, + }) + ) + const embedded = new SimClient(profile({ transport })) + + await embedded.request('/api/v2/catalog') + + const init = transport.mock.calls[0]?.[1] + expect(new Headers(init?.headers).has('accept-encoding')).toBe(false) + }) +}) diff --git a/packages/sim-cli/src/http/client.ts b/packages/sim-cli/src/http/client.ts index 53a2b9a229e..5990b79db07 100644 --- a/packages/sim-cli/src/http/client.ts +++ b/packages/sim-cli/src/http/client.ts @@ -109,6 +109,9 @@ export function buildUrl( return url.toString() } +/** The encodings the CLI asks for; Node's fetch would otherwise offer only gzip and deflate. */ +const ACCEPT_ENCODING = 'br, gzip' + /** * Statuses `fetch` would otherwise follow for us, and must not. * @@ -731,6 +734,10 @@ export class SimClient { ? { authorization: `Bearer ${credential.oauth.accessToken}` } : {}), accept: 'application/json', + // A server or proxy that compresses answers may send the large ones (the block catalog, + // a workspace's source) as brotli or gzip; fetch decodes them. An in-process transport + // reads the handler's response as it is, so it never asks for an encoding. + ...(this.profile.transport ? {} : { 'accept-encoding': ACCEPT_ENCODING }), ...identityHeaders(), ...(hasBody ? { 'content-type': 'application/json' } : {}), ...options.headers, @@ -799,7 +806,7 @@ export class SimClient { const error = response.status === 401 ? new SimApiError( - `${refused.message} — run: sim login --profile ${this.profile.authProfile}`, + `${refused.message} — ${this.credentialRemedy()}`, refused.status, refused.code ?? AUTH_REQUIRED_CODE, refused.details, @@ -809,7 +816,7 @@ export class SimClient { : refused error.retryAfterMs = parseRetryAfter(response.headers.get('retry-after'), 300_000) if (namesKeyScopeRefusal(error)) { - error.message = `${error.message} — this operation does not support workspace API keys; use an OAuth login or personal API key: sim login --profile ${this.profile.authProfile}` + error.message = `${error.message} — this operation does not support workspace API keys; use an OAuth login or personal API key: ${this.credentialRemedy()}` } if (response.status === 429) throw rateLimitedError(error, rateLimitResends) throw error @@ -826,6 +833,20 @@ export class SimClient { * endpoint. A `Location` that is missing or unparseable still has to produce a * sentence — the redirect is the finding either way. */ + /** + * What to do about a credential the server refused, by where it came from: `sim login` replaces + * a stored login, but a key from `SIM_API_KEY` or `--api-key` overrides any login, so logging in + * again would change nothing. + */ + private credentialRemedy(): string { + const login = `sim login --profile ${this.profile.authProfile}` + if (this.profile.sources.credential === 'env') { + return `the key comes from SIM_API_KEY, which overrides any login: fix it, or unset it and run ${login}` + } + if (this.profile.sources.credential === 'flag') return 'pass a valid key with --api-key' + return `run: ${login}` + } + private toRedirectError(url: string, path: string, response: Response): SimApiError { const requestId = responseRequestId(response) const location = response.headers.get('location')?.trim() diff --git a/packages/sim-cli/src/output/io.ts b/packages/sim-cli/src/output/io.ts index 33c2724bf1b..76a90041f4f 100644 --- a/packages/sim-cli/src/output/io.ts +++ b/packages/sim-cli/src/output/io.ts @@ -1,29 +1,86 @@ import { format } from 'node:util' import { embedStore } from '../embed-context' +/** + * Bun writes `console.log` to a piped stdout outside `process.stdout`, and once that stream + * exists (anything reading `process.stdout.isTTY` creates it) the console's writes past the + * pipe buffer are dropped when the process ends. The stream's own writes are kept, so under + * Bun every line goes through it. + */ +const writeLinesThroughStream = process.versions.bun !== undefined + +/** + * A drain still owed per standard stream: set when a write is refused (`false`), settled by + * `drain`, or by `error`/`close` when the reader went away. Bun reports `writableLength` as `0` + * while it still holds the refused bytes, so this is the only signal it gives. + */ +const pendingDrains = new Map>() + +function writeStream(stream: NodeJS.WriteStream, chunk: string | Uint8Array): boolean { + const accepted = stream.write(chunk) + if (!accepted && !pendingDrains.has(stream)) { + pendingDrains.set( + stream, + new Promise((resolve) => { + const settle = () => { + stream.off('drain', settle) + stream.off('error', settle) + stream.off('close', settle) + pendingDrains.delete(stream) + resolve() + } + stream.once('drain', settle) + stream.once('error', settle) + stream.once('close', settle) + }) + ) + } + return accepted +} + /** Only CLI-owned output enters the invocation result; host logs retain their normal sinks. */ export function printLine(...args: unknown[]): void { const context = embedStore.getStore() if (context) context.stdout.write(`${format(...args)}\n`) + else if (writeLinesThroughStream) writeStream(process.stdout, `${format(...args)}\n`) else console.log(...args) } export function printError(...args: unknown[]): void { const context = embedStore.getStore() if (context) context.stderr.write(`${format(...args)}\n`) + else if (writeLinesThroughStream) writeStream(process.stderr, `${format(...args)}\n`) else console.error(...args) } export function writeStdout(chunk: string | Uint8Array): boolean { const context = embedStore.getStore() - if (!context) return process.stdout.write(chunk) + if (!context) return writeStream(process.stdout, chunk) context.stdout.write(chunk) return true } export function writeStderr(chunk: string | Uint8Array): boolean { const context = embedStore.getStore() - if (!context) return process.stderr.write(chunk) + if (!context) return writeStream(process.stderr, chunk) context.stderr.write(chunk) return true } + +/** Resolves once `stream` has handed everything written to it to the OS. */ +async function drained(stream: NodeJS.WriteStream): Promise { + await pendingDrains.get(stream) + if (stream.writableLength === 0 || stream.destroyed) return + await new Promise((resolve) => { + stream.write('', () => resolve()) + }) +} + +/** + * Waits until stdout and stderr have written out what the CLI printed. `process.exit` ends the + * process at once, and a pipe holds only 64 KiB, so exiting without this cuts a large result + * short for whatever reads it (`sim … | jq`) while the exit status still says success. + */ +export async function flushOutput(): Promise { + await Promise.all([drained(process.stdout), drained(process.stderr)]) +} diff --git a/packages/sim-cli/src/terminal-output.test.ts b/packages/sim-cli/src/terminal-output.test.ts new file mode 100644 index 00000000000..82ebcd4e3de --- /dev/null +++ b/packages/sim-cli/src/terminal-output.test.ts @@ -0,0 +1,145 @@ +import { execFileSync, spawn } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type Server } from 'node:http' +import { createRequire } from 'node:module' +import type { AddressInfo } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +/** + * Output past the 64 KiB a pipe buffers is where an exit that does not wait + * for stdout loses data, so every case here writes megabytes into a pipe whose + * reader starts late. + */ +const ROWS = 20_000 +const READER_DELAY_MS = 400 + +interface PipedRun { + exitCode: number | null + stdout: string + stderr: string +} + +/** Spawns `command`, holds its stdout pipe unread for a moment, then drains it. */ +function runPiped(command: string, args: string[], env: NodeJS.ProcessEnv): Promise { + const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }) + let stdout = '' + let stderr = '' + child.stdout.setEncoding('utf8').pause() + setTimeout(() => child.stdout.resume(), READER_DELAY_MS) + child.stdout.on('data', (chunk: string) => { + stdout += chunk + }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk + }) + return new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (exitCode) => resolve({ exitCode, stdout, stderr })) + }) +} + +describe('stdout piped to a slow reader', () => { + let dir: string + let server: Server + let endpoint: string + const rows = Array.from({ length: ROWS }, (_, index) => ({ + id: `row_${index}`, + data: { name: 'x'.repeat(50), index }, + })) + + beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'sim-cli-pipe-')) + server = createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1') + const limit = Number(url.searchParams.get('limit') ?? 100) + const start = Number(url.searchParams.get('cursor') ?? 0) + const next = start + limit < rows.length ? String(start + limit) : null + response.setHeader('content-type', 'application/json') + response.end(JSON.stringify({ data: rows.slice(start, start + limit), nextCursor: next })) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + }) + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())) + rmSync(dir, { recursive: true, force: true }) + }) + + function cliEnv(): NodeJS.ProcessEnv { + return { + ...process.env, + SIM_CONFIG_DIR: join(dir, 'config'), + SIM_ENDPOINT: endpoint, + SIM_API_KEY: 'sk-test', + SIM_WORKSPACE: 'ws_1', + SIM_NO_UPDATE_CHECK: '1', + SIM_CLI_TELEMETRY_DISABLED: '1', + NO_PROXY: '127.0.0.1', + } + } + + it('writes every row of a long list before exiting 0', async () => { + const entrypoint = join(import.meta.dirname, 'index.ts') + const list = (output: string) => + runPiped( + 'bun', + [entrypoint, '--output', output, 'tables', 'rows', 'list', 'tbl_1', '--limit', '0'], + cliEnv() + ) + + const json = await list('json') + expect(json.stderr).toBe('') + expect(json.exitCode).toBe(0) + expect(json.stdout.length).toBeGreaterThan(1024 * 1024) + expect(JSON.parse(json.stdout).data).toHaveLength(ROWS) + + const table = await list('table') + expect(table.exitCode).toBe(0) + expect(table.stdout.trimEnd().split('\n').length).toBeGreaterThan(ROWS) + }, 60_000) + + /** + * The failing command writes its output first and its error after, the order + * a failed `workflows run` uses, and exits with the error's code. + */ + function failingCommandSource(terminalModule: string): string { + return ` + import { Command, runTerminalCli, printLine, SimApiError } from ${JSON.stringify(terminalModule)} + const program = new Command('sim') + program.command('fail').action(() => { + printLine('y'.repeat(${2 * 1024 * 1024})) + throw new SimApiError('the run failed', 0, 'RUN_FAILED', undefined, 5) + }) + process.argv = [process.argv[0], 'sim', 'fail'] + await runTerminalCli(program) + ` + } + + it('drains stdout before exiting with an error code, under Bun and Node', async () => { + const harness = join(dir, 'harness.ts') + writeFileSync( + harness, + `export { Command } from ${JSON.stringify(createRequire(import.meta.url).resolve('commander'))} +export { runTerminalCli } from ${JSON.stringify(join(import.meta.dirname, 'terminal.ts'))} +export { printLine } from ${JSON.stringify(join(import.meta.dirname, 'output/io.ts'))} +export { SimApiError } from ${JSON.stringify(join(import.meta.dirname, 'http/client.ts'))} +` + ) + const bundle = join(dir, 'harness.mjs') + execFileSync('bun', ['build', harness, '--target=node', '--format=esm', '--outfile', bundle], { + stdio: 'pipe', + }) + const script = join(dir, 'fail.mjs') + writeFileSync(script, failingCommandSource(bundle)) + + for (const runtime of ['bun', process.execPath]) { + const run = await runPiped(runtime, [script], cliEnv()) + expect(run.exitCode).toBe(5) + expect(run.stdout.length).toBe(2 * 1024 * 1024 + 1) + expect(run.stderr).toContain('the run failed') + } + }, 60_000) +}) diff --git a/packages/sim-cli/src/terminal.ts b/packages/sim-cli/src/terminal.ts index d5d8f217d43..1bd094917ed 100644 --- a/packages/sim-cli/src/terminal.ts +++ b/packages/sim-cli/src/terminal.ts @@ -4,12 +4,14 @@ import { styles } from '#sim-cli/output/presentation' import { CliUpdateError } from '#sim-cli/update/install' import { ProfileConfigError } from './config/index' import { profileFrom } from './context' +import { CliExit } from './embed-context' import { formatApiErrorDetails, isRequestTimeout, RAISE_TIMEOUT_HINT, SimApiError, } from './http/client' +import { flushOutput, printError, writeStderr } from './output/io' import { sanitize } from './output/render' import { buildProgram } from './program' import { createCommandTelemetry } from './telemetry/index' @@ -18,11 +20,12 @@ import { createCommandTelemetry } from './telemetry/index' * Prints the one-line explanation for an error the CLI understands and returns * the exit code it deserves, or `null` for an error it does not: that is a bug * in the CLI, and hiding it behind a friendly message would make it - * unreportable, so the caller lets it keep its stack trace. + * unreportable, so the caller prints it with its stack trace. */ function explainFailure(error: unknown, program: Command): number | null { + if (error instanceof CliExit) return error.code if (error instanceof ProfileConfigError || error instanceof CliUpdateError) { - console.error(styles().red(`Error: ${sanitize(error.message)}`)) + printError(styles().red(`Error: ${sanitize(error.message)}`)) return 1 } // `AbortSignal.timeout` keeps firing after `fetch` resolves, so a bound that @@ -30,7 +33,7 @@ function explainFailure(error: unknown, program: Command): number | null { // surfaces here rather than inside the client. A user's own Ctrl-C raises // `AbortError` instead, which is deliberately left alone. if (isRequestTimeout(error)) { - console.error(styles().red(`Error: the request timed out. ${RAISE_TIMEOUT_HINT}`)) + printError(styles().red(`Error: the request timed out. ${RAISE_TIMEOUT_HINT}`)) return 1 } if (error instanceof SimApiError) { @@ -49,17 +52,17 @@ function explainFailure(error: unknown, program: Command): number | null { ...(error.requestId ? { requestId: error.requestId } : {}), }, } - process.stderr.write(output === 'json' ? `${JSON.stringify(payload)}\n` : dump(payload)) + writeStderr(output === 'json' ? `${JSON.stringify(payload)}\n` : dump(payload)) return error.exitCode } - console.error(styles().red(`Error: ${sanitize(error.message)}`)) - if (error.code) console.error(styles().dim(` code: ${sanitize(error.code)}`)) + printError(styles().red(`Error: ${sanitize(error.message)}`)) + if (error.code) printError(styles().dim(` code: ${sanitize(error.code)}`)) if (error.details !== undefined) { for (const line of formatApiErrorDetails(error.details)) { - console.error(styles().dim(sanitize(line))) + printError(styles().dim(sanitize(line))) } } - if (error.requestId) console.error(styles().dim(` request id: ${sanitize(error.requestId)}`)) + if (error.requestId) printError(styles().dim(` request id: ${sanitize(error.requestId)}`)) return error.exitCode } return null @@ -69,6 +72,7 @@ function explainFailure(error: unknown, program: Command): number | null { * Anything the CLI can explain prints as one line and exits with its code. A * failure is reported here, where its class and code are known; every other * way the process ends is reported from the exit listener telemetry installs. + * Every exit waits for stdout and stderr to drain first (see {@link flushOutput}). */ export async function runTerminalCli(suppliedProgram?: Command): Promise { const telemetry = suppliedProgram ? undefined : createCommandTelemetry() @@ -78,8 +82,11 @@ export async function runTerminalCli(suppliedProgram?: Command): Promise { await program.parseAsync(process.argv) } catch (error) { const exitCode = explainFailure(error, program) - telemetry?.complete({ exitCode: exitCode ?? 1, error }) - if (exitCode === null) throw error - process.exit(exitCode) + telemetry?.complete( + error instanceof CliExit ? { exitCode: error.code } : { exitCode: exitCode ?? 1, error } + ) + if (exitCode === null) printError(error) + await flushOutput() + process.exit(exitCode ?? 1) } } diff --git a/packages/sim-cli/src/terminal/interrupts.test.ts b/packages/sim-cli/src/terminal/interrupts.test.ts new file mode 100644 index 00000000000..f9dccc313c6 --- /dev/null +++ b/packages/sim-cli/src/terminal/interrupts.test.ts @@ -0,0 +1,52 @@ +import { spawn } from 'node:child_process' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' + +const OUTPUT_BYTES = 4_000_000 + +/** + * Runs a deferred write in a real process that prints {@link OUTPUT_BYTES} on stdout after a + * Ctrl-C arrived, with the reader holding stdout back for a moment, as a slow pipe does. + */ +function interruptedWithBackedUpPipe(): Promise<{ bytes: number; signal: string | null }> { + const interrupts = join(import.meta.dirname, 'interrupts.ts') + const io = join(import.meta.dirname, '../output/io.ts') + const script = ` + const { deferInterrupts } = await import(${JSON.stringify(interrupts)}) + const { writeStdout } = await import(${JSON.stringify(io)}) + const { sleep } = await import('@sim/utils/helpers') + await deferInterrupts(async () => { + process.stderr.write('ready\\n') + await sleep(300) + writeStdout('x'.repeat(${OUTPUT_BYTES})) + }, { waiting: 'waiting', abandoned: 'abandoned' }) + ` + const child = spawn('bun', ['-e', script], { + cwd: join(import.meta.dirname, '../..'), + stdio: ['ignore', 'pipe', 'pipe'], + }) + let bytes = 0 + child.stdout.pause() + child.stdout.on('data', (chunk: Buffer) => { + bytes += chunk.length + }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + if (chunk.includes('ready')) { + child.kill('SIGINT') + setTimeout(() => child.stdout.resume(), 1_000) + } + }) + return new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', (_code, signal) => resolve({ bytes, signal })) + }) +} + +describe('deferInterrupts', () => { + it('ends by the held interrupt only after what the write printed reached the pipe', async () => { + const run = await interruptedWithBackedUpPipe() + + expect(run.bytes).toBe(OUTPUT_BYTES) + expect(run.signal).toBe('SIGINT') + }, 15_000) +}) diff --git a/packages/sim-cli/src/terminal/interrupts.ts b/packages/sim-cli/src/terminal/interrupts.ts new file mode 100644 index 00000000000..a7d4d0908f0 --- /dev/null +++ b/packages/sim-cli/src/terminal/interrupts.ts @@ -0,0 +1,84 @@ +import { embedStore } from '#sim-cli/embed-context' +import { flushOutput, printError } from '#sim-cli/output/io' + +const INTERRUPT_SIGNALS: readonly NodeJS.Signals[] = ['SIGINT', 'SIGTERM'] + +/** How long an interrupt waits for a deferred write before the command stops without it. */ +export const DEFERRED_INTERRUPT_BOUND_MS = 60_000 + +interface Deferral { + pending: NodeJS.Signals | null +} + +let active: Deferral | null = null + +/** + * Whether an interrupt arriving now is held by {@link deferInterrupts}. A signal handler that + * cleans up and ends the process returns without acting while this holds: the deferral raises + * the signal again once its write is done (or abandoned), and the handler runs then. + */ +export function interruptsDeferred(): boolean { + return active !== null +} + +export interface DeferInterruptsOptions { + /** Printed at the first interrupt: what the command finishes before it stops. */ + waiting: string + /** Printed when the command stops without finishing: a second interrupt, or the bound elapsed. */ + abandoned: string + boundMs?: number +} + +/** + * Runs `run`, a write the server may already be committing followed by saving its answer + * locally, with Ctrl-C (and `SIGTERM`) held until it ends: stopping in between leaves a write the + * server made that the directory never recorded. The first interrupt prints `waiting`; when + * `run` settles the process ends by that signal (exit 130 for Ctrl-C). A second interrupt, or + * `boundMs` passing, stops at once and prints `abandoned`, which names how to recover. + */ +export async function deferInterrupts( + run: () => Promise, + options: DeferInterruptsOptions +): Promise { + if (embedStore.getStore() || active) return run() + const deferral: Deferral = { pending: null } + let bound: ReturnType | undefined + + const release = () => { + for (const signal of INTERRUPT_SIGNALS) process.off(signal, onSignal) + clearTimeout(bound) + if (active === deferral) active = null + } + /** + * Ends the process by `signal` once stdout and stderr have drained, as every other exit does, + * so a backed-up pipe still receives what was printed. The handler is already released: an + * interrupt while they drain ends the process at once. + */ + const reraise = (signal: NodeJS.Signals) => { + void flushOutput().finally(() => process.kill(process.pid, signal)) + } + const abandon = (signal: NodeJS.Signals) => { + release() + printError(options.abandoned) + reraise(signal) + } + function onSignal(signal: NodeJS.Signals) { + if (deferral.pending) { + abandon(signal) + return + } + deferral.pending = signal + printError(options.waiting) + bound = setTimeout(() => abandon(signal), options.boundMs ?? DEFERRED_INTERRUPT_BOUND_MS) + } + + active = deferral + for (const signal of INTERRUPT_SIGNALS) process.on(signal, onSignal) + try { + return await run() + } finally { + const held = active === deferral ? deferral.pending : null + release() + if (held) reraise(held) + } +} diff --git a/packages/sim-cli/src/terminal/line-input.test.ts b/packages/sim-cli/src/terminal/line-input.test.ts new file mode 100644 index 00000000000..acf59604c02 --- /dev/null +++ b/packages/sim-cli/src/terminal/line-input.test.ts @@ -0,0 +1,43 @@ +import { PassThrough } from 'node:stream' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { askLine } from './line-input' + +/** A terminal's stdin, so readline reads keys (Ctrl-C, Ctrl-D) rather than plain text. */ +function terminalInput(): PassThrough { + const input = new PassThrough() + Object.assign(input, { isTTY: true, setRawMode: () => input }) + return input +} + +let input: PassThrough + +beforeEach(() => { + input = terminalInput() + vi.spyOn(process, 'stdin', 'get').mockReturnValue(input as unknown as typeof process.stdin) + vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + Object.defineProperty(process.stderr, 'isTTY', { value: true, configurable: true }) +}) + +afterEach(() => { + Object.defineProperty(process.stderr, 'isTTY', { value: undefined, configurable: true }) +}) + +describe('askLine', () => { + it('resolves the line typed', async () => { + const answer = askLine('Name? ') + input.write('sim\r') + await expect(answer).resolves.toBe('sim') + }) + + it('cancels with the interrupt status 130 at Ctrl-C', async () => { + const answer = askLine('Name? ') + input.write('\u0003') + await expect(answer).rejects.toMatchObject({ code: 'CANCELLED', exitCode: 130 }) + }) + + it('resolves null at Ctrl-D, for the caller to cancel as it chooses', async () => { + const answer = askLine('Name? ') + input.write('\u0004') + await expect(answer).resolves.toBeNull() + }) +}) diff --git a/packages/sim-cli/src/terminal/line-input.ts b/packages/sim-cli/src/terminal/line-input.ts new file mode 100644 index 00000000000..5fc53f647a5 --- /dev/null +++ b/packages/sim-cli/src/terminal/line-input.ts @@ -0,0 +1,46 @@ +import { createInterface } from 'node:readline/promises' +import { SimApiError } from '#sim-cli/http/client' +import { writeStderr } from '#sim-cli/output/io' + +/** The shell's status for a command the user interrupted with Ctrl-C. */ +const INTERRUPTED_EXIT_CODE = 130 + +/** + * Whether a pending question failed because its input ended: Bun rejects it with an + * `AbortError` at Ctrl-D, where Node leaves it pending and closes the interface instead. + */ +function isInputEnded(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === 'AbortError' || ('code' in error && error.code === 'ABORT_ERR')) + ) +} + +/** + * Asks `question` on stderr and resolves the line typed, or `null` when the input ends first + * (Ctrl-D, or a stdin closed under the prompt), so a caller cancels cleanly instead of hanging + * or printing a runtime's stack trace. Ctrl-C at a terminal, which readline reads as a key rather + * than delivering the signal, rejects with a `CANCELLED` error that exits 130, as an interrupt + * anywhere else does. + */ +export async function askLine(question: string): Promise { + const prompt = createInterface({ input: process.stdin, output: process.stderr }) + const ended = new Promise((resolve) => prompt.once('close', () => resolve(null))) + const interrupted = new Promise((_resolve, reject) => + prompt.once('SIGINT', () => + reject(new SimApiError('Interrupted.', 0, 'CANCELLED', undefined, INTERRUPTED_EXIT_CODE)) + ) + ) + try { + const answer = await Promise.race([prompt.question(question), ended, interrupted]) + if (answer === null) writeStderr('\n') + return answer + } catch (error) { + if (error instanceof SimApiError) writeStderr('\n') + if (!isInputEnded(error)) throw error + writeStderr('\n') + return null + } finally { + prompt.close() + } +} diff --git a/packages/sim-cli/src/test/catalog-cache.ts b/packages/sim-cli/src/test/catalog-cache.ts new file mode 100644 index 00000000000..07f01dab955 --- /dev/null +++ b/packages/sim-cli/src/test/catalog-cache.ts @@ -0,0 +1,20 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +/** + * Caches catalog JSON `text` in config directory `configDir` as the CLI does: the catalog under its + * hash, and `workspaceId`'s pointer to it unless `workspaceId` is `null`. + */ +export function cacheCatalogText( + configDir: string, + workspaceId: string | null, + text: string +): void { + const { hash } = JSON.parse(text) as { hash: string } + const root = join(configDir, 'cache/source-catalog') + mkdirSync(join(root, 'workspaces'), { recursive: true }) + writeFileSync(join(root, `${hash}.json`), text) + if (workspaceId !== null) { + writeFileSync(join(root, 'workspaces', `${workspaceId}.json`), JSON.stringify({ hash })) + } +} diff --git a/packages/sim-cli/src/test/context-mock.ts b/packages/sim-cli/src/test/context-mock.ts index 21cac9375e4..712a194220d 100644 --- a/packages/sim-cli/src/test/context-mock.ts +++ b/packages/sim-cli/src/test/context-mock.ts @@ -16,10 +16,23 @@ export const contextMockState: { workspaceSource: 'config', } +/** The resolved profile `clientFrom` and `profileFrom` return, built from {@link contextMockState}. */ +export function mockedProfile() { + return { + workspaceId: contextMockState.workspaceId, + output: contextMockState.output, + name: 'default', + authProfile: 'default', + apiKey: 'k', + endpoint: 'https://sim.example', + sources: { workspaceId: contextMockState.workspaceSource }, + } +} + /** * The `SimClient` methods protocol commands call, plus the context entry * points. Non-bare defaults: `mockRequireWorkspace` returns - * `contextMockState.workspaceId`; `mockClientFrom` returns + * the workspace it is given, else `contextMockState.workspaceId`; `mockClientFrom` returns * `{ client: { request, requestRaw, requireWorkspace }, profile }` with the * profile built from {@link contextMockState} (name `default`, api key `k`, * endpoint `https://sim.example`). @@ -27,24 +40,16 @@ export const contextMockState: { export const contextMockFns = { mockRequest: vi.fn(), mockRequestRaw: vi.fn(), - mockRequireWorkspace: vi.fn(() => contextMockState.workspaceId), + mockRequireWorkspace: vi.fn((explicit?: string) => explicit ?? contextMockState.workspaceId), mockGlobalsOf: vi.fn(), - mockProfileFrom: vi.fn(), + mockProfileFrom: vi.fn(mockedProfile), mockClientFrom: vi.fn(() => ({ client: { request: contextMockFns.mockRequest, requestRaw: contextMockFns.mockRequestRaw, requireWorkspace: contextMockFns.mockRequireWorkspace, }, - profile: { - workspaceId: contextMockState.workspaceId, - output: contextMockState.output, - name: 'default', - authProfile: 'default', - apiKey: 'k', - endpoint: 'https://sim.example', - sources: { workspaceId: contextMockState.workspaceSource }, - }, + profile: mockedProfile(), })), } diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index 4088972298c..1007c3d7a15 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -38,7 +38,9 @@ stored workflow (see "Preserved state"). There is no verbose mode. Two escape hatches carry state that runs but has no SDK syntax: - `meta.extra`: stored params of a schema-agnostic custom block, the agent's legacy prompt fields, - a container's stored params. + a container's stored params other than those `block.data` supersedes (the config inputs the + editing engine copies into `data`, `SUPERSEDED_CONTAINER_PARAMS`), which never run and are + preserved state. - `meta.subflowData`: a loop or parallel's `block.data` subflow keys canonicalize could not read as config, which the serializer passes on. Kept apart from `extra` because canvas containers often store params under the same names. @@ -197,7 +199,15 @@ key-keyed lock; `mergeSourceMetaFromLock` stores a write's hints back by id. 3. `db-name`: the prev block with the same key (the lockless and stale-lock path); 4. `inferred`: exactly one unmatched prev block and one unmatched compiled block with the same type and parent and agreeing edge neighbourhoods, so renaming the variable and the display - name together keeps the id (`W_MAT_INFERRED_RENAME`; to get a new block instead, delete the old one in one apply and add the new one in the next); + name together keeps the id; failing that, a prev block the lock binds (the source dropped its + key and its ident) pairs with a new block of its type and parent only when exactly one such + pair shows a strong identity signal: equal settings (references to the old key read as the + new), most bound neighbours shared, or every block that referenced the old key now referencing + the new one. A trigger (webhook paths, schedules) pairs only on equal settings, under either + rule (`W_MAT_INFERRED_RENAME`; to get a new block instead, delete the old one in one apply and + add the new one in the next). A delete and an add with no signal stay a delete, refused as + `E_APPLY_BLOCK_DELETE` (naming the look-alike new blocks and how to keep the ids) unless the + caller passes `allowDelete`; 5. otherwise created: the lock's id when unclaimed, else minted (`idSource`). A match across types is a delete plus a create (`W_MAT_RETYPE_RECREATED`), except a catalog @@ -310,8 +320,8 @@ print compiles with zero errors. ## Compile -`compileWorkflow({ entry, files, catalog, sdkMajor, limits?, knownKeys?, stored?, storedUnread? })` → `{ workflow, -diagnostics, spans, hints, sdkMajor }`. It parses with Babel and interprets the AST statically; it +`compileWorkflow({ entry, files, catalog, sdkMajor, limits?, knownKeys?, stored?, storedUnread?, +storedUnreadEdges? })` → `{ workflow, diagnostics, spans, hints, sdkMajor, envReferences, edges }`. It parses with Babel and interprets the AST statically; it never executes the file. `workflow` is `null` iff any diagnostic is an error. It accepts exactly what the printer emits plus the conveniences SYNTAX.md lists, checks names, scope, wiring and catalog fields, fills defaults, resolves `stored()` from `CompileInput.stored`, and returns `spans` @@ -326,9 +336,12 @@ Compile also refuses or flags source the runtime would silently mishandle: of edges the stored workflow holds is `W_EDGE_CYCLE`, so stored state keeps round-tripping (`storedUnread`, the CLI's offline check, which has the lock but not the stored workflow, takes every edge between two blocks its lock binds, by key or by ident for a block renamed in source, - as one it may hold; the same goes for `W_ERROR_PORT_STORED`); -- a reference into an agent whose `responseFormat` declares its fields names a declared field or - a static output (`W_UNKNOWN_OUTPUT_PATH` otherwise); a format with references inside is unread; + as one it may hold, or with `storedUnreadEdges`, the edges the get printed by lock key, only + those; the same goes for `W_ERROR_PORT_STORED`); +- a reference into an agent whose `responseFormat` declares its fields, or into a Start, API or + Input trigger (whose `inputFormat` declares its inputs), names a declared field or a static + output (`W_UNKNOWN_OUTPUT_PATH` otherwise; a warning, since a caller may pass an undeclared + input the unified Start passes through); a format with references inside is unread; - a dropdown value outside its static options is `W_UNKNOWN_OPTION`, a `model` the catalog does not list `W_UNKNOWN_MODEL` (free text stays valid), and an unknown `operation` `E_UNKNOWN_OPERATION` unless the stored block already holds it; @@ -365,7 +378,8 @@ then writes the stored graph so that a no-op is byte-identical (O1): filled into fields that never run are dropped); - the fields of the loop or parallel types not in effect, and prev's whole config when it runs the same; -- unknown `block.data` source does not carry. +- unknown `block.data` source does not carry; +- a container's subBlocks `block.data` supersedes (`iterations`, `loopType`, …). Canonicalize's output already holds all of this, so restoring onto it changes nothing. diff --git a/packages/workflow-code/src/canonicalize/subflows.ts b/packages/workflow-code/src/canonicalize/subflows.ts index d85ff1f58f0..77f88100cb4 100644 --- a/packages/workflow-code/src/canonicalize/subflows.ts +++ b/packages/workflow-code/src/canonicalize/subflows.ts @@ -32,6 +32,18 @@ export const SUBFLOW_DATA_KEYS: ReadonlySet = new Set([ 'batchSize', ]) +/** + * Container subBlocks `block.data` supersedes: the config inputs the editing engine copies into + * `data` (`iterations` → `count`, `condition` → `whileCondition`) and may also leave stored as + * subBlocks. The serializer reads a container's config from `data` only, so these never run: + * preserved state, which print leaves out and apply restores from the stored block. + */ +export const SUPERSEDED_CONTAINER_PARAMS: ReadonlySet = new Set([ + ...SUBFLOW_DATA_KEYS, + 'iterations', + 'condition', +]) + /** A subflow config with the `block.data` keys whose stored value it could not represent. */ interface SubflowResult { spec: S diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index cb7c733d100..a2fdb95a20e 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -84,7 +84,9 @@ export default workflow('Name', { description: '…', variables: { … } }, ({ v `E_UNSUPPORTED_SYNTAX`. - A pinned param (`operation`, or a builder's `fixed` entry) is `E_FIXED_PARAM`; the compiler adds pinned values. -- A param not visible under the block's values is `W_FIELD_NOT_APPLICABLE`. A required field left +- A param not visible under the block's values is `W_FIELD_NOT_APPLICABLE`, naming the value that + hides it: it never runs, so apply keeps the stored value (preserved state) and the written-back + source drops the line. A required field left empty (or `todo()`) is `W_REQUIRED_MISSING`. A value whose JSON type differs from the field's is `W_VALUE_KIND`; it is never coerced. @@ -190,8 +192,9 @@ reordered tool never takes another tool's secret. - `blk.a.b` → ``, `blk['x-y']` → ``, `blk.items[0]` → ``, `blk.items['0']` → ``. Paths are typed from the block's outputs; a path no output has is `W_UNKNOWN_OUTPUT_PATH`. For an agent with a `responseFormat`, its declared fields count - beside the static outputs (skipped when the format holds references); other dynamic outputs - skip the check. + beside the static outputs (skipped when the format holds references); for a Start, API or Input + trigger its declared `inputFormat` inputs do, so `start.nope`, or a reference to an input since + removed, warns at the reference; other dynamic outputs skip the check. - A ref to a block declared later is `W_FORWARD_REF`; it still resolves. A ref into a sibling subflow's child is `E_OUT_OF_SCOPE`. An unknown identifier is `E_UNKNOWN_IDENTIFIER` with a did-you-mean. Wiring resolves its identifiers the same way. @@ -215,7 +218,9 @@ A JSON code field (subBlock type `code`: `responseFormat`, table row `data`, a r written as an object or array literal compiles to `JSON.stringify(value, null, 2)` text. Each bare reference is stored by its output type: quoted (`""`) when the catalog or the workflow types it `string`, bare (``) otherwise, because the executor splices a string raw and any other value as -JSON. A reference nothing types is stored bare with `W_JSON_REF_UNTYPED`. A template is a JSON +JSON. A reference nothing types is stored bare with `W_JSON_REF_UNTYPED`, which is not raised for a +Function `result` field its JavaScript returns as a non-string literal (`return { total: a + b }`), +nor, against a stored workflow, for a field the source left as stored. A template is a JSON string, which is how a value becomes text on purpose. A resource (`table('x')`, `credential()`, `file()`) or `raw(node)` is not data, so inside the literal it is `E_UNSUPPORTED_SYNTAX`. The printer writes a field's text back as the literal only when serializing that literal gives the same bytes; otherwise the text prints as a @@ -415,7 +420,7 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `E_UNKNOWN_BUILDER` | A builder path the catalog does not have, a bare builder-shaped call (`apii('Name', …)`) nothing binds, or a namespace the SDK lacks or the file did not import; `hint` names the builder a common other name stands for (`http.request` → `api`), then prefers a synonymous verb (`postMessage` → `sendMessage`), then the closest path, then the namespace's builders. | | `E_UNKNOWN_PARAM` | A param the block does not have; the hint has a did-you-mean. | | `E_FIXED_PARAM` | A param the builder already pins. | -| `W_FIELD_NOT_APPLICABLE` | A param not visible under the block's values. | +| `W_FIELD_NOT_APPLICABLE` | A param not visible under the block's values; apply keeps the stored value and the written-back source drops it. | | `W_REQUIRED_MISSING` | A required field left empty, including `todo()`; not repeated for a field an `E_UNKNOWN_PARAM` did-you-mean names, or one written with a value that failed to compile. | | `W_UNKNOWN_OPTION` | A dropdown value none of its options names; `hint` lists them. | | `E_UNKNOWN_OPERATION` | An `operation` the block does not offer; `hint` lists them. A retired one the stored block holds is `W_UNKNOWN_OPTION`. | @@ -511,7 +516,8 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `W_APPLY_TRIGGER_REMOVED` | A deleted or retyped block still serves a live webhook, schedule, chat output or MCP tool until the next deploy. | | `W_APPLY_CATALOG_MOVED` | Compile failed and the block catalog changed since the read: get the source again. | | `E_APPLY_LINT`, `W_APPLY_LINT` | A workflow lint finding, mapped to a source line. | -| `E_APPLY_BLOCK_DELETE` | The apply deletes blocks the source no longer declares; pass `--allow-delete`. | +| `E_APPLY_BLOCK_DELETE` | The apply deletes blocks the source no longer declares; pass `--allow-delete`. When it also adds blocks of a deleted one's type, it names them as possible renames and says how to keep the ids. | +| `W_APPLY_PROBABLE_RENAME` | With `--allow-delete`, a deleted block and an added one share a type and parent: a probable rename that gets a new id. | | `E_NAME_TAKEN` | The `workflow('…')` name renames the workflow to one another workflow in its folder holds. | | `E_WORKFLOW_META_INVALID` | The `workflow('…')` name is empty or over 255 characters, or its description is over 50,000 characters: limits the workflow API holds every workflow to. | | `E_APPLY_INVALID_SCHEDULE` | An added or changed schedule whose cron, timezone or interval deploy would refuse. | diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 06d2973bcbd..0a3dc59abf8 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -76,6 +76,7 @@ import type { Range } from './report' import { compileToolList } from './tools' import { fieldFor, + hiddenReason, isVisible, missingRequiredFields, paramLookup, @@ -1521,6 +1522,15 @@ function valueKindHint(sub: CatalogSubBlock): string { } } +/** + * `W_FIELD_NOT_APPLICABLE`: a hidden field never runs, and source holds only what runs, so an + * apply restores the stored value over it (preserved state) and the printed source drops it. + */ +function notApplicableMessage(param: string, reason: string | null): string { + const hidden = reason ? `hidden (${reason})` : "hidden under the block's current values" + return `${param} is ${hidden}, so it never runs: an apply does not save it over the stored value, and the source it writes back drops this line.` +} + /** Catalog warnings: visibility, required fields, value kinds, slider ranges. */ function checkFields(ctx: CompileContext, index: FieldIndex, state: BlockState): void { const { decl, params } = state @@ -1532,10 +1542,10 @@ function checkFields(ctx: CompileContext, index: FieldIndex, state: BlockState): if (!isVisible(sub, values)) { ctx.reporter.report( 'W_FIELD_NOT_APPLICABLE', - `${entry.param} is hidden under the block's current values; it is kept but not used.`, + notApplicableMessage(entry.param, hiddenReason(sub, values)), rangeOf(ctx, entry.property), { - hint: 'Remove it; apply keeps the stored value.', + hint: `Remove ${entry.param}, or change the values that hide it so it runs.`, blockKey: decl.key, param: entry.param, } @@ -1806,7 +1816,9 @@ export function compileBlock(ctx: CompileContext, decl: BlockDecl): CompiledBloc /** * `W_LEGACY_BLOCK_TYPE` for a new block of a legacy or deprecated type, naming its replacement and - * that type's builders. A block the stored workflow already holds under that type stays quiet. + * that type's builders. A block the stored workflow already holds under that type stays quiet; an + * offline check, which has only the lock, stays quiet on a block the lock binds by key or ident, + * as the server does for the stored block it continues. */ function checkLegacyType(ctx: CompileContext, decl: BlockDecl): void { const block = catalogBlockOf(decl) @@ -1814,6 +1826,14 @@ function checkLegacyType(ctx: CompileContext, decl: BlockDecl): void { if (!block || !sunset) return const stored = ctx.stored ? getOwn(ctx.stored.blocks, decl.key) : undefined if (stored?.type === block.type) return + const lock = ctx.storedUnread + if ( + lock && + (Object.hasOwn(lock.blocks, decl.key) || + Object.values(lock.blocks).some((entry) => entry.ident === decl.ident)) + ) { + return + } const replacement = sunset.replacedBy const namespaces = replacement ? [ diff --git a/packages/workflow-code/src/compile/checks.test.ts b/packages/workflow-code/src/compile/checks.test.ts index c4f9f781a67..c433b00a8c2 100644 --- a/packages/workflow-code/src/compile/checks.test.ts +++ b/packages/workflow-code/src/compile/checks.test.ts @@ -104,6 +104,129 @@ describe('cycles', () => { }) }) +describe("references into a start trigger's declared inputs", () => { + const START_WITH = (inputs: string) => + `const start = trigger.start('Start', { inputFormat: ${inputs} })` + const unknown = (inputs: string, ref: string, extra = '') => + findings( + run( + [ + START_WITH(inputs), + `const next = api('Next', { url: t\`https://x/\${${ref}}\` }${extra})`, + 'start.to(next)', + ].join('\n') + ), + 'W_UNKNOWN_OUTPUT_PATH' + ) + + it('warns on an input the start does not declare, at the reference', () => { + const [finding] = unknown("{ label: 'string' }", 'start.nope') + expect(finding?.message).toBe('Start declares no input nope: its inputFormat declares label.') + expect(finding?.span).toMatchObject({ file: 'workflow.ts', line: 6 }) + }) + + it('warns on an input that was removed while a reference still reads it', () => { + const [finding] = unknown('{}', 'start.label') + expect(finding?.message).toBe('Start declares no input label: its inputFormat declares none.') + expect(finding?.hint).toBe( + "Add label to start's inputFormat, or reference an input it declares." + ) + }) + + it('suggests a declared input an unknown one is close to', () => { + expect(unknown("{ label: 'string' }", 'start.lable')[0]?.hint).toBe('Did you mean label?') + }) + + it.each([ + ['a declared input', 'start.label'], + ['a path under a declared input', 'start.label.length'], + ['a static output', 'start.input'], + ])('accepts %s', (_label, ref) => { + expect(unknown("{ label: 'string' }", ref)).toEqual([]) + }) +}) + +describe("a field hidden under the block's values", () => { + const DROPPED = + 'so it never runs: an apply does not save it over the stored value, and the source it writes back drops this line.' + + it('says the value is dropped, and names the condition it shows under', () => { + const [hidden] = findings( + run("const s = slack.sendMessage('S', { authMethod: 'oauth', text: 'x', limit: 3 })"), + 'W_FIELD_NOT_APPLICABLE' + ) + expect(hidden?.message).toBe( + `limit is hidden (it shows only when operation is list_channels), ${DROPPED}` + ) + expect(hidden?.hint).toBe('Remove limit, or change the values that hide it so it runs.') + }) + + it('names the model a model-gated field does not apply to', () => { + const agentBlock = TEST_CATALOG.blocks.agent as CatalogBlock + const gated: BlockCatalog = { + ...TEST_CATALOG, + blocks: { + ...TEST_CATALOG.blocks, + agent: { + ...agentBlock, + subBlocks: agentBlock.subBlocks.map((sub) => + sub.id === 'temperature' + ? { + ...sub, + condition: { opaque: true }, + visibleForModel: { 'gpt-5': false, 'claude-sonnet-4-5': true }, + } + : sub + ), + }, + }, + } + const written = (model: string) => + findings( + run(`const a = agent('A', { model: '${model}', temperature: 0.5 })`, undefined, gated), + 'W_FIELD_NOT_APPLICABLE' + ) + expect(written('gpt-5')[0]?.message).toBe( + `temperature is hidden (model gpt-5 does not use it), ${DROPPED}` + ) + expect(written('claude-sonnet-4-5')).toEqual([]) + }) + + it('names the clause an opaque condition still asks for the operation, before the model', () => { + const slackBlock = TEST_CATALOG.blocks.slack as CatalogBlock + const gated: BlockCatalog = { + ...TEST_CATALOG, + blocks: { + ...TEST_CATALOG.blocks, + slack: { + ...slackBlock, + subBlocks: slackBlock.subBlocks.map((sub) => + sub.id === 'limit' + ? { + ...sub, + condition: { opaque: true }, + visibleFor: { send: true, list_channels: true }, + visibleWhen: { send: { field: 'authMethod', value: 'bot_token' } }, + } + : sub + ), + }, + }, + } + const [hidden] = findings( + run( + "const s = slack.sendMessage('S', { authMethod: 'oauth', text: 'x', limit: 3 })", + undefined, + gated + ), + 'W_FIELD_NOT_APPLICABLE' + ) + expect(hidden?.message).toBe( + `limit is hidden (it shows only when authMethod is bot_token), ${DROPPED}` + ) + }) +}) + describe('references into an agent responseFormat', () => { const FORMAT = "responseFormat: { name: 'classify', schema: { type: 'object', properties: { priority: { type: 'string' }, score: { type: 'number' } } } }" @@ -264,6 +387,24 @@ describe('legacy block types', () => { if (!stored) throw new Error('expected a workflow') expect(findings(run(body, stored, withLegacy), 'W_LEGACY_BLOCK_TYPE')).toEqual([]) }) + + it('stays quiet offline for a legacy block the lock binds, as the server does', () => { + const offline = (lockKey: string, ident: string) => + compileWorkflow({ + entry: { path: 'workflow.ts', text: source("const s = block('slack_legacy', 'S', {})") }, + files: new Map(), + catalog: withLegacy, + sdkMajor: 1, + storedUnread: { + version: 1, + blocks: { [lockKey]: { id: 'b-1', parent: null, ident, order: 0 } }, + variables: {}, + }, + }) + expect(findings(offline('s', 's'), 'W_LEGACY_BLOCK_TYPE')).toEqual([]) + expect(findings(offline('old', 's'), 'W_LEGACY_BLOCK_TYPE')).toEqual([]) + expect(findings(offline('other', 'other'), 'W_LEGACY_BLOCK_TYPE')).toHaveLength(1) + }) }) describe('unwired branches', () => { diff --git a/packages/workflow-code/src/compile/context.ts b/packages/workflow-code/src/compile/context.ts index e628bf928d5..e898733b931 100644 --- a/packages/workflow-code/src/compile/context.ts +++ b/packages/workflow-code/src/compile/context.ts @@ -11,6 +11,7 @@ import type { CodeFileRef } from '../code-files' import type { RefSegment } from '../code-files/ref-path' import type { SourceSpan } from '../diagnostics/frame' import type { JsonTree } from '../json-fields' +import type { WorkflowLockV1 } from '../lock' import type { Anchor, AnchorKey, @@ -18,6 +19,7 @@ import type { CanonicalParam, CanonicalVariable, CanonicalWorkflow, + CanonicalWorkflowMeta, LoopType, ParallelType, SubBlockId, @@ -143,10 +145,17 @@ export interface CompileContext { wiring: WiringStatement[] implied: ImpliedEdge[] spans: Map + /** Where `workflow()` writes the name and the description: `CompileResult.metaSpans`. */ + metaSpans: Partial> /** `CompileInput.knownKeys`: the only keys a `meta.key` override may name. */ knownKeys: ReadonlySet /** `CompileInput.stored`: the stored workflow `stored()` reads its values from. */ stored: CanonicalWorkflow | undefined + /** + * `CompileInput.storedUnread` when `stored` is absent: the lock of a stored workflow an offline + * check continues without reading it. + */ + storedUnread: WorkflowLockV1 | undefined /** `CompileInput.envNames`: the env names a run resolves, when known. */ envNames: ReadonlySet | undefined /** `CompileInput.envDeclaredIn`: the file the env names are declared in, when there is one. */ @@ -159,12 +168,13 @@ export interface CompileContext { */ jsonFields: JsonFieldEntry[] /** - * References into blocks whose outputs come from a `responseFormat` param, checked once every - * block is compiled, since the format is JSON field text written last (`checkResponseFormatRefs`). + * References into blocks whose outputs a param declares (`responseFormat`, an input trigger's + * `inputFormat`), checked once every block is compiled, since the format is JSON field text + * written last (`checkDynamicOutputRefs`). */ - responseFormatRefs: ResponseFormatRef[] + dynamicOutputRefs: DynamicOutputRef[] /** `x.error` references, checked once the edges are known (`checkErrorOutputRefs`). */ - errorOutputRefs: Array> + errorOutputRefs: Array> /** * Branch keys already refused with an error, by block key. Wiring and branch meta that name one * stay quiet instead of adding `E_UNKNOWN_BRANCH` for the same mistake. @@ -178,7 +188,7 @@ export interface CompileContext { } /** A reference path into a block whose outputs its `responseFormat` declares. */ -interface ResponseFormatRef { +interface DynamicOutputRef { at: Range target: BlockDecl segments: RefSegment[] diff --git a/packages/workflow-code/src/compile/edges.ts b/packages/workflow-code/src/compile/edges.ts index a7c6110de1e..15e29283d77 100644 --- a/packages/workflow-code/src/compile/edges.ts +++ b/packages/workflow-code/src/compile/edges.ts @@ -51,6 +51,12 @@ function portForHandle(handle: string, from: BlockDecl): Port { /** Implicit branch keys of a never-persisted branch list. */ const IMPLICIT_BRANCHES = { condition: ['if', 'else'], router: ['route0'] } as const +/** What `EdgeCollector` knows of a stored workflow it continues but cannot read. */ +interface UnreadBound { + lockKeys: ReadonlyMap + edges?: ReadonlySet +} + /** Pass 2 of wiring: validates each edge a wiring statement denotes and collects accepted ones. */ export class EdgeCollector { readonly edges: CanonicalEdge[] = [] @@ -65,10 +71,12 @@ export class EdgeCollector { private readonly ctx: CompileContext, private readonly blocks: ReadonlyMap, /** - * Under `CompileInput.storedUnread`, the source keys of the blocks its lock binds, renamed - * ones included; `undefined` when the compile holds the stored workflow or continues none. + * Under `CompileInput.storedUnread`, the lock key of each source block its lock binds, by + * source key (a renamed block's is its vacated lock key), and with + * `CompileInput.storedUnreadEdges` the tuple keys of the stored edges by lock key; + * `undefined` when the compile holds the stored workflow or continues none. */ - private readonly unreadBound: ReadonlySet | undefined + private readonly unreadBound: UnreadBound | undefined ) { for (const decl of ctx.decls) { if (decl.duplicate) continue @@ -129,7 +137,7 @@ export class EdgeCollector { const stored = (this.ctx.stored?.edges ?? []).some( (edge) => edge.from === from.key && edge.to === to.key && edge.port.kind === 'error' ) - const unread = !stored && this.maybeStored(from.key, to.key) + const unread = !stored && this.maybeStored(from.key, port, to.key) if (!stored && !unread) { this.error('E_ERROR_PORT_UNSUPPORTED', `${from.name} has no error port.`, node, { hint: containerKindOf(from) @@ -237,7 +245,7 @@ export class EdgeCollector { return ( edge && !storedTuples.has(edgeTupleKey(edge.from, portKey(edge.port), edge.to)) && - !this.maybeStored(edge.from, edge.to) + !this.maybeStored(edge.from, edge.port, edge.to) ) }) const candidates = fresh.length > 0 ? fresh : inside @@ -304,11 +312,16 @@ export class EdgeCollector { } /** - * Whether an edge between `from` and `to` may be one the stored workflow holds although this - * compile cannot read it (`CompileInput.storedUnread`): both ends are blocks its lock binds. + * Whether an edge may be one the stored workflow holds although this compile cannot read it + * (`CompileInput.storedUnread`): both ends are blocks its lock binds, and with + * `CompileInput.storedUnreadEdges` the stored workflow held this edge between them. */ - private maybeStored(from: BlockKey, to: BlockKey): boolean { - return this.unreadBound?.has(from) === true && this.unreadBound.has(to) + private maybeStored(from: BlockKey, port: Port, to: BlockKey): boolean { + const lockFrom = this.unreadBound?.lockKeys.get(from) + const lockTo = this.unreadBound?.lockKeys.get(to) + if (lockFrom === undefined || lockTo === undefined) return false + const stored = this.unreadBound?.edges + return stored === undefined || stored.has(edgeTupleKey(lockFrom, portKey(port), lockTo)) } /** diff --git a/packages/workflow-code/src/compile/function-returns.ts b/packages/workflow-code/src/compile/function-returns.ts new file mode 100644 index 00000000000..38af616f1a3 --- /dev/null +++ b/packages/workflow-code/src/compile/function-returns.ts @@ -0,0 +1,471 @@ +/** + * What a Function block's `result` fields hold, read from the object literal its JavaScript code + * returns (`return { total: subtotal + tax, tier: 'gold' }`), so a bare reference to one in a JSON + * field is not reported as untyped when the code says it is a number, boolean, array or object. + * + * The reading is deliberately narrow: every `return` outside a nested function must return an + * object literal, and a field counts only when each of them sets it to an expression whose kind + * is certain (a literal, arithmetic, a comparison, `String(…)`, the `.length` of a string or an + * array, a local variable never + * reassigned to another kind, …). Anything else stays unknown, and the reference keeps its + * warning: a wrong "typed" would store a string bare and break the JSON at run time. + */ + +import { parse } from '@babel/parser' +import type { Expression, Node, ObjectExpression, PatternLike } from '@babel/types' +import { childNodes } from '../babel' +import { parseStoredPath } from '../code-files/ref-path' +import { toCodeFile } from '../code-files/to-file' +import type { BlockKey, CanonicalBlock } from '../model' +import { getOwn } from '../records' + +/** A value's kind as far as JSON splicing goes: text, or any other JSON value. */ +export type ReturnedKind = 'string' | 'typed' + +/** The JavaScript kinds an expression can be inferred to have; `typed` is a mix of non-strings. */ +type JsKind = 'string' | 'number' | 'boolean' | 'typed' + +const FUNCTION_TYPE = 'function' + +const NUMBER_FUNCTIONS: ReadonlySet = new Set(['Number', 'parseInt', 'parseFloat']) +const BOOLEAN_FUNCTIONS: ReadonlySet = new Set(['Boolean', 'isNaN', 'isFinite']) +const STRING_METHODS: ReadonlySet = new Set([ + 'toFixed', + 'toPrecision', + 'toString', + 'toUpperCase', + 'toLowerCase', + 'trim', + 'trimStart', + 'trimEnd', + 'padStart', + 'padEnd', + 'repeat', + 'toISOString', + 'toLocaleString', + 'join', + 'charAt', + 'replace', + 'replaceAll', + 'normalize', + 'substring', +]) +const BOOLEAN_METHODS: ReadonlySet = new Set([ + 'includes', + 'startsWith', + 'endsWith', + 'some', + 'every', + 'test', + 'hasOwnProperty', +]) +const NUMBER_METHODS: ReadonlySet = new Set([ + 'indexOf', + 'lastIndexOf', + 'findIndex', + 'getTime', + 'charCodeAt', +]) +const ARRAY_METHODS: ReadonlySet = new Set(['map', 'filter', 'split', 'flat', 'flatMap']) +const STATIC_KINDS: Readonly> = { + 'Number.isFinite': 'boolean', + 'Number.isInteger': 'boolean', + 'Number.isNaN': 'boolean', + 'Number.isSafeInteger': 'boolean', + 'Number.parseFloat': 'number', + 'Number.parseInt': 'number', + 'Array.isArray': 'boolean', + 'Object.keys': 'typed', + 'Object.values': 'typed', + 'Object.entries': 'typed', + 'JSON.stringify': 'string', + 'Date.now': 'number', +} +const ARITHMETIC: ReadonlySet = new Set([ + '-', + '*', + '/', + '%', + '**', + '|', + '&', + '^', + '<<', + '>>', + '>>>', +]) +const COMPARISON: ReadonlySet = new Set([ + '==', + '===', + '!=', + '!==', + '<', + '<=', + '>', + '>=', + 'in', + 'instanceof', +]) + +/** Each function block's returned field kinds, read once per compiled block object. */ +const cache = new WeakMap | null>() + +function isFunctionNode(node: Node): boolean { + return ( + node.type === 'FunctionDeclaration' || + node.type === 'FunctionExpression' || + node.type === 'ArrowFunctionExpression' || + node.type === 'ObjectMethod' || + node.type === 'ClassMethod' || + node.type === 'ClassPrivateMethod' + ) +} + +/** Every node under `root`, calling `visit` on each; `enterFunctions` false skips nested bodies. */ +function walk(root: Node, enterFunctions: boolean, visit: (node: Node) => void): void { + const stack: Node[] = [root] + while (stack.length > 0) { + const node = stack.pop() + if (!node) break + visit(node) + if (!enterFunctions && node !== root && isFunctionNode(node)) continue + stack.push(...childNodes(node)) + } +} + +/** The names a declaration pattern binds; `null` for a destructuring, whose kinds are unknown. */ +function boundName(pattern: PatternLike | Node): string | null { + return pattern.type === 'Identifier' ? pattern.name : null +} + +/** The local variables of the code and what each is set to, for {@link Inference.identifier}. */ +interface Bindings { + /** The initializer of each name declared once with one; `null` for any other declaration. */ + initial: Map + /** Every assignment to each name, anywhere in the code (callbacks included). */ + writes: Map> +} + +function collectBindings(program: Node): Bindings { + const initial = new Map() + const writes = new Map>() + const declare = (name: string, value: Expression | null) => { + initial.set(name, initial.has(name) ? null : value) + } + walk(program, true, (node) => { + if (node.type === 'VariableDeclarator') { + const name = boundName(node.id) + if (name !== null) declare(name, node.init ?? null) + else for (const child of childNodes(node.id)) walkPatternNames(child, (n) => declare(n, null)) + } else if (node.type === 'FunctionDeclaration' && node.id) { + declare(node.id.name, null) + } else if (isFunctionNode(node) && 'params' in node) { + for (const param of node.params) walkPatternNames(param, (n) => declare(n, null)) + } else if (node.type === 'CatchClause' && node.param) { + walkPatternNames(node.param, (n) => declare(n, null)) + } else if (node.type === 'AssignmentExpression' && node.left.type === 'Identifier') { + const list = writes.get(node.left.name) ?? [] + list.push({ operator: node.operator, value: node.right }) + writes.set(node.left.name, list) + } else if (node.type === 'AssignmentExpression') { + walkPatternNames(node.left, (n) => { + const list = writes.get(n) ?? [] + list.push({ operator: '=', value: null }) + writes.set(n, list) + }) + } else if ( + (node.type === 'ForOfStatement' || node.type === 'ForInStatement') && + node.left.type !== 'VariableDeclaration' + ) { + // `for (x of …)` reassigns an existing `x` each pass, with no assignment expression. + walkPatternNames(node.left, (n) => { + const list = writes.get(n) ?? [] + list.push({ operator: '=', value: null }) + writes.set(n, list) + }) + } else if (node.type === 'UpdateExpression' && node.argument.type === 'Identifier') { + const list = writes.get(node.argument.name) ?? [] + list.push({ operator: '++', value: null }) + writes.set(node.argument.name, list) + } + }) + return { initial, writes } +} + +/** Calls `found` with every identifier a pattern binds. */ +function walkPatternNames(pattern: Node, found: (name: string) => void): void { + walk(pattern, false, (node) => { + if (node.type === 'Identifier') found(node.name) + }) +} + +/** Two kinds of the same value (both branches of `a ? b : c`), as one kind, or unknown. */ +function merge(a: JsKind | undefined, b: JsKind | undefined): JsKind | undefined { + if (a === undefined || b === undefined) return undefined + if (a === b) return a + return a !== 'string' && b !== 'string' ? 'typed' : undefined +} + +/** The member name a non-computed (or string-keyed) member expression or call reads. */ +function memberName(node: Node): string | null { + if (node.type !== 'MemberExpression' && node.type !== 'OptionalMemberExpression') return null + if (!node.computed && node.property.type === 'Identifier') return node.property.name + return node.property.type === 'StringLiteral' ? node.property.value : null +} + +/** Infers expression kinds over one program's bindings, guarding against self-reference. */ +class Inference { + private readonly resolving = new Set() + + constructor(private readonly bindings: Bindings) {} + + kind(node: Node | null | undefined): JsKind | undefined { + if (!node) return undefined + switch (node.type) { + case 'NumericLiteral': + case 'BigIntLiteral': + return 'number' + case 'StringLiteral': + case 'TemplateLiteral': + return 'string' + case 'BooleanLiteral': + return 'boolean' + case 'NullLiteral': + case 'ArrayExpression': + case 'ObjectExpression': + return 'typed' + case 'UnaryExpression': + if (node.operator === '!' || node.operator === 'delete') return 'boolean' + if (node.operator === 'typeof') return 'string' + if (node.operator === '-' || node.operator === '+' || node.operator === '~') return 'number' + return undefined + case 'BinaryExpression': + return this.binary(node.operator, node.left, node.right) + case 'LogicalExpression': + return merge(this.kind(node.left), this.kind(node.right)) + case 'ConditionalExpression': + return merge(this.kind(node.consequent), this.kind(node.alternate)) + case 'SequenceExpression': + return this.kind(node.expressions.at(-1)) + case 'AssignmentExpression': + return node.operator === '=' ? this.kind(node.right) : undefined + case 'UpdateExpression': + return 'number' + case 'MemberExpression': + case 'OptionalMemberExpression': + return memberName(node) === 'length' && + (this.kind(node.object) === 'string' || this.isArray(node.object)) + ? 'number' + : undefined + case 'CallExpression': + case 'OptionalCallExpression': + return this.call(node.callee) + case 'Identifier': + return this.identifier(node.name) + default: + return undefined + } + } + + private binary(operator: string, left: Node, right: Node): JsKind | undefined { + if (COMPARISON.has(operator)) return 'boolean' + if (ARITHMETIC.has(operator)) return 'number' + if (operator !== '+') return undefined + const [a, b] = [this.kind(left), this.kind(right)] + if (a === 'string' || b === 'string') return 'string' + return a === 'number' && b === 'number' ? 'number' : undefined + } + + private call(callee: Node): JsKind | undefined { + if (callee.type === 'Identifier') { + if (callee.name === 'String') return 'string' + if (NUMBER_FUNCTIONS.has(callee.name)) return 'number' + if (BOOLEAN_FUNCTIONS.has(callee.name)) return 'boolean' + return undefined + } + const method = memberName(callee) + if (method === null || !('object' in callee)) return undefined + if (callee.object.type === 'Identifier') { + if (callee.object.name === 'Math') return 'number' + const known = STATIC_KINDS[`${callee.object.name}.${method}`] + if (known) return known + } + if (STRING_METHODS.has(method)) return 'string' + if (BOOLEAN_METHODS.has(method)) return 'boolean' + if (NUMBER_METHODS.has(method)) return 'number' + if (ARRAY_METHODS.has(method)) return 'typed' + return undefined + } + + /** + * Whether `node` is certainly an array: a literal, an array method's result, `Object.keys` + * (`values`, `entries`), or a local variable set to one and never reassigned. Any other object + * can hold anything under `length`. + */ + private isArray(node: Node): boolean { + if (node.type === 'ArrayExpression') return true + if (node.type === 'CallExpression' || node.type === 'OptionalCallExpression') { + const method = memberName(node.callee) + if (method === null || !('object' in node.callee)) return false + if (node.callee.object.type === 'Identifier' && node.callee.object.name === 'Object') { + return method === 'keys' || method === 'values' || method === 'entries' + } + return ARRAY_METHODS.has(method) + } + if (node.type !== 'Identifier') return false + const init = this.bindings.initial.get(node.name) + if (!init || this.bindings.writes.has(node.name) || this.resolving.has(node.name)) return false + this.resolving.add(node.name) + try { + return this.isArray(init) + } finally { + this.resolving.delete(node.name) + } + } + + /** + * A local variable's kind: its initializer's, kept only when every later assignment keeps it + * (`+=` of a number to a number, of anything to a string; `++`; `=` of the same kind). + */ + private identifier(name: string): JsKind | undefined { + const init = this.bindings.initial.get(name) + if (!init || this.resolving.has(name)) return undefined + this.resolving.add(name) + try { + const declared = this.kind(init) + if (declared === undefined) return undefined + for (const { operator, value } of this.bindings.writes.get(name) ?? []) { + if (operator === '++') { + if (declared !== 'number') return undefined + } else if (operator === '=') { + if (merge(declared, this.kind(value)) !== declared) return undefined + } else if (operator === '+=') { + const added = this.kind(value) + if (!(declared === 'string' || (declared === 'number' && added === 'number'))) { + return undefined + } + } else if (ARITHMETIC.has(operator.slice(0, -1))) { + if (declared !== 'number') return undefined + } else { + return undefined + } + } + return declared + } finally { + this.resolving.delete(name) + } + } +} + +/** + * The fields one returned object literal sets, by name. A spread, or a key not known statically + * (computed, or any other key syntax), may replace any field before it, so it makes earlier ones + * unknown; a method or accessor makes its own field unknown. + */ +function returnedFields( + object: ObjectExpression, + inference: Inference +): Map { + const fields = new Map() + for (const property of object.properties) { + const key = + property.type === 'SpreadElement' || property.computed + ? null + : property.key.type === 'Identifier' + ? property.key.name + : property.key.type === 'StringLiteral' + ? property.key.value + : property.key.type === 'NumericLiteral' + ? String(property.key.value) + : null + if (key === null) { + for (const known of fields.keys()) fields.set(known, undefined) + continue + } + fields.set(key, property.type === 'ObjectProperty' ? inference.kind(property.value) : undefined) + } + return fields +} + +/** The field kinds a function body's returns agree on, or `null` when any return is no literal. */ +function readReturns(code: string): ReadonlyMap | null { + let program: Node + try { + const file = parse(toCodeFile(code, 'javascript'), { + sourceType: 'unambiguous', + allowReturnOutsideFunction: true, + allowAwaitOutsideFunction: true, + errorRecovery: true, + }) + if (file.errors && file.errors.length > 0) return null + program = file.program + } catch { + return null + } + const returns: Array = [] + walk(program, false, (node) => { + if (node.type === 'ReturnStatement') returns.push(node.argument) + }) + if (returns.length === 0) return null + const inference = new Inference(collectBindings(program)) + let agreed: Map | null = null + for (const argument of returns) { + if (argument?.type !== 'ObjectExpression') return null + const fields = returnedFields(argument, inference) + if (agreed === null) { + agreed = fields + continue + } + for (const [key, kind] of agreed) { + agreed.set(key, fields.has(key) ? merge(kind, fields.get(key)) : undefined) + } + } + const kinds = new Map() + for (const [key, kind] of agreed ?? []) { + if (kind !== undefined) kinds.set(key, kind === 'string' ? 'string' : 'typed') + } + return kinds +} + +/** + * The kind of `result.` of a JavaScript Function block, from the object literal its code + * returns; `undefined` for any other block, path or language, or when the code does not say. + */ +function functionResultKind( + block: CanonicalBlock, + path: readonly string[] +): ReturnedKind | undefined { + if (block.type !== FUNCTION_TYPE || path.length !== 2 || path[0] !== 'result') return undefined + let fields = cache.get(block) + if (fields === undefined) { + const code = getOwn(block.params, 'code') + const language = getOwn(block.params, 'language') + const javascript = + language === undefined || + (language.kind === 'value' && (language.value === 'javascript' || language.value === '')) + fields = + javascript && code?.kind === 'value' && typeof code.value === 'string' + ? readReturns(code.value) + : null + cache.set(block, fields) + } + return fields?.get(path[1]) +} + +/** + * What a stored reference (``) holds when it names a field a Function block's + * code returns as a literal: see {@link functionResultKind}. `block` finds a compiled block by key. + */ +export function referencedResultKind( + token: string, + block: (key: BlockKey) => CanonicalBlock | undefined +): ReturnedKind | undefined { + if (!token.startsWith('<') || !token.endsWith('>')) return undefined + const path = parseStoredPath(token.slice(1, -1)) + if (!path) return undefined + const names = path.map((segment) => (segment.kind === 'name' ? segment.name : null)) + const [head, ...rest] = names + if (head === null || head === undefined || rest.some((name) => name === null)) return undefined + const target = block(head) + return target ? functionResultKind(target, rest as string[]) : undefined +} diff --git a/packages/workflow-code/src/compile/grammar.test.ts b/packages/workflow-code/src/compile/grammar.test.ts index 63b32b6b411..cb7792dac36 100644 --- a/packages/workflow-code/src/compile/grammar.test.ts +++ b/packages/workflow-code/src/compile/grammar.test.ts @@ -1062,6 +1062,48 @@ describe('errors, each with its code, location and hint', () => { ]) }) + it('resolves a code file $ref to the stored key of a block the source renamed', () => { + const text = source( + [ + START, + "const lookup = api('Fetch Page', { url: 'x' })", + "const f = fn('F', { code: codeFile('./fns/f.js') })", + ].join('\n') + ) + const files = { 'workflows/fns/f.js': 'return $ref.lookup.data + $ref.lookup.nope' } + const lockBlock = (id: string, ident: string, order: number) => ({ + id, + parent: null, + ident, + order, + }) + const storedUnread = { + version: 1 as const, + blocks: { + start: lockBlock('b-start', 'start', 0), + lookup: lockBlock('b-lookup', 'lookup', 1), + f: lockBlock('b-f', 'f', 2), + }, + variables: {}, + } + const compile = (input: Partial[0]>) => + compileWorkflow({ + entry: { path: ENTRY, text }, + files: new Map(Object.entries(files)), + catalog: TEST_CATALOG, + sdkMajor: 1, + ...input, + }) + const codes = (result: CompileResult) => + result.diagnostics + .filter((d) => d.span.file === 'workflows/fns/f.js') + .map((d) => `${d.code} ${d.span.column}`) + const expected = ['W_UNKNOWN_OUTPUT_PATH 27'] + expect(codes(compile({ storedUnread }))).toEqual(expected) + expect(codes(compile({ renamedKeys: new Map([['lookup', 'fetchpage']]) }))).toEqual(expected) + expect(codes(compile({}))).toEqual(['W_UNRESOLVED_REF 8', 'W_UNRESOLVED_REF 27']) + }) + it('reports a syntax error inside a code file at its own location', () => { const result = run(source("const f = fn('F', { code: codeFile('./fns/bad.js') })"), { 'workflows/fns/bad.js': 'return (', diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 1677338661d..06024dd4426 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -6,7 +6,7 @@ import { VARIABLE_TYPES, type VariableType, } from '@sim/workflow-types/workflow' -import { encodeAnchor } from '../anchors' +import { edgeTupleKey, encodeAnchor, portKey } from '../anchors' import { literalText } from '../babel' import { compareEdges } from '../canonicalize/edges' import type { BlockCatalog } from '../catalog/types' @@ -21,23 +21,26 @@ import type { CanonicalBlock, CanonicalVariable, CanonicalWorkflow, + CanonicalWorkflowMeta, + PortKey, VarKey, } from '../model' -import { setOwn } from '../records' +import { getOwn, setOwn } from '../records' import { OLDEST_SDK_MAJOR, SDK_MODULE } from '../sdk' import { compileBlock } from './blocks' import { checkStorableLiterals, checkStorableText } from './characters' import { CommentCollector } from './comments' import { type CompileContext, rangeOf, recordSpan, type Scope, unsupported } from './context' import { EdgeCollector } from './edges' +import { referencedResultKind } from './function-returns' import { didYouMean, HINTS, hintForNode, nodeLabel } from './hints' import { type CompileLimits, DEFAULT_LIMITS, parseWorkflowSource, sdkHeader } from './parse' import { ErrorLimitReached, Reporter } from './report' import { checkSuspiciousName, collectBody, collisionSuffix } from './scope' import { checkCodeFileRefs, + checkDynamicOutputRefs, checkErrorOutputRefs, - checkResponseFormatRefs, compileJsonLiteral, objectEntries, } from './values' @@ -73,6 +76,21 @@ export interface CompileInput { * workflow holds it, not an error. Ignored when `stored` is given. */ storedUnread?: WorkflowLockV1 + /** + * With `storedUnread`, the edges the stored workflow held when that lock was written, by lock + * key (a get's printed source compiled: `CompileResult.edges`). An edge between bound blocks + * then counts as possibly stored only when it is one of these, so a cycle or an error edge the + * source adds between bound blocks is the error the server reports. Absent, any such edge may + * be stored. + */ + storedUnreadEdges?: ReadonlyArray + /** + * Stored block keys the source renamed, stored key → source key (`lockIdentRenames`, which the + * server computes against `stored`; with `storedUnread` the compile computes them itself). An + * apply rewrites every `` to the new key, so a code file's `$ref.` + * still resolves and is checked against the renamed block instead of warning. + */ + renamedKeys?: ReadonlyMap /** * The env variable names a run resolves (the workspace's plus the caller's personal ones). * `env.NAME` outside it compiles with `W_ENV_UNKNOWN`; absent, no name is checked. @@ -93,6 +111,11 @@ export interface CompileResult { diagnostics: LocatedDiagnostic[] /** Where each anchor sits, to map later server findings back to lines. */ spans: Partial> + /** + * Where `workflow()` writes the name and the description, so a server finding about the + * workflow itself (a name another workflow holds, a block delete) points there. + */ + metaSpans: Partial> /** Idents as written, `order` as declaration index, and code files, for the lock. */ hints: SourceHints /** The header's major, or `null` when the header is missing. */ @@ -102,6 +125,19 @@ export interface CompileResult { * workspace apply checks against the secrets its directory declares. */ envReferences: string[] + /** + * Every edge the wiring denotes that passed validation, by block key, whether or not the + * compile failed elsewhere: what an offline check records of a get's printed source as + * `CompileInput.storedUnreadEdges`. + */ + edges: StoredEdgeRef[] +} + +/** One edge as `CompileInput.storedUnreadEdges` names it: block keys and the port's `portKey`. */ +export interface StoredEdgeRef { + from: BlockKey + port: PortKey + to: BlockKey } const WORKFLOW_OPTIONS = ['description', 'variables'] @@ -283,6 +319,7 @@ function compileWorkflowCall( } ) } else if (nameNode) { + ctx.metaSpans.name = ctx.reporter.span(rangeOf(ctx, nameNode)) checkSuspiciousName(ctx, name, nameNode, 'workflow') } const optionsNode: Node | undefined = third ? second : undefined @@ -300,6 +337,7 @@ function compileWorkflowCall( } else { for (const [key, property] of objectEntries(ctx, optionsNode)) { if (key === 'description') { + ctx.metaSpans.description = ctx.reporter.span(rangeOf(ctx, property.value)) description = literalText(property.value) if (description === null) unsupported( @@ -384,14 +422,32 @@ function emptyResult(reporter: Reporter, sdkMajor: number | null): CompileResult workflow: null, diagnostics: reporter.diagnostics, spans: {}, + metaSpans: {}, hints: { blocks: {} }, sdkMajor, envReferences: [], + edges: [], } } const BYTE_ORDER_MARK = '\uFEFF' +/** + * Whether the stored workflow holds `value` in `param` of block `key` already: a JSON field the + * source left as it was printed, whose untyped references the edit did not write, so + * `W_JSON_REF_UNTYPED` is not repeated on every apply of a workflow it was saved with. + */ +function storedHolds( + stored: CanonicalWorkflow | undefined, + key: BlockKey, + param: string, + value: string +): boolean { + const held = stored && getOwn(stored.blocks, key) + const storedParam = held ? getOwn(held.params, param) : undefined + return storedParam?.kind === 'value' && storedParam.value === value +} + /** * Statically interprets a workflow source file into the canonical model. Pass 1 collects * imports and every block declaration, nested bodies included; pass 2 evaluates values, @@ -473,13 +529,15 @@ function compileEntry( wiring: [], implied: [], spans: new Map(), + metaSpans: {}, knownKeys: input.knownKeys ?? new Set(), stored: input.stored, + storedUnread: input.stored === undefined ? input.storedUnread : undefined, envNames: input.envNames, envDeclaredIn: input.envDeclaredIn, envReferences: new Set(), jsonFields: [], - responseFormatRefs: [], + dynamicOutputRefs: [], errorOutputRefs: [], refusedBranches: new Map(), valueConsts: new Set(), @@ -592,7 +650,11 @@ function compileEntry( }) } - checkCodeFileRefs(ctx) + const unread = input.stored === undefined ? input.storedUnread : undefined + const unreadRenames = unread + ? lockIdentRenames(hints, unread, (key) => Object.hasOwn(unread.blocks, key)) + : new Map() + checkCodeFileRefs(ctx, new Map([...unreadRenames, ...(input.renamedKeys ?? [])])) const refTypes: RefTypeScope = { catalog: input.catalog, @@ -601,9 +663,23 @@ function compileEntry( } const isString = (token: string) => refValueKind(token, refTypes) === 'string' for (const { params, id, tree, refs } of ctx.jsonFields) { - setOwn(params, id, { kind: 'value', value: jsonFieldText(tree, isString) }) + const value = jsonFieldText(tree, isString) + setOwn(params, id, { kind: 'value', value }) for (const ref of refs) { if (refValueKind(ref.token, refTypes) !== 'untyped') continue + const returned = referencedResultKind(ref.token, refTypes.block) + if (returned === 'typed') continue + // An unchanged field stays quiet only while what it reads is too: a Function that now + // returns text where it returned a number makes the same bare token break the JSON. + if ( + storedHolds(input.stored, ref.blockKey, id, value) && + referencedResultKind( + ref.token, + (key) => input.stored && getOwn(input.stored.blocks, key) + ) === returned + ) { + continue + } reporter.report( 'W_JSON_REF_UNTYPED', `Nothing declares the type of ${ref.token}; it is stored bare, so a string value would break the JSON.`, @@ -616,17 +692,24 @@ function compileEntry( } } - checkResponseFormatRefs(ctx, (key) => compiled.get(key)) + checkDynamicOutputRefs(ctx, (key) => compiled.get(key)) - const unread = input.stored === undefined ? input.storedUnread : undefined const edges = new EdgeCollector( ctx, compiled, - unread && - new Set([ - ...Object.keys(unread.blocks), - ...lockIdentRenames(hints, unread, (key) => Object.hasOwn(unread.blocks, key)).values(), - ]) + unread && { + lockKeys: new Map([ + ...Object.keys(unread.blocks).map((key) => [key, key] as const), + ...[...unreadRenames].map(([lockKey, key]) => [key, lockKey] as const), + ]), + ...(input.storedUnreadEdges + ? { + edges: new Set( + input.storedUnreadEdges.map((edge) => edgeTupleKey(edge.from, edge.port, edge.to)) + ), + } + : {}), + } ) for (const entry of ctx.wiring) edges.statement(entry) for (const implied of ctx.implied) edges.implied(implied.container, implied.child, implied.body) @@ -664,8 +747,10 @@ function compileEntry( workflow, diagnostics: reporter.diagnostics, spans, + metaSpans: ctx.metaSpans, hints, sdkMajor: parsed.sdkMajor, envReferences: [...ctx.envReferences].sort(compareStrings), + edges: sortedEdges.map((edge) => ({ from: edge.from, port: portKey(edge.port), to: edge.to })), } } diff --git a/packages/workflow-code/src/compile/validate-catalog.ts b/packages/workflow-code/src/compile/validate-catalog.ts index 2b03684aef4..6b9429b0919 100644 --- a/packages/workflow-code/src/compile/validate-catalog.ts +++ b/packages/workflow-code/src/compile/validate-catalog.ts @@ -1,7 +1,13 @@ -import { type ConditionLookup, evaluateClause, evaluateCondition } from '../catalog/conditions' +import { + type ConditionClause, + type ConditionLookup, + evaluateClause, + evaluateCondition, +} from '../catalog/conditions' import type { CatalogField, FieldIndex } from '../catalog/surface' import type { CatalogSubBlock } from '../catalog/types' import { isPlainRecord, type SubBlockId, type Value } from '../model' +import { getOwn } from '../records' /** The field a written param name targets: its written name or, when aliased, its own name. */ export function fieldFor(index: FieldIndex, key: string): CatalogField | undefined { @@ -18,6 +24,55 @@ export function isVisible(sub: CatalogSubBlock, values: ReadonlyMap +): string | null { + const { condition, visibleFor, visibleWhen, visibleForModel } = sub + const rejected = (field: string) => { + const value = values.get(field) + return `${field} ${value === undefined || value === null ? 'unset' : String(value)} does not use it` + } + if (!condition) return null + const lookup = paramLookup(values) + /** The part of `clause` that hides the field, described; `null` when none does. */ + const failingPart = (clause: ConditionClause): string | null => { + const failing = [clause, clause.and].find( + (part): part is NonNullable => + part !== undefined && + evaluateClause({ field: part.field, value: part.value, not: part.not }, lookup) === false + ) + if (!failing) return null + if (Array.isArray(failing.value) && failing.value.length > LISTED_VALUES) { + return rejected(failing.field) + } + const wanted = Array.isArray(failing.value) + ? failing.value.map(String).join(' or ') + : String(failing.value) + return `it shows only when ${failing.field} is ${failing.not ? 'not ' : ''}${wanted}` + } + if (!('opaque' in condition)) return failingPart(condition) + // The order `evaluateCondition` decides an opaque condition in: the operation, then the clause + // `visibleWhen` still asks for it, and the model only for an operation `visibleFor` omits. + const operation = values.get('operation') + if (typeof operation === 'string' && visibleFor && Object.hasOwn(visibleFor, operation)) { + if (!visibleFor[operation]) return rejected('operation') + const asked = visibleWhen && getOwn(visibleWhen, operation) + return asked ? failingPart(asked) : null + } + const model = values.get('model') + if (typeof model === 'string' && visibleForModel?.[model] === false) return rejected('model') + return null +} + /** * The fields a block leaves empty although they are required under the values it runs with * (`W_REQUIRED_MISSING`). `values` are the block's compiled params, which already hold each diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index e1e4e1e625c..8461ab6a913 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -47,6 +47,7 @@ import { } from '../model' import { getOwn, setOwn } from '../records' import { isTemplateTag, SDK_RESOURCE_CALLS, type SdkCoreExport, TEMPLATE_TAGS } from '../sdk' +import { inputFormatFields } from '../start-inputs' import { type BlockDecl, type CompileContext, @@ -700,8 +701,12 @@ function checkOutputPath( return } const block = catalogBlockOf(target) - if (block?.dynamicOutputs?.source === 'responseFormat') { - ctx.responseFormatRefs.push({ at, target, segments, blockKey }) + const dynamicSource = block?.dynamicOutputs?.source + if ( + dynamicSource === 'responseFormat' || + (dynamicSource === 'inputFormat' && block && DECLARED_INPUT_TRIGGERS.has(block.type)) + ) { + ctx.dynamicOutputRefs.push({ at, target, segments, blockKey }) return } // Table rows only refine the element type of `rows`/`row`; the outputs themselves are static. @@ -750,23 +755,59 @@ export function checkErrorOutputRefs(ctx: CompileContext, erroring: ReadonlySet< } /** - * Checks each reference into a block whose outputs a `responseFormat` param declares, once every - * block is compiled: the first path segment must be a field the format declares or one of the - * block's static outputs (`W_UNKNOWN_OUTPUT_PATH` otherwise). A format the compiler cannot read - * (not JSON, or holding references) leaves every path allowed; an absent one leaves the static - * outputs. + * The input triggers whose outputs are exactly their declared inputs plus static outputs, as the + * editor's tag picker lists them (`getEffectiveBlockOutputs`). The legacy starter's outputs + * depend on its mode, so it is left out. */ -export function checkResponseFormatRefs( +const DECLARED_INPUT_TRIGGERS: ReadonlySet = new Set([ + 'start_trigger', + 'api_trigger', + 'input_trigger', +]) + +/** The unified Start's run-metadata output, present while its `runMetadata` switch is on. */ +const START_RUN_METADATA = 'metadata' + +/** + * The output fields a block's dynamic-outputs param declares: an agent's `responseFormat` fields, + * or an input trigger's `inputFormat` inputs (plus the unified Start's `metadata` while run + * metadata is on). `null` when the param cannot be read statically. + */ +function declaredOutputFields( + source: 'responseFormat' | 'inputFormat', + compiled: CanonicalBlock, + param: string +): string[] | null { + if (source === 'responseFormat') return responseFormatFields(getOwn(compiled.params, param)) + const inputs = inputFormatFields(getOwn(compiled.params, param)) + if (inputs === null) return null + const names = inputs.map((input) => input.name) + const metadata = getOwn(compiled.params, 'runMetadata') + return metadata?.kind === 'value' && metadata.value === true + ? [...names, START_RUN_METADATA] + : names +} + +/** + * Checks each reference into a block whose outputs a param declares (an agent's + * `responseFormat`, an input trigger's `inputFormat`), once every block is compiled: the first + * path segment must be a declared field or one of the block's static outputs + * (`W_UNKNOWN_OUTPUT_PATH` otherwise). A warning, as for any unknown output path: a caller may + * still pass an undeclared input, which the unified Start passes through. A param the compiler + * cannot read (not JSON, or holding references) leaves every path allowed; an absent one leaves + * the static outputs. + */ +export function checkDynamicOutputRefs( ctx: CompileContext, blockOf: (key: BlockKey) => CanonicalBlock | undefined ): void { - for (const { at, target, segments, blockKey } of ctx.responseFormatRefs) { + for (const { at, target, segments, blockKey } of ctx.dynamicOutputRefs) { const block = catalogBlockOf(target) const compiled = blockOf(target.key) const dynamic = block?.dynamicOutputs if (!block || !compiled || !dynamic) continue - const param = getOwn(compiled.params, dynamic.param) - const fields = responseFormatFields(param) + if (dynamic.source !== 'responseFormat' && dynamic.source !== 'inputFormat') continue + const fields = declaredOutputFields(dynamic.source, compiled, dynamic.param) const [first] = segments if (fields === null || first?.kind !== 'name' || fields.includes(first.name)) continue const builder = target.builder.kind === 'catalog' ? target.builder.builder : undefined @@ -777,11 +818,28 @@ export function checkResponseFormatRefs( continue } const known = [...fields, ...Object.keys(outputs)] + const path = formatStoredPath(segments) + if (dynamic.source === 'inputFormat') { + ctx.reporter.report( + 'W_UNKNOWN_OUTPUT_PATH', + fields.length > 0 + ? `${target.name} declares no input ${first.name}: its inputFormat declares ${fields.join(', ')}.` + : `${target.name} declares no input ${first.name}: its inputFormat declares none.`, + at, + { + hint: () => + didYouMean(first.name, known) ?? + `Add ${first.name} to ${target.ident ?? target.name}'s inputFormat, or reference an input it declares.`, + blockKey, + } + ) + continue + } ctx.reporter.report( 'W_UNKNOWN_OUTPUT_PATH', fields.length > 0 - ? `${target.name} has no output ${formatStoredPath(segments)}: its responseFormat declares ${fields.join(', ')}.` - : `${target.name} has no output ${formatStoredPath(segments)}, and declares no responseFormat field of that name.`, + ? `${target.name} has no output ${path}: its responseFormat declares ${fields.join(', ')}.` + : `${target.name} has no output ${path}, and declares no responseFormat field of that name.`, at, { hint: () => @@ -799,9 +857,14 @@ export function checkResponseFormatRefs( * that names no block (a block's ident, a renamed block's old key, a typo) or a variable that does * not exist is `W_UNRESOLVED_REF`, and a path no output has is `W_UNKNOWN_OUTPUT_PATH`. Warnings, * not errors, because stored code may already hold a dangling reference and must round-trip; at - * run time such a token is spliced into the code as text, which fails as JavaScript. + * run time such a token is spliced into the code as text, which fails as JavaScript. A head that + * is the stored key of a block the source renamed (`renamed`, stored key → source key) resolves: + * an apply rewrites it to the new key, so its path is checked against the renamed block. */ -export function checkCodeFileRefs(ctx: CompileContext): void { +export function checkCodeFileRefs( + ctx: CompileContext, + renamed: ReadonlyMap +): void { for (const { path, blockKey, refs } of ctx.codeFiles.chains) { for (const ref of refs) { const at: Range = { file: path, start: ref.start, end: ref.end } @@ -829,7 +892,10 @@ export function checkCodeFileRefs(ctx: CompileContext): void { continue } if (isReservedWorkflowBlockName(head.name)) continue - const target = ctx.declsByKey.get(head.name) + const renamedTo = renamed.get(head.name) + const target = + ctx.declsByKey.get(head.name) ?? + (renamedTo === undefined ? undefined : ctx.declsByKey.get(renamedTo)) if (target) { checkOutputPath(ctx, at, target, rest, blockKey) continue diff --git a/packages/workflow-code/src/diagnostics/codes.ts b/packages/workflow-code/src/diagnostics/codes.ts index d4f6574686a..6dd3ae5eb1d 100644 --- a/packages/workflow-code/src/diagnostics/codes.ts +++ b/packages/workflow-code/src/diagnostics/codes.ts @@ -127,7 +127,8 @@ const COMPILE_EDGE_CODES = [ /** * `.onError()` on a block with no error port (a loop or parallel) that the stored workflow * already wires: kept so it round-trips; source never adds one. Unread (`storedUnread`), any such - * edge between two bound blocks, which an apply keeps only if the stored workflow holds it. + * edge between two bound blocks (with `storedUnreadEdges`, one the get printed), which an apply + * keeps only if the stored workflow holds it. */ 'W_ERROR_PORT_STORED', /** `x.key.to()` where `key` is not a branch of `x`. */ @@ -447,6 +448,11 @@ const APPLY_DIAGNOSTIC_CODES = [ 'W_APPLY_TRIGGER_REMOVED', /** Compile failed and the catalog changed since the source was read: get the source again. */ 'W_APPLY_CATALOG_MOVED', + /** + * An apply passed `allowDelete` deletes a block and adds one of the same type and parent: a + * probable rename whose name and const both changed, which gets a new id. + */ + 'W_APPLY_PROBABLE_RENAME', /** A blocking workflow lint finding, mapped to a source line. */ 'E_APPLY_LINT', /** A non-blocking workflow lint finding, mapped to a source line. */ diff --git a/packages/workflow-code/src/equivalence/executable.ts b/packages/workflow-code/src/equivalence/executable.ts index e7d3398e015..a50b2e1e5d5 100644 --- a/packages/workflow-code/src/equivalence/executable.ts +++ b/packages/workflow-code/src/equivalence/executable.ts @@ -16,7 +16,7 @@ */ import { compareEdges } from '../canonicalize/edges' -import { SUBFLOW_DATA_KEYS } from '../canonicalize/subflows' +import { SUBFLOW_DATA_KEYS, SUPERSEDED_CONTAINER_PARAMS } from '../canonicalize/subflows' import { executableInactiveParams } from '../catalog/activity' import { isCanonicalPair } from '../catalog/canonical-index' import { serializedCanonicalIndex } from '../catalog/surface' @@ -89,7 +89,14 @@ function executableParams( declared: CatalogBlock | undefined, catalog: BlockCatalog ): Record { - if (!declared) return { ...block.params } + if (!declared) { + if (!block.subflow) return { ...block.params } + const params: Record = {} + for (const [id, param] of Object.entries(block.params)) { + if (!SUPERSEDED_CONTAINER_PARAMS.has(id)) setOwn(params, id, param) + } + return params + } const inactive = new Set(executableInactiveParams(block, catalog)) const params: Record = {} for (const [id, param] of Object.entries(block.params)) { diff --git a/packages/workflow-code/src/index.ts b/packages/workflow-code/src/index.ts index 07445c505fa..1cc4ce34d88 100644 --- a/packages/workflow-code/src/index.ts +++ b/packages/workflow-code/src/index.ts @@ -36,9 +36,9 @@ export { fromCodeFile, isIdentifierName, } from './code-files' -export { type CompileResult, compileWorkflow } from './compile' +export { type CompileResult, compileWorkflow, type StoredEdgeRef } from './compile' export { codePointLabel, scriptName } from './compile/characters' -export { didYouMean } from './compile/hints' +export { didYouMean, suggestBuilder } from './compile/hints' export { babelMessage, bracketDepthOverflow, diff --git a/packages/workflow-code/src/json-fields.test.ts b/packages/workflow-code/src/json-fields.test.ts index 96ea924d69a..80f76840906 100644 --- a/packages/workflow-code/src/json-fields.test.ts +++ b/packages/workflow-code/src/json-fields.test.ts @@ -3,7 +3,7 @@ import type { BlockCatalog, CatalogBlock } from './catalog/types' import { compileWorkflow } from './compile' import { TEST_CATALOG } from './compile/catalog.test-helpers' import { refValueKind, valueTree } from './json-fields' -import type { CanonicalBlock } from './model' +import type { CanonicalBlock, CanonicalWorkflow } from './model' const IMPORTS = "import { agent, api, raw, table, trigger, workflow } from '@sim/workflow-sdk'" @@ -142,3 +142,100 @@ describe('refValueKind for a block acting as a trigger', () => { expect(kind('', block)).toBe('untyped') }) }) + +describe('W_JSON_REF_UNTYPED on a Function block output', () => { + const FN_IMPORTS = "import { api, fn, trigger, workflow } from '@sim/workflow-sdk'" + + /** Compiles a Function block with `code` whose result `field` a JSON body uses bare. */ + function compile(code: string, field: string, stored?: CanonicalWorkflow) { + const text = `// @sim-sdk 1\n${FN_IMPORTS}\n\nexport default workflow('Test', {}, () => { + const start = trigger.start('Start') + const score = fn('Score', { code: ${JSON.stringify(code)} }) + const next = api('Next', { url: 'https://x', body: { v: score.result.${field} } }) + start.to(score) + score.to(next) +})\n` + return compileWorkflow({ + entry: { path: 'workflow.ts', text }, + files: new Map(), + catalog: TEST_CATALOG, + sdkMajor: 1, + ...(stored ? { stored } : {}), + }) + } + + function untyped(code: string, field: string, stored?: CanonicalWorkflow) { + return compile(code, field, stored).diagnostics.filter((d) => d.code === 'W_JSON_REF_UNTYPED') + } + + it('stays quiet on a field the code returns as a number, boolean, array or object literal', () => { + const code = [ + 'const items = $ref.start.items || []', + 'const tax = Math.round(items.length * 10) / 100', + 'let score = $ref.start.total > 2000 ? 90 : 20', + "if ($ref.start.tier === 'new') score += 10", + "const names = 'a,b'.split(',')", + 'return { tax, score, ok: score >= 70, count: names.length, chars: "ab".length, list: items.map((i) => i.sku), meta: {} }', + ].join('\n') + for (const field of ['tax', 'score', 'ok', 'count', 'chars', 'list', 'meta']) { + expect(untyped(code, field), field).toEqual([]) + } + }) + + it('keeps the warning on a field returned as text, which stored bare breaks the JSON', () => { + const code = "return { tier: 'gold', label: String(1) + 'x' }" + expect(untyped(code, 'tier')).toHaveLength(1) + expect(untyped(code, 'label')).toHaveLength(1) + }) + + it('keeps the warning when nothing certain says what the field holds', () => { + // A reference, a variable reassigned to text, a field only one return sets, a non-literal + // return, and a spread that may replace the field. + expect(untyped('return { v: $ref.start.v }', 'v')).toHaveLength(1) + expect(untyped("let v = 1\nif ($ref.start.x) v = 'one'\nreturn { v }", 'v')).toHaveLength(1) + expect(untyped('if ($ref.start.x) return { v: 1 }\nreturn { w: 2 }', 'v')).toHaveLength(1) + expect(untyped('if ($ref.start.x) return { v: 1 }\nreturn $ref.start.o', 'v')).toHaveLength(1) + expect(untyped('return { v: 1, ...$ref.start.o }', 'v')).toHaveLength(1) + expect(untyped('return { v: 1 ', 'v')).toHaveLength(1) + }) + + it('keeps the warning when a computed key, a loop or an object length may make the field text', () => { + // A computed key or a getter of the same name replaces the field written before it. + expect(untyped("const key = 'v'\nreturn { v: 1, [key]: 'text' }", 'v')).toHaveLength(1) + expect(untyped("return { v: 1, get v() { return 'text' } }", 'v')).toHaveLength(1) + // A for-of or for-in loop over an existing variable reassigns it without an assignment. + expect(untyped("let v = 1\nfor (v of ['a']) {}\nreturn { v }", 'v')).toHaveLength(1) + expect(untyped('let v = 1\nfor (v in { a: 1 }) {}\nreturn { v }', 'v')).toHaveLength(1) + expect(untyped("let v = 1\nfor ([v] of [['a']]) {}\nreturn { v }", 'v')).toHaveLength(1) + // `length` is a number only on a string or an array: an object can hold anything there. + expect(untyped("return { v: ({ length: 'oops' }).length }", 'v')).toHaveLength(1) + expect( + untyped("let list = []\nlist = { length: 'oops' }\nreturn { v: list.length }", 'v') + ).toHaveLength(1) + expect( + untyped('const items = $ref.start.items || []\nreturn { v: items.length }', 'v') + ).toHaveLength(1) + }) + + it('reads only the returns of the body, not those of a callback inside it', () => { + const code = "const f = (x) => { return { v: 'text' } }\nreturn { v: 1 }" + expect(untyped(code, 'v')).toEqual([]) + }) + + it('on an apply, warns only on a JSON field the source changed from the stored workflow', () => { + const code = 'return { v: $ref.start.v, w: $ref.start.w }' + const printed = compile(code, 'v').workflow + if (!printed) throw new Error('the source did not compile') + + expect(untyped(code, 'v', printed)).toEqual([]) + expect(untyped(code, 'w', printed)).toHaveLength(1) + }) + + it('on an apply, warns on an unchanged JSON field when the Function it reads now returns text there', () => { + const printed = compile('return { v: 1 }', 'v').workflow + if (!printed) throw new Error('the source did not compile') + + expect(untyped('return { v: 2 }', 'v', printed)).toEqual([]) + expect(untyped("return { v: 'one' }", 'v', printed)).toHaveLength(1) + }) +}) diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index b23cba01984..0feb02e8e76 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -1,12 +1,14 @@ -import type { BlockState } from '@sim/workflow-types/workflow' +import { findWorkflowReferenceTokens } from '@sim/utils/workflow-references' +import { type BlockState, normalizeWorkflowBlockName } from '@sim/workflow-types/workflow' import type { CanonicalizeResult } from '../canonicalize' import type { BlockCatalog } from '../catalog/types' import { catalogBlock } from '../catalog/view' +import { parseStoredPath } from '../code-files/ref-path' import type { MaterializeDiagnosticCode } from '../diagnostics/codes' import type { Diagnostic } from '../diagnostics/frame' import { pointer, type WorkflowLockBlock, type WorkflowLockV1 } from '../lock' -import type { BlockKey, CanonicalWorkflow } from '../model' -import { getOwn } from '../records' +import type { BlockKey, CanonicalBlock, CanonicalWorkflow } from '../model' +import { deepEqual, getOwn } from '../records' import type { MaterializeHooks } from './context' /** How identity rule matched a compiled block to a stored one (rules 1–4). */ @@ -248,6 +250,12 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { match, settled, diagnostics, + catalog, + lockedIds: new Set( + Object.entries(lock?.blocks ?? {}).flatMap(([key, entry]) => + duplicates.has(key) ? [] : [entry.id] + ) + ), }) } @@ -319,6 +327,12 @@ interface InferenceState { match: (key: BlockKey, prevId: string, via: IdentityMatchRule) => boolean settled: Map diagnostics: IdentityInputs['diagnostics'] + /** + * The ids the lock binds: blocks the author's source held. After rules 1–3 such a block is + * unmatched only when the source dropped both its key and its ident. + */ + lockedIds: ReadonlySet + catalog: BlockCatalog } /** `in:` / `out:` for every already-bound neighbour of a block. */ @@ -345,14 +359,98 @@ function sameSet(left: Set, right: Set): boolean { return left.size === right.size && [...left].every((item) => right.has(item)) } +/** A block's params with every `` reference token rewritten to `to`. */ +function withKeyRenamed(params: CanonicalBlock['params'], from: BlockKey, to: BlockKey) { + const rewrite = (value: unknown): unknown => { + if (typeof value === 'string') { + return value.split(`<${from}.`).join(`<${to}.`).split(`<${from}>`).join(`<${to}>`) + } + if (Array.isArray(value)) return value.map(rewrite) + if (value === null || typeof value !== 'object') return value + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewrite(item)])) + } + return rewrite(params) +} + +/** + * Whether two blocks' settings are equal, references to the renamed block's old key read as its + * new one: equal on every param both hold, and holding at least one in common. A param only one + * side holds is a default compile filled in or canvas state, which says nothing about identity. + */ +function settingsAgree( + target: CanonicalBlock, + prev: CanonicalBlock, + key: BlockKey, + prevKey: BlockKey +): boolean { + const left = target.params + const right = withKeyRenamed(prev.params, prevKey, key) as CanonicalBlock['params'] + const shared = Object.keys(left).filter((id) => Object.hasOwn(right, id)) + return shared.length > 0 && shared.every((id) => deepEqual(left[id], right[id])) +} + +/** Every string under `value`, depth first. */ +function* strings(value: unknown): Generator { + if (typeof value === 'string') yield value + else if (Array.isArray(value)) for (const item of value) yield* strings(item) + else if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) yield* strings(item) + } +} + +/** + * Whether a block holds a reference to `key` in any field a reference can sit in (its params, a + * condition or router's branches, a loop or parallel's collection), its head read as the resolver + * reads it: normalized, so `` references `fetch`. + */ +function referencesKey(block: CanonicalBlock, key: BlockKey): boolean { + for (const text of strings([block.params, block.branches, block.subflow])) { + for (const token of findWorkflowReferenceTokens(text)) { + if (token.kind !== 'workflow') continue + const head = parseStoredPath(token.value.slice(1, -1))?.[0] + if (head?.kind === 'name' && normalizeWorkflowBlockName(head.name) === key) return true + } + } + return false +} + +/** + * The strong identity signal that ties a new block to a vacated stored one, if any: `settings`, + * equal settings (references to the old key read as the new); `connections`, most of their + * already-bound neighbours shared; `references`, every bound block that referenced the old key + * now references the new one (the author renamed the references). + */ +type InferenceBasis = 'connections' | 'settings' | 'references' + +const INFERENCE_REASON: Record = { + connections: 'same type, parent and connections', + settings: 'same type, parent and settings', + references: 'same type and parent, and the references to it renamed with it', +} + /** - * Identity rule 4, repeated until it binds nothing new so a renamed container lets its - * renamed children pair next. A group is `(type, parent id)`; only a group with exactly one - * unmatched block on each side pairs, and only when both blocks have the same already-bound - * neighbours in the same directions. Each pairing is flagged with `W_MAT_INFERRED_RENAME`. + * Whether a block acts as a trigger: its id serves webhook paths, schedules and run history, so + * rule 4 binds one only on equal settings. + */ +function isTriggerLike(block: CanonicalBlock, catalog: BlockCatalog): boolean { + const declared = catalogBlock(catalog, block.type) + return ( + block.triggerMode || declared?.category === 'triggers' || declared?.graph.canBeTarget === false + ) +} + +/** + * Identity rule 4, repeated until it binds nothing new so a renamed container lets its renamed + * children pair next. A group is `(type, parent id)`. A group with exactly one unmatched block on + * each side pairs when both have the same already-bound neighbours in the same directions; then, + * once no group pairs that way, a new block pairs with a stored block the lock binds (its key and + * ident both left the source) when exactly one such block of its group, and no other new block, + * shows a strong identity signal ({@link InferenceBasis}). A trigger pairs only on equal + * settings. Each pairing is flagged with `W_MAT_INFERRED_RENAME`; a delete and an add that show + * no signal stay a delete, which `--allow-delete` guards. */ function inferRenames(state: InferenceState): void { - const { workflow, prev, resolved, claimed } = state + const { workflow, prev, resolved, claimed, lockedIds, catalog } = state const prevCanonical = prev.canonical.workflow const prevIdentity = prev.canonical.identity.blocks /** Prev blocks a settled key accounts for: a retype's own prev, else the prev under its key. */ @@ -368,9 +466,7 @@ function inferRenames(state: InferenceState): void { return id !== undefined && claimed.has(id) ? id : undefined } - let progressed = true - while (progressed) { - progressed = false + const groups = () => { const targetGroups = new Map() for (const key of state.open()) { const block = workflow.blocks[key] @@ -392,25 +488,90 @@ function inferRenames(state: InferenceState): void { if (keys) keys.push(key) else prevGroups.set(group, [key]) } - for (const [group, targets] of targetGroups) { + return [...targetGroups].flatMap(([group, targets]) => { const prevKeys = prevGroups.get(group) - if (targets.length !== 1 || prevKeys?.length !== 1) continue + return prevKeys ? [{ targets, prevKeys }] : [] + }) + } + + const pair = (key: BlockKey, prevKey: BlockKey, basis: InferenceBasis): boolean => { + if (!state.match(key, prevIdentity[prevKey].id, 'inferred')) return false + state.diagnostics.push({ + code: 'W_MAT_INFERRED_RENAME', + severity: 'warning', + message: `Block "${key}" is taken as a rename of "${prevKey}" (${INFERENCE_REASON[basis]}), so it keeps that block's id and run history. To make it a new block instead, delete "${prevKey}" in one apply and add "${key}" in the next`, + blockKey: key, + }) + return true + } + + const neighbours = (key: BlockKey, prevKey: BlockKey) => ({ + target: neighbourhood(workflow.edges, key, idOfTarget), + prev: neighbourhood(prevCanonical.edges, prevKey, idOfPrev), + }) + + const byConnections = (): boolean => { + let progressed = false + for (const { targets, prevKeys } of groups()) { + if (targets.length !== 1 || prevKeys.length !== 1) continue const [key] = targets const [prevKey] = prevKeys - const agree = sameSet( - neighbourhood(workflow.edges, key, idOfTarget), - neighbourhood(prevCanonical.edges, prevKey, idOfPrev) - ) - if (!agree) continue - const prevId = prevIdentity[prevKey].id - if (!state.match(key, prevId, 'inferred')) continue - progressed = true - state.diagnostics.push({ - code: 'W_MAT_INFERRED_RENAME', - severity: 'warning', - message: `Block "${key}" is taken as a rename of "${prevKey}" (same type, parent and connections), so it keeps that block's id and run history. To make it a new block instead, delete "${prevKey}" in one apply and add "${key}" in the next`, - blockKey: key, + const sides = neighbours(key, prevKey) + if (!sameSet(sides.target, sides.prev)) continue + const triggerHeld = + isTriggerLike(workflow.blocks[key], catalog) && + !settingsAgree(workflow.blocks[key], prevCanonical.blocks[prevKey], key, prevKey) + if (!triggerHeld && pair(key, prevKey, 'connections')) progressed = true + } + return progressed + } + + /** The strong signal tying new block `key` to vacated stored block `prevKey`, if any. */ + const signal = (key: BlockKey, prevKey: BlockKey): InferenceBasis | null => { + const target = workflow.blocks[key] + if (settingsAgree(target, prevCanonical.blocks[prevKey], key, prevKey)) return 'settings' + if (isTriggerLike(target, catalog)) return null + const sides = neighbours(key, prevKey) + const shared = [...sides.target].filter((item) => sides.prev.has(item)).length + const union = new Set([...sides.target, ...sides.prev]).size + if (shared > 0 && shared * 2 > union) return 'connections' + const referrers = Object.keys(prevCanonical.blocks).filter( + (other) => other !== prevKey && referencesKey(prevCanonical.blocks[other], prevKey) + ) + const renamedAtEverySite = + referrers.length > 0 && + referrers.every((other) => { + const id = idOfPrev(other) + const now = id === undefined ? undefined : [...resolved].find(([, b]) => b.id === id)?.[0] + return now !== undefined && referencesKey(workflow.blocks[now], key) }) + return renamedAtEverySite ? 'references' : null + } + + const byVacatedLock = (): boolean => { + let progressed = false + for (const { targets, prevKeys } of groups()) { + const vacated = prevKeys.filter((prevKey) => lockedIds.has(prevIdentity[prevKey].id)) + if (vacated.length === 0) continue + const signals = new Map>() + for (const key of targets) { + const found = new Map() + for (const prevKey of vacated) { + const basis = signal(key, prevKey) + if (basis) found.set(prevKey, basis) + } + signals.set(key, found) + } + for (const [key, found] of signals) { + if (found.size !== 1) continue + const [[prevKey, basis]] = [...found] + const rivals = [...signals.values()].filter((other) => other.has(prevKey)).length + if (rivals === 1 && pair(key, prevKey, basis)) progressed = true + } } + return progressed } + + let progressed = true + while (progressed) progressed = byConnections() || byVacatedLock() } diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 3d5f89ba0c9..14ad4a7e684 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -566,6 +566,45 @@ describe('materialize: identity', () => { expect(codes(result)).not.toContain('W_MAT_INFERRED_RENAME') }) + it.each([ + [' > 1', ' > 1'], + [' > 1', ' > 1'], + ])( + 'infers a rename whose every reference (%s in a condition branch) was renamed with it', + (before, after) => { + const prev = workflowState( + [ + blockState('a1', 'function', 'One', { code: 'return 1' }), + blockState('c1', 'condition', 'Gate', { + conditions: JSON.stringify([ + { id: 'c1-if', title: 'if', value: before }, + { id: 'c1-else', title: 'else', value: '' }, + ]), + }), + blockState('u1', 'function', 'Use', { code: 'return 0' }), + ], + { edges: [edge('e1', 'u1', 'a1')] } + ) + // Neither settings nor connections tie the two: only the renamed references do. + const target = edit(canonical(prev).workflow, (draft) => { + draft.edges = [] + draft.blocks.uno = { + ...draft.blocks.one, + key: 'uno', + name: 'Uno', + params: { code: { kind: 'value', value: 'return 2' } }, + } + draft.blocks = omit(draft.blocks, ['one']) + const rows = draft.blocks.gate.branches?.rows ?? [] + rows[0].value = after + }) + const result = run(target, lockFor(prev), { prev, lockFreshness: 'fresh' }) + expect(result.entries).toContainEqual( + expect.objectContaining({ status: 'matched', key: 'uno', id: 'a1', via: 'inferred' }) + ) + } + ) + it('infers a rename under a fresh lock when the ident changed too', () => { const prev = base() const lock = lockFor(prev) diff --git a/packages/workflow-code/src/materialize/preserve.ts b/packages/workflow-code/src/materialize/preserve.ts index 4146f83a46b..7b07c7c3eb8 100644 --- a/packages/workflow-code/src/materialize/preserve.ts +++ b/packages/workflow-code/src/materialize/preserve.ts @@ -9,13 +9,14 @@ * editor defaults compile filled into fields that never run; * - the config fields of the loop or parallel types not in effect, and prev's spelling of a config * that runs the same (`count: 0` and `count: 5`); - * - unknown `block.data` extra data source does not carry. + * - unknown `block.data` extra data source does not carry; + * - a container's subBlocks `block.data` supersedes (`SUPERSEDED_CONTAINER_PARAMS`). * * Canonicalize's output already holds all of this, so restoring onto it changes nothing and the * no-op round trip stays byte-identical. */ -import { SUBFLOW_DATA_KEYS } from '../canonicalize/subflows' +import { SUBFLOW_DATA_KEYS, SUPERSEDED_CONTAINER_PARAMS } from '../canonicalize/subflows' import { canonicalActivityInputs, isInactive, withStoredValue } from '../catalog/activity' import { type CanonicalGroup, isCanonicalPair } from '../catalog/canonical-index' import { serializedCanonicalIndex } from '../catalog/surface' @@ -245,7 +246,10 @@ export function restorePreserved( restorePairs(next, prev, declared, catalog) restoreInactive(next, prev, declared, catalog) } - if (target.subflow) next.subflow = restoreSubflow(target.subflow, prev.subflow) + if (target.subflow) { + next.subflow = restoreSubflow(target.subflow, prev.subflow) + for (const id of SUPERSEDED_CONTAINER_PARAMS) setParam(next.params, id, getOwn(prev.params, id)) + } setOrDelete(next, 'extra', restoreExtra(target, prev)) return next } diff --git a/packages/workflow-code/src/print/printer.ts b/packages/workflow-code/src/print/printer.ts index 8a4f3460f3e..b4d08a207d0 100644 --- a/packages/workflow-code/src/print/printer.ts +++ b/packages/workflow-code/src/print/printer.ts @@ -11,7 +11,7 @@ import { isReservedWorkflowBlockName, normalizeWorkflowBlockName, } from '@sim/workflow-types/workflow' -import { SUBFLOW_DATA_KEYS } from '../canonicalize/subflows' +import { SUBFLOW_DATA_KEYS, SUPERSEDED_CONTAINER_PARAMS } from '../canonicalize/subflows' import { builderPins } from '../catalog/surface' import { catalogBlock } from '../catalog/view' import type { PrintDiagnosticCode } from '../diagnostics/codes' @@ -235,7 +235,7 @@ function containerCall(ctx: PrintContext, block: CanonicalBlock): Expr { ctx, block, catalogBlock(ctx.catalog, block.type), - new Set() + new Set(SUPERSEDED_CONTAINER_PARAMS) ) const subflowData: Record = {} for (const [key, value] of Object.entries(block.extra ?? {})) { diff --git a/packages/workflow-code/src/start-inputs.ts b/packages/workflow-code/src/start-inputs.ts index dfb621fdfb1..362c62bce76 100644 --- a/packages/workflow-code/src/start-inputs.ts +++ b/packages/workflow-code/src/start-inputs.ts @@ -3,7 +3,7 @@ * workflow call's `inputMapping` fills and a run's input carries. */ import { isRecordLike } from '@sim/utils/object' -import type { CanonicalWorkflow } from './model' +import type { CanonicalParam, CanonicalWorkflow } from './model' /** * The trigger types whose `inputFormat` defines a workflow's inputs. Webhook and schedule @@ -32,9 +32,17 @@ export function startInputs(workflow: CanonicalWorkflow): WorkflowInput[] | null INPUT_TRIGGER_TYPES.some((type) => type === block.type) ) if (!start) return null - const format = start.params.inputFormat - if (format?.kind !== 'value' || !Array.isArray(format.value)) return [] - return format.value.flatMap((field) => { + return inputFormatFields(start.params.inputFormat) ?? [] +} + +/** + * The fields an `inputFormat` param declares, in order: `[]` when it is absent, `null` when its + * value is not a list (a reference or other text), so nothing can be said about its fields. + */ +export function inputFormatFields(param: CanonicalParam | undefined): WorkflowInput[] | null { + if (param === undefined) return [] + if (param.kind !== 'value' || !Array.isArray(param.value)) return null + return param.value.flatMap((field) => { if (!isRecordLike(field) || typeof field.name !== 'string' || field.name === '') return [] return [{ name: field.name, type: typeof field.type === 'string' ? field.type : 'string' }] }) diff --git a/packages/workflow-sdk/README.md b/packages/workflow-sdk/README.md index 23de41ae4a1..c2106c43538 100644 --- a/packages/workflow-sdk/README.md +++ b/packages/workflow-sdk/README.md @@ -26,10 +26,13 @@ have the CLI confirm the directory holds that workflow. cannot (duplicate names, cycles, literal secrets). It exits 1 on errors and on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value or a literal secret in a block the lock does not bind yet (`--strict`: in every block; `--no-strict`: never on a - warning). It also adds the blocks the source now uses to `tsconfig.json`. Then - `bunx -p typescript@7.0.2 tsc -p ./wf` type-checks with the TypeScript release the - declarations are built for. `tsconfig.json` includes only the blocks the source used at the - last check, get or apply (fast); `tsconfig.full.json` includes every current block. In a + warning). It also adds the blocks the source now uses to `tsconfig.json`. Then always run + `bunx -p typescript@7.0.2 tsc -p ./wf` after an edit and fix what it reports: it takes about a + second and catches what check and the dry run do not, such as a reference to a start input the + trigger does not declare. It type-checks with the TypeScript release the declarations are built + for. `tsconfig.json` includes only the blocks the source used at the + last check, get or apply (fast); `tsconfig.full.json` includes every current block, from the CLI's cache. A builder of a block + the directory does not hold yet type-checks once `check` has run: it writes that block's file. In a workspace directory, `sim workspaces source check --dir .` checks every workflow against the directory's own secrets, credentials, resource keys and table columns, and `sim workflows source check` run in a `workflows/` folder does the same for one. @@ -37,17 +40,19 @@ have the CLI confirm the directory holds that workflow. until it exits 0. 4. **Apply** saves the draft. It rewrites `./wf` from what was stored, in canonical form (normalized values, order and quoting show up there, so re-read `workflow.ts` before the next edit), and - refreshes `.sim/lock.json`, `.sim/base.json` and `tsconfig.json`. Apply never deploys. + refreshes `.sim/lock.json`, `.sim/base.json` and `tsconfig.json`. It deploys only with `--deploy`. 5. **On a 409** (`SOURCE_STALE`, exit 5) the workflow changed since you read it and nothing was saved. Run `get` into a fresh directory, reapply your edit there (diff your copy against the old source to find it), dry-run, apply. Keep the fresh `.sim/` files: `.sim/base.json` holds the fingerprint that makes the next apply safe. 6. **New workflow**: write `workflow.ts` in an empty directory and run - `sim workflows source create --dir ./wf [--name ] [--folder ] --yes`. It prints the + `sim workflows source create --dir ./wf --workspace [--name ] [--folder ] --yes`. It prints the new workflow id and writes the full tree a `get` writes: the `.sim/sdk` types, `tsconfig.json`, `.sim/AGENTS.md` and the lock files. Continue with the loop above. Run on a directory without - `workflow.ts` and without `--dry-run` or `--yes`, `create --dir ./wf` writes a starter - `workflow.ts` and the types instead, and creates nothing. + `workflow.ts` and without `--dry-run` or `--yes`, `create --dir ./wf --workspace ` writes a + starter `workflow.ts` and that workspace's types instead, records the workspace in + `.sim/starter.json`, and creates nothing; the create that follows targets that workspace, and + one into another workspace is refused unless `--workspace` names it. `apply` and `create` ask for confirmation on a terminal; elsewhere pass `--yes` (without it they refuse with `CONFIRMATION_REQUIRED`). `--dry-run` never asks. @@ -57,17 +62,18 @@ refuse with `CONFIRMATION_REQUIRED`). `--dry-run` never asks. | Path | Contents | |------|----------| | `workflow.ts`, `functions/*` | The source. Edit these. | -| `.sim/lock.json` | Binds each block to its stored id. Renaming a const or a display name keeps the id. Do not edit. | +| `.sim/lock.json` | Binds each block to its stored id. Renaming a display name keeps the id; renaming the const too keeps it only when settings, neighbours or references still tie the block. Do not edit. | | `.sim/base.json` | The workflow, workspace, fingerprint and catalog hash the source was read at. Do not edit. | | `.sim/written.json` | The CLI's record of each file as it last wrote it, so `status`, `diff` and a later `get` can tell your unapplied edits from files it may replace. Commit it with the lock and base. Do not edit. | | `.sim/sdk/core.d.ts` | This package: `workflow`, `t`, `env`, `credential`, `table`, `BlockMeta`, ... | -| `.sim/sdk/INDEX.md` | One line per builder: `slack.sendMessage — Slack: Send Message → blocks/slack_v2.d.ts (SlackSendMessageParams)`, legacy builders apart, and the block type → builder naming rule. `grep` it to find a builder. | -| `.sim/sdk/blocks/{type}.d.ts` | One block type's builders with their params and outputs, for every block, not only the ones in use. `grep` the params type `INDEX.md` names rather than reading a file whole. | -| `.sim/sdk/index.d.ts` | Imports every current block (not the legacy-only ones); `tsconfig.full.json` includes it. | +| `.sim/sdk/INDEX.md` | The block type → builder naming rule, how a workflow runs, and one line per builder of the blocks the directory holds: `slack.sendMessage — Slack: Send Message → blocks/slack_v2.d.ts (SlackSendMessageParams)`. Any other builder: `sim workflows source params `. | +| `.sim/sdk/blocks/{type}.d.ts` | One block type's builders with their params and outputs, for the blocks the source uses and the core builders' (`check` adds a block an edit starts using). `sim workflows source params` prints one builder rather than a file whole. | +| `.sim/sdk/index.d.ts` | Names the catalog and generator the declarations come from, and imports the blocks the directory holds. | +| `~/.sim/cache/source-catalog//sdk-/` | Every block's declarations and the full `INDEX.md`, generated once per catalog and shared by every directory; `tsconfig.full.json` includes its `index.d.ts` (every current block). | | `.sim/sdk/workspace.d.ts` | This workspace's tables (with columns), knowledge bases, credentials, workflows (with inputs), skills, custom tools and env var names, so `table('tickets')`, `credential('support-slack')`, `skill('triage')` and `env.X` are checked. | | `tsconfig.json` | Resolves `@sim/workflow-sdk` to `.sim/sdk` and type-checks `workflow.ts` with the blocks it uses (from the source and the server's `blockTypes`) and the core builders' blocks: a few dozen files for `tsc` instead of every block. | -| `tsconfig.full.json` | The same with every current block, plus any legacy-only block the workflow uses. | -| `.sim/AGENTS.md` | The authoring guide for agents, linking [the docs](https://docs.sim.ai/workflows/code). A top-level `AGENTS.md` of your own is never touched. | +| `tsconfig.full.json` | The same with every current block (the cached index, an absolute path on this machine), plus any legacy-only block the workflow uses. | +| `.sim/AGENTS.md` | The authoring guide for agents: the primer `get` prints (edit loop, identity, references, deletes, deploy, exit codes, and a snippet for each builder the source calls), then a reference linking [the docs](https://docs.sim.ai/workflows/code). A top-level `AGENTS.md` of your own is never touched. | | `.sim/examples/*.ts` | Example workflows, together one correct example of each common block: `cookbook.ts` (agent, condition, function, table, knowledge, Slack, human in the loop, loop, response), `integrations.ts` (webhook, weekday cron, HTTP headers, table update by id, Gmail, `onError`), `agents.ts` (router, agent tools, child workflow, parallel). Read them before writing. They name example resources, so the tsconfig files leave them out. | | `.gitattributes` | `text eol=lf` for the source files, merged into any file already there. | @@ -99,14 +105,23 @@ Every code and its meaning: the "Diagnostics" table of `SYNTAX.md` in the Sim re ## Exit codes +Every `sim workflows source` and `sim workspaces source` command exits with one of these: + | Code | Meaning | Do | |---|---|---| -| 0 | Applied, or a clean dry run | Done | -| 1 | The source has errors, or the command failed | Fix the printed findings | -| 5 | `SOURCE_STALE`: the workflow changed since `get` | Get again, reapply the edit (step 5) | -| 6 | The workflow is locked | Ask someone to unlock it in the workflow builder | -| 7 | The source is too large | Split function code or remove blocks | -| 9 | Still rate limited (HTTP 429) after the CLI's retries; nothing was written | Wait a minute and run the same command again | +| 0 | Applied or created, a dry run or check with no errors, or a clean status | Done | +| 1 | The source has errors, a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed | Fix what it printed | +| 5 | `SOURCE_STALE`: the workflow changed since `get`, and nothing was written | Get again, reapply the edit (step 5) | +| 6 | The workflow or its folder is locked (`E_APPLY_LOCKED`) | Ask someone to unlock it in the workflow builder | +| 7 | The source is too large (on a get, status or diff too) | Split function code or remove blocks | +| 8 | `sim workspaces source apply` stopped part-way; what was written is saved into the directory | Apply again until it exits 0 | +| 9 | Still rate limited (HTTP 429) after the CLI's retries; nothing more was written | Wait a minute and run the same command again | +| 10 | Busy: another apply is writing the workspace, or another `sim` command is using the directory; nothing was written | Run it again once the other finished | +| 11 | `status`: only your local edits | Dry-run, then apply | +| 12 | `status`: the server changed since the get (drift, or a conflict with your edits) | Follow the `Next:` line it prints | +| 13 | Not logged in, or the server refused the credential (`AUTH_REQUIRED`); nothing was sent | `sim login`, or fix or unset `SIM_API_KEY` | +| 14 | `apply` or `create` without `--yes` and no terminal to ask: it printed the plan, found no errors, and wrote nothing | Run the same command with `--yes` | +| 130 | Interrupted (Ctrl-C). An apply or create already sent is finished and saved first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and says it may have been written | `sim workflows source status --dir ./wf` | ## Write declarations, not programs @@ -153,7 +168,7 @@ files are stored verbatim. One file per block; imports between function files ar | `src/core.d.ts` | Authoring types. Exports exactly `SDK_CORE_EXPORTS` from `@sim/workflow-code`; after editing run `bun scripts/generate-embedded-text.ts`. | | `cookbook/examples/*.ts` | The examples the CLI writes into `.sim/examples/`. After editing run `bun scripts/generate-embedded-text.ts`; `apps/sim/lib/workflows/code/real-catalog.test.ts` type-checks and compiles each against the real catalog (its `COOKBOOK_WORKSPACE` lists the resources they name). | | `src/runtime.ts` | The runtime entry: every export throws `SimSdkNotExecutable`. | -| `src/codegen/` | `generateSdkTypes(catalog)` (catalog → `.d.ts` and the `.sim/sdk/INDEX.md` builder list, deterministic, sorted by path), `generateWorkspaceTypes(manifest)` (`workspace.d.ts`), and `layout.ts`: which declaration files a directory's tsconfig files include. | +| `src/codegen/` | `generateSdkTypes(catalog, { blockTypes })` (catalog → `.d.ts` and the `INDEX.md` builder list, for every block or a directory's subset, deterministic, sorted by path), `describeBuilder`/`lookupBuilder` (one builder's params and outputs, for `sim workflows source params`), `generateWorkspaceTypes(manifest)` (`workspace.d.ts`), and `layout.ts`: which declaration files a directory's tsconfig files include. | | `test/examples` | Five example workflows, also compiled against the real catalog with zero diagnostics in `apps/sim/lib/workflows/code/sdk-examples.test.ts`. | | `test/negative` | Code each generated type must reject. | diff --git a/packages/workflow-sdk/cookbook/examples/agents.ts b/packages/workflow-sdk/cookbook/examples/agents.ts index 3d708abd73e..32e8859d7ea 100644 --- a/packages/workflow-sdk/cookbook/examples/agents.ts +++ b/packages/workflow-sdk/cookbook/examples/agents.ts @@ -64,8 +64,10 @@ export default workflow( inputMapping: { email: start.email }, }) - // Parallel: one branch per item, run at once. p.currentItem is the item; each.results lists - // every branch's output once all finish. (loop.forEach is the sequential form.) + // Parallel: one branch per item, run at once. p.currentItem is the item and p.index its + // position; each.results lists every branch's output once all finish. (loop.forEach is the + // sequential form.) In a function code file inside the body, the same values are + // $ref.parallel.currentItem and $ref.parallel.index. const each = parallel.collection('Each Domain', { collection: start.domains }, (p) => { const brief = agent('Company Brief', { model: 'claude-sonnet-4-6', diff --git a/packages/workflow-sdk/cookbook/examples/cookbook.ts b/packages/workflow-sdk/cookbook/examples/cookbook.ts index bd4acf69498..14ed8d9ed9f 100644 --- a/packages/workflow-sdk/cookbook/examples/cookbook.ts +++ b/packages/workflow-sdk/cookbook/examples/cookbook.ts @@ -103,9 +103,12 @@ export default workflow('Cookbook', { description: 'One example of each common b else: true, }) - // Loop over a list: p.currentItem is one element; the loop's results is the list of - // iteration outputs. In js`` code a reference is the value itself, not text spliced in: write - // it where an expression goes, never inside quotes ('${x}' would give the JSON, quotes included). + // Loop over a list: p.currentItem is one element and p.index its position; the loop's results + // is the list of iteration outputs. In js`` code a reference is the value itself, not text + // spliced in: write it where an expression goes, never inside quotes ('${x}' would give the + // JSON, quotes included). A function code file (code: codeFile('./functions/line.js')) reads + // other blocks as $ref.. ($ref.recentorders.rows) and, inside the loop body, + // the iteration as $ref.loop.currentItem and $ref.loop.index. const each = loop.forEach('Each Order', { collection: recent.rows }, (p) => { const line = fn('Line', { code: js`return ${p.currentItem.data.orderId} + ': ' + ${p.currentItem.data.status}`, diff --git a/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts b/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts index feaf003c963..dfec30cef72 100644 --- a/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts +++ b/packages/workflow-sdk/src/codegen/cookbook-text.generated.ts @@ -1,4 +1,4 @@ // Generated by scripts/generate-embedded-text.ts from cookbook/examples; do not edit. /** The example workflows, file name → text, written as `.sim/examples/`: together one correct example of each common block. */ -export const COOKBOOK_EXAMPLES: Readonly> = {"agents.ts":"// @sim-sdk 1\nimport {\n agent,\n childWorkflow,\n credential,\n customTool,\n env,\n mcp,\n parallel,\n response,\n router,\n runWorkflow,\n slack,\n t,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `lookup-order`, `linear`, `ops-slack`,\n// `enrich-lead` and ANTHROPIC_API_KEY stand in for this workspace\u0027s resources\n// (.sim/sdk/workspace.d.ts; `sim workflows source resources` lists the real keys).\nexport default workflow(\n \u0027Cookbook: agents and subflows\u0027,\n { description: \u0027Router, agent tools, a child workflow and a parallel fan-out.\u0027 },\n () => {\n const start = trigger.start(\u0027Start\u0027, {\n inputFormat: { message: \u0027string\u0027, email: \u0027string\u0027, domains: \u0027array\u0027 },\n })\n\n // Router: the model picks one route by its description; wire with triage..to(...).\n const triage = router(\u0027Triage\u0027, {\n context: start.message,\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n routes: {\n support: \u0027A customer asking about an existing order\u0027,\n sales: \u0027A prospect asking about pricing or a demo\u0027,\n },\n })\n\n // Agent tools: a workspace custom tool, one tool of an MCP server, an integration operation\n // (.tool.; the params given here are fixed, the model fills the rest) and\n // another workflow.\n const support = agent(\u0027Support Agent\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Answer the customer. Use your tools when they help.\u0027 },\n { role: \u0027user\u0027, content: start.message },\n ],\n tools: [\n customTool(\u0027lookup-order\u0027),\n mcp(\u0027linear\u0027).tool(\u0027create_issue\u0027),\n slack.tool.sendMessage({ credential: credential(\u0027ops-slack\u0027), channel: \u0027support-urgent\u0027 }),\n childWorkflow(\u0027enrich-lead\u0027).asTool(),\n ],\n })\n\n // Child workflow: inputMapping is checked against the child\u0027s start inputs. Its output is\n // enrich.result (untyped JSON): the child\u0027s response block output, so a field of the child\u0027s\n // response body is enrich.result.data..\n const enrich = runWorkflow(\u0027Enrich Lead\u0027, {\n workflowId: childWorkflow(\u0027enrich-lead\u0027),\n inputMapping: { email: start.email },\n })\n\n // Parallel: one branch per item, run at once. p.currentItem is the item; each.results lists\n // every branch\u0027s output once all finish. (loop.forEach is the sequential form.)\n const each = parallel.collection(\u0027Each Domain\u0027, { collection: start.domains }, (p) => {\n const brief = agent(\u0027Company Brief\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Describe ${p.currentItem} in two sentences.` }],\n })\n p.to(brief)\n })\n\n const answer = response(\u0027Answer\u0027, { body: { answer: support.content } })\n const briefs = response(\u0027Briefs\u0027, {\n body: { lead: t`${enrich.result}`, briefs: t`${each.results}` },\n })\n\n start.to(triage)\n triage.support.to(support)\n support.to(answer)\n triage.sales.to(enrich)\n enrich.to(each)\n each.to(briefs)\n }\n)\n","cookbook.ts":"// @sim-sdk 1\nimport {\n agent,\n condition,\n credential,\n env,\n fn,\n human,\n js,\n knowledge,\n knowledgeBase,\n loop,\n response,\n slack,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `orders`, `policies`, `ops-slack` and\n// ANTHROPIC_API_KEY stand in for this workspace\u0027s resources. `sim workflows source resources`\n// lists the real keys, and .sim/sdk/workspace.d.ts types them.\nexport default workflow(\u0027Cookbook\u0027, { description: \u0027One example of each common block.\u0027 }, () => {\n // Triggers: trigger.start (API/manual input), trigger.schedule, trigger.webhook.\n const start = trigger.start(\u0027Start\u0027, { inputFormat: { orderId: \u0027string\u0027, amount: \u0027number\u0027 } })\n const nightly = trigger.schedule(\u0027Nightly\u0027, {\n scheduleType: \u0027daily\u0027,\n dailyTime: \u002702:00\u0027,\n timezone: \u0027UTC\u0027,\n })\n\n // Agent: structured output via responseFormat; its fields become outputs (check.risk).\n // apiKey: needed on self-hosted Sim; hosted Sim supplies one.\n const check = agent(\u0027Check Order\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Rate the fraud risk of an order as low, med or high.\u0027 },\n { role: \u0027user\u0027, content: t`Order ${start.orderId} for ${start.amount}` },\n ],\n responseFormat: {\n name: \u0027risk\u0027,\n strict: true,\n schema: {\n type: \u0027object\u0027,\n properties: {\n risk: { type: \u0027string\u0027, enum: [\u0027low\u0027, \u0027med\u0027, \u0027high\u0027] },\n reason: { type: \u0027string\u0027 },\n },\n required: [\u0027risk\u0027, \u0027reason\u0027],\n additionalProperties: false,\n },\n },\n })\n\n // Condition: one branch per key, `else: true` last; wire with route..to(...).\n const route = condition(\u0027Route Risk\u0027, {\n high: js`${check.risk} === \u0027high\u0027`,\n med: js`${check.risk} === \u0027med\u0027`,\n else: true,\n })\n\n // Function: inline JavaScript; reference outputs with ${...}. Result is fn.result.\n const shape = fn(\u0027Shape\u0027, {\n code: js`return { id: ${start.orderId}, cents: Math.round(${start.amount} * 100) }`,\n })\n\n // Tables: query rows (each row is { id, data: { column } }) and insert a row.\n const recent = table.queryRows(\u0027Recent Orders\u0027, {\n tableId: table(\u0027orders\u0027),\n filterInput: { all: [{ field: \u0027status\u0027, op: \u0027eq\u0027, value: \u0027open\u0027 }] },\n limit: 50,\n })\n const save = table.insertRow(\u0027Save Order\u0027, {\n tableId: table(\u0027orders\u0027),\n data: { orderId: start.orderId, status: \u0027open\u0027, risk: check.risk },\n })\n\n // Knowledge base search: results is the list of matching chunks.\n const docs = knowledge.search(\u0027Policy Search\u0027, {\n knowledgeBaseId: knowledgeBase(\u0027policies\u0027),\n query: check.reason,\n topK: 3,\n })\n\n // Slack: a saved credential by key (see .sim/sdk/workspace.d.ts).\n const alert = slack.sendMessage(\u0027Alert Ops\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-alerts\u0027,\n messageFormat: \u0027text\u0027,\n text: t`High-risk order ${start.orderId}: ${check.reason}`,\n })\n\n // Human in the loop: inputFormat is the form; answers are approval.submission..\n const approval = human(\u0027Manager Review\u0027, {\n builderData: [{ name: \u0027orderId\u0027, type: \u0027string\u0027, value: start.orderId }],\n inputFormat: { approved: { type: \u0027boolean\u0027, required: true }, note: \u0027string\u0027 },\n })\n const decided = condition(\u0027Approved\u0027, {\n yes: js`${approval.submission.approved} === true`,\n else: true,\n })\n\n // Loop over a list: p.currentItem is one element; the loop\u0027s results is the list of\n // iteration outputs. In js`` code a reference is the value itself, not text spliced in: write\n // it where an expression goes, never inside quotes (\u0027${x}\u0027 would give the JSON, quotes included).\n const each = loop.forEach(\u0027Each Order\u0027, { collection: recent.rows }, (p) => {\n const line = fn(\u0027Line\u0027, {\n code: js`return ${p.currentItem.data.orderId} + \u0027: \u0027 + ${p.currentItem.data.status}`,\n })\n p.to(line)\n })\n const report = agent(\u0027Report\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Summarize these orders in 3 lines:\\n${each.results}` }],\n })\n const post = slack.sendMessage(\u0027Post Report\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-daily\u0027,\n messageFormat: \u0027text\u0027,\n text: report.content,\n })\n\n // Response: the API reply body. Wrap an untyped value (fn.result.x) in t`` when it is text.\n const reply = response(\u0027Respond\u0027, { body: { risk: check.risk, id: t`${shape.result.id}` } })\n const rejected = response(\u0027Respond Rejected\u0027, { body: { status: \u0027rejected\u0027 } })\n\n // Wiring: a.to(b) chains; a block can fan out to several targets.\n start.to(check)\n check.to(save)\n save.to(route)\n route.high.to(alert)\n alert.to(approval)\n approval.to(decided)\n decided.yes.to(shape)\n decided.else.to(rejected)\n route.med.to(docs)\n docs.to(shape)\n route.else.to(shape)\n shape.to(reply)\n nightly.to(recent)\n recent.to(each)\n each.to(report)\n report.to(post)\n})\n","integrations.ts":"// @sim-sdk 1\nimport {\n api,\n credential,\n env,\n gmail,\n response,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `leads`, `ops-gmail`, CLEARBIT_KEY and\n// LEADS_WEBHOOK_TOKEN stand in for this workspace\u0027s resources (.sim/sdk/workspace.d.ts).\n// Integration builders are named by service, not block type: `gmail_v2` is `gmail.*`, and a\n// hidden older version lives under `legacy.*`. .sim/sdk/INDEX.md lists every builder.\nexport default workflow(\n \u0027Cookbook: integrations\u0027,\n { description: \u0027Webhook, weekday schedule, HTTP with headers, table update, Gmail, error path.\u0027 },\n () => {\n // Webhook trigger: inputFormat types the posted JSON fields (hook.leadId). With requireAuth\n // the caller must send the token; keep it in an env var.\n const hook = trigger.webhook(\u0027Lead Webhook\u0027, {\n requireAuth: true,\n token: env.LEADS_WEBHOOK_TOKEN,\n inputFormat: { leadId: \u0027string\u0027, email: \u0027string\u0027, domain: \u0027string\u0027 },\n })\n\n // Schedule: no weekday preset exists, so scheduleType \u0027custom\u0027 takes a 5-field cron.\n const weekdays = trigger.schedule(\u0027Weekdays 8am\u0027, {\n scheduleType: \u0027custom\u0027,\n cronExpression: \u00270 8 * * 1-5\u0027,\n timezone: \u0027America/New_York\u0027,\n })\n\n // HTTP request: headers (and query params) are table rows, { cells: { Key, Value } }.\n const lookup = api(\u0027Company Lookup\u0027, {\n url: t`https://company.clearbit.com/v2/companies/find?domain=${hook.domain}`,\n method: \u0027GET\u0027,\n headers: [{ cells: { Key: \u0027Authorization\u0027, Value: t`Bearer ${env.CLEARBIT_KEY}` } }],\n })\n\n // Update one row by id: data holds only the columns to change. lookup.data is untyped JSON,\n // so a text field from it is wrapped in t``.\n const save = table.updateRowById(\u0027Save Company\u0027, {\n tableId: table(\u0027leads\u0027),\n rowId: hook.leadId,\n data: { company: t`${lookup.data.name}`, enriched: true },\n })\n\n // Gmail (gmail_v2): send one email with a saved Google credential.\n const welcome = gmail.sendEmail(\u0027Welcome Email\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: hook.email,\n subject: \u0027Welcome aboard\u0027,\n body: t`Thanks for signing up from ${hook.domain}.`,\n })\n\n // Query rows, then email them as one digest. A typed column filter: { field, op, value }.\n const pending = table.queryRows(\u0027Pending Leads\u0027, {\n tableId: table(\u0027leads\u0027),\n filterInput: { all: [{ field: \u0027enriched\u0027, op: \u0027eq\u0027, value: false }] },\n limit: 100,\n })\n const digest = gmail.sendEmail(\u0027Pending Digest\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: \u0027ops@example.com\u0027,\n subject: \u0027Leads still to enrich\u0027,\n body: t`${pending.rows}`,\n })\n\n // Error path: x.onError(y) runs y when x fails, instead of failing the run; y reads the\n // failure message as x.error.\n const failed = response(\u0027Lookup Failed\u0027, {\n body: { ok: false, error: lookup.error },\n status: 502,\n })\n const done = response(\u0027Done\u0027, { body: { ok: true, row: save.row } })\n\n hook.to(lookup)\n lookup.to(save)\n lookup.onError(failed)\n save.to(welcome)\n welcome.to(done)\n weekdays.to(pending)\n pending.to(digest)\n }\n)\n"} +export const COOKBOOK_EXAMPLES: Readonly> = {"agents.ts":"// @sim-sdk 1\nimport {\n agent,\n childWorkflow,\n credential,\n customTool,\n env,\n mcp,\n parallel,\n response,\n router,\n runWorkflow,\n slack,\n t,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `lookup-order`, `linear`, `ops-slack`,\n// `enrich-lead` and ANTHROPIC_API_KEY stand in for this workspace\u0027s resources\n// (.sim/sdk/workspace.d.ts; `sim workflows source resources` lists the real keys).\nexport default workflow(\n \u0027Cookbook: agents and subflows\u0027,\n { description: \u0027Router, agent tools, a child workflow and a parallel fan-out.\u0027 },\n () => {\n const start = trigger.start(\u0027Start\u0027, {\n inputFormat: { message: \u0027string\u0027, email: \u0027string\u0027, domains: \u0027array\u0027 },\n })\n\n // Router: the model picks one route by its description; wire with triage..to(...).\n const triage = router(\u0027Triage\u0027, {\n context: start.message,\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n routes: {\n support: \u0027A customer asking about an existing order\u0027,\n sales: \u0027A prospect asking about pricing or a demo\u0027,\n },\n })\n\n // Agent tools: a workspace custom tool, one tool of an MCP server, an integration operation\n // (.tool.; the params given here are fixed, the model fills the rest) and\n // another workflow.\n const support = agent(\u0027Support Agent\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Answer the customer. Use your tools when they help.\u0027 },\n { role: \u0027user\u0027, content: start.message },\n ],\n tools: [\n customTool(\u0027lookup-order\u0027),\n mcp(\u0027linear\u0027).tool(\u0027create_issue\u0027),\n slack.tool.sendMessage({ credential: credential(\u0027ops-slack\u0027), channel: \u0027support-urgent\u0027 }),\n childWorkflow(\u0027enrich-lead\u0027).asTool(),\n ],\n })\n\n // Child workflow: inputMapping is checked against the child\u0027s start inputs. Its output is\n // enrich.result (untyped JSON): the child\u0027s response block output, so a field of the child\u0027s\n // response body is enrich.result.data..\n const enrich = runWorkflow(\u0027Enrich Lead\u0027, {\n workflowId: childWorkflow(\u0027enrich-lead\u0027),\n inputMapping: { email: start.email },\n })\n\n // Parallel: one branch per item, run at once. p.currentItem is the item and p.index its\n // position; each.results lists every branch\u0027s output once all finish. (loop.forEach is the\n // sequential form.) In a function code file inside the body, the same values are\n // $ref.parallel.currentItem and $ref.parallel.index.\n const each = parallel.collection(\u0027Each Domain\u0027, { collection: start.domains }, (p) => {\n const brief = agent(\u0027Company Brief\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Describe ${p.currentItem} in two sentences.` }],\n })\n p.to(brief)\n })\n\n const answer = response(\u0027Answer\u0027, { body: { answer: support.content } })\n const briefs = response(\u0027Briefs\u0027, {\n body: { lead: t`${enrich.result}`, briefs: t`${each.results}` },\n })\n\n start.to(triage)\n triage.support.to(support)\n support.to(answer)\n triage.sales.to(enrich)\n enrich.to(each)\n each.to(briefs)\n }\n)\n","cookbook.ts":"// @sim-sdk 1\nimport {\n agent,\n condition,\n credential,\n env,\n fn,\n human,\n js,\n knowledge,\n knowledgeBase,\n loop,\n response,\n slack,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `orders`, `policies`, `ops-slack` and\n// ANTHROPIC_API_KEY stand in for this workspace\u0027s resources. `sim workflows source resources`\n// lists the real keys, and .sim/sdk/workspace.d.ts types them.\nexport default workflow(\u0027Cookbook\u0027, { description: \u0027One example of each common block.\u0027 }, () => {\n // Triggers: trigger.start (API/manual input), trigger.schedule, trigger.webhook.\n const start = trigger.start(\u0027Start\u0027, { inputFormat: { orderId: \u0027string\u0027, amount: \u0027number\u0027 } })\n const nightly = trigger.schedule(\u0027Nightly\u0027, {\n scheduleType: \u0027daily\u0027,\n dailyTime: \u002702:00\u0027,\n timezone: \u0027UTC\u0027,\n })\n\n // Agent: structured output via responseFormat; its fields become outputs (check.risk).\n // apiKey: needed on self-hosted Sim; hosted Sim supplies one.\n const check = agent(\u0027Check Order\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [\n { role: \u0027system\u0027, content: \u0027Rate the fraud risk of an order as low, med or high.\u0027 },\n { role: \u0027user\u0027, content: t`Order ${start.orderId} for ${start.amount}` },\n ],\n responseFormat: {\n name: \u0027risk\u0027,\n strict: true,\n schema: {\n type: \u0027object\u0027,\n properties: {\n risk: { type: \u0027string\u0027, enum: [\u0027low\u0027, \u0027med\u0027, \u0027high\u0027] },\n reason: { type: \u0027string\u0027 },\n },\n required: [\u0027risk\u0027, \u0027reason\u0027],\n additionalProperties: false,\n },\n },\n })\n\n // Condition: one branch per key, `else: true` last; wire with route..to(...).\n const route = condition(\u0027Route Risk\u0027, {\n high: js`${check.risk} === \u0027high\u0027`,\n med: js`${check.risk} === \u0027med\u0027`,\n else: true,\n })\n\n // Function: inline JavaScript; reference outputs with ${...}. Result is fn.result.\n const shape = fn(\u0027Shape\u0027, {\n code: js`return { id: ${start.orderId}, cents: Math.round(${start.amount} * 100) }`,\n })\n\n // Tables: query rows (each row is { id, data: { column } }) and insert a row.\n const recent = table.queryRows(\u0027Recent Orders\u0027, {\n tableId: table(\u0027orders\u0027),\n filterInput: { all: [{ field: \u0027status\u0027, op: \u0027eq\u0027, value: \u0027open\u0027 }] },\n limit: 50,\n })\n const save = table.insertRow(\u0027Save Order\u0027, {\n tableId: table(\u0027orders\u0027),\n data: { orderId: start.orderId, status: \u0027open\u0027, risk: check.risk },\n })\n\n // Knowledge base search: results is the list of matching chunks.\n const docs = knowledge.search(\u0027Policy Search\u0027, {\n knowledgeBaseId: knowledgeBase(\u0027policies\u0027),\n query: check.reason,\n topK: 3,\n })\n\n // Slack: a saved credential by key (see .sim/sdk/workspace.d.ts).\n const alert = slack.sendMessage(\u0027Alert Ops\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-alerts\u0027,\n messageFormat: \u0027text\u0027,\n text: t`High-risk order ${start.orderId}: ${check.reason}`,\n })\n\n // Human in the loop: inputFormat is the form; answers are approval.submission..\n const approval = human(\u0027Manager Review\u0027, {\n builderData: [{ name: \u0027orderId\u0027, type: \u0027string\u0027, value: start.orderId }],\n inputFormat: { approved: { type: \u0027boolean\u0027, required: true }, note: \u0027string\u0027 },\n })\n const decided = condition(\u0027Approved\u0027, {\n yes: js`${approval.submission.approved} === true`,\n else: true,\n })\n\n // Loop over a list: p.currentItem is one element and p.index its position; the loop\u0027s results\n // is the list of iteration outputs. In js`` code a reference is the value itself, not text\n // spliced in: write it where an expression goes, never inside quotes (\u0027${x}\u0027 would give the\n // JSON, quotes included). A function code file (code: codeFile(\u0027./functions/line.js\u0027)) reads\n // other blocks as $ref.. ($ref.recentorders.rows) and, inside the loop body,\n // the iteration as $ref.loop.currentItem and $ref.loop.index.\n const each = loop.forEach(\u0027Each Order\u0027, { collection: recent.rows }, (p) => {\n const line = fn(\u0027Line\u0027, {\n code: js`return ${p.currentItem.data.orderId} + \u0027: \u0027 + ${p.currentItem.data.status}`,\n })\n p.to(line)\n })\n const report = agent(\u0027Report\u0027, {\n model: \u0027claude-sonnet-4-6\u0027,\n apiKey: env.ANTHROPIC_API_KEY,\n messages: [{ role: \u0027user\u0027, content: t`Summarize these orders in 3 lines:\\n${each.results}` }],\n })\n const post = slack.sendMessage(\u0027Post Report\u0027, {\n credential: credential(\u0027ops-slack\u0027),\n destinationType: \u0027channel\u0027,\n channel: \u0027ops-daily\u0027,\n messageFormat: \u0027text\u0027,\n text: report.content,\n })\n\n // Response: the API reply body. Wrap an untyped value (fn.result.x) in t`` when it is text.\n const reply = response(\u0027Respond\u0027, { body: { risk: check.risk, id: t`${shape.result.id}` } })\n const rejected = response(\u0027Respond Rejected\u0027, { body: { status: \u0027rejected\u0027 } })\n\n // Wiring: a.to(b) chains; a block can fan out to several targets.\n start.to(check)\n check.to(save)\n save.to(route)\n route.high.to(alert)\n alert.to(approval)\n approval.to(decided)\n decided.yes.to(shape)\n decided.else.to(rejected)\n route.med.to(docs)\n docs.to(shape)\n route.else.to(shape)\n shape.to(reply)\n nightly.to(recent)\n recent.to(each)\n each.to(report)\n report.to(post)\n})\n","integrations.ts":"// @sim-sdk 1\nimport {\n api,\n credential,\n env,\n gmail,\n response,\n t,\n table,\n trigger,\n workflow,\n} from \u0027@sim/workflow-sdk\u0027\n\n// An example to copy from, not a workflow of yours: `leads`, `ops-gmail`, CLEARBIT_KEY and\n// LEADS_WEBHOOK_TOKEN stand in for this workspace\u0027s resources (.sim/sdk/workspace.d.ts).\n// Integration builders are named by service, not block type: `gmail_v2` is `gmail.*`, and a\n// hidden older version lives under `legacy.*`. .sim/sdk/INDEX.md lists every builder.\nexport default workflow(\n \u0027Cookbook: integrations\u0027,\n { description: \u0027Webhook, weekday schedule, HTTP with headers, table update, Gmail, error path.\u0027 },\n () => {\n // Webhook trigger: inputFormat types the posted JSON fields (hook.leadId). With requireAuth\n // the caller must send the token; keep it in an env var.\n const hook = trigger.webhook(\u0027Lead Webhook\u0027, {\n requireAuth: true,\n token: env.LEADS_WEBHOOK_TOKEN,\n inputFormat: { leadId: \u0027string\u0027, email: \u0027string\u0027, domain: \u0027string\u0027 },\n })\n\n // Schedule: no weekday preset exists, so scheduleType \u0027custom\u0027 takes a 5-field cron.\n const weekdays = trigger.schedule(\u0027Weekdays 8am\u0027, {\n scheduleType: \u0027custom\u0027,\n cronExpression: \u00270 8 * * 1-5\u0027,\n timezone: \u0027America/New_York\u0027,\n })\n\n // HTTP request: headers (and query params) are table rows, { cells: { Key, Value } }.\n const lookup = api(\u0027Company Lookup\u0027, {\n url: t`https://company.clearbit.com/v2/companies/find?domain=${hook.domain}`,\n method: \u0027GET\u0027,\n headers: [{ cells: { Key: \u0027Authorization\u0027, Value: t`Bearer ${env.CLEARBIT_KEY}` } }],\n })\n\n // Update one row by id: data holds only the columns to change. lookup.data is untyped JSON,\n // so a text field from it is wrapped in t``.\n const save = table.updateRowById(\u0027Save Company\u0027, {\n tableId: table(\u0027leads\u0027),\n rowId: hook.leadId,\n data: { company: t`${lookup.data.name}`, enriched: true },\n })\n\n // Gmail (gmail_v2): send one email with a saved Google credential.\n const welcome = gmail.sendEmail(\u0027Welcome Email\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: hook.email,\n subject: \u0027Welcome aboard\u0027,\n body: t`Thanks for signing up from ${hook.domain}.`,\n })\n\n // Query rows, then email them as one digest. A typed column filter: { field, op, value }.\n const pending = table.queryRows(\u0027Pending Leads\u0027, {\n tableId: table(\u0027leads\u0027),\n filterInput: { all: [{ field: \u0027enriched\u0027, op: \u0027eq\u0027, value: false }] },\n limit: 100,\n })\n const digest = gmail.sendEmail(\u0027Pending Digest\u0027, {\n credential: credential(\u0027ops-gmail\u0027),\n to: \u0027ops@example.com\u0027,\n subject: \u0027Leads still to enrich\u0027,\n body: t`${pending.rows}`,\n })\n\n // Error path: x.onError(y) runs y when x fails, instead of failing the run; y reads the\n // failure message as x.error.\n const failed = response(\u0027Lookup Failed\u0027, {\n body: { ok: false, error: lookup.error },\n status: 502,\n })\n const done = response(\u0027Done\u0027, { body: { ok: true, row: save.row } })\n\n hook.to(lookup)\n lookup.to(save)\n lookup.onError(failed)\n save.to(welcome)\n welcome.to(done)\n weekdays.to(pending)\n pending.to(digest)\n }\n)\n"} diff --git a/packages/workflow-sdk/src/codegen/generate.ts b/packages/workflow-sdk/src/codegen/generate.ts index 5ae0c5a82ac..9cc59794e5d 100644 --- a/packages/workflow-sdk/src/codegen/generate.ts +++ b/packages/workflow-sdk/src/codegen/generate.ts @@ -1,5 +1,6 @@ /** - * `generateSdkTypes(catalog)`: the block catalog → `.d.ts` files. + * `generateSdkTypes(catalog, options)`: the block catalog → `.d.ts` files, for every block (the + * CLI's cached copy of a catalog) or only the blocks a source directory holds. * * - `blocks/{type}.d.ts` per block type with builders: every builder's params and output types, * and the augmentation of `@sim/workflow-sdk` with the builder functions, tool forms and @@ -8,8 +9,9 @@ * - `models.d.ts`: model ids, apart so a model-list change touches one file. * - `index.d.ts`: imports the models and every current block (not the legacy-only ones); the only * file that names the catalog hash. - * - `INDEX.md`: one line per builder naming its block, operation, file and params type, so an - * agent finds a builder with one `grep` instead of reading declarations. + * - `INDEX.md`: the naming rule, how a workflow runs, and one line per builder naming its block, + * operation, file and params type (in a directory's copy, only its blocks' builders; any other + * builder is one `sim workflows source params` away). * * Output is deterministic: the same catalog gives byte-identical files in path order. */ @@ -17,7 +19,7 @@ import { compareStrings } from '@sim/utils/string' import type { BlockCatalog, CatalogBlock, CatalogOutput, CatalogSubBlock } from '@sim/workflow-code' import { builderOutputSchemas, printKey, quoteString } from '@sim/workflow-code' -import { subBlockEditorDefault } from '@sim/workflow-code/catalog/defaults' +import { type EditorDefault, subBlockEditorDefault } from '@sim/workflow-code/catalog/defaults' import { builderPins, hasDefaultValue, @@ -56,7 +58,16 @@ export interface SdkTypeFile { text: string } -export type GenerateSdkTypesFn = (catalog: BlockCatalog) => SdkTypeFile[] +/** Which declarations `generateSdkTypes` writes. */ +interface SdkTypesOptions { + /** + * Only these block types' files, with `index.d.ts` importing them and `INDEX.md` listing their + * builders: a source directory's own copy. Without it, every block (the cached full catalog). + */ + blockTypes?: ReadonlySet +} + +export type GenerateSdkTypesFn = (catalog: BlockCatalog, options?: SdkTypesOptions) => SdkTypeFile[] /** Names `core.d.ts` exports that generated files may reference; imported only when used. */ const CORE_TYPE_NAMES = [ @@ -184,7 +195,34 @@ interface BuilderOutput { body: string } -interface BuilderTypes { +/** One param of a builder, as `describeBuilder` reports it. */ +export interface ParamFact { + /** What source writes. */ + key: string + /** The TypeScript type, `| null` included where a stored empty value prints as `null`. */ + type: string + required: boolean + /** The param's doc lines: title, bounds, selector, dependencies. */ + doc: string[] + /** The value an omitted param means (the editor default), when it has one. */ + default?: EditorDefault +} + +/** + * The params of a builder as facts: the ones every call takes, and when one field (the gate) + * decides which others apply, its values and the params of each. + */ +export interface ParamFacts { + common: ParamFact[] + gate: { + key: string + /** The gate's value when it is omitted, if any. */ + default?: Scalar + variants: Array<{ values: Scalar[]; params: ParamFact[] }> + } | null +} + +export interface BuilderTypes { /** The params type declaration, exported. */ params: string output: BuilderOutput @@ -198,6 +236,8 @@ interface BuilderTypes { summary: string /** The params type's name. */ paramsName: string + /** The params as facts. */ + facts: ParamFacts } /** @@ -225,7 +265,8 @@ function operationOutputs( return { outputs: picked, nullable: new Set(returned.nullable ?? []) } } -function operationLabel(block: CatalogBlock, operation: string): string | undefined { +/** The label of `operation` in the block's operation dropdown, if it has one. */ +export function operationLabel(block: CatalogBlock, operation: string): string | undefined { const dropdown = block.subBlocks.find((subBlock) => subBlock.id === 'operation') if (!dropdown || !Array.isArray(dropdown.options)) return undefined return dropdown.options.find((option) => option.id === operation)?.label @@ -265,6 +306,52 @@ function requiredInVariant(param: Param, requiredByVisibility: boolean): boolean return requiredByVisibility && !hasDefaultValue(param.members) } +/** The params of a builder as {@link ParamFacts}, required as the params type renders them. */ +function paramFacts( + params: readonly Param[], + gate: Gate | null, + candidates: readonly CatalogSubBlock[], + pinned: VisibilityContext['pinned'] +): ParamFacts { + const fact = (param: Param, required: boolean): ParamFact => { + const editorDefault = param.members + .map((member) => subBlockEditorDefault(member)) + .find((value) => value !== undefined) + const nullable = param.members.some((member) => subBlockEditorDefault(member) !== undefined) + return { + key: param.key, + type: nullable ? `${param.type} | null` : param.type, + required, + doc: param.doc.filter((line) => !line.startsWith('@when')), + ...(editorDefault !== undefined ? { default: editorDefault } : {}), + } + } + const gated = new Set() + if (gate) { + gated.add(gate.field) + for (const variant of gate.variants) for (const p of variant.params) gated.add(p.name) + } + const common = params + .filter((param) => !gated.has(param.name)) + .map((param) => fact(param, requiredForBuilder(param.members, pinned))) + if (!gate) return { common, gate: null } + const fallback = defaultGateValue(candidates, gate.field) + return { + common, + gate: { + key: params.find((param) => param.name === gate.field)?.key ?? gate.field, + ...(fallback !== undefined ? { default: fallback } : {}), + variants: gate.variants.map((variant) => ({ + values: [...variant.values], + params: variant.params.flatMap(({ name, required }) => { + const param = params.find((candidate) => candidate.name === name) + return param ? [fact(param, requiredInVariant(param, required))] : [] + }), + })), + }, + } +} + function renderParamsType( params: readonly Param[], gate: Gate | null, @@ -366,7 +453,8 @@ function buildParams( return { params, gate, candidates, context } } -function builderTypes(catalog: BlockCatalog, entry: BuilderEntry): BuilderTypes { +/** A builder's generated types: its params and output declarations, signatures and facts. */ +export function builderTypes(catalog: BlockCatalog, entry: BuilderEntry): BuilderTypes { const block = catalogBlock(catalog, entry.builder.type) if (!block) { throw new Error( @@ -433,6 +521,7 @@ function builderTypes(catalog: BlockCatalog, entry: BuilderEntry): BuilderTypes special, summary: [block.name, label].filter(Boolean).join(': '), paramsName, + facts: paramFacts(params, gate, candidates, context.pinned), } } @@ -648,12 +737,20 @@ const NAMING_RULE = [ '`.tool.(params)` (`slack.tool.sendMessage({ … })`).', ] +/** The lookup command for any builder, which the directory's `INDEX.md` points to. */ +const PARAMS_LOOKUP = + 'Look up any builder, used here or not: `sim workflows source params ` (`table.upsertRow`, `trigger.schedule`; a namespace such as `gmail` lists its builders) prints its params and outputs offline.' + /** * `INDEX.md`: one line per builder, `key — Block: Operation → file (ParamsType)`, current builders - * first and legacy ones in their own section, so `grep -i gmail .sim/sdk/INDEX.md` finds a builder - * and the params type to read. + * first and legacy ones in their own section. The full catalog's lists every builder; a + * directory's (`subset`) only the builders of the blocks it holds, beside the lookup command. */ -function generateSdkIndex(catalog: BlockCatalog, entries: readonly IndexEntry[]): SdkTypeFile { +function generateSdkIndex( + catalog: BlockCatalog, + entries: readonly IndexEntry[], + subset: boolean +): SdkTypeFile { const line = (entry: IndexEntry) => `- \`${entry.key}\` — ${entry.summary} → \`${blockTypesFile(entry.type)}\` (\`${entry.paramsName}\`)` const sorted = [...entries].sort((a, b) => compareStrings(a.key, b.key)) @@ -668,10 +765,10 @@ function generateSdkIndex(catalog: BlockCatalog, entries: readonly IndexEntry[]) '', ...NAMING_RULE, '', - 'Find a builder: `grep -i "gmail" .sim/sdk/INDEX.md`. Read its params and outputs: `grep -n -A40 "export type GmailSendEmailParams" .sim/sdk/blocks/gmail_v2.d.ts` (the output type follows as `GmailSendEmailOutput`).', + PARAMS_LOOKUP, '', ...RUNTIME_SEMANTICS, - '## Builders', + subset ? '## Builders of the blocks in this directory' : '## Builders', '', ...current.map(line), '', @@ -689,22 +786,27 @@ function generateSdkIndex(catalog: BlockCatalog, entries: readonly IndexEntry[]) } } -/** Generates every declaration file and `INDEX.md` for a catalog, sorted by path. */ -export const generateSdkTypes: GenerateSdkTypesFn = (catalog) => { +/** + * Generates the declaration files and `INDEX.md` for a catalog, sorted by path: every block, or + * only `options.blockTypes` (a directory's own copy, whose `index.d.ts` imports just those). + */ +export const generateSdkTypes: GenerateSdkTypesFn = (catalog, options = {}) => { const byType = new Map() for (const entry of builderEntries(catalog)) { + if (options.blockTypes && !options.blockTypes.has(entry.builder.type)) continue const list = byType.get(entry.builder.type) if (list) list.push(entry) else byType.set(entry.builder.type, [entry]) } const types = [...byType.keys()].sort(compareStrings) - const legacyOnly = legacyOnlyBlockTypes(catalog) + // A directory's index imports every block it holds: it holds a legacy-only one only when used. + const legacyOnly = options.blockTypes ? new Set() : legacyOnlyBlockTypes(catalog) const index: IndexEntry[] = [] const files: SdkTypeFile[] = [generateModelsFile(catalog)] for (const type of types) { files.push(generateBlockFile(catalog, type, byType.get(type) ?? [], index)) } - files.push(generateSdkIndex(catalog, index)) + files.push(generateSdkIndex(catalog, index, options.blockTypes !== undefined)) files.push({ path: SDK_INDEX_FILE, text: [ diff --git a/packages/workflow-sdk/src/codegen/index.ts b/packages/workflow-sdk/src/codegen/index.ts index 54672b4ded8..57668009355 100644 --- a/packages/workflow-sdk/src/codegen/index.ts +++ b/packages/workflow-sdk/src/codegen/index.ts @@ -7,15 +7,25 @@ export { sdkIndexHeader, } from './generate' export { + checkedBlockTypes, defaultBlockFiles, fullBlockFiles, referencedBlockTypes, + referencedBuilders, SDK_GUIDE_FILE, SDK_INDEX_FILE, SDK_MODELS_FILE, } from './layout' export { SDK_MAJOR } from './major' export { SPECIAL_BUILDER_KEYS } from './names' +export { + type BuilderLookup, + type BuilderReference, + builderSnippet, + describeBuilder, + lookupBuilder, + renderBuilderReference, +} from './reference' export { SOURCE_COMPILER_OPTIONS, typecheckCommand } from './tsconfig' export { type GenerateWorkspaceTypesFn, diff --git a/packages/workflow-sdk/src/codegen/layout.ts b/packages/workflow-sdk/src/codegen/layout.ts index ff7e523d8d1..ce65bb8d80e 100644 --- a/packages/workflow-sdk/src/codegen/layout.ts +++ b/packages/workflow-sdk/src/codegen/layout.ts @@ -1,8 +1,9 @@ /** * The layout of the generated declarations (`generateSdkTypes`) and which of them a source - * directory type-checks. A whole catalog is about 500 block files; `tsc` over all of them costs - * hundreds of megabytes, so a directory's `tsconfig.json` includes only the blocks its source - * uses, and `tsconfig.full.json` every current block for adding a new one. + * directory holds and type-checks. A whole catalog is about 500 block files (over 13 MB), so a + * directory holds and its `tsconfig.json` includes only the blocks its source uses; the whole set + * lives once per catalog in the CLI's cache, which `tsconfig.full.json` includes for adding any + * block. */ import { compareStrings } from '@sim/utils/string' @@ -64,31 +65,44 @@ export function legacyOnlyBlockTypes(catalog: BlockCatalog): Set { const CALL = /([A-Za-z_$][\w$]*(?:\s*\.\s*[A-Za-z_$][\w$]*)*)\s*\(/g /** - * The block types `sources` (workflow files) name through a builder: every dotted call whose path - * is a catalog builder (`slack.sendMessage(`) or its tool form (`slack.tool.sendMessage(`). A - * text scan, not a compile, so it reads any source, broken or not; a builder named in a comment - * only adds a file to the type-check. `block('type', …)` has untyped params, so needs no file. + * The catalog builders `sources` (workflow files) call, sorted: every dotted call whose path is a + * catalog builder (`slack.sendMessage(`) or its tool form (`slack.tool.sendMessage(`, read as + * `slack.sendMessage`). A text scan, not a compile, so it reads any source, broken or not; a + * builder named in a comment counts too. `block('type', …)` names no builder. */ -export function referencedBlockTypes(catalog: BlockCatalog, sources: Iterable): string[] { - const types = new Set() +export function referencedBuilders(catalog: BlockCatalog, sources: Iterable): string[] { + const keys = new Set() for (const text of sources) { for (const match of text.matchAll(CALL)) { const path = match[1].replace(/\s+/g, '').split('.') if (path.length > 1 && path[path.length - 2] === TOOL_NAMESPACE) path.splice(path.length - 2, 1) else if (path.length > 1 && path[path.length - 1] === TOOL_NAMESPACE) path.pop() - const builder = catalogBuilder(catalog, path.join('.')) - if (builder) types.add(builder.type) + const key = path.join('.') + if (catalogBuilder(catalog, key)) keys.add(key) } } + return [...keys].sort(compareStrings) +} + +/** + * The block types `sources` name through a builder ({@link referencedBuilders}): the blocks a + * directory's `tsconfig.json` type-checks. + */ +export function referencedBlockTypes(catalog: BlockCatalog, sources: Iterable): string[] { + const types = new Set() + for (const key of referencedBuilders(catalog, sources)) { + const builder = catalogBuilder(catalog, key) + if (builder) types.add(builder.type) + } return [...types].sort(compareStrings) } /** - * The block files a directory's `tsconfig.json` includes, relative to the SDK directory: those of - * the `used` types that have one, plus the {@link ALWAYS_CHECKED_BUILDERS}' blocks. + * The block types a directory holds declarations for and its `tsconfig.json` type-checks: the + * `used` types that have builders, plus the {@link ALWAYS_CHECKED_BUILDERS}' blocks. */ -export function defaultBlockFiles(catalog: BlockCatalog, used: Iterable): string[] { +export function checkedBlockTypes(catalog: BlockCatalog, used: Iterable): string[] { const declared = declaredBlockTypes(catalog) const types = new Set() for (const key of ALWAYS_CHECKED_BUILDERS) { @@ -96,7 +110,15 @@ export function defaultBlockFiles(catalog: BlockCatalog, used: Iterable) if (type) types.add(type) } for (const type of used) if (declared.has(type)) types.add(type) - return [...types].sort(compareStrings).map(blockTypesFile) + return [...types].sort(compareStrings) +} + +/** + * The block files a directory's `tsconfig.json` includes, relative to the SDK directory: those of + * the {@link checkedBlockTypes}. + */ +export function defaultBlockFiles(catalog: BlockCatalog, used: Iterable): string[] { + return checkedBlockTypes(catalog, used).map(blockTypesFile) } /** diff --git a/packages/workflow-sdk/src/codegen/names.ts b/packages/workflow-sdk/src/codegen/names.ts index 74280ab02d7..ce148d1a525 100644 --- a/packages/workflow-sdk/src/codegen/names.ts +++ b/packages/workflow-sdk/src/codegen/names.ts @@ -84,3 +84,15 @@ export function builderEntries(catalog: BlockCatalog): BuilderEntry[] { } return entries } + +/** + * The entry of the catalog builder `key`, or `null` when the catalog has none, named as + * {@link builderEntries} names it. It does not validate the rest of the map. + */ +export function builderEntry(catalog: BlockCatalog, key: string): BuilderEntry | null { + const builder = catalogBuilder(catalog, key) + if (!builder || !catalogBlock(catalog, builder.type)) return null + const path = key.split('.') + const special = path.length === 1 ? SDK_SPECIAL_BUILDERS[path[0]] : undefined + return { key, path, builder, special, typeName: pascal(path) } +} diff --git a/packages/workflow-sdk/src/codegen/reference.test.ts b/packages/workflow-sdk/src/codegen/reference.test.ts new file mode 100644 index 00000000000..ef6f88fcb7d --- /dev/null +++ b/packages/workflow-sdk/src/codegen/reference.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import type { BlockCatalog } from '@sim/workflow-code' +import { catalogBuilderPaths } from '@sim/workflow-code/catalog/view' +import { describe, expect, it } from 'vitest' +import { builderSnippet, describeBuilder } from './reference' + +/** A real catalog projected from registry blocks, shared with the SDK's own type checks. */ +const CATALOG = JSON.parse( + readFileSync(join(__dirname, '../../test/fixtures/catalog.json'), 'utf8') +) as BlockCatalog + +describe('builder reference', () => { + it('gives each special builder the call core.d.ts declares, with its own params and return type', () => { + expect(describeBuilder(CATALOG, 'condition')?.signature).toBe( + "condition>(name: string, branches: B & ConditionBranches, meta?: BlockMeta): Block> & Branches>" + ) + expect(describeBuilder(CATALOG, 'note')?.signature).toBe( + "note(name: string, params: SpecialParams<'note'>, meta?: BlockMeta): Annotation" + ) + expect(describeBuilder(CATALOG, 'response')?.signature.split('\n')).toEqual([ + "response(name: string, params: ResponseBody, meta?: BlockMeta): Block>", + "response(name: string, params: SpecialParams<'response'>, meta?: BlockMeta): Block>", + ]) + }) + + it('fills a text, number or boolean param of a snippet with an example value, not its type', () => { + const snippets = catalogBuilderPaths(CATALOG).flatMap((key) => { + const reference = describeBuilder(CATALOG, key) + return reference ? [builderSnippet(reference)] : [] + }) + expect(snippets.join('\n')).toContain("'…'") + for (const snippet of snippets) { + expect(snippet).not.toContain('` prints, and the snippet the source primer carries + * for each builder a directory already uses. The params, their required-ness and the outputs are + * the generated declarations' own (`builderTypes`), so the two never disagree. + */ + +import { compareStrings } from '@sim/utils/string' +import type { BlockCatalog } from '@sim/workflow-code' +import { suggestBuilder } from '@sim/workflow-code' +import { catalogBlock, catalogBuilder, catalogBuilderPaths } from '@sim/workflow-code/catalog/view' +import { SDK_CORE_EXPORTS } from '@sim/workflow-code/sdk' +import { CORE_DTS } from './core-text.generated' +import { builderTypes, operationLabel, type ParamFact, type ParamFacts } from './generate' +import { isLegacyBuilder } from './layout' +import { builderEntry, TOOL_NAMESPACE } from './names' + +/** A builder as `sim workflows source params` reports it. */ +export interface BuilderReference { + /** The dotted builder path (`slack.sendMessage`). */ + key: string + blockType: string + /** `Block: Operation`. */ + summary: string + operation?: string + /** An older block version's builder, kept so stored workflows print; write the current one. */ + legacy: boolean + /** The call, as the declarations type it. */ + signature: string + /** The agent tool form (`slack.tool.sendMessage(…)`), when the builder has one. */ + tool: string | null + paramsName: string + params: ParamFacts + /** Params `core.d.ts` types for the builder itself (`fn`'s `code`), not listed in `params`. */ + coreParams: readonly string[] + outputName: string + /** The output type's members as the declaration holds them, one line each. */ + outputs: string[] +} + +/** A builder of a namespace, in a lookup's list. */ +interface BuilderSummary { + key: string + summary: string +} + +/** What a builder lookup found. */ +export type BuilderLookup = + | { kind: 'builder'; reference: BuilderReference } + | { kind: 'namespace'; namespace: string; builders: BuilderSummary[] } + | { kind: 'core'; name: string; declaration: string } + | { kind: 'missing'; hint?: string } + +/** `Block: Operation` for a catalog builder. */ +function summaryOf(catalog: BlockCatalog, key: string): string { + const builder = catalogBuilder(catalog, key) + const block = builder ? catalogBlock(catalog, builder.type) : undefined + if (!builder || !block) return key + const label = builder.operation ? operationLabel(block, builder.operation) : undefined + return [block.name, label].filter(Boolean).join(': ') +} + +/** `slack.tool.sendMessage` → `slack.sendMessage`: the tool form names the same builder. */ +function withoutToolSegment(query: string): string { + const path = query.split('.') + const at = path.indexOf(TOOL_NAMESPACE) + if (at > 0 && at === path.length - 2) path.splice(at, 1) + return path.join('.') +} + +/** The reference of the catalog builder `key`, or `null` when the catalog has none. */ +export function describeBuilder(catalog: BlockCatalog, key: string): BuilderReference | null { + const entry = builderEntry(catalog, key) + if (!entry) return null + const types = builderTypes(catalog, entry) + const signature = entry.special + ? coreSignatures(key) + : `${entry.path.slice(0, -1).join('.')}.${(types.signature.at(-1) ?? '').replace(/^export function /, '')}`.replace( + /^\./, + '' + ) + const tool = types.tool + ? `${entry.path.slice(0, -1).join('.')}.${TOOL_NAMESPACE}.${(types.tool.at(-1) ?? '').replace(/^export function /, '')}` + : null + return { + key, + blockType: entry.builder.type, + summary: types.summary, + ...(entry.builder.operation ? { operation: entry.builder.operation } : {}), + legacy: isLegacyBuilder(key), + signature, + tool, + paramsName: types.paramsName, + params: types.facts, + coreParams: entry.special?.coreParams ?? [], + outputName: types.output.name, + outputs: types.output.body === '' ? [] : types.output.body.split('\n'), + } +} + +/** + * Looks `query` up as a builder (`table.upsertRow`, `slack.tool.sendMessage`), a namespace + * (`gmail`, `trigger`: its builders listed), or a construct `core.d.ts` declares (`loop`, `t`); + * otherwise reports the closest builder or namespace as a hint. + */ +export function lookupBuilder(catalog: BlockCatalog, query: string): BuilderLookup { + const key = withoutToolSegment(query.trim()) + const reference = describeBuilder(catalog, key) + if (reference) return { kind: 'builder', reference } + const paths = catalogBuilderPaths(catalog).sort(compareStrings) + const builders = paths + .filter((path) => path.startsWith(`${key}.`)) + .map((path) => ({ key: path, summary: summaryOf(catalog, path) })) + if (builders.length > 0) return { kind: 'namespace', namespace: key, builders } + const root = key.split('.')[0] ?? key + if ((SDK_CORE_EXPORTS as readonly string[]).includes(root)) { + return { kind: 'core', name: key, declaration: coreDeclaration(root) } + } + const hint = suggestBuilder( + key, + paths.filter((path) => !isLegacyBuilder(path)) + ) + return hint ? { kind: 'missing', hint } : { kind: 'missing' } +} + +const MAX_CORE_LINES = 80 + +/** Each top-level declaration of `name` in `core.d.ts` (overloads included): its line range. */ +function coreDeclarationRanges(name: string): Array<{ start: number; end: number }> { + const lines = CORE_DTS.split('\n') + const start = new RegExp( + `^export (?:declare )?(?:const|function|namespace|interface|type) ${name}\\b` + ) + const declarations: Array<{ start: number; end: number }> = [] + for (let index = 0; index < lines.length; index++) { + if (!start.test(lines[index])) continue + let depth = 0 + let end = index + for (; end < lines.length; end++) { + for (const char of lines[end]) { + if ('({['.includes(char)) depth++ + else if (')}]'.includes(char)) depth-- + } + const continues = /[=:,|&]\s*$/.test(lines[end]) || /^\s+[?:|&]/.test(lines[end + 1] ?? '') + if (depth <= 0 && !continues) break + } + declarations.push({ start: index, end: end + 1 }) + index = end + } + return declarations +} + +/** + * A special builder's call as `core.d.ts` declares it (`condition` takes its branches, `note` + * returns an `Annotation`), one line per overload. + */ +function coreSignatures(name: string): string { + const lines = CORE_DTS.split('\n') + return coreDeclarationRanges(name) + .map(({ start, end }) => + lines + .slice(start, end) + .join(' ') + .replace(/^export (?:declare )?function /, '') + .replace(/\s+/g, ' ') + .replace(/([(<]) /g, '$1') + .replace(/,? ([)>])/g, '$1') + .trim() + ) + .join('\n') +} + +/** + * Every top-level declaration of `name` in `core.d.ts` (overloads included) with the doc comment + * above it, as the file holds it: what a construct the catalog does not generate takes. + */ +function coreDeclaration(name: string): string { + const lines = CORE_DTS.split('\n') + const out: string[] = [] + for (const { start, end } of coreDeclarationRanges(name)) { + let first = start + if (lines[start - 1]?.trim().endsWith('*/')) { + while (first > 0 && !lines[first].trim().startsWith('/**')) first-- + } + if (out.length > 0) out.push('') + out.push(...lines.slice(first, end)) + } + return out.slice(0, MAX_CORE_LINES).join('\n') +} + +const MAX_SNIPPET_TYPE = 60 +const MAX_SNIPPET_OPTIONAL = 12 + +/** A type short enough for one snippet line. */ +function shortType(type: string): string { + const flat = type.replace(/\s+/g, ' ') + return flat.length > MAX_SNIPPET_TYPE ? `${flat.slice(0, MAX_SNIPPET_TYPE - 1)}…` : flat +} + +/** A value of `type` for an example call: a literal it allows, else a placeholder. */ +function exampleValue(type: string): string { + const literal = /^\s*('(?:[^'\\]|\\.)*'|-?\d+(?:\.\d+)?|true|false)\s*(?:\||$)/.exec(type) + if (literal) return literal[1] + if (/^CredentialRef\b/.test(type)) return "credential('')" + if (/^TableRef\b/.test(type)) return "table('')" + if (/^TableData\b/.test(type)) return "{ column: '…' }" + if (/^KnowledgeBaseRef\b/.test(type)) return "knowledgeBase('')" + if (/^WorkflowRef\b/.test(type)) return "childWorkflow('')" + if (/^(?:Template|string|Value]+\[\])/.test(type)) return '[]' + return `<${shortType(type.split(' | ')[0] ?? type)}>` +} + +/** The names of an output type's top-level members. */ +function outputNames(outputs: readonly string[]): string[] { + return outputs.flatMap((line) => { + const member = /^ {2}(?:readonly )?'?([\w$-]+)'?\??:/.exec(line) + return member ? [member[1]] : [] + }) +} + +function paramLine(param: ParamFact): string { + const fallback = param.default !== undefined ? ` = ${JSON.stringify(param.default)}` : '' + return `${param.key}: ${shortType(param.type)}${fallback}` +} + +/** + * A few lines for the source primer: the call with its required params filled in, the required + * params with their types, the optional ones by name, and the outputs. + */ +export function builderSnippet(reference: BuilderReference): string { + const required = reference.params.common.filter((param) => param.required) + const optional = reference.params.common.filter((param) => !param.required) + const gate = reference.params.gate + const variant = gate + ? (gate.variants.find((v) => gate.default !== undefined && v.values.includes(gate.default)) ?? + gate.variants[0]) + : undefined + const args = [ + ...required.map((param) => `${param.key}: ${exampleValue(param.type)}`), + ...(gate && variant && gate.default === undefined + ? [`${gate.key}: ${JSON.stringify(variant.values[0])}`] + : []), + ...(variant?.params ?? []) + .filter((param) => param.required) + .map((param) => `${param.key}: ${exampleValue(param.type)}`), + ] + const call = `${reference.key}('Name', ${args.length > 0 ? `{ ${args.join(', ')} }` : '{}'})` + const lines = [`- \`${reference.key}\` (${reference.summary}): \`const x = ${call}\``] + if (required.length > 0) lines.push(` required: ${required.map(paramLine).join('; ')}`) + if (gate) { + const variants = gate.variants.map((variant) => { + const needed = variant.params.filter((param) => param.required).map((param) => param.key) + return `${variant.values.map((value) => JSON.stringify(value)).join('|')}${needed.length > 0 ? ` → ${needed.join(', ')}` : ''}` + }) + const fallback = gate.default !== undefined ? ` (default ${JSON.stringify(gate.default)})` : '' + lines.push(` ${gate.key}${fallback} picks the rest: ${variants.join('; ')}`) + } + if (optional.length > 0) { + const shown = optional.slice(0, MAX_SNIPPET_OPTIONAL).map((param) => param.key) + const more = optional.length > shown.length ? `, +${optional.length - shown.length} more` : '' + lines.push(` optional: ${shown.join(', ')}${more}`) + } + if (reference.coreParams.length > 0) { + lines.push(` core.d.ts types: ${reference.coreParams.join(', ')}`) + } + const outputs = outputNames(reference.outputs) + if (outputs.length > 0) lines.push(` outputs: ${outputs.join(', ')}`) + return lines.join('\n') +} + +function describeParam(param: ParamFact, indent: string): string[] { + const fallback = param.default !== undefined ? ` (default ${JSON.stringify(param.default)})` : '' + const head = `${indent}${param.key}${param.required ? '' : '?'}: ${param.type}${fallback}` + return [head, ...param.doc.map((line) => `${indent} ${line}`)] +} + +/** The full text `sim workflows source params ` prints for one builder. */ +export function renderBuilderReference(reference: BuilderReference): string { + const where = [ + `block ${reference.blockType}`, + reference.operation && `operation ${reference.operation}`, + ] + .filter(Boolean) + .join(', ') + const lines = [`${reference.key} — ${reference.summary} (${where})`] + if (reference.legacy) { + lines.push('Legacy: an older block version. Write the current builder instead.') + } + lines.push('', reference.signature) + if (reference.tool) lines.push(`${reference.tool} (as an agent tool)`) + const required = reference.params.common.filter((param) => param.required) + const optional = reference.params.common.filter((param) => !param.required) + lines.push('', `Params (${reference.paramsName}):`) + if (reference.params.common.length === 0 && !reference.params.gate) lines.push(' (none)') + for (const param of [...required, ...optional]) lines.push(...describeParam(param, ' ')) + const gate = reference.params.gate + if (gate) { + const fallback = gate.default !== undefined ? ` (default ${JSON.stringify(gate.default)})` : '' + lines.push(` ${gate.key}${fallback} picks which of these apply:`) + for (const variant of gate.variants) { + lines.push( + ` ${gate.key}: ${variant.values.map((value) => JSON.stringify(value)).join(' | ')}` + ) + for (const param of variant.params) lines.push(...describeParam(param, ' ')) + } + } + if (reference.coreParams.length > 0) { + lines.push(` Also typed in core.d.ts: ${reference.coreParams.join(', ')}`) + } + lines.push('', `Outputs (${reference.outputName}):`) + lines.push(...(reference.outputs.length > 0 ? reference.outputs : [' (none)'])) + return lines.join('\n') +} diff --git a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts index 0c3c7975a9d..724b21ef94e 100644 --- a/packages/workflow-sdk/src/codegen/workspace-text.generated.ts +++ b/packages/workflow-sdk/src/codegen/workspace-text.generated.ts @@ -11,7 +11,7 @@ export const WORKSPACE_SDK_DTS: Readonly> = { 'index.d.ts': '/**\n * The workspace declaration SDK: `import { … } from \u0027@sim/workflow-sdk/workspace\u0027` in the\n * declaration files of a workspace source directory (`tables/.ts`, …). Like the workflow\n * SDK, nothing here runs: the server reads each file statically.\n */\nexport * from \u0027./credentials\u0027\nexport * from \u0027./deployment\u0027\nexport * from \u0027./file\u0027\nexport * from \u0027./knowledge\u0027\nexport * from \u0027./mcp\u0027\nexport * from \u0027./secrets\u0027\nexport * from \u0027./skill\u0027\nexport * from \u0027./table\u0027\nexport * from \u0027./tool\u0027\n', 'knowledge.d.ts': - '/**\n * Knowledge base declarations: `knowledge//knowledge.ts` in a workspace source directory,\n * beside `knowledge//docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload, so its tag values are not kept), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n', + '/**\n * Knowledge base declarations: `knowledge//knowledge.ts` in a workspace source directory,\n * beside `knowledge//docs/`, which holds one file per document.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineKnowledgeBase } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineKnowledgeBase({\n * name: \u0027Policies\u0027,\n * description: \u0027Refund and shipping policies\u0027,\n * chunking: { maxSize: 1024, minSize: 100, overlap: 200 },\n * })\n * ```\n *\n * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored\n * document (a new upload that takes the old one\u0027s tag values), and removing one deletes it only with\n * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a\n * connector syncs are never part of source.\n */\n\n/** How documents are split into chunks before they are embedded. */\nexport interface KnowledgeChunking {\n /** Largest chunk, in tokens (100–4000; 1024 when omitted). */\n maxSize?: number\n /** Smallest chunk, in characters (1–2000; 100 when omitted). */\n minSize?: number\n /** Tokens neighbouring chunks share (0–500; 200 when omitted). */\n overlap?: number\n strategy?: \u0027auto\u0027 | \u0027text\u0027 | \u0027regex\u0027 | \u0027recursive\u0027 | \u0027sentence\u0027 | \u0027token\u0027\n strategyOptions?: {\n /** The split pattern of the `regex` strategy. */\n pattern?: string\n separators?: string[]\n recipe?: \u0027plain\u0027 | \u0027markdown\u0027 | \u0027code\u0027\n /** `regex` only: every match is its own chunk. */\n strictBoundaries?: boolean\n }\n}\n\n/** A knowledge base declaration. */\nexport interface KnowledgeBaseDeclaration {\n name: string\n description?: string\n /** The knowledge base folder path (`/support`); the workspace root when omitted. */\n folder?: string\n /**\n * Applies to documents indexed from now on; stored chunks keep the settings they were made\n * with.\n */\n chunking?: KnowledgeChunking\n /** The embedding model, fixed when the knowledge base is created. */\n embeddingModel?: string\n}\n\n/** Declares a knowledge base. Its default export is the whole file. */\nexport declare function defineKnowledgeBase(\n knowledgeBase: KnowledgeBaseDeclaration\n): KnowledgeBaseDeclaration\n', 'mcp.d.ts': '/**\n * MCP server declarations: `mcp/.ts` declares one server the workspace\u0027s agents call tools\n * on.\n *\n * ```ts\n * // @sim-sdk 1\n * import { defineMcpServer, env } from \u0027@sim/workflow-sdk/workspace\u0027\n *\n * export default defineMcpServer({\n * name: \u0027GitHub\u0027,\n * url: \u0027https://api.githubcopilot.com/mcp/\u0027,\n * headers: { Authorization: \u0027Bearer {{GITHUB_TOKEN}}\u0027, \u0027X-Team\u0027: env.GITHUB_TEAM },\n * })\n * ```\n *\n * A workflow names the server by its key and one of its tools by name:\n * `mcp(\u0027github\u0027).tool(\u0027create_issue\u0027)`. Secrets never sit in source: a credential header\n * (`Authorization`, `Cookie`, or a name with `key`, `token`, `secret`, `auth`, …) holds env\n * references (`env.NAME`, or `{{NAME}}` in its text after a scheme such as `Bearer`) or\n * `stored()`, which keeps the value the server holds. The server\u0027s id derives from its URL, so a\n * `url` with another host or path replaces the server; a new query string updates it in place.\n */\nimport type { EnvRef } from \u0027@sim/workflow-sdk\u0027\n\nexport { env, stored } from \u0027@sim/workflow-sdk\u0027\n\n/** A header value: text (`{{NAME}}` names an env variable), `env.NAME`, or `stored()`. */\nexport type McpHeaderValue = string | EnvRef\n\n/** How the server authenticates. */\nexport interface McpServerAuth {\n /** `headers` (the default) sends the declared headers; `oauth` signs in with an OAuth grant. */\n type: \u0027none\u0027 | \u0027headers\u0027 | \u0027oauth\u0027\n /** A pre-registered OAuth client, for a server without dynamic client registration. */\n clientId?: string\n}\n\n/** An MCP server declaration. */\nexport interface McpServerDeclaration {\n /** The display name; the key when left out. */\n name?: string\n description?: string\n /** An `http://` or `https://` URL. A new host or path replaces the server; a new query does not. */\n url: string\n transport?: \u0027streamable-http\u0027\n headers?: { [name: string]: McpHeaderValue }\n /** `{ type: \u0027headers\u0027 }` when left out. The OAuth client secret is set in the server settings. */\n auth?: McpServerAuth\n /** Milliseconds a request waits; 30000 when left out. */\n timeout?: number\n /** 3 when left out. */\n retries?: number\n /** `true` when left out. */\n enabled?: boolean\n}\n\n/** Declares an MCP server. Its default export is the whole file. */\nexport declare function defineMcpServer(server: McpServerDeclaration): McpServerDeclaration\n', 'secrets.d.ts': diff --git a/packages/workflow-sdk/src/workspace/knowledge.d.ts b/packages/workflow-sdk/src/workspace/knowledge.d.ts index c3345960a1f..b97a78ec431 100644 --- a/packages/workflow-sdk/src/workspace/knowledge.d.ts +++ b/packages/workflow-sdk/src/workspace/knowledge.d.ts @@ -14,7 +14,7 @@ * ``` * * Documents sync by content: adding a file to `docs/` uploads it, editing one replaces the stored - * document (a new upload, so its tag values are not kept), and removing one deletes it only with + * document (a new upload that takes the old one's tag values), and removing one deletes it only with * `--allow-delete`. Every uploaded or replaced document is indexed after the apply. Documents a * connector syncs are never part of source. */ diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index 3ce623354cf..2b1a173b773 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -73,6 +73,13 @@ deleted. A workflow still names a workspace file by its resource key (`file('rep its path: a file an apply creates claims the key a get would first allocate it, so a workflow of the same apply resolves it. +A table's column keys persist with the table too: every table write records each declared column +whose key is not its name in `metadata.sourceColumnKeys` (column id → key), and a read prints a +column under the base's binding, else that record, else its name. So a fresh get, and the target's +read in a promotion, print the key the declaration gave the column (also after a rename in Sim), and +a promotion's managed rows match the stored ones by the same keys. An apply that finds a column a +read would print under another key records the keys alone (`fields: ['column keys']`). + ## Fingerprints and the three-way rule A resource's fingerprint is `fingerprintOf(canonical(decl))` = `rf1:` + SHA-256 of the canonical @@ -131,7 +138,10 @@ apart from the declarations'). - `{ mode: 'seed' }` writes the rows into a table the apply creates, or adopts while it holds no rows and the base has no entry for it (`I_TABLE_SEED_SKIPPED` otherwise). Seed rows are outside - every fingerprint, and a get never prints them. + every fingerprint, and a get never prints them: they are the directory's. The table's + `metadata.sourceRows` records `{ mode: 'seed' }`, so a get prints the declaration's `rows` and + the CLI keeps the rows file it finds beside it (`seedRowsFiles`), and a promotion from a + re-read directory still seeds the tables it creates. - `{ mode: 'managed', key }` makes the file the table's whole content, each row matched to its stored row by the unique `key` column. The table's `canonical()` carries only `{ mode, key }` (so tables without managed rows fingerprint as before); the rows have their own fingerprint @@ -238,7 +248,10 @@ resolves keys an earlier kind creates), check references across kinds, authorize cap the plan. Any error refuses the whole apply before the first write, so the plan refuses what a write would refuse: a delete of a locked resource (`E_APPLY_LOCKED`), of one a stored workflow still uses (`E_PLAN_REF_DELETED` for a declared caller, `W_RESOURCE_IN_USE` for one outside the -directory, workflows included), and a table change whose data breaks a new column rule. A plan +directory, workflows included), and a table change whose data breaks a new column rule. A column +rename warns `W_TABLE_COLUMN_RENAMED_IN_USE` on each Table block of a stored workflow that still writes, +filters or sorts by the old name (the run drops it), at the line of the declared workflow's file; an +edited workflow folder is checked by its own lint against the declared columns instead. A plan reports at most 1000 findings, errors first, then warnings, then infos, and an info counts the rest, so cutting findings never refuses a plan. @@ -267,7 +280,9 @@ the write is still caught only where the child use case takes a precondition (a **Staged blobs.** Blob bytes never travel in an apply. A client first asks which of the blobs it will send it has not staged (`POST /source/blobs/missing`, by SHA-256 and size), uploads only those (`POST /source/blobs`, multipart, one part per blob named by its hash, at most -`maxBlobBytesPerRequest` per request, `stageBatches` in `src/limits.ts`), then sends one apply +`maxBlobBytesPerRequest` per request framing included, a larger blob alone, `stageBatches` in +`src/limits.ts`; every body stays under the 10 MiB Next.js proxy ceiling, past which the route +would receive a truncated body, and the route answers a larger one `413 PAYLOAD_TOO_LARGE`), then sends one apply whose manifest names each blob by path, hash and size. The server checks every uploaded part against its hash before staging it, so a staged blob is verified bytes. The plan resolves the blobs its writes need against the staging area once, before any write: a missing one (never @@ -311,9 +326,19 @@ each folder's `.sim/lock.json` and `.sim/base.json` replaced by the target's fol directory declares unless `--allow-delete`. The server sees an ordinary apply against its own base, so every rule holds unchanged: a resource bound under the key is updated in place (a workflow keeps its id and block ids through the target's lock), an unbound one of the same name is adopted, a -missing one created, a change in the target between the read and the apply is `E_APPLY_STALE`, and -secrets and credentials bind by name and key to the target's own. The answer is not saved: the -directory stays bound to the workspace it was read from. Without `--to`, the CLI refuses a +missing one created, and secrets and credentials bind by name and key to the target's own. The +directory stays bound to the workspace it was read from; the one thing a promotion saves is the +target's promotion record, `.sim/targets/.json` (`promotionRecord`): the target's +base entries after the apply and, per workflow folder, the target workflow's fingerprint and a +digest of the folder as promoted. The next promotion composes with it: each base entry the record +holds for the same id stands in for the target's live one, so the server's three-way rule compares +with what the last promotion left (a target edit since is kept when the directory left the resource +alone, `E_APPLY_STALE` when it changed it too); a workflow folder whose digest is unchanged is sent +as its base alone (a no-op), and an edited one whole with the recorded fingerprint, so a target edit +since is the folder's 409. Without a record entry (the first promotion, or a resource created in +the target since) the live entry is the base and the target is updated to match the directory; +deleting the record re-takes the target as it stands. Read alone, the target's live base could +never look changed. Without `--to`, the CLI refuses a directory read from another workspace (`SOURCE_OTHER_WORKSPACE`); `--workspace ` instead applies it as a copy whose base the server ignores (`W_BASE_OTHER_WORKSPACE`), which only a first apply to an empty target survives (a second is `E_APPLY_NO_BASE` for everything it created). diff --git a/packages/workspace-code/src/declaration/fields.ts b/packages/workspace-code/src/declaration/fields.ts index a33d2e75a9c..1db74cf767a 100644 --- a/packages/workspace-code/src/declaration/fields.ts +++ b/packages/workspace-code/src/declaration/fields.ts @@ -41,7 +41,7 @@ const SECRET_VALUE_CODES: ReadonlySet = new Set([ /** * A {@link FieldReport} that appends to `diagnostics`, located through the reader's spans. Once it * has appended `MAX_FINDINGS_PER_FILE` findings (findings already in `diagnostics`, such as ones - * about other files of the resource, do not count) it records one `E_LIMIT_EXCEEDED` (with a span + * about other files of the resource, do not count) it records one `I_FINDINGS_TRUNCATED` (with a span * but no frame, since the value it points at may be a secret) and drops the rest. */ export function fieldReporter( diff --git a/packages/workspace-code/src/declaration/read.test.ts b/packages/workspace-code/src/declaration/read.test.ts index 97f9a36b09a..f79eb3a091f 100644 --- a/packages/workspace-code/src/declaration/read.test.ts +++ b/packages/workspace-code/src/declaration/read.test.ts @@ -59,7 +59,10 @@ describe('readDeclaration on large files', () => { code: 'E_DECL_UNSUPPORTED', span: { line: MAX_FINDINGS_PER_FILE + 2, column: 1 }, }) - expect(read.diagnostics[MAX_FINDINGS_PER_FILE]).toMatchObject({ code: 'E_LIMIT_EXCEEDED' }) + expect(read.diagnostics[MAX_FINDINGS_PER_FILE]).toMatchObject({ + code: 'I_FINDINGS_TRUNCATED', + severity: 'info', + }) expect(fullTextScans()).toBeLessThanOrEqual(1) }) @@ -84,7 +87,7 @@ describe('readDeclaration on large files', () => { expect(parsed.diagnostics).toHaveLength(MAX_FINDINGS_PER_FILE + 1) const options = parsed.diagnostics.filter((finding) => finding.code === 'E_TABLE_OPTIONS') expect(options.at(-1)?.span).toMatchObject({ line: options.length + 3 }) - expect(parsed.diagnostics.at(-1)).toMatchObject({ code: 'E_LIMIT_EXCEEDED' }) + expect(parsed.diagnostics.at(-1)).toMatchObject({ code: 'I_FINDINGS_TRUNCATED' }) expect(fullTextScans()).toBeLessThanOrEqual(2) }) diff --git a/packages/workspace-code/src/diagnostics/codes.ts b/packages/workspace-code/src/diagnostics/codes.ts index dce974fd914..647ae6b209a 100644 --- a/packages/workspace-code/src/diagnostics/codes.ts +++ b/packages/workspace-code/src/diagnostics/codes.ts @@ -26,6 +26,12 @@ const DECLARATION_CODES = [ 'E_DECL_UNKNOWN_FIELD', /** A path inside a kind's folder that no resource owns. */ 'E_DECL_UNKNOWN_FILE', + /** + * A file under a kind's folder whose path no workspace holds (a hidden name such as `.env`, or a + * character such as `:` or `?`): the CLI does not send it, so an apply neither writes nor + * deletes anything for it. + */ + 'W_PATH_NOT_SENT', /** A blob manifest entry listed twice, or at a path that holds no binary file: the apply writes nothing. */ 'E_BLOB_MANIFEST', /** @@ -33,7 +39,7 @@ const DECLARATION_CODES = [ * apply writes nothing. Stage it, then apply again. */ 'E_BLOB_MISSING', - /** A size, count or nesting limit exceeded (a file, a kind, the staging area, findings). */ + /** A size, count or nesting limit exceeded (a file, a kind, the staging area). */ 'E_LIMIT_EXCEEDED', /** A resource key outside `^[a-z][a-z0-9_-]{0,63}$`. */ 'E_KEY_INVALID', @@ -239,6 +245,11 @@ const PLAN_CODES = [ 'E_PLAN_REF_HELD', /** A resource this apply deletes is still used by stored workflows. */ 'W_RESOURCE_IN_USE', + /** + * A table column this apply renames is still written, filtered or sorted by its old name in a + * stored workflow's Table block, which the run then drops. + */ + 'W_TABLE_COLUMN_RENAMED_IN_USE', /** Another resource of the kind holds this name or key. */ 'E_NAME_TAKEN', /** A workflow folder sent without its files whose base names no workflow of this workspace. */ @@ -262,7 +273,10 @@ const PLAN_CODES = [ 'I_KEY_REBOUND', /** A declaration under a new key for a resource the base read under another: keys never move. */ 'E_KEY_RENAMED', - /** The plan had more findings than one answer reports; this one counts those left out. */ + /** + * More findings than one answer (or one file's list) reports; this one says where the list + * stopped. Informational: the findings it follows decide whether the apply is refused. + */ 'I_FINDINGS_TRUNCATED', ] as const diff --git a/packages/workspace-code/src/diagnostics/finding.ts b/packages/workspace-code/src/diagnostics/finding.ts index a42e66c07ac..b6f866905c8 100644 --- a/packages/workspace-code/src/diagnostics/finding.ts +++ b/packages/workspace-code/src/diagnostics/finding.ts @@ -41,7 +41,7 @@ export type WorkspaceDiagnostic & ResourceFields /** - * Findings one declaration file lists; past it the file gets one `E_LIMIT_EXCEEDED` and the rest + * Findings one declaration file lists; past it the file gets one `I_FINDINGS_TRUNCATED` and the rest * are dropped, so a file of bad values costs no more than its first hundred. */ export const MAX_FINDINGS_PER_FILE = 100 @@ -51,7 +51,7 @@ const MAX_MESSAGE_LENGTH = 1_000 /** The finding that ends a file's list at {@link MAX_FINDINGS_PER_FILE}. */ export function findingLimitReached(extras: DiagnosticExtras = {}): WorkspaceDiagnostic { return diagnostic( - 'E_LIMIT_EXCEEDED', + 'I_FINDINGS_TRUNCATED', `Stopped after ${MAX_FINDINGS_PER_FILE} findings in this file.`, { ...extras, hint: 'Fix these and apply again to see the rest.' } ) diff --git a/packages/workspace-code/src/kinds/knowledge.test.ts b/packages/workspace-code/src/kinds/knowledge.test.ts index 34820e64766..f3fed3f8001 100644 --- a/packages/workspace-code/src/kinds/knowledge.test.ts +++ b/packages/workspace-code/src/kinds/knowledge.test.ts @@ -192,6 +192,21 @@ describe('knowledgeCodec', () => { ]) }) + it('still refuses when the error is past the first hundred document findings', () => { + const suspicious = Array.from({ length: 120 }, (_, index) => ({ + path: `knowledge/policies/docs/a${index}\u200b.md`, + sha256: 'a'.repeat(64), + size: 1, + })) + const parsed = knowledgeCodec.parse('policies', declaration("{ name: 'Policies' }"), [ + ...suspicious, + { path: 'knowledge/policies/docs/z.md', sha256: 'abc', size: 1 }, + ]) + + expect(parsed.decl).toBeUndefined() + expect(parsed.diagnostics.map((finding) => finding.code)).toContain('E_DECL_SHAPE') + }) + it('caps document findings without spending the declaration file its own findings', () => { const bad = Array.from({ length: 150 }, (_, index) => ({ path: `knowledge/policies/docs/d${index}.md`, @@ -209,7 +224,7 @@ describe('knowledgeCodec', () => { } expect(Object.fromEntries(counts)).toEqual({ E_DECL_SHAPE: 100, - E_LIMIT_EXCEEDED: 1, + I_FINDINGS_TRUNCATED: 1, E_DECL_UNKNOWN_FIELD: 1, }) expect(parsed.diagnostics.at(-1)).toMatchObject({ diff --git a/packages/workspace-code/src/kinds/knowledge.ts b/packages/workspace-code/src/kinds/knowledge.ts index 962317364e0..96717afe5e2 100644 --- a/packages/workspace-code/src/kinds/knowledge.ts +++ b/packages/workspace-code/src/kinds/knowledge.ts @@ -238,14 +238,18 @@ function fileFinding( /** * `findings` about a knowledge base's document files, past {@link MAX_FINDINGS_PER_FILE} folded - * into one `E_LIMIT_EXCEEDED`: a folder of bad documents costs no more than its first hundred. + * into one `I_FINDINGS_TRUNCATED`: a folder of bad documents costs no more than its first + * hundred, and refuses for what those hundred are, not as a size limit. Errors are kept before + * warnings, so a refusal is never folded away behind a hundred warnings. */ function cappedFileFindings(key: string, findings: WorkspaceDiagnostic[]): WorkspaceDiagnostic[] { if (findings.length <= MAX_FINDINGS_PER_FILE) return findings + const errors = findings.filter((finding) => finding.severity === 'error') + const rest = findings.filter((finding) => finding.severity !== 'error') return [ - ...findings.slice(0, MAX_FINDINGS_PER_FILE), + ...[...errors, ...rest].slice(0, MAX_FINDINGS_PER_FILE), diagnostic( - 'E_LIMIT_EXCEEDED', + 'I_FINDINGS_TRUNCATED', `Stopped after ${MAX_FINDINGS_PER_FILE} findings about the files in knowledge/${key}/.`, { kind: 'knowledge-base', key, hint: 'Fix these and apply again to see the rest.' } ), diff --git a/packages/workspace-code/src/kinds/literal-secrets.test.ts b/packages/workspace-code/src/kinds/literal-secrets.test.ts index 5b97d99159c..386b98b9b6a 100644 --- a/packages/workspace-code/src/kinds/literal-secrets.test.ts +++ b/packages/workspace-code/src/kinds/literal-secrets.test.ts @@ -24,7 +24,7 @@ describe('findings about a literal secret', () => { const source = `${HEADER}import { defineMcpServer } from '@sim/workflow-sdk/workspace'\n\nexport default defineMcpServer({\n name: 'Docs',\n url: 'https://mcp.example.com/mcp',\n headers: {\n Authorization: 'Bearer ${SECRET}',\n ${headers},\n },\n})\n` const parsed = mcpCodec.parse('docs', { 'mcp/docs.ts': source }, []) const codes = new Set(parsed.diagnostics.map((finding) => finding.code)) - expect(codes).toEqual(new Set(['E_MCP_LITERAL_SECRET', 'E_LIMIT_EXCEEDED'])) + expect(codes).toEqual(new Set(['E_MCP_LITERAL_SECRET', 'I_FINDINGS_TRUNCATED'])) expect(parsed.diagnostics[0].span).toMatchObject({ file: 'mcp/docs.ts', line: 8 }) expect(echoed(parsed.diagnostics)).not.toContain(SECRET) }) diff --git a/packages/workspace-code/src/limits.ts b/packages/workspace-code/src/limits.ts index 5c035593dde..c788d84709d 100644 --- a/packages/workspace-code/src/limits.ts +++ b/packages/workspace-code/src/limits.ts @@ -21,8 +21,14 @@ export const WORKSPACE_SOURCE_LIMITS = { maxDirectoryWorkflowBytes: 32_000_000, /** Bytes of one blob (a knowledge document or a workspace file). */ maxBlobBytes: 10_000_000, - /** Blob bytes one staging request carries (and one request may download to hash). */ - maxBlobBytesPerRequest: 24_000_000, + /** + * Blob bytes, multipart framing included, one staging request carries when it holds more than + * one blob; a blob up to `maxBlobBytes` goes in a request of its own. Either body stays under + * {@link SOURCE_REQUEST_BODY_CEILING_BYTES}. + */ + maxBlobBytesPerRequest: 8_000_000, + /** Stored bytes one request (a get, or one apply) may download to hash. */ + maxHashBytesPerRequest: 24_000_000, /** Blobs one staging request carries. */ maxBlobsPerRequest: 1_000, /** Blobs one caller may hold staged in one workspace at once. */ @@ -60,13 +66,38 @@ export const WORKSPACE_SOURCE_REQUEST_MAX_BYTES = WORKSPACE_SOURCE_LIMITS.maxWorkflowBytes) + 6_000_000 -/** Body cap for one staging request: its blobs plus multipart framing. */ +/** + * The most bytes a request body may carry and still reach a route whole: Next.js buffers each + * `/api` body for `apps/sim/proxy.ts` and silently truncates it past + * `experimental.proxyClientMaxBodySize` (its 10 MiB default; `PROXY_CLIENT_MAX_BODY_BYTES` in + * apps/sim/lib/api/server/validation.ts), so a larger staging body arrives as a broken multipart + * prefix. + */ +export const SOURCE_REQUEST_BODY_CEILING_BYTES = 10 * 1024 * 1024 + +/** + * Multipart framing budgeted per staged part: its boundary line and the headers naming its SHA-256 + * as field and file name (about 270 bytes as `FormData` serializes them). + */ +export const STAGE_PART_FRAMING_BYTES = 512 + +/** + * Body cap for one staging request: the larger of a full batch and one blob of `maxBlobBytes` + * alone, with its framing and the closing boundary. Below + * {@link SOURCE_REQUEST_BODY_CEILING_BYTES}, so a body the proxy truncated (whose declared or read + * length reaches the ceiling) is refused as too large rather than misread as malformed. + */ export const WORKSPACE_SOURCE_STAGE_MAX_BYTES = - WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest + 2_000_000 + Math.max( + WORKSPACE_SOURCE_LIMITS.maxBlobBytesPerRequest, + WORKSPACE_SOURCE_LIMITS.maxBlobBytes + STAGE_PART_FRAMING_BYTES + ) + STAGE_PART_FRAMING_BYTES /** * Splits `blobs` into staging requests, in order: each carries at most `maxBlobsPerRequest` blobs - * and `maxBlobBytesPerRequest` bytes, and always at least one blob, so every request progresses. + * and `maxBlobBytesPerRequest` bytes counting {@link STAGE_PART_FRAMING_BYTES} per blob, and always + * at least one blob, so every request progresses and a blob too large to share a request goes + * alone. */ export function stageBatches(blobs: readonly T[]): T[][] { const { maxBlobBytesPerRequest, maxBlobsPerRequest } = WORKSPACE_SOURCE_LIMITS @@ -74,16 +105,17 @@ export function stageBatches(blobs: readonly T[]): T let batch: T[] = [] let bytes = 0 for (const blob of blobs) { + const framed = blob.size + STAGE_PART_FRAMING_BYTES if ( batch.length > 0 && - (bytes + blob.size > maxBlobBytesPerRequest || batch.length >= maxBlobsPerRequest) + (bytes + framed > maxBlobBytesPerRequest || batch.length >= maxBlobsPerRequest) ) { batches.push(batch) batch = [] bytes = 0 } batch.push(blob) - bytes += blob.size + bytes += framed } if (batch.length > 0) batches.push(batch) return batches diff --git a/packages/workspace-code/src/promote.test.ts b/packages/workspace-code/src/promote.test.ts new file mode 100644 index 00000000000..ac47d5c83ac --- /dev/null +++ b/packages/workspace-code/src/promote.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest' +import type { WorkspaceBaseV1 } from './base' +import { + type PromotionRecord, + parsePromotionRecord, + promotionRecord, + promotionSubmission, +} from './promote' + +const folderBase = (workflowId: string, fingerprint: string) => + JSON.stringify({ version: 1, workflowId, workspaceId: 'ws_prod', fingerprint, catalogHash: 'c' }) + +const DIR = { + 'workflows/ingest/workflow.ts': '// @sim-sdk 1\n', + 'workflows/ingest/.sim/base.json': folderBase('wf_dev', 'sf1:dev'), +} + +const TARGET = { + files: { + 'workflows/ingest/workflow.ts': '// @sim-sdk 1\n// prod hotfix\n', + 'workflows/ingest/.sim/base.json': folderBase('wf_prod', 'sf1:hotfix'), + }, + base: { + version: 1, + workspaceId: 'ws_prod', + resources: { 'workflow:ingest': { id: 'wf_prod' } }, + } satisfies WorkspaceBaseV1, +} + +/** The record the first promotion of {@link DIR} leaves, with `plan` as the apply's. */ +function firstRecord(plan: Array<{ kind: string; key: string; action: string; held?: boolean }>) { + const submission = promotionSubmission({ + files: DIR, + blobPaths: [], + kinds: ['workflow'], + target: TARGET, + allowDelete: false, + }) + return promotionRecord({ + workspaceId: 'ws_prod', + previous: undefined, + kinds: ['workflow'], + files: DIR, + submission, + applied: { base: TARGET.base, files: {}, plan }, + }) +} + +describe('promotion records', () => { + it('does not record a workflow the apply held back, so the next promotion sends it again', () => { + const record = firstRecord([{ kind: 'workflow', key: 'ingest', action: 'update', held: true }]) + + expect(record.workflows).toEqual({}) + const next = promotionSubmission({ + files: DIR, + blobPaths: [], + kinds: ['workflow'], + target: TARGET, + record, + allowDelete: false, + }) + expect(next.files['workflows/ingest/workflow.ts']).toBe(DIR['workflows/ingest/workflow.ts']) + }) + + it('sends an unchanged folder as its base alone, and an edited one with the recorded fingerprint', () => { + const record = firstRecord([{ kind: 'workflow', key: 'ingest', action: 'noop' }]) + const promote = (files: Record) => + promotionSubmission({ + files, + blobPaths: [], + kinds: ['workflow'], + target: { + ...TARGET, + files: { + ...TARGET.files, + 'workflows/ingest/.sim/base.json': folderBase('wf_prod', 'sf1:edited-in-prod'), + }, + }, + record, + allowDelete: false, + }) + + expect(Object.keys(promote(DIR).files)).toEqual(['workflows/ingest/.sim/base.json']) + const edited = promote({ ...DIR, 'workflows/ingest/workflow.ts': '// @sim-sdk 1\n// v2\n' }) + expect(JSON.parse(edited.files['workflows/ingest/.sim/base.json']).fingerprint).toBe( + 'sf1:hotfix' + ) + }) + + it('ignores a record for another target, or one this build cannot read', () => { + const record: PromotionRecord = { + version: 1, + workspaceId: 'ws_prod', + resources: {}, + workflows: {}, + } + + expect(parsePromotionRecord(record, 'ws_prod')).toEqual(record) + expect(parsePromotionRecord(record, 'ws_other')).toBeUndefined() + expect(parsePromotionRecord({ ...record, workflows: { ingest: 1 } }, 'ws_prod')).toBeUndefined() + expect(parsePromotionRecord({ ...record, version: 2 }, 'ws_prod')).toBeUndefined() + }) +}) diff --git a/packages/workspace-code/src/promote.ts b/packages/workspace-code/src/promote.ts index 2dd67473e81..a66370cff73 100644 --- a/packages/workspace-code/src/promote.ts +++ b/packages/workspace-code/src/promote.ts @@ -3,13 +3,29 @@ * resources by key there. The target's own get answer supplies everything that records ids (the * root base and each workflow folder's lock and base), so the target is planned exactly as an * edit of its own checkout would be: a resource the target holds under the same key is updated - * in place (its ids, a workflow's block ids and so its webhook paths stay), a key it lacks is - * created or adopted by name, and a change made in the target since its get is stale. + * in place (its ids, a workflow's block ids and so its webhook paths stay), and a key it lacks is + * created or adopted by name. + * + * What the target held after the last promotion from this directory is its + * {@link PromotionRecord} (`.sim/targets/.json`), and it is the base the three-way + * rule compares with: a resource changed in the target since then is kept when the directory left + * it alone and refused as stale when the directory changed it too. A resource the record does not + * hold for the target's id (the first promotion, or one created there since) takes the target as + * it stands. * * Client-safe and pure: the CLI composes the submission, the server sees an ordinary apply. */ +import { isRecordLike } from '@sim/utils/object' +import { compareStrings } from '@sim/utils/string' +import { parseSourceBase, sha256Hex, stableStringify } from '@sim/workflow-code' import { SOURCE_BASE_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' -import { parseResourceAddress, type WorkspaceBaseV1 } from './base' +import { + type BaseResource, + parseResourceAddress, + parseWorkspaceBase, + resourceAddress, + type WorkspaceBaseV1, +} from './base' import { isFileKind, KIND_DIRS, type WorkspaceKind } from './kind' import { fileKindGroupKey } from './kinds/entries' import { FILE_KIND_CODECS, KIND_LAYOUTS } from './kinds/registry' @@ -33,6 +49,8 @@ export interface PromotionInput { kinds: readonly WorkspaceKind[] /** The target's get answer for those kinds. */ target: { files: Readonly>; base: WorkspaceBaseV1 } + /** What the last promotion from this directory left in the target, when one is recorded. */ + record?: PromotionRecord /** * Whether the promotion may delete what the target holds and the directory does not declare. * Without it, those resources leave the base, so the apply keeps them with a warning instead of @@ -66,12 +84,124 @@ function declaredKeys( return new Set(read.decl ? Object.keys(read.decl) : []) } +/** + * `.sim/targets/.json`: what one target workspace held after the last promotion from + * this directory, so the next promotion compares with it rather than with the target as it is + * read then (which would never look changed). + */ +export interface PromotionRecord { + version: 1 + /** The target workspace. */ + workspaceId: string + /** The target's root base entries after the last promotion, by resource address. */ + resources: WorkspaceBaseV1['resources'] + /** Each promoted workflow folder, by key. */ + workflows: Record +} + +/** A workflow folder as the last promotion left it in the target. */ +interface PromotedWorkflow { + /** The target's workflow id. */ + workflowId: string + /** The target workflow's source fingerprint after the promotion (its folder base's). */ + fingerprint: string + /** {@link folderDigest} of the directory's folder as promoted. */ + digest: string +} + +/** The directory path of a target's promotion record. */ +export function promotionRecordPath(workspaceId: string): string | null { + return /^[A-Za-z0-9_-]{1,128}$/.test(workspaceId) ? `.sim/targets/${workspaceId}.json` : null +} + +/** The digest of a workflow folder's source files (its records excluded), by path and text. */ +function folderDigest(key: string, files: Readonly>): string { + const prefix = `${workflowFolder(key)}/` + const entries = Object.entries(files) + .filter(([path]) => path.startsWith(prefix) && !path.slice(prefix.length).startsWith('.sim/')) + .sort(([a], [b]) => compareStrings(a, b)) + return sha256Hex(stableStringify(entries)) +} + +/** A workflow folder base's text with `fingerprint` in place of its own, or `null` if unreadable. */ +function withFingerprint(text: string, fingerprint: string): string | null { + const parsed = parseSourceBase(text) + if (!('base' in parsed)) return null + return `${JSON.stringify({ ...parsed.base, fingerprint }, null, 2)}\n` +} + +/** + * The folder base of a workflow the last promotion left in target `workspaceId`, from its record: + * it names that workflow and the fingerprint the promotion left. The catalog it was printed with + * is not recorded, and a base naming a workflow the target no longer holds is never compiled. + */ +function recordedFolderBase(promoted: PromotedWorkflow, workspaceId: string): string { + return `${JSON.stringify( + { + version: 1, + workflowId: promoted.workflowId, + workspaceId, + fingerprint: promoted.fingerprint, + catalogHash: '', + }, + null, + 2 + )}\n` +} + +/** The workflow a folder base's text names, or `null` if unreadable. */ +function folderWorkflow( + text: string | undefined +): { workflowId: string; fingerprint: string } | null { + if (text === undefined) return null + const parsed = parseSourceBase(text) + return 'base' in parsed + ? { workflowId: parsed.base.workflowId, fingerprint: parsed.base.fingerprint } + : null +} + +/** + * Reads a promotion record for `workspaceId`; `undefined` for anything else (no record, another + * target's, an unreadable one), which makes the promotion take the target as it stands. + */ +export function parsePromotionRecord( + input: unknown, + workspaceId: string +): PromotionRecord | undefined { + if (!isRecordLike(input) || input.version !== 1 || input.workspaceId !== workspaceId) { + return undefined + } + const base = parseWorkspaceBase({ version: 1, workspaceId, resources: input.resources }).base + if (!base || !isRecordLike(input.workflows)) return undefined + const workflows: Record = {} + for (const [key, entry] of Object.entries(input.workflows)) { + if ( + !isRecordLike(entry) || + typeof entry.workflowId !== 'string' || + typeof entry.fingerprint !== 'string' || + typeof entry.digest !== 'string' + ) { + return undefined + } + workflows[key] = { + workflowId: entry.workflowId, + fingerprint: entry.fingerprint, + digest: entry.digest, + } + } + return { version: 1, workspaceId, resources: base.resources, workflows } +} + /** * The submission that promotes a directory to the target its get answer describes: the * directory's source with every workflow folder's lock and base replaced by the target's folder * of the same key (or dropped, so a key the target lacks creates a workflow), and the target's - * base. Values never travel: secrets and credentials are names and keys the target binds to its - * own. + * base, narrowed to what the directory declares unless deletes are allowed. With a + * {@link PromotionRecord}, each entry the record holds for the same id stands in for the target's + * own, and a workflow folder records the fingerprint the last promotion left: sent as its base + * alone (a no-op) when the directory has not changed it since, else whole, so a target edit since + * is refused as stale. Values never travel: secrets and credentials are names and keys the target + * binds to its own. */ export function promotionSubmission(input: PromotionInput): PromotionSubmission { const files: Record = {} @@ -83,20 +213,117 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission input.kinds.map((kind) => [kind, declaredKeys(kind, files, paths)] as const) ) for (const key of declared.get('workflow') ?? []) { - for (const record of [SOURCE_LOCK_FILE, SOURCE_BASE_FILE]) { - const path = `${workflowFolder(key)}/${record}` - const text = input.target.files[path] - if (text !== undefined) files[path] = text + const folder = workflowFolder(key) + const basePath = `${folder}/${SOURCE_BASE_FILE}` + const lockPath = `${folder}/${SOURCE_LOCK_FILE}` + const targetBase = input.target.files[basePath] + const live = folderWorkflow(targetBase) + const promoted = input.record?.workflows[key] + if (targetBase === undefined && promoted) { + // Deleted in the target since the last promotion: the recorded base names the workflow it + // left there, so the apply keeps the deletion, or refuses an edit since as stale. + const unchanged = promoted.digest === folderDigest(key, input.files) + if (unchanged) { + for (const path of Object.keys(files)) { + if (path.startsWith(`${folder}/`)) delete files[path] + } + } + files[basePath] = recordedFolderBase(promoted, input.target.base.workspaceId) + continue + } + if (targetBase === undefined || !live || promoted?.workflowId !== live.workflowId) { + for (const path of [basePath, lockPath]) { + const text = input.target.files[path] + if (text !== undefined) files[path] = text + } + continue } + if (promoted.digest === folderDigest(key, input.files)) { + for (const path of Object.keys(files)) { + if (path.startsWith(`${folder}/`)) delete files[path] + } + files[basePath] = targetBase + continue + } + files[basePath] = withFingerprint(targetBase, promoted.fingerprint) ?? targetBase + const lock = input.target.files[lockPath] + if (lock !== undefined) files[lockPath] = lock } const resources = Object.fromEntries( - Object.entries(input.target.base.resources).filter(([address]) => { + Object.entries(input.target.base.resources).flatMap(([address, live]) => { const parsed = parseResourceAddress(address) - if (!parsed) return false + if (!parsed) return [] const keys = declared.get(parsed.kind) - if (keys === undefined) return false - return input.allowDelete || keys.has(parsed.key) + if (keys === undefined || !(input.allowDelete || keys.has(parsed.key))) return [] + const recorded: BaseResource | undefined = + input.record?.resources[resourceAddress(parsed.kind, parsed.key)] + return [[address, recorded?.id === live.id ? recorded : live] as const] }) ) + // A resource the last promotion left in the target that the target no longer holds was deleted + // there: its recorded entry stays in the base, so the apply keeps the deletion when the + // directory left it alone and refuses an edit since as stale, rather than creating it again. + for (const [address, recorded] of Object.entries(input.record?.resources ?? {})) { + if (Object.hasOwn(input.target.base.resources, address)) continue + const parsed = parseResourceAddress(address) + if (parsed && declared.get(parsed.kind)?.has(parsed.key)) resources[address] = recorded + } return { files, base: { ...input.target.base, resources } } } + +/** + * The record a promotion leaves for its target, from the apply's answer (`applied`: the target's + * base after it, the files it reprinted, among them each written workflow folder's base, and its + * plan). A resource's entry is the answer's base entry; a workflow folder's is what the apply + * wrote, or what was sent whole and planned as a no-op, else the `previous` entry (a folder sent + * as its base alone, or held back, or never reached). The record of a kind the promotion did not + * manage is kept. + */ +export function promotionRecord(input: { + workspaceId: string + previous: PromotionRecord | undefined + kinds: readonly WorkspaceKind[] + /** The directory's text files, as {@link PromotionInput} took them. */ + files: Readonly> + submission: PromotionSubmission + applied: { + base: WorkspaceBaseV1 + files: Readonly> + plan: ReadonlyArray<{ kind: string; key: string; action: string; held?: boolean }> + } +}): PromotionRecord { + const managed = new Set(input.kinds) + const resources: WorkspaceBaseV1['resources'] = {} + for (const [address, entry] of Object.entries(input.previous?.resources ?? {})) { + const parsed = parseResourceAddress(address) + if (parsed && !managed.has(parsed.kind)) { + resources[resourceAddress(parsed.kind, parsed.key)] = entry + } + } + Object.assign(resources, input.applied.base.resources) + const workflows: Record = {} + if (!managed.has('workflow')) { + Object.assign(workflows, input.previous?.workflows) + return { version: 1, workspaceId: input.workspaceId, resources, workflows } + } + const paths = Object.keys(input.files).filter((path) => kindOfPath(path) === 'workflow') + for (const key of KIND_LAYOUTS.workflow.group(paths).groups.keys()) { + const folder = `${workflowFolder(key)}/` + const basePath = `${folder}${SOURCE_BASE_FILE}` + const change = input.applied.plan.find( + (planned) => planned.kind === 'workflow' && planned.key === key + ) + const sentWhole = Object.keys(input.submission.files).some( + (path) => path.startsWith(folder) && path !== basePath + ) + const left = + folderWorkflow(input.applied.files[basePath]) ?? + (sentWhole && change?.action === 'noop' && !change.held + ? folderWorkflow(input.submission.files[basePath]) + : null) + const previous = input.previous?.workflows[key] + if (left) workflows[key] = { ...left, digest: folderDigest(key, input.files) } + else if (previous) workflows[key] = previous + } + return { version: 1, workspaceId: input.workspaceId, resources, workflows } +}