diff --git a/.claude/rules/sim-workflow-code.md b/.claude/rules/sim-workflow-code.md index 752c3d67731..8bf5712ad06 100644 --- a/.claude/rules/sim-workflow-code.md +++ b/.claude/rules/sim-workflow-code.md @@ -4,13 +4,23 @@ paths: - "packages/workflow-code/**" - "packages/workspace-code/**" - "packages/workflow-sdk/**" + - "packages/table-cells/**" + - "packages/workflow-persistence/src/save.ts" - "packages/sim-cli/src/commands/protocol/*source*" + - "packages/sim-cli/src/commands/protocol/source-dir/**" + - "packages/sim-cli/src/commands/protocol/workflow-dir.ts" + - "packages/sim-cli/src/commands/protocol/stored-edges.ts" + - "packages/sim-cli/src/commands/protocol/local-errors.ts" - "apps/sim/lib/workflows/code/**" - "apps/sim/lib/workflows/application/*source*" + - "apps/sim/lib/workflows/application/source-pipeline/**" + - "apps/sim/lib/workflows/api/source-*" - "apps/sim/lib/workspace-source/**" - "apps/sim/blocks/sdk-names/**" - "apps/sim/app/workspace/**/panel/components/code/**" + - "apps/sim/app/api/workflows/*/source/**" - "apps/sim/app/api/v2/**/source/**" + - "apps/sim/lib/api/contracts/workflow-source.ts" - "apps/sim/lib/api/contracts/v2/*source*" - "apps/sim/app/api/cron/cleanup-source-staging/**" --- @@ -33,7 +43,7 @@ workspace directory). | `packages/workflow-sdk` | Authoring declarations (`src/core.d.ts`, `src/workspace/*.d.ts`), type codegen from the catalog (`src/codegen/`), the cookbook | | `packages/sim-cli/src/commands/protocol/` | `workflow-source-*` and `workspace-source-*` commands, which never import each other's command modules; `source-dir/`, `workflow-dir.ts`, `source-check.ts`, `source-write.ts`, `source-tree.ts`, `source-ui.ts` and `workspace-source-dir.ts` (the workspace directory reader a workflow check also reads declarations through) are shared | | `apps/sim/lib/workflows/code/` | Server glue: catalog projection from the registry (`catalog/`), source-apply gates (`apply/`), stored canonical form, the feature flag, the proof corpus (`corpus/`) | -| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources | +| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources; `source-pipeline/` holds the one plan → preview/commit flow an apply and a create share | | `apps/sim/lib/workspace-source/` | Workspace source use cases (`application/`), one adapter per kind (`application/kinds//`), route policies (`api/`) | | `apps/sim/blocks/sdk-names/.json` | Frozen SDK builder names, one file per block type | | `apps/sim/app/workspace/**/panel/components/code/` | The read-only Code tab | diff --git a/.cursor/rules/sim-workflow-code.mdc b/.cursor/rules/sim-workflow-code.mdc index aaf39757082..c631f32be1b 100644 --- a/.cursor/rules/sim-workflow-code.mdc +++ b/.cursor/rules/sim-workflow-code.mdc @@ -1,6 +1,6 @@ --- description: "Workflows and workspaces as code (workflow source, workspace source, the workflow SDK, the Code tab)" -globs: ["packages/workflow-code/**","packages/workspace-code/**","packages/workflow-sdk/**","packages/sim-cli/src/commands/protocol/*source*","apps/sim/lib/workflows/code/**","apps/sim/lib/workflows/application/*source*","apps/sim/lib/workspace-source/**","apps/sim/blocks/sdk-names/**","apps/sim/app/workspace/**/panel/components/code/**","apps/sim/app/api/v2/**/source/**","apps/sim/lib/api/contracts/v2/*source*","apps/sim/app/api/cron/cleanup-source-staging/**"] +globs: ["packages/workflow-code/**","packages/workspace-code/**","packages/workflow-sdk/**","packages/table-cells/**","packages/workflow-persistence/src/save.ts","packages/sim-cli/src/commands/protocol/*source*","packages/sim-cli/src/commands/protocol/source-dir/**","packages/sim-cli/src/commands/protocol/workflow-dir.ts","packages/sim-cli/src/commands/protocol/stored-edges.ts","packages/sim-cli/src/commands/protocol/local-errors.ts","apps/sim/lib/workflows/code/**","apps/sim/lib/workflows/application/*source*","apps/sim/lib/workflows/application/source-pipeline/**","apps/sim/lib/workflows/api/source-*","apps/sim/lib/workspace-source/**","apps/sim/blocks/sdk-names/**","apps/sim/app/workspace/**/panel/components/code/**","apps/sim/app/api/workflows/*/source/**","apps/sim/app/api/v2/**/source/**","apps/sim/lib/api/contracts/workflow-source.ts","apps/sim/lib/api/contracts/v2/*source*","apps/sim/app/api/cron/cleanup-source-staging/**"] --- @@ -23,7 +23,7 @@ workspace directory). | `packages/workflow-sdk` | Authoring declarations (`src/core.d.ts`, `src/workspace/*.d.ts`), type codegen from the catalog (`src/codegen/`), the cookbook | | `packages/sim-cli/src/commands/protocol/` | `workflow-source-*` and `workspace-source-*` commands, which never import each other's command modules; `source-dir/`, `workflow-dir.ts`, `source-check.ts`, `source-write.ts`, `source-tree.ts`, `source-ui.ts` and `workspace-source-dir.ts` (the workspace directory reader a workflow check also reads declarations through) are shared | | `apps/sim/lib/workflows/code/` | Server glue: catalog projection from the registry (`catalog/`), source-apply gates (`apply/`), stored canonical form, the feature flag, the proof corpus (`corpus/`) | -| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources | +| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources; `source-pipeline/` holds the one plan → preview/commit flow an apply and a create share | | `apps/sim/lib/workspace-source/` | Workspace source use cases (`application/`), one adapter per kind (`application/kinds//`), route policies (`api/`) | | `apps/sim/blocks/sdk-names/.json` | Frozen SDK builder names, one file per block type | | `apps/sim/app/workspace/**/panel/components/code/` | The read-only Code tab | diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index f3c7c515d3d..b91e32700b9 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8586,7 +8586,7 @@ sim workflows source apply [workflowId] [options] Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment), [getSourceResources](/api-reference/workflows/getSourceResources). ### sim workflows source create @@ -8614,7 +8614,7 @@ sim workflows source create [options] Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment). ### sim workflows source check @@ -8641,7 +8641,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -9982,7 +9982,8 @@ sim workspaces source apply [options] | `--dir ` | No | Workspace source directory written by `workspaces source get`. Defaults to `.`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | -| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | +| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares (with --to, only what it deleted since the last promotion: the target's own resources are kept). | +| `--delete-target-only` | No | With --to and --allow-delete, also delete the target's resources this directory never declared and no promotion from it left there (prod-only tables and their rows, skills made in prod). Without it a promotion keeps them, naming each with a warning. | | `--only ` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | | `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | @@ -10017,7 +10018,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index a376ef22740..7e76fc8249d 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1367,7 +1367,7 @@ sim workflows source create [options] Exit codes: 0 created (or a clean dry run, or a starter written); 1 the source has errors (each printed as file:line:column with its frame and hint), the workflow name is taken (SOURCE_NAME_TAKEN: pass --name), or the command failed (a starter is created in the workspace its types were written for unless --workspace names another); 6 the folder is locked; 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was created: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was created (run the same command with --yes). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow). +API request and response schemas: [createWorkflowFromSource](/api-reference/workflows/createWorkflowFromSource), [getSourceCatalog](/api-reference/catalog/getSourceCatalog), [getSourceResources](/api-reference/workflows/getSourceResources), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment). ### Apply a workflow source directory to a workflow draft @@ -1402,7 +1402,7 @@ sim workflows source apply [workflowId] [options] Exit codes: 0 applied (or a clean dry run); 1 the source has errors (each printed as file:line:column with its frame and hint) or the command failed; 5 the workflow changed since the directory was read (SOURCE_STALE, on a dry run too: get it again, reapply the edit, retry); 6 the workflow or its folder is locked (E_APPLY_LOCKED; a locked block exits 1, E_APPLY_LOCKED_BLOCK); 7 the source is too large; 9 rate limited (HTTP 429) after the CLI's retries, so nothing was written: wait a minute and run it again; 10 (busy, retry) another sim command is using --dir, or the workspace directory it is a workflow folder of (SOURCE_DIR_BUSY): nothing was read or written, run it again once the other finished; 14 without --yes and with no terminal to ask: the plan was printed with no errors and nothing was written (run the same command with --yes to apply it). Exit 13: not logged in, or the server refused the credential (AUTH_REQUIRED); nothing was sent. Run `sim login`, or fix or unset SIM_API_KEY when the key comes from it. Exit 130: interrupted (Ctrl-C). A write already sent is finished and saved into the directory first, waiting up to 60s for its answer; a second Ctrl-C, or no answer by then, stops at once and prints that it may have been written and how to check. -API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getSourceResources](/api-reference/workflows/getSourceResources). +API request and response schemas: [applyWorkflowSource](/api-reference/workflows/applyWorkflowSource), [deployWorkflow](/api-reference/workflows/deployWorkflow), [getWorkflowDeployment](/api-reference/workflows/getWorkflowDeployment), [getSourceResources](/api-reference/workflows/getSourceResources). ### Show the block catalog workflow source is checked against @@ -1445,7 +1445,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index e0002bfa79b..64e39c047d7 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -753,7 +753,8 @@ sim workspaces source apply [options] | `--dir ` | No | Workspace source directory written by `workspaces source get`. Defaults to `.`. | | `--dry-run` | No | Plan and check without writing anything. | | `-y, --yes` | No | Apply without asking for confirmation. | -| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares. | +| `--allow-delete` | No | Delete resources, columns and data the directory no longer declares (with --to, only what it deleted since the last promotion: the target's own resources are kept). | +| `--delete-target-only` | No | With --to and --allow-delete, also delete the target's resources this directory never declared and no promotion from it left there (prod-only tables and their rows, skills made in prod). Without it a promotion keeps them, naming each with a warning. | | `--only ` | No | Apply only these kinds (comma-separated folders: secrets.ts, credentials.ts, tools, skills, mcp, tables, knowledge, files, workflows, deployments). | | `--to ` | No | Promote the directory to another workspace (dev to prod): its resources bind by key to that workspace's own and are updated in place. Only .sim/targets/<workspaceId>.json changes, the record the next promotion compares with: a resource changed there since is kept, or refused as stale (exit 5) when the directory changed it too. | @@ -786,7 +787,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/workflows/workspace-code.mdx b/apps/docs/content/docs/workflows/workspace-code.mdx index bfe38d33471..c19b2711db9 100644 --- a/apps/docs/content/docs/workflows/workspace-code.mdx +++ b/apps/docs/content/docs/workflows/workspace-code.mdx @@ -93,7 +93,7 @@ sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --dry sim workspaces source apply --dir ./my-workspace --to "$PROD_WORKSPACE_ID" --yes ``` -Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. Resources only the target holds are kept unless you pass `--allow-delete`, which deletes every one of them: the plan names each with a `W_PROMOTE_DELETES_TARGET_ONLY` warning. The directory stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. +Each resource binds by key to the target's own. One the target holds under the same key is updated in place, so a workflow keeps its id and its webhook URLs. A key the target lacks is created. Secrets and credentials bind by name to the target's own, and no value is copied. A resource you deleted from the directory since the last promotion is deleted in the target too, and like any delete it needs `--allow-delete`. Resources only the target holds, such as a table created in prod and the rows its runs wrote, are kept, with or without `--allow-delete`: the plan names each with a `W_UNDECLARED` warning. To delete them as well, pass `--delete-target-only` beside `--allow-delete`; the plan then names each with a `W_PROMOTE_DELETES_TARGET_ONLY` warning, so run it with `--dry-run` first. The directory stays bound to the workspace it was read from. Without `--to`, `check`, `status`, `diff` and `apply` act on the workspace the directory was read from, whatever your default workspace; a `--workspace` naming another one is refused. Each promotion records what it left in the target in `.sim/targets/.json`, and the next promotion compares with that record: @@ -102,7 +102,7 @@ Each promotion records what it left in the target in `.sim/targets/ - A resource deleted in the target since the last promotion stays deleted, with a `W_PROMOTE_TARGET_DELETED` warning. Remove its entry from the record to create it again. - The first promotion, and any resource the record does not hold (one created in the target since), updates the target to match the directory. Review the dry run first. -Commit the record with the rest of the directory, so a promotion from CI or another clone compares with it. Deleting it makes the next promotion take the target as it stands. +Commit the record with the rest of the directory, so a promotion from CI or another clone compares with it. Don't delete it to get past a stale refusal: without it, the next promotion overwrites every change made in the target since, including rows its workflows wrote. ## Git and CI diff --git a/apps/docs/openapi-v2-resources.json b/apps/docs/openapi-v2-resources.json index 09a62bdb1e0..06b645fde37 100644 --- a/apps/docs/openapi-v2-resources.json +++ b/apps/docs/openapi-v2-resources.json @@ -32568,6 +32568,11 @@ } ] }, + "optionsVary": { + "description": "The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.", + "type": "boolean", + "const": true + }, "hidden": { "description": "Hidden from the editor.", "type": "boolean" @@ -33643,6 +33648,11 @@ } ] }, + "optionsVary": { + "description": "The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.", + "type": "boolean", + "const": true + }, "hidden": { "description": "Hidden from the editor.", "type": "boolean" @@ -34491,6 +34501,11 @@ } ] }, + "optionsVary": { + "description": "The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.", + "type": "boolean", + "const": true + }, "hidden": { "description": "Hidden from the editor.", "type": "boolean" diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts index 8838546e7d2..c9710866aa6 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.test.ts @@ -41,17 +41,17 @@ const auth = { keyType: 'workspace' as const, } -function apply(query = '') { +function apply( + query = '', + files: Record = { + 'workflows/a/workflow.ts': "// @sim-sdk 1\nexport default workflow('A', () => {})\n", + } +) { return POST( new NextRequest(`http://localhost:3000/api/v2/workspaces/${WORKSPACE_ID}/source${query}`, { method: 'POST', headers: { 'x-api-key': 'key', 'content-type': 'application/json' }, - body: JSON.stringify({ - files: { - 'workflows/a/workflow.ts': "// @sim-sdk 1\nexport default workflow('A', () => {})\n", - }, - kinds: ['workflow'], - }), + body: JSON.stringify({ files, kinds: ['workflow'] }), }), { params: Promise.resolve({ workspaceId: WORKSPACE_ID }) } ) @@ -76,6 +76,36 @@ describe('POST /api/v2/workspaces/{workspaceId}/source', () => { v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) }) + it('charges a dry run, which writes nothing, to the plan bucket per folder sent, and a real apply to the write bucket', async () => { + mocks.apply.mockResolvedValue({ + workspaceId: WORKSPACE_ID, + dryRun: true, + plan: [], + applied: [], + needBlobs: [], + files: {}, + base: { version: 1, workspaceId: WORKSPACE_ID, resources: {} }, + diagnostics: [], + }) + const folders = Object.fromEntries( + Array.from({ length: 61 }, (_, index) => [ + `workflows/w${index}/workflow.ts`, + "// @sim-sdk 1\nexport default workflow('W', () => {})\n", + ]) + ) + const charges = () => + v2RouteMocks.operationRate.mock.calls + .filter(([key]) => String(key).endsWith(':api-key:key-1')) + .map(([key, , cost]) => `${String(key).split(':')[1]} ${cost}`) + + expect((await apply('?dryRun=true', folders)).status).toBe(200) + expect(charges()).toEqual(['source.plan 1', 'source.plan 39']) + + v2RouteMocks.operationRate.mockClear() + expect((await apply('', folders)).status).toBe(200) + expect(charges()).toEqual(['workflows.source.write 1', 'workflows.source.write 39']) + }) + it('answers a plan refused before any write with at most 1000 findings, and one that counts the rest', async () => { mocks.apply.mockRejectedValue(new WorkspaceSourceRejectedError(folderErrors(1_500))) diff --git a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts index 4afda03230c..dd95b4209ea 100644 --- a/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts +++ b/apps/sim/app/api/v2/workspaces/[workspaceId]/source/route.ts @@ -43,14 +43,16 @@ export const GET = defineV2JsonRoute({ * POST /api/v2/workspaces/[workspaceId]/source — applies a workspace source directory. Any error * in the plan writes nothing (400, 409 stale, 423 locked, 413 over a limit); a dry run answers * 200 with its findings. A write that fails mid-apply answers 200 with `failed` and what was - * written before it. Draws on the source-write budget (see `workspaceApplyRateCost`); a server - * already running as many applies as it allows answers 429. + * written before it. A real apply draws on the source-write budget and a dry run, which writes + * nothing, on the source-plan budget the status plan route uses (`v2RateLimits.sourceApply`), each + * charged per workflow folder sent with its files (`workspaceApplyRateCost`); a server already + * running as many applies as it allows answers 429. */ export const POST = defineV2JsonRoute({ contract: v2ApplyWorkspaceSourceContract, auth: v2ApiKeyAuth, operation: workspaceSourceOperations.apply, - rateLimit: v2RateLimits.sourceWrite, + rateLimit: v2RateLimits.sourceApply, errorPolicy: v2WorkspaceSourceErrorPolicy, parseOptions: { maxBodyBytes: WORKSPACE_SOURCE_REQUEST_MAX_BYTES }, mapInput: ({ params, query, body }) => ({ diff --git a/apps/sim/executor/execution/condition-join-loop.test.ts b/apps/sim/executor/execution/condition-join-loop.test.ts new file mode 100644 index 00000000000..2552774177d --- /dev/null +++ b/apps/sim/executor/execution/condition-join-loop.test.ts @@ -0,0 +1,105 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { + createSerializedBlock, + createSerializedWorkflow, +} from '@sim/testing/factories/serialized-block.factory' +import { toolsMock, toolsMockFns } from '@sim/testing/mocks/tools.mock' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { BlockType } from '@/executor/constants' +import { DAGExecutor } from '@/executor/execution/executor' +import { ConditionBlockHandler } from '@/executor/handlers/condition/handler' +import type { SerializedWorkflow } from '@/serializer/types' + +vi.mock('@/tools', () => toolsMock) + +/** Real condition handler; every other block answers with its own name. */ +vi.mock('@/executor/handlers/registry', () => ({ + createBlockHandlers: () => [ + new ConditionBlockHandler(), + { + canHandle: () => true, + execute: async (_ctx: unknown, block: { metadata?: { name?: string } }) => ({ + result: block.metadata?.name, + }), + }, + ], +})) + +const PRINCIPAL: SessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } + +function condition(id: string, conditions: Array<{ id: string; title: string; value: string }>) { + return createSerializedBlock({ + id, + name: id, + type: BlockType.CONDITION, + params: { conditions: JSON.stringify(conditions) }, + }) +} + +/** + * A two-iteration loop where `lc1` sends iteration 0 to `la` and iteration 1 to `lb`, both + * joining at `lc2`, whose passthrough output carries the branch it read. + */ +function joinInLoop(branchEdgeOrder: ['la' | 'lb', 'la' | 'lb']): SerializedWorkflow { + const blocks = [ + createSerializedBlock({ id: 'start', name: 'start', type: BlockType.STARTER }), + createSerializedBlock({ id: 'loop', name: 'loop', type: BlockType.LOOP }), + condition('lc1', [ + { id: 'first', title: 'if', value: ' == 0' }, + { id: 'rest', title: 'else', value: '' }, + ]), + createSerializedBlock({ id: 'la', name: 'LA', type: BlockType.FUNCTION }), + createSerializedBlock({ id: 'lb', name: 'LB', type: BlockType.FUNCTION }), + condition('lc2', [{ id: 'always', title: 'if', value: 'true' }]), + createSerializedBlock({ id: 'lout', name: 'LOut', type: BlockType.FUNCTION }), + ] + return { + ...createSerializedWorkflow(blocks, [ + { source: 'start', target: 'loop' }, + { source: 'loop', target: 'lc1', sourceHandle: 'loop-start-source' }, + { source: 'lc1', target: 'la', sourceHandle: 'condition-first' }, + { source: 'lc1', target: 'lb', sourceHandle: 'condition-rest' }, + ...branchEdgeOrder.map((source) => ({ source, target: 'lc2' })), + { source: 'lc2', target: 'lout', sourceHandle: 'condition-always' }, + ]), + loops: { + loop: { + id: 'loop', + nodes: ['lc1', 'la', 'lb', 'lc2', 'lout'], + iterations: 2, + loopType: 'for', + }, + }, + } +} + +describe('a condition joining two branches inside a loop', () => { + /** Condition code runs as plain JavaScript instead of through the sandboxed function tool. */ + beforeEach(() => { + toolsMockFns.mockExecuteTool.mockImplementation( + async (_toolId: string, params: { code: string }) => ({ + success: true, + output: { result: new Function(params.code)() }, + }) + ) + }) + + it.each([ + ['la', 'lb'], + ['lb', 'la'], + ] as const)( + 'reads the branch that ran in each iteration (edges into the join: %s first)', + async (first, second) => { + const result = await new DAGExecutor({ + workflow: joinInLoop([first, second]), + contextExtensions: { workspaceId: 'ws', executionId: 'exec-1', principal: PRINCIPAL }, + }).execute('wf') + + expect(result.success).toBe(true) + const joinReads = (result.logs ?? []) + .filter((log) => log.blockId.startsWith('lc2')) + .map((log) => log.output?.result) + expect(joinReads).toEqual(['LA', 'LB']) + } + ) +}) diff --git a/apps/sim/executor/handlers/condition/handler.test.ts b/apps/sim/executor/handlers/condition/handler.test.ts index 6535bb2ec7f..1f49b471731 100644 --- a/apps/sim/executor/handlers/condition/handler.test.ts +++ b/apps/sim/executor/handlers/condition/handler.test.ts @@ -811,6 +811,7 @@ describe('ConditionBlockHandler', () => { workspaceId: 'test-workspace-id', workflow: parallelWorkflow, blockStates: parallelBlockStates, + executedBlocks: new Set(parallelBlockStates.keys()), blockLogs: [], completedBlocks: new Set(), decisions: { diff --git a/apps/sim/executor/handlers/condition/handler.ts b/apps/sim/executor/handlers/condition/handler.ts index eb9076af9f7..52f4694676e 100644 --- a/apps/sim/executor/handlers/condition/handler.ts +++ b/apps/sim/executor/handlers/condition/handler.ts @@ -569,7 +569,8 @@ export class ConditionBlockHandler implements BlockHandler { * The block whose output the condition evaluates against and passes on: of the blocks with an * edge into it (each as its own branch's copy inside a parallel, when that copy ran), the first * that ran in this execution, else the first. A condition joining an if/else so reads the - * branch that ran, whichever edge was drawn first. + * branch that ran, whichever edge was drawn first. "Ran" reads `executedBlocks`, which a loop + * clears between iterations; a block state keeps `executed` from earlier iterations. */ private resolveSourceBlockId( ctx: ExecutionContext, @@ -583,7 +584,7 @@ export class ConditionBlockHandler implements BlockHandler { const virtualSourceId = buildBranchNodeId(conn.source, branchIndex) return ctx.blockStates.has(virtualSourceId) ? virtualSourceId : conn.source }) - return sources.find((id) => ctx.blockStates.get(id)?.executed) ?? sources[0] + return sources.find((id) => ctx.executedBlocks.has(id)) ?? sources[0] } private findConnectionForCondition( diff --git a/apps/sim/lib/api/contracts/v2/source-catalog.ts b/apps/sim/lib/api/contracts/v2/source-catalog.ts index b157471e323..78091e623d3 100644 --- a/apps/sim/lib/api/contracts/v2/source-catalog.ts +++ b/apps/sim/lib/api/contracts/v2/source-catalog.ts @@ -168,6 +168,12 @@ const catalogSubBlockSchema = z ]) .optional() .describe('Selectable options.'), + optionsVary: z + .literal(true) + .optional() + .describe( + 'The options depend on deployment settings and were evaluated where this catalog was built; the write-time check evaluates them again.' + ), hidden: z.boolean().optional().describe('Hidden from the editor.'), readOnly: z.boolean().optional().describe('Display-only.'), password: z.boolean().optional().describe('Holds a secret.'), diff --git a/apps/sim/lib/api/server/routes/v2-json-route.ts b/apps/sim/lib/api/server/routes/v2-json-route.ts index dee705ef039..502c86a20ca 100644 --- a/apps/sim/lib/api/server/routes/v2-json-route.ts +++ b/apps/sim/lib/api/server/routes/v2-json-route.ts @@ -4,6 +4,7 @@ import { setRequestAuth } from '@sim/logger' import { LRUCache } from 'lru-cache' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' +import { booleanQueryFlagSchema } from '@/lib/api/contracts/primitives' import { recordRateLimitSnapshot } from '@/lib/api/server/rate-limit-context' import { type CopilotRouteUseCase, @@ -164,7 +165,8 @@ const VALIDATED_BODY_CACHE_MAX = 4 * The budget for writing workflow source (apply, dry run, and create from source), per subject * and on every plan. Each request parses, compiles, canonicalizes and diffs a body of up to * `SOURCE_REQUEST_MAX_BYTES`, so it costs far more than the row reads the generic budget is sized - * for; a dry run costs nearly as much as an apply, so it is charged the same. + * for; a dry run costs nearly as much as an apply, so it is charged the same amount (a workspace + * dry run on the plan bucket instead, see `sourceApply`). */ const V2_SOURCE_WRITE_RATE_LIMIT: TokenBucketConfig = { maxTokens: 40, @@ -195,6 +197,14 @@ const V2_SOURCE_PLAN_RATE_LIMIT: TokenBucketConfig = { refillIntervalMs: 60_000, } +/** Whether `request` asks for a dry run (`?dryRun=true`), as the source apply contract reads it. */ +function isDryRunRequest(request: NextRequest): boolean { + const flag = booleanQueryFlagSchema.safeParse( + request.nextUrl.searchParams.get('dryRun') ?? undefined + ) + return flag.success && flag.data +} + /** * Staging the binary files of a workspace source apply: each request carries up to 24 MB, so a * 1 GB directory stages in about 45 requests. Its own bucket, so staging never spends the @@ -243,6 +253,26 @@ export const v2RateLimits = { return enforceV2Buckets(request, auth, 'source.plan', V2_SOURCE_PLAN_RATE_LIMIT, cost) }, } satisfies V2RateLimitPolicy, + /** + * A source apply route that also serves dry runs (`?dryRun=true`): a dry run writes nothing and + * runs what a plan runs, so it draws on the {@link V2_SOURCE_PLAN_RATE_LIMIT} bucket as the plan + * routes do, and only a real apply spends the {@link V2_SOURCE_WRITE_RATE_LIMIT} bucket applies + * need. Capped at the smaller bucket, so the largest request fits either. + */ + sourceApply: { + maxCost: Math.min(V2_SOURCE_WRITE_RATE_LIMIT.maxTokens, V2_SOURCE_PLAN_RATE_LIMIT.maxTokens), + async enforce(request, auth, _operation, cost = 1) { + return isDryRunRequest(request) + ? enforceV2Buckets(request, auth, 'source.plan', V2_SOURCE_PLAN_RATE_LIMIT, cost) + : enforceV2Buckets( + request, + auth, + 'workflows.source.write', + V2_SOURCE_WRITE_RATE_LIMIT, + cost + ) + }, + } satisfies V2RateLimitPolicy, /** {@link V2_SOURCE_BLOB_STAGE_RATE_LIMIT}, shared by staging and asking what is staged. */ sourceBlobStage: { async enforce(request, auth) { diff --git a/apps/sim/lib/table/application/reshape.ts b/apps/sim/lib/table/application/reshape.ts index f61f12172e4..2d19b0ac2e9 100644 --- a/apps/sim/lib/table/application/reshape.ts +++ b/apps/sim/lib/table/application/reshape.ts @@ -33,7 +33,7 @@ import { rowDataNameToId, } from '@/lib/table/column-keys' import { columnTypeById } from '@/lib/table/column-types' -import { addTableColumn, deleteColumn } from '@/lib/table/columns/service' +import { addTableColumn, deleteColumn, stripDroppedColumnData } from '@/lib/table/columns/service' import { signalTableRowsChanged, signalTableSchemaChanged } from '@/lib/table/events' import { performUpdateTableColumn } from '@/lib/table/orchestration' import { readCurrentRowsVersion } from '@/lib/table/row-changes' @@ -357,9 +357,24 @@ export const reshapeTableUseCase = defineAuthorizedTableUseCase({ const progress: TableFieldsProgress = { current: locked, applied: [], changed: [] } if (input.table) await writeTableFields(progress, input.table, resolution?.folderId, tx) let table = progress.current + // Managed rows' version is the base source compares with, so the drop's cell strip + // commits with it rather than moving the version after the apply recorded it. + const managed = + input.sourceRows === undefined + ? locked.metadata?.sourceRows?.mode === 'managed' + : input.sourceRows?.mode === 'managed' + const droppedIds: string[] = [] for (const columnName of input.drop ?? []) { - table = await deleteColumn({ tableId: table.id, columnName }, requestId, options) + const dropped = table.schema.columns.find((column) => + columnMatchesRef(column, columnName) + ) + table = await deleteColumn({ tableId: table.id, columnName }, requestId, { + ...options, + callerStripsRows: managed, + }) + if (managed && dropped) droppedIds.push(getColumnId(dropped)) } + await stripDroppedColumnData(tx, table, droppedIds) // With rows to write, a column made required is made so after them (an add, below, too). const deferredUpdates: string[] = [] const updates = (input.update ?? []).flatMap((entry) => { diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index 961d8ca0736..14fb8a0a5fb 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -20,7 +20,7 @@ import { userTableDefinitions, userTableRows } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { resolveCurrencyCode } from '@sim/table-cells/currency' import { omit } from '@sim/utils/object' -import { and, asc, count, eq, gt, sql } from 'drizzle-orm' +import { and, asc, count, eq, gt, type SQL, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { columnMatchesRef, generateColumnId, getColumnId } from '@/lib/table/column-keys' import { @@ -387,6 +387,43 @@ function stripColumnIdsFromMetadata( return next } +/** Removes `columnIds`' cells from every row of the table, in `trx`. */ +async function stripColumnData( + trx: DbTransaction, + tableId: string, + workspaceId: string, + columnIds: string[] +): Promise { + await updateTableRowsWithDerivedSecretProvenance(trx, { + rowWhere: and( + eq(userTableRows.tableId, tableId), + eq(userTableRows.workspaceId, workspaceId) + ) as SQL, + transformation: { mode: 'remove-columns', columnIds }, + }) +} + +/** The statement timeout a strip of `columnCount` columns from `rowCount` rows gets. */ +function stripTimeoutMs(rowCount: number, columnCount: number): number { + return scaledStatementTimeoutMs(rowCount, { baseMs: 60_000, perRowMs: 2 * columnCount }) +} + +/** + * Removes the cells of `columnIds`, columns {@link deleteColumn} dropped in `tx` with + * `callerStripsRows`, from every row of `table`, in one pass under a row-count-scaled timeout, so + * the row version the commit leaves already holds that write. + */ +export async function stripDroppedColumnData( + tx: TableWriteTx, + table: Pick, + columnIds: string[] +): Promise { + if (columnIds.length === 0) return + const timeoutMs = stripTimeoutMs(table.rowCount ?? 0, columnIds.length) + await setTableTxTimeouts(tx.trx, { statementMs: timeoutMs, idleMs: timeoutMs }) + await stripColumnData(tx.trx, table.id, table.workspaceId, columnIds) +} + /** * Fire-and-forget reclamation of a deleted column's row storage. The column is * already gone from the schema, so reads never surface the orphaned id — @@ -404,18 +441,8 @@ function stripColumnDataInBackground( void (async () => { try { await db.transaction(async (trx) => { - const statementMs = scaledStatementTimeoutMs(rowCount, { - baseMs: 60_000, - perRowMs: 2 * columnIds.length, - }) - await setTableTxTimeouts(trx, { statementMs }) - await updateTableRowsWithDerivedSecretProvenance(trx, { - rowWhere: and( - eq(userTableRows.tableId, tableId), - eq(userTableRows.workspaceId, workspaceId) - )!, - transformation: { mode: 'remove-columns', columnIds }, - }) + await setTableTxTimeouts(trx, { statementMs: stripTimeoutMs(rowCount, columnIds.length) }) + await stripColumnData(trx, tableId, workspaceId, columnIds) }) logger.info( `Background-stripped deleted column data [${columnIds.join(', ')}] from table ${tableId}`, @@ -432,9 +459,10 @@ function stripColumnDataInBackground( } /** - * Deletes a column from a table's schema. When id-keyed, returns once the schema - * is updated and reclaims the column's row-data storage in the background - * (fire-and-forget); the legacy path strips the row key synchronously. + * Deletes a column from a table's schema. Returns once the schema is updated and + * reclaims the column's row-data storage in the background (fire-and-forget), + * unless `callerStripsRows` leaves that to the caller's transaction + * ({@link stripDroppedColumnData}). * * @param data - Delete column data * @param requestId - Request ID for logging @@ -444,7 +472,14 @@ function stripColumnDataInBackground( export async function deleteColumn( data: DeleteColumnData, requestId: string, - options?: ColumnMutationOptions + options?: ColumnMutationOptions & { + /** + * Leave the column's cells for the caller to strip in its `tx` with + * {@link stripDroppedColumnData}, once for every column it drops, so the row version the + * commit leaves already holds that write (workspace source records it for managed rows). + */ + callerStripsRows?: boolean + } ): Promise { const { def, stripKey } = await withLockedTable( data.tableId, @@ -534,9 +569,9 @@ export async function deleteColumn( def.rowCount ?? 0, requestId ) + if (!options?.tx) strip() // Inside a caller's transaction the drop may still roll back, so the cells stay until it commits. - if (options?.tx) options.tx.afterCommit(strip) - else strip() + else if (!options.callerStripsRows) options.tx.afterCommit(strip) return def } diff --git a/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts new file mode 100644 index 00000000000..ac5b0a3ae4e --- /dev/null +++ b/apps/sim/lib/workflows/application/apply-workflow-operations-edits.integration.ts @@ -0,0 +1,223 @@ +/** + * Real PostgreSQL coverage for what an operations batch persists when it renames a block or + * sets a loop or parallel count from a string, read back from the normalized tables. + */ +import { db } from '@sim/db' +import { auditLog, permissions, user, workflow, workspace } from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { featureFlagsMock } from '@sim/testing/mocks/feature-flags.mock' +import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' +import { generateId } from '@sim/utils/id' +import { saveWorkflowToNormalizedTables } from '@sim/workflow-persistence/save' +import type { BlockState, WorkflowState } from '@sim/workflow-types/workflow' +import { eq } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' +import { applyWorkflowOperations } from '@/lib/workflows/application/apply-workflow-operations' +import type { EditWorkflowOperation } from '@/lib/workflows/editing/types' +import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) +vi.mock('@/lib/realtime/notify', () => realtimeNotifyMock) + +const ownerId = generateId() +const workspaceId = generateId() +const principal = createSessionPrincipal({ userId: ownerId, sessionId: generateId() }) + +function block(id: string, type: string, name: string, code?: string): BlockState { + return { + id, + type, + name, + position: { x: 0, y: 0 }, + subBlocks: code === undefined ? {} : { code: { id: 'code', type: 'code', value: code } }, + outputs: {}, + enabled: true, + data: {}, + } +} + +async function seedWorkflow(blocks: BlockState[]): Promise { + const id = generateId() + const now = new Date() + await db.insert(workflow).values({ + id, + userId: ownerId, + workspaceId, + name: `Operations edits ${id}`, + lastSynced: now, + createdAt: now, + updatedAt: now, + }) + const state: WorkflowState = { + blocks: Object.fromEntries(blocks.map((entry) => [entry.id, entry])), + edges: [], + loops: {}, + parallels: {}, + } + const saved = await saveWorkflowToNormalizedTables(id, state) + if (!saved.success) throw new Error(`Seeding the workflow failed: ${saved.error}`) + return id +} + +function apply(workflowId: string, operations: EditWorkflowOperation[], atomic = false) { + return withPermissionGroupScope(() => + applyWorkflowOperations.execute({ principal, input: { workflowId, operations, atomic } }) + ) +} + +async function stored(workflowId: string) { + const loaded = await loadWorkflowFromNormalizedTables(workflowId) + if (!loaded) throw new Error('The workflow did not load') + return loaded +} + +beforeAll(async () => { + const now = new Date() + await db.insert(user).values({ + id: ownerId, + name: 'Operations edits fixture', + email: `${ownerId}@operations-edits.invalid`, + emailVerified: true, + createdAt: now, + updatedAt: now, + }) + await insertWorkspaceFixture(db, { + id: workspaceId, + name: 'Operations edits', + ownerId, + billedAccountUserId: ownerId, + }) + await db.insert(permissions).values({ + id: generateId(), + userId: ownerId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) +}) + +afterAll(async () => { + await db.delete(auditLog).where(eq(auditLog.workspaceId, workspaceId)) + await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId)) + await db.delete(user).where(eq(user.id, ownerId)) +}) + +describe('renaming a block through an operations batch', () => { + it('rewrites the references other blocks hold to it, as the editor does', async () => { + const save = generateId() + const respond = generateId() + const unrelated = generateId() + const workflowId = await seedWorkflow([ + block(generateId(), 'starter', 'Start'), + block(save, 'function', 'Save Order', 'return { row: 1 }'), + block( + respond, + 'function', + 'Respond', + 'return { saved: , all: }' + ), + block(unrelated, 'function', 'Unrelated', 'return '), + ]) + + await apply(workflowId, [ + { operation_type: 'edit', block_id: save, params: { name: 'Save Order Edited' } }, + ]) + + const { blocks } = await stored(workflowId) + expect(blocks[save].name).toBe('Save Order Edited') + expect(blocks[respond].subBlocks.code.value).toBe( + 'return { saved: , all: }' + ) + expect(blocks[unrelated].subBlocks.code.value).toBe('return ') + }) +}) + +describe('a loop or parallel count given as a string', () => { + it('stores a numeric string as its number on add and on edit', async () => { + const workflowId = await seedWorkflow([block(generateId(), 'starter', 'Start')]) + const loop = generateId() + const parallel = generateId() + + await apply( + workflowId, + [ + { + operation_type: 'add', + block_id: loop, + params: { type: 'loop', name: 'Repeat', inputs: { loopType: 'for', iterations: '3' } }, + }, + { + operation_type: 'add', + block_id: parallel, + params: { + type: 'parallel', + name: 'Fan Out', + inputs: { parallelType: 'count', count: '2' }, + }, + }, + ], + true + ) + + const added = await stored(workflowId) + expect(added.loops[loop]).toMatchObject({ iterations: 3 }) + expect(added.parallels[parallel]).toMatchObject({ count: 2 }) + + await apply(workflowId, [ + { operation_type: 'edit', block_id: loop, params: { inputs: { iterations: ' 4 ' } } }, + ]) + + expect((await stored(workflowId)).loops[loop]).toMatchObject({ iterations: 4 }) + }) + + it('refuses a non-numeric count instead of silently resetting it', async () => { + const workflowId = await seedWorkflow([block(generateId(), 'starter', 'Start')]) + + await expect( + apply( + workflowId, + [ + { + operation_type: 'add', + block_id: generateId(), + params: { + type: 'loop', + name: 'Repeat', + inputs: { loopType: 'for', iterations: 'three' }, + }, + }, + ], + true + ) + ).rejects.toMatchObject({ + droppedInputs: [expect.objectContaining({ field: 'iterations', value: 'three' })], + }) + expect(Object.keys((await stored(workflowId)).loops)).toEqual([]) + }) + + it.each(['1e3', '0x10', '2.5', '-1', '0', 2.5, -1])( + 'refuses %j, which is no whole number of at least 1', + async (iterations) => { + const workflowId = await seedWorkflow([block(generateId(), 'starter', 'Start')]) + + await expect( + apply( + workflowId, + [ + { + operation_type: 'add', + block_id: generateId(), + params: { type: 'loop', name: 'Repeat', inputs: { loopType: 'for', iterations } }, + }, + ], + true + ) + ).rejects.toMatchObject({ + droppedInputs: [expect.objectContaining({ field: 'iterations', value: iterations })], + }) + expect(Object.keys((await stored(workflowId)).loops)).toEqual([]) + } + ) +}) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index 8f72c8c20cb..4a3abb65403 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -1542,12 +1542,60 @@ describe('source apply edge cases', () => { 'E_APPLY_INVALID_SUBBLOCK_FIELD' ) - expect( - error.diagnostics.find((d) => d.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD') - ).toMatchObject({ blockId: score.id, param: 'language' }) + const refusals = error.diagnostics.filter((d) => d.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD') + expect(refusals).toHaveLength(1) + expect(refusals[0]).toMatchObject({ blockId: score.id, param: 'language' }) + expect(error.diagnostics.map((d) => d.code)).not.toContain('W_UNKNOWN_OPTION') expect(await snapshot(workflowId)).toEqual(before) }) + check( + 'an edited block keeps a value outside its options the stored block already holds', + async () => { + const start = seedBlock('start_trigger', 'Start') + const score = seedBlock('function', 'Score', { + code: ['code', 'return 1'], + language: ['dropdown', 'cobol'], + }) + const workflowId = await seedGraph([start, score]) + const source = await getInlineSource(workflowId) + + const result = await apply(source, edit(source.files, 'return 1', 'return 2')) + + expect(result.applied).toBe(true) + const stored = (await snapshot(workflowId)).state?.blocks[score.id] + expect(stored?.subBlocks.code?.value).toBe('return 2') + expect(stored?.subBlocks.language?.value).toBe('cobol') + } + ) + + check( + "an edited block's stored out-of-options value written bare again is not refused: the edit does not set that field", + async () => { + const start = seedBlock('start_trigger', 'Start') + const score = seedBlock('function', 'Score', { + code: ['code', 'return 1'], + language: ['dropdown', 'cobol'], + }) + const workflowId = await seedGraph([start, score]) + const source = await getInlineSource(workflowId) + + const result = await apply( + source, + edit( + edit(source.files, "language: raw('cobol')", "language: 'cobol'"), + 'return 1', + 'return 2' + ) + ) + + expect(result.applied).toBe(true) + expect(result.diagnostics.map((d) => d.code)).not.toContain('E_APPLY_INVALID_SUBBLOCK_FIELD') + const after = await snapshot(workflowId) + expect(after.state?.blocks[score.id]?.subBlocks.language?.value).toBe('cobol') + } + ) + check( 'a value the field normalizes is stored normalized, and untouched blocks keep their bytes', async () => { @@ -2035,6 +2083,33 @@ export default workflow('Fresh', {}, () => { } ) + check( + 'a lock copied from another workflow is ignored where it claims ids, keeping the stored ones', + async () => { + const target = await seedWorkflow() + const other = await seedWorkflow() + const source = await getSource(target.workflowId) + const foreign = await getSource(other.workflowId) + + // Block ids only: the foreign lock's variable ids are another matter. + const result = await apply(source, edit(source.files, '41 + 1', '41 + 2'), { + lock: { ...foreign.lock, variables: source.lock.variables }, + }) + + expect(result.applied).toBe(true) + const claims = result.diagnostics.filter((d) => d.code === 'W_MAT_LOCK_ID_CLAIMED') + expect(claims.length).toBeGreaterThan(0) + for (const claim of claims) { + expect(claim.message).toContain('the block keeps the id of the stored block it matches') + } + expect(result.mintedIds).toEqual({}) + const after = await snapshot(target.workflowId) + expect(Object.keys(after.state?.blocks ?? {}).sort()).toEqual( + [target.startId, target.scoreId].sort() + ) + } + ) + check('a lock id another workflow already stores is dropped and a new id is minted', async () => { const target = await seedWorkflow() const other = await seedWorkflow() @@ -2045,7 +2120,11 @@ export default workflow('Fresh', {}, () => { expect(result.applied).toBe(true) expect(result.diagnostics).toContainEqual( - expect.objectContaining({ code: 'W_MAT_LOCK_ID_CLAIMED', blockKey: 'extra' }) + expect.objectContaining({ + code: 'W_MAT_LOCK_ID_CLAIMED', + blockKey: 'extra', + message: expect.stringContaining('a new id was minted'), + }) ) const minted = result.mintedIds.extra expect(minted).toBeDefined() diff --git a/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts b/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts index 602bb458739..81c802b1ef2 100644 --- a/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts +++ b/apps/sim/lib/workflows/application/execute-workflow-selected-outputs.integration.ts @@ -221,6 +221,17 @@ describe('selectedOutputs on a run that calls a child workflow', () => { }) }) + it('matches a child block name regardless of case and spacing, as the parent does', async () => { + const childSelector = `${childWorkflowId}.Child Done.data.answer` + const result = await run([childSelector, 'Done.data.answer']) + + expect(result).toMatchObject({ + ok: true, + status: 'completed', + blockOutputs: { [childSelector]: 'from child', 'Done.data.answer': 'from child' }, + }) + }) + it('returns a child block output selected by child block ID', async () => { const childSelector = `${childWorkflowId}.${blockIds.childDone}.data.answer` const result = await run([childSelector]) diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts index 6ede61f3432..9ac87868483 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/finalize.ts @@ -28,6 +28,7 @@ import type { } from '@/lib/workflows/application/source-pipeline/evaluate/types' import { foreignResourceRefusals } from '@/lib/workflows/application/source-resource-scope' import { normalizeWorkflowVariables } from '@/lib/workflows/application/workflow-variables' +import { changedSubBlockFields } from '@/lib/workflows/code/apply/gates' import { mergeSourceMetaFromLock } from '@/lib/workflows/code/source-lock' import type { WorkflowLintReport, @@ -189,7 +190,15 @@ export async function finalizePlan( const changedIds = new Set([...graphDiff.added, ...graphDiff.modified.map((m) => m.id)]) const changedKeys = new Set([...changedIds].flatMap((id) => keyById.get(id) ?? [])) const keyOfId = (id: string) => keyById.get(id) - const compileDiagnostics = editedCompileFindings(refusals, changedKeys) + const changedFields = new Map( + [...changedSubBlockFields(storedGraph, { blocks: gated.blocks, edges }, graphDiff)].flatMap( + ([id, fields]) => { + const key = keyById.get(id) + return key === undefined ? [] : [[key, fields] as const] + } + ) + ) + const compileDiagnostics = editedCompileFindings(refusals, changedKeys, changedFields) const compiledLint = withoutCompiledReferences( withoutPendingReferences(lint, pending), compileDiagnostics, @@ -202,16 +211,31 @@ export async function finalizePlan( pending, keyOf: keyOfId, }) + // A field the compile already refused (with its span and options) needs no second refusal from + // the value gate. + const refusedFields = new Set( + compileDiagnostics.flatMap((finding) => + finding.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD' + ? [`${finding.blockKey}\u0000${finding.param}`] + : [] + ) + ) + const refusedAgain = (finding: { code: string; blockId?: string; param?: string }) => + finding.code === 'E_APPLY_INVALID_SUBBLOCK_FIELD' && + finding.blockId !== undefined && + refusedFields.has(`${keyOfId(finding.blockId)}\u0000${finding.param}`) const diagnostics = [ ...compileDiagnostics, - ...resolved.keptDiagnostics, - ...resolved.materializeDiagnostics, - ...refusals.sourceRefusals, - ...refusals.renameWarnings, - ...gated.gateFindings, - ...normalized.diagnostics, - ...foreignIds, - ...lintDiagnostics(compiledLint, changedIds, keyOfId), + ...[ + ...resolved.keptDiagnostics, + ...resolved.materializeDiagnostics, + ...refusals.sourceRefusals, + ...refusals.renameWarnings, + ...gated.gateFindings, + ...normalized.diagnostics, + ...foreignIds, + ...lintDiagnostics(compiledLint, changedIds, keyOfId), + ].filter((finding) => !refusedAgain(finding)), ].map(locate) if (hasErrors(diagnostics)) { return { diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts index 94d2a521c2b..3cfa38064d1 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/refusals.ts @@ -94,7 +94,7 @@ function variableDeleteRefusal( const [next] = added const rename = removed.length === 1 && added.length === 1 && former.type === next.type - ? `If ${next.name} renames ${former.name}, apply the new key with the value ${former.name} holds first and change the value in a second apply, so the variable keeps its id; or pass --allow-delete (allowDelete=true) to replace it.` + ? `If ${next.name} renames ${former.name}, keep the value ${former.name} holds and move every vars reference to the new key in this apply, then change the value in a second apply, so the variable keeps its id; a variable no block reads that holds 0, false or an empty value cannot be tied, so pass --allow-delete (allowDelete=true) to replace it.` : undefined return diagnostic( 'E_APPLY_VARIABLE_DELETE', @@ -359,14 +359,18 @@ export async function sourceRefusals( /** * The compile's findings, each on a block this write changes (or whose code it edits) refused as - * the shared edited-block rule says, so a warning the author introduced cannot be written. + * the shared edited-block rule says, so a warning the author introduced cannot be written. A + * field finding is refused only on a field the write sets (`changedFields`, by block key), the + * fields the value gate judges. */ export function editedCompileFindings( refusals: Pick, - changedKeys: ReadonlySet + changedKeys: ReadonlySet, + changedFields: ReadonlyMap> ): LocatedDiagnostic[] { return refuseEditedFindings(refusals.compileFindings, { changed: (key) => changedKeys.has(key), codeEdited: refusals.codeEdited, + paramChanged: (key, param) => changedFields.get(key)?.has(param) === true, }) } diff --git a/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts b/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts index 6cacb3d0f07..7e744f3a846 100644 --- a/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts +++ b/apps/sim/lib/workflows/application/source-pipeline/evaluate/resolve.ts @@ -276,20 +276,25 @@ export async function resolveSubmission( const claimed = new Set(claimedIds) if (claimed.size > 0) { const blocks: Record = {} + const claimedKeys: string[] = [] for (const [key, entry] of Object.entries(lock.blocks)) { - if (claimed.has(entry.id)) { - claimedDiagnostics.push( - diagnostic( - 'W_MAT_LOCK_ID_CLAIMED', - `The lock id for block "${key}" belongs to another workflow; a new id was minted`, - { blockKey: key } - ) - ) - } else { - blocks[key] = entry - } + if (claimed.has(entry.id)) claimedKeys.push(key) + else blocks[key] = entry } materialized = materializeWith({ ...lock, blocks }) + // Without its lock id the block binds as a lockless one would: to the stored block of its + // name, or to a new id. + const outcomes = new Map(materialized.entries.map((entry) => [entry.key, entry])) + for (const key of claimedKeys) { + const kept = outcomes.get(key)?.status === 'matched' + claimedDiagnostics.push( + diagnostic( + 'W_MAT_LOCK_ID_CLAIMED', + `The lock id for block "${key}" belongs to another workflow, so it was ignored; ${kept ? 'the block keeps the id of the stored block it matches' : 'a new id was minted'}`, + { blockKey: key } + ) + ) + } } const idByKey = new Map( diff --git a/apps/sim/lib/workflows/code/apply/gates.ts b/apps/sim/lib/workflows/code/apply/gates.ts index cf220ef7ecd..3ef777b6116 100644 --- a/apps/sim/lib/workflows/code/apply/gates.ts +++ b/apps/sim/lib/workflows/code/apply/gates.ts @@ -190,6 +190,23 @@ function collectGatedBlocks(base: SourceGraph, target: SourceGraph, diff: GraphD return gated } +/** + * Per block id the diff adds or modifies, the subblock ids whose value it sets: the fields the + * value gate judges ({@link GatedBlock.changedFields}). + */ +export function changedSubBlockFields( + base: SourceGraph, + target: SourceGraph, + diff: GraphDiff +): Map> { + return new Map( + collectGatedBlocks(base, target, diff).map((entry) => [ + entry.block.id, + new Set(entry.changedFields), + ]) + ) +} + /** * The placement gates on a new or retyped block, judged against the target graph's other blocks so * replacing a trigger in one apply (remove the old, add the new) passes. Returns whether the diff --git a/apps/sim/lib/workflows/code/catalog/project.ts b/apps/sim/lib/workflows/code/catalog/project.ts index 2b049d269f4..99e52c9f4b7 100644 --- a/apps/sim/lib/workflows/code/catalog/project.ts +++ b/apps/sim/lib/workflows/code/catalog/project.ts @@ -349,7 +349,9 @@ function projectSubBlock(subBlock: SubBlockConfig, ctx: SubBlockContext): Catalo if (writable && typeof subBlock.value === 'function') { assign(projected, 'valueDefault', projectValueDefault(subBlock.value, subBlock.valueScope)) } - assign(projected, 'options', projectOptions(subBlock)) + const options = projectOptions(subBlock) + assign(projected, 'options', options) + if (typeof subBlock.options === 'function' && Array.isArray(options)) projected.optionsVary = true assign(projected, 'hidden', subBlock.hidden) assign(projected, 'readOnly', subBlock.readOnly) diff --git a/apps/sim/lib/workflows/editing/operations.test.ts b/apps/sim/lib/workflows/editing/operations.test.ts index cf8c4f02e4e..1ac10156166 100644 --- a/apps/sim/lib/workflows/editing/operations.test.ts +++ b/apps/sim/lib/workflows/editing/operations.test.ts @@ -606,6 +606,91 @@ describe('handleEditOperation container inputs', () => { }) }) +describe('a falsy loop or parallel count', () => { + it.each([0, false, null, ''])('refuses iterations %j on an added loop', (iterations) => { + const { state, validationErrors } = applyOperationsToWorkflowState(makeLoopWorkflow(), [ + { + operation_type: 'add', + block_id: 'loop-2', + params: { type: 'loop', name: 'Loop 2', inputs: { loopType: 'for', iterations } }, + }, + ]) + + expect(validationErrors).toEqual([ + expect.objectContaining({ blockType: 'loop', field: 'iterations', value: iterations }), + ]) + expect(Object.values(state.blocks)).toContainEqual( + expect.objectContaining({ + name: 'Loop 2', + data: expect.not.objectContaining({ count: expect.anything() }), + }) + ) + }) + + it('refuses count 0 on an added parallel', () => { + const { validationErrors } = applyOperationsToWorkflowState(makeLoopWorkflow(), [ + { + operation_type: 'add', + block_id: 'parallel-2', + params: { + type: 'parallel', + name: 'Parallel 2', + inputs: { parallelType: 'count', count: 0 }, + }, + }, + ]) + + expect(validationErrors).toEqual([ + expect.objectContaining({ blockType: 'parallel', field: 'count', value: 0 }), + ]) + }) + + it('refuses iterations 0 on a loop merged in as an existing nested child, keeping its count', () => { + const workflow = makeParallelWorkflow() + const blocks: Record = workflow.blocks + blocks['inner-loop'] = { + id: 'inner-loop', + type: 'loop', + name: 'Inner', + position: { x: 0, y: 0 }, + enabled: true, + subBlocks: {}, + outputs: {}, + data: { parentId: 'loop-1', extent: 'parent', loopType: 'for', count: 4 }, + } + + const { state, validationErrors } = applyOperationsToWorkflowState(workflow, [ + { + operation_type: 'edit', + block_id: 'loop-1', + params: { + nestedNodes: { + inner: { type: 'loop', name: 'Inner', inputs: { loopType: 'for', iterations: 0 } }, + }, + }, + }, + ]) + + expect(validationErrors).toContainEqual( + expect.objectContaining({ blockId: 'inner-loop', field: 'iterations', value: 0 }) + ) + expect(state.blocks['inner-loop'].data.count).toBe(4) + }) +}) + +describe('a block rename', () => { + it('writes a new name holding `$` replacement patterns into references literally', () => { + const workflow = makeLoopWorkflow() + workflow.blocks['agent-1'].subBlocks.systemPrompt.value = 'Use ' + + const { state } = applyOperationsToWorkflowState(workflow, [ + { operation_type: 'edit', block_id: 'condition-1', params: { name: "Cond$&$'" } }, + ]) + + expect(state.blocks['agent-1'].subBlocks.systemPrompt.value).toBe("Use ") + }) +}) + describe('handleEditOperation nestedNodes merge', () => { it('preserves existing child block IDs when editing a loop with nestedNodes', () => { const workflow = makeLoopWorkflow() diff --git a/apps/sim/lib/workflows/editing/operations.ts b/apps/sim/lib/workflows/editing/operations.ts index c0879c35537..fb3ccd57de9 100644 --- a/apps/sim/lib/workflows/editing/operations.ts +++ b/apps/sim/lib/workflows/editing/operations.ts @@ -4,6 +4,7 @@ import { isNonEmptyValue } from '@sim/workflow-code/catalog/canonical-index' import type { BlockState } from '@sim/workflow-types/workflow' import { isEqual } from 'es-toolkit' import { checkBlockPlacement } from '@/lib/workflows/editing/gates' +import { rewriteBlockNameReferences } from '@/lib/workflows/references/block-rename' import { isValidKey } from '@/lib/workflows/sanitization/key-validation' import { getTransitiveSubBlockDependents } from '@/lib/workflows/subblocks/dependencies' import { buildSubBlockValues } from '@/lib/workflows/subblocks/visibility' @@ -22,7 +23,7 @@ import { updateCanonicalModesForInputs, } from './builders' import { VALID_LOOP_TYPES, VALID_PARALLEL_TYPES } from './container-types' -import type { EditWorkflowOperation, OperationContext } from './types' +import type { EditWorkflowOperation, OperationContext, ValidationError } from './types' import { logSkippedItem } from './types' import { findBlockWithDuplicateNormalizedName, validateInputsForBlock } from './validation' @@ -65,10 +66,60 @@ function refuseNestedPlacement( return true } +/** + * Renames a block and rewrites the `, blockId: string, name: string): void { + const nameMap = new Map([[normalizeName(blocks[blockId].name), normalizeName(name)]]) + blocks[blockId].name = name + for (const [id, block] of Object.entries(blocks)) { + if (id === blockId) continue + for (const subBlock of Object.values(block.subBlocks ?? {})) { + if (subBlock.value === undefined || subBlock.value === null) continue + subBlock.value = rewriteBlockNameReferences(subBlock.value, nameMap) as typeof subBlock.value + } + } +} + +/** + * A loop's `iterations` or a parallel's `count` as stored: a whole number of at least 1, given as + * a number or as a string of digits (the way a number input coerces), the bound source compile + * holds a count to. Anything else (`0`, `-1`, `2.5`, `"1e3"`, `"0x10"`) is refused as an input + * validation error, keeping `current`, since the loader would otherwise reset it to its default. + */ +function containerCount( + block: BlockState, + field: 'iterations' | 'count', + value: unknown, + validationErrors: ValidationError[], + current?: unknown +): unknown { + const parsed = + typeof value === 'number' + ? value + : typeof value === 'string' && /^\s*\d+\s*$/.test(value) + ? Number(value) + : Number.NaN + if (Number.isSafeInteger(parsed) && parsed >= 1) return parsed + validationErrors.push({ + blockId: block.id, + blockType: block.type, + field, + value, + error: `Invalid ${field} "${String(value)}" for ${block.type} "${block.name}" - must be a whole number of at least 1`, + }) + return current +} + /** * Applies loop/parallel container config from `inputs` onto a block state (data.loopType, etc.). */ -function applyLoopOrParallelContainerData(block: any, params: Record): void { +function applyLoopOrParallelContainerData( + block: any, + params: Record, + validationErrors: ValidationError[] +): void { if (params.type === 'loop') { const loopType = params.inputs?.loopType && VALID_LOOP_TYPES.includes(params.inputs.loopType) @@ -79,7 +130,16 @@ function applyLoopOrParallelContainerData(block: any, params: Record): void { +function updateLoopOrParallelContainerData( + block: any, + params: Record, + validationErrors: ValidationError[] +): void { if (block.type === 'loop') { block.data = block.data || {} if (params.inputs?.loopType) { @@ -177,8 +250,14 @@ function updateLoopOrParallelContainerData(block: any, params: Record | undefined { + const selectedName = normalizeName(selector.blockId) let found: Record | undefined for (const span of spans) { if ( inSelectedWorkflow && span.output !== undefined && - (span.blockId === selector.blockId || normalizeName(span.name) === selector.blockId) + (span.blockId === selector.blockId || normalizeName(span.name) === selectedName) ) { found = span.output } diff --git a/apps/sim/lib/workflows/references/block-rename.ts b/apps/sim/lib/workflows/references/block-rename.ts new file mode 100644 index 00000000000..09accb96ceb --- /dev/null +++ b/apps/sim/lib/workflows/references/block-rename.ts @@ -0,0 +1,47 @@ +import { escapeRegExp } from '@sim/utils/string' + +/** + * `value` with every ` +): unknown { + if (typeof value === 'string') { + let updatedValue = value + nameMap.forEach((newName, oldName) => { + /** + * A rename to itself is a no-op, so skip the scan entirely. This is the + * whole map on the import path (`regenerateWorkflowIds` seeds it with + * `name -> name`), which turns an O(names x values) rescan of every + * sub-block string into nothing. + */ + if (oldName === newName) return + + /** + * `oldName` is a block name, which reaches this function straight from + * imported workflow JSON — `normalizeWorkflowBlockName` only lowercases + * and strips whitespace/dots, so regex metacharacters survive. Without + * escaping, a name like `a*a*a*a*b` compiles to a catastrophically + * backtracking pattern that pins the event loop on a sub-kilobyte input. + */ + const regex = new RegExp(`<${escapeRegExp(oldName)}\\.`, 'g') + // A callback, so `$&` or `$'` in a new name is written literally, not as a pattern. + updatedValue = updatedValue.replace(regex, () => `<${newName}.`) + }) + return updatedValue + } + if (Array.isArray(value)) { + return value.map((item) => rewriteBlockNameReferences(item, nameMap)) + } + if (value && typeof value === 'object') { + const result: Record = {} + for (const [key, val] of Object.entries(value)) { + result[key] = rewriteBlockNameReferences(val, nameMap) + } + return result + } + return value +} diff --git a/apps/sim/lib/workspace-source/application/apply-workspace-source.ts b/apps/sim/lib/workspace-source/application/apply-workspace-source.ts index 7b92a8a36b0..ae8924f956d 100644 --- a/apps/sim/lib/workspace-source/application/apply-workspace-source.ts +++ b/apps/sim/lib/workspace-source/application/apply-workspace-source.ts @@ -19,7 +19,7 @@ import { refusalDiagnostic, WorkspaceSourceRejectedError, } from '@/lib/workspace-source/application/errors' -import { finding, GET_HINT } from '@/lib/workspace-source/application/findings' +import { finding, findingIdentity, GET_HINT } from '@/lib/workspace-source/application/findings' import { claimKey } from '@/lib/workspace-source/application/keys' import { workspaceSourceOperations } from '@/lib/workspace-source/application/operations' import { @@ -396,12 +396,25 @@ async function writeAll( base: WorkspaceBaseV1, result: ApplyWorkspaceSourceResult ): Promise { + // A write re-reports what its plan already found: each finding is listed once, with the ids + // the write gave it. + const reported = new Map( + result.diagnostics.map((diagnostic, at) => [findingIdentity(diagnostic), at]) + ) for (const change of writeOrder(changes)) { try { await lease.confirm() const written = await write(change, writeCtx) Object.assign(result.files, written.text) - result.diagnostics.push(...(written.diagnostics ?? [])) + for (const diagnostic of written.diagnostics ?? []) { + const at = reported.get(findingIdentity(diagnostic)) + if (at !== undefined) { + result.diagnostics[at] = diagnostic + continue + } + reported.set(findingIdentity(diagnostic), result.diagnostics.length) + result.diagnostics.push(diagnostic) + } result.applied.push(summarize(change, written.id)) if (written.keyTaken) { // Written but not bound to its key, so the base keeps what it held, and a later write diff --git a/apps/sim/lib/workspace-source/application/apply.integration.ts b/apps/sim/lib/workspace-source/application/apply.integration.ts index ce7f98db0e2..9b55df147d5 100644 --- a/apps/sim/lib/workspace-source/application/apply.integration.ts +++ b/apps/sim/lib/workspace-source/application/apply.integration.ts @@ -252,6 +252,54 @@ describe('workspace source apply', () => { } } + check( + 'a real apply reports each finding of a written workflow once, as its dry run does', + async () => { + const workspaceId = await newWorkspace() + const files = { + ...STARTER, + 'workflows/ingest/workflow.ts': INGEST.replace( + 'data: { email: start.email },', + 'data: { email: start.email, note: env.QA_NOT_SET },' + ).replace('import { table,', 'import { env, table,'), + } + const once = (diagnostics: ReadonlyArray<{ code: string; severity: string }>) => + codes(diagnostics, 'warning').filter((code) => code === 'W_ENV_UNKNOWN') + + const planned = await apply(workspaceId, files, { dryRun: true }) + expect(once(planned.diagnostics)).toEqual(['W_ENV_UNKNOWN']) + const written = await apply(workspaceId, files) + expect(written.failed).toBeUndefined() + expect(once(written.diagnostics)).toEqual(['W_ENV_UNKNOWN']) + } + ) + + check( + 'a workflow edited on both sides is refused as stale with the hint every other kind gives', + async () => { + const workspaceId = await newWorkspace() + await apply(workspaceId, STARTER) + const read = await get(workspaceId) + const entry = 'workflows/ingest/workflow.ts' + const renamedTo = (name: string) => ({ + ...read.files, + [entry]: read.files[entry].replace("'Save Lead'", `'${name}'`), + }) + await apply(workspaceId, renamedTo('Store Lead'), { base: read.base }) + + const stale = await refusal(apply(workspaceId, renamedTo('Keep Lead'), { base: read.base })) + + expect(stale.diagnostics).toEqual([ + expect.objectContaining({ + code: 'E_APPLY_STALE', + kind: 'workflow', + key: 'ingest', + hint: expect.stringContaining('sim workspaces source get'), + }), + ]) + } + ) + check( 'every audit entry an apply writes carries its apply id, and the next apply has another', async () => { @@ -2255,6 +2303,15 @@ export default workflow('Probe', {}, () => { key: 'ingest', file: entry, }) + const created = 'workflows/copy/workflow.ts' + const createTaken = await apply( + workspaceId, + { ...read.files, [created]: INGEST.replace("workflow('Ingest'", "workflow('Intake'") }, + { base: read.base, dryRun: true } + ) + expect( + createTaken.diagnostics.find((finding) => finding.code === 'E_NAME_TAKEN') + ).toMatchObject({ kind: 'workflow', key: 'copy', file: created, hint: expect.any(String) }) const result = await apply(workspaceId, renamedTo('Ingest v2'), { base: read.base }) expect(result.failed).toBeUndefined() diff --git a/apps/sim/lib/workspace-source/application/errors.test.ts b/apps/sim/lib/workspace-source/application/errors.test.ts index 65816ad1de5..07817945138 100644 --- a/apps/sim/lib/workspace-source/application/errors.test.ts +++ b/apps/sim/lib/workspace-source/application/errors.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it } from 'vitest' import { OrchestrationError } from '@/lib/core/orchestration/types' import { TableConflictError } from '@/lib/table/service' +import { FileConflictError } from '@/lib/uploads/contexts/workspace/file-manager' +import { WorkflowSourceNameTakenError } from '@/lib/workflows/application/source-error' import { refusalDiagnostic, WorkspaceSourceRejectedError, @@ -16,6 +18,28 @@ describe('refusalDiagnostic', () => { ) }) + it('scopes a taken workflow name to its folder, where the uniqueness check runs', () => { + const finding = refusalDiagnostic(new WorkflowSourceNameTakenError('Ingest'), { + kind: 'workflow', + key: 'ingest', + }) + + expect(finding.code).toBe('E_NAME_TAKEN') + expect(finding.message).not.toContain('workspace') + expect(finding.message).toContain('folder') + }) + + it('hints a taken file name be renamed locally, as a file has no declaration', () => { + const finding = refusalDiagnostic(new FileConflictError('notes.txt'), { + kind: 'file', + key: 'notes.txt', + }) + + expect(finding.code).toBe('E_NAME_TAKEN') + expect(finding.hint).not.toContain('declaration') + expect(finding.hint).toContain('locally') + }) + it('rethrows an outage, so the request answers 503 rather than blaming the caller', () => { const outage = new OrchestrationError('unavailable', 'The egress gateway is unreachable') expect(() => refusalDiagnostic(outage, { kind: 'table', key: 'leads' })).toThrow(outage) diff --git a/apps/sim/lib/workspace-source/application/errors.ts b/apps/sim/lib/workspace-source/application/errors.ts index 08589fe64f9..82ca6b4730b 100644 --- a/apps/sim/lib/workspace-source/application/errors.ts +++ b/apps/sim/lib/workspace-source/application/errors.ts @@ -1,5 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' +import { SOURCE_ENTRY_FILE } from '@sim/workflow-code/limits' import { refusalClass, type WorkspaceKind } from '@sim/workspace-code' +import { workflowFolder } from '@sim/workspace-code/kinds/workflow' import { OrchestrationError } from '@/lib/core/orchestration/types' import { KnowledgeBaseConflictError } from '@/lib/knowledge/service' import { TableRowsChangedError } from '@/lib/table/application/errors' @@ -113,11 +115,28 @@ export function refusalDiagnostic( ) { return finding('E_APPLY_STALE', getErrorMessage(error), { hint: GET_HINT, ...resource }) } + if (error instanceof WorkflowSourceNameTakenError && resource.key !== undefined) { + // Workflow folders are not placed in a folder by source, so the use case's "or in another + // folder" is advice a directory cannot follow. + const file = `${workflowFolder(resource.key)}/${SOURCE_ENTRY_FILE}` + return finding( + 'E_NAME_TAKEN', + `A workflow named ${JSON.stringify(error.workflowName)} already exists in the folder this workflow is saved to.`, + { + hint: `Rename the workflow in ${file}, or rename, move or delete the one that holds the name in Sim.`, + file, + ...resource, + } + ) + } if (error instanceof ResourceKeyTakenError || isNameTaken(error)) { return finding('E_NAME_TAKEN', error.message, { - ...(error instanceof ResourceKeyTakenError - ? { hint: 'Rename the key, or delete the resource that holds it.' } - : {}), + hint: + error instanceof ResourceKeyTakenError + ? 'Rename the key, or delete the resource that holds it.' + : resource.kind === 'file' + ? 'Rename it locally, or rename or delete the file that holds the name in Sim.' + : 'Rename it in its declaration, or rename or delete the resource that holds the name in Sim.', ...resource, }) } diff --git a/apps/sim/lib/workspace-source/application/findings.test.ts b/apps/sim/lib/workspace-source/application/findings.test.ts new file mode 100644 index 00000000000..ecd7d41a267 --- /dev/null +++ b/apps/sim/lib/workspace-source/application/findings.test.ts @@ -0,0 +1,25 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { findingIdentity } from '@/lib/workspace-source/application/findings' + +const UNKNOWN_ENV = { + code: 'W_ENV_UNKNOWN', + severity: 'warning', + message: 'env.QA_NOT_SET is not set in this workspace.', +} as const + +describe('findingIdentity', () => { + it('matches a write finding to its plan finding by block key, whatever id the write gave the block', () => { + expect(findingIdentity({ ...UNKNOWN_ENV, blockKey: 'save', blockId: 'blk_new' })).toBe( + findingIdentity({ ...UNKNOWN_ENV, blockKey: 'save' }) + ) + }) + + it('keeps apart findings that name different blocks only by id', () => { + expect(findingIdentity({ ...UNKNOWN_ENV, blockId: 'blk_a' })).not.toBe( + findingIdentity({ ...UNKNOWN_ENV, blockId: 'blk_b' }) + ) + }) +}) diff --git a/apps/sim/lib/workspace-source/application/findings.ts b/apps/sim/lib/workspace-source/application/findings.ts index 8f93d232cb8..8b967624682 100644 --- a/apps/sim/lib/workspace-source/application/findings.ts +++ b/apps/sim/lib/workspace-source/application/findings.ts @@ -2,6 +2,7 @@ * The findings every kind reports the same way, so the plan reads alike across kinds. */ +import { stableStringify } from '@sim/workflow-code' import { diagnostic } from '@sim/workflow-code/diagnostics' import { type BaseResource, @@ -24,6 +25,22 @@ import type { WorkspaceSourceDiagnostic, } from '@/lib/workspace-source/application/types' +/** + * What makes two findings the same one, for a write that re-reports what its plan found (a + * workflow's commit runs its pipeline again) with the block ids the write gave it (a workflow + * folder's findings carry the pipeline's `blockKey` and `blockId` beside the flat fields): a finding + * naming its block by key matches on the key, whatever id it carries, and one naming its block + * only by id keeps the id, so two such findings on different blocks never merge. + */ +export function findingIdentity( + diagnostic: WorkspaceSourceDiagnostic & { blockKey?: string; blockId?: string } +): string { + return stableStringify({ + ...diagnostic, + blockId: diagnostic.blockKey === undefined ? diagnostic.blockId : undefined, + }) +} + /** * One workspace source finding, its severity fixed by the code's prefix. Every finding the app * reports with a workspace code goes through here, so a misspelled code fails the type-check; @@ -145,7 +162,7 @@ export function missingFolderFinding( kind === 'table' ? 'E_TABLE_FOLDER' : 'E_KB_FOLDER', `The folder ${folder} does not exist in this workspace.`, { - hint: `Create the folder first, or drop \`folder\` to keep the ${WORKSPACE_KIND_SPECS[kind].noun} at the root.`, + hint: `Create it first (\`sim ${kind === 'table' ? 'tables' : 'knowledge'} folders create "${folder.replace(/^\//, '')}"\`), or drop \`folder\` to keep the ${WORKSPACE_KIND_SPECS[kind].noun} at the root.`, } ) } diff --git a/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts index 9926380a5e4..fc5c2d5b869 100644 --- a/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/credentials/adapter.ts @@ -106,7 +106,7 @@ function unavailable( hiddenBinding: boolean ): CredentialChange { const { key } = change - const wanted = `a ${entry.provider} credential${entry.name !== undefined ? ` named "${entry.name}"` : ''}` + const wanted = `${entry.provider} credential${entry.name !== undefined ? ` named "${entry.name}"` : ''}` const ambiguous = candidates.length > 1 const holders = heldOutside .slice(0, NAMED_CREDENTIALS) diff --git a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts index 1d1f072379f..8b12136e809 100644 --- a/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/workflow/adapter.ts @@ -83,6 +83,7 @@ import { planUndeclared, refuseDeclaredTwice, serverGoneFinding, + staleFinding, } from '@/lib/workspace-source/application/findings' import { bindStored, claimKey, listUpTo } from '@/lib/workspace-source/application/keys' import { liveTriggerCounts } from '@/lib/workspace-source/application/kinds/deployment/triggers' @@ -434,7 +435,13 @@ async function planUpdate( }, }) const result = summarizePlan(outcome) - change.diagnostics.push(...inFolder(key, result.diagnostics)) + // The pipeline words a stale draft for a single-workflow directory; a workspace names the + // folder and its own next step, as every other kind's stale finding does. + change.diagnostics.push( + ...inFolder(key, result.diagnostics).map((finding) => + finding.code === 'E_APPLY_STALE' ? staleFinding('workflow', key) : finding + ) + ) if (result.workflowName !== undefined && result.diff.meta?.includes('name')) { renamed.set(key, result.workflowName) } diff --git a/apps/sim/lib/workspace-source/application/promote.integration.ts b/apps/sim/lib/workspace-source/application/promote.integration.ts index 6f878503ba4..9e151e20077 100644 --- a/apps/sim/lib/workspace-source/application/promote.integration.ts +++ b/apps/sim/lib/workspace-source/application/promote.integration.ts @@ -93,24 +93,39 @@ const DEV: Record = { /** The text files a get wrote (no blobs here), as a directory holds them. */ type Read = Awaited> +/** `apply --to`'s `--allow-delete` and `--delete-target-only`. */ +interface PromoteOptions { + allowDelete?: boolean + deleteTargetOnly?: boolean +} + /** What `apply --to` sends for `dev` (a get of the dev workspace) to `target`. */ -async function promote(dev: Read, target: string, options: { allowDelete?: boolean } = {}) { +async function promote(dev: Read, target: string, options: PromoteOptions = {}) { const prod = await runner.get(target) const composed = promotionSubmission({ files: dev.files, blobPaths: [], kinds: dev.kinds, target: prod, + deleteTargetOnly: options.deleteTargetOnly === true, + }) + return runner.apply(target, { + files: composed.files, + base: composed.base, allowDelete: options.allowDelete === true, }) - return runner.apply(target, { files: composed.files, base: composed.base, ...options }) } /** * Promotes `dev` to `target` as `apply --to` does with the target's promotion record, returning * the apply's result and the record it leaves. */ -async function promoteRecorded(dev: Read, target: string, record: PromotionRecord | undefined) { +async function promoteRecorded( + dev: Read, + target: string, + record: PromotionRecord | undefined, + options: PromoteOptions = {} +) { const prod = await runner.get(target) const composed = promotionSubmission({ files: dev.files, @@ -118,11 +133,16 @@ async function promoteRecorded(dev: Read, target: string, record: PromotionRecor kinds: dev.kinds, target: prod, record, - allowDelete: false, + deleteTargetOnly: options.deleteTargetOnly === true, + }) + const result = await runner.apply(target, { + files: composed.files, + base: composed.base, + allowDelete: options.allowDelete === true, }) - const result = await runner.apply(target, { files: composed.files, base: composed.base }) return { result, + composed, record: promotionRecord({ workspaceId: target, previous: record, @@ -384,7 +404,7 @@ export default defineTable({ }) check( - "the target's own resources are kept, with a warning, unless the promotion allows deletes", + "the target's own resources are kept, with a warning, even with --allow-delete, and deleted only with --delete-target-only", async () => { const dev = await workspaces.create({ name: 'Dev', ownerId }) const prod = await workspaces.create({ name: 'Prod', ownerId }) @@ -412,10 +432,67 @@ export default defineTable({ ) expect(prodTables.map((table) => table.name).sort()).toEqual(['audit', 'leads']) - const deleted = await promote(await runner.get(dev), prod, { allowDelete: true }) + const allowed = await promote(await runner.get(dev), prod, { allowDelete: true }) + expect(allowed.applied).toEqual([]) + expect( + allowed.diagnostics + .filter((finding) => finding.code === 'W_UNDECLARED') + .map((finding) => finding.key) + ).toEqual(['audit']) + + const deleted = await promote(await runner.get(dev), prod, { + allowDelete: true, + deleteTargetOnly: true, + }) expect( deleted.applied.map((change) => `${change.action} ${change.kind}:${change.key}`) ).toEqual(['delete table:audit']) } ) + + check( + 'a resource deleted in the directory since the last promotion is a delete: refused without --allow-delete, and --allow-delete deletes it alone, keeping what only the target holds', + async () => { + const dev = await workspaces.create({ name: 'Dev', ownerId }) + const prod = await workspaces.create({ name: 'Prod', ownerId }) + await runner.apply(dev, { files: DEV }) + const first = await promoteRecorded(await runner.get(dev), prod, undefined) + expect(first.result.failed).toBeUndefined() + const prodRead = await runner.get(prod) + await runner.apply(prod, { + files: { + 'tables/leads.ts': prodRead.files['tables/leads.ts'], + 'tables/audit.ts': LEADS.replace("name: 'leads'", "name: 'audit'"), + }, + base: prodRead.base, + kinds: ['table'], + }) + const devRead = await runner.get(dev) + await runner.apply(dev, { + files: Object.fromEntries( + Object.entries(devRead.files).filter(([path]) => path !== 'skills/tone.md') + ), + base: devRead.base, + allowDelete: true, + }) + const deletedInDev = await runner.get(dev) + + const refused = await refusal(promoteRecorded(deletedInDev, prod, first.record)) + expect(codes(refused.diagnostics)).toEqual(['E_APPLY_DELETE_NOT_ALLOWED']) + expect((await runner.get(prod)).files['skills/tone.md']).toContain('Be brief.') + + const allowed = await promoteRecorded(deletedInDev, prod, first.record, { allowDelete: true }) + expect( + allowed.result.applied.map((change) => `${change.action} ${change.kind}:${change.key}`) + ).toEqual(['delete skill:tone']) + expect(allowed.composed.targetOnly).toEqual(['table:audit']) + expect( + allowed.result.diagnostics + .filter((finding) => finding.code === 'W_UNDECLARED') + .map((finding) => finding.key) + ).toEqual(['audit']) + expect(allowed.record.resources['table:audit']).toBeUndefined() + expect((await runner.get(prod)).files['tables/audit.ts']).toBeDefined() + } + ) }) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index f528ff4d4b3..f46f9ddca1d 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -1115,4 +1115,99 @@ export default defineTable({ const settled = await apply(workspaceId, result.files, { base: result.base }) expect(settled.applied).toEqual([]) }) + + check( + "columns one apply drops from managed rows are stripped in one pass, under a timeout scaled past the table write's 10 s default", + async () => { + const workspaceId = await newWorkspace() + const read = await managedPlans(workspaceId) + const table = await storedTable(workspaceId) + // A probe on this table's row updates records each one that rewrites cells, and the timeout it + // ran under. + const probe = `strip_probe_${table.id.replace(/[^a-z0-9]/gi, '').toLowerCase()}` + await db.execute( + sql.raw(`create table ${probe} (row_id text not null, timeout_ms numeric not null)`) + ) + await db.execute( + sql.raw( + `create function ${probe}_fn() returns trigger language plpgsql as $$ begin insert into ${probe} values (new.id, extract(epoch from current_setting('statement_timeout')::interval) * 1000); return new; end $$` + ) + ) + await db.execute( + sql.raw( + `create trigger ${probe}_trg before update on user_table_rows for each row when (new.table_id = '${table.id}' and old.data is distinct from new.data) execute function ${probe}_fn()` + ) + ) + try { + const dropped = await apply( + workspaceId, + { + 'tables/plans.ts': read.files['tables/plans.ts'] + .replace(' seats: column.number(),\n', '') + .replace(" tier: column.select({ options: ['basic', 'pro'] }),\n", ''), + 'tables/plans.rows.jsonl': '{"code":"solo"}\n{"code":"team"}\n', + }, + { base: read.base, allowDelete: true } + ) + expect(dropped.failed).toBeUndefined() + const updates = [ + ...(await db.execute<{ passes: number; timeout_ms: number }>( + sql.raw( + `select count(*)::int as passes, min(timeout_ms)::int as timeout_ms from ${probe} group by row_id` + ) + )), + ] + expect(updates).toHaveLength(2) + for (const update of updates) { + expect(update.passes).toBe(1) + expect(update.timeout_ms).toBeGreaterThanOrEqual(60_000) + } + expect(Object.keys((await storedRows(workspaceId))[0])).toEqual(['code']) + } finally { + await db.execute(sql.raw(`drop trigger if exists ${probe}_trg on user_table_rows`)) + await db.execute(sql.raw(`drop function if exists ${probe}_fn()`)) + await db.execute(sql.raw(`drop table if exists ${probe}`)) + } + } + ) + + check( + 'a column the apply drops leaves no row drift: the next plan sees no server change, and a delete of the table is not stale', + async () => { + const workspaceId = await newWorkspace() + const read = await managedPlans(workspaceId) + const tierId = + (await storedTable(workspaceId)).schema.columns.find((column) => column.name === 'tier') + ?.id ?? '' + const dropped = await apply( + workspaceId, + { + 'tables/plans.ts': read.files['tables/plans.ts'].replace( + " tier: column.select({ options: ['basic', 'pro'] }),\n", + '' + ), + 'tables/plans.rows.jsonl': '{"code":"solo","seats":1}\n{"code":"team","seats":10}\n', + }, + { base: read.base, allowDelete: true } + ) + expect(dropped.failed).toBeUndefined() + // The apply strips the dropped column's cells in its own transaction, so none are left. + const table = await storedTable(workspaceId) + const [{ holding }] = await db + .select({ holding: sql`count(*)::int` }) + .from(userTableRows) + .where(and(eq(userTableRows.tableId, table.id), sql`${userTableRows.data} ? ${tierId}`)) + expect(holding).toBe(0) + + const planned = await apply(workspaceId, dropped.files, { + base: dropped.base, + dryRun: true, + }) + expect(codes(planned.diagnostics, 'warning')).not.toContain('W_SERVER_CHANGED') + const deleted = await apply(workspaceId, {}, { base: dropped.base, allowDelete: true }) + expect(deleted.applied.map((change) => `${change.action} ${change.key}`)).toEqual([ + 'delete plans', + ]) + } + ) }) diff --git a/apps/sim/lib/workspace-source/application/tools-skills.integration.ts b/apps/sim/lib/workspace-source/application/tools-skills.integration.ts index ea3d93b5496..289010517dd 100644 --- a/apps/sim/lib/workspace-source/application/tools-skills.integration.ts +++ b/apps/sim/lib/workspace-source/application/tools-skills.integration.ts @@ -10,7 +10,7 @@ import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' import { realtimeNotifyMock } from '@sim/testing/mocks/realtime-notify.mock' import { generateId } from '@sim/utils/id' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' -import { eq, inArray, sql } from 'drizzle-orm' +import { and, eq, inArray, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, vi } from 'vitest' import { createWorkspaceCustomToolUseCase, @@ -820,13 +820,17 @@ describe('workspace source: custom tools and skills', () => { const created = await apply(workspaceId, TOOLS, { kinds: ['custom-tool'] }) const toolId = created.base.resources['custom-tool:lookup']?.id ?? '' const authorize = deleteWorkspaceCustomToolUseCase.authorize + // Only this test's delete races: a call for another workspace (a test still running after its + // timeout) passes through untouched. const racing = vi .spyOn(deleteWorkspaceCustomToolUseCase, 'authorize') .mockImplementation(async (args) => { - await db - .update(customTools) - .set({ code: 'return "edited in the UI"' }) - .where(eq(customTools.id, toolId)) + if (args.input.workspaceId === workspaceId && args.input.toolId === toolId) { + await db + .update(customTools) + .set({ code: 'return "edited in the UI"' }) + .where(and(eq(customTools.id, toolId), eq(customTools.workspaceId, workspaceId))) + } return authorize.call(deleteWorkspaceCustomToolUseCase, args) }) diff --git a/apps/sim/stores/workflows/utils.ts b/apps/sim/stores/workflows/utils.ts index ede20204e4a..ff64e4885e0 100644 --- a/apps/sim/stores/workflows/utils.ts +++ b/apps/sim/stores/workflows/utils.ts @@ -1,11 +1,11 @@ import { generateId } from '@sim/utils/id' -import { escapeRegExp } from '@sim/utils/string' import { mergeSubblockStateWithValues } from '@sim/workflow-persistence/subblocks' import { filterUniqueWorkflowEdges } from '@sim/workflow-types/workflow' import type { Edge } from '@xyflow/react' import { DEFAULT_DUPLICATE_OFFSET } from '@/lib/workflows/autolayout/constants' import { remapConditionBlockIds, remapConditionEdgeHandle } from '@/lib/workflows/condition-ids' import { isDynamicHandleSubblock } from '@/lib/workflows/dynamic-handle-topology' +import { rewriteBlockNameReferences } from '@/lib/workflows/references/block-rename' import { normalizeName } from '@/executor/constants' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' @@ -112,43 +112,6 @@ export function mergeSubblockState( return mergeSubblockStateWithValues(blocks, workflowSubblockValues, blockId) } -function updateValueReferences(value: unknown, nameMap: Map): unknown { - if (typeof value === 'string') { - let updatedValue = value - nameMap.forEach((newName, oldName) => { - /** - * A rename to itself is a no-op, so skip the scan entirely. This is the - * whole map on the import path (`regenerateWorkflowIds` seeds it with - * `name -> name`), which turns an O(names x values) rescan of every - * sub-block string into nothing. - */ - if (oldName === newName) return - - /** - * `oldName` is a block name, which reaches this function straight from - * imported workflow JSON — `normalizeWorkflowBlockName` only lowercases - * and strips whitespace/dots, so regex metacharacters survive. Without - * escaping, a name like `a*a*a*a*b` compiles to a catastrophically - * backtracking pattern that pins the event loop on a sub-kilobyte input. - */ - const regex = new RegExp(`<${escapeRegExp(oldName)}\\.`, 'g') - updatedValue = updatedValue.replace(regex, `<${newName}.`) - }) - return updatedValue - } - if (Array.isArray(value)) { - return value.map((item) => updateValueReferences(item, nameMap)) - } - if (value && typeof value === 'object') { - const result: Record = {} - for (const [key, val] of Object.entries(value)) { - result[key] = updateValueReferences(val, nameMap) - } - return result - } - return value -} - /** * Clears a cloned block's `triggerPath` so it derives a fresh webhook URL from its own block id. * @@ -198,7 +161,7 @@ function updateBlockReferences( } if (subBlock.value !== undefined && subBlock.value !== null) { - const updatedValue = updateValueReferences( + const updatedValue = rewriteBlockNameReferences( subBlock.value, nameMap ) as SubBlockState['value'] @@ -494,7 +457,7 @@ export function regenerateBlockIds( Object.entries(newSubBlockValues).forEach(([_, blockValues]) => { Object.keys(blockValues).forEach((subBlockId) => { - blockValues[subBlockId] = updateValueReferences(blockValues[subBlockId], nameMap) + blockValues[subBlockId] = rewriteBlockNameReferences(blockValues[subBlockId], nameMap) }) }) diff --git a/apps/sim/stores/workflows/workflow/store.ts b/apps/sim/stores/workflows/workflow/store.ts index 716ade55cd5..ea211a357bd 100644 --- a/apps/sim/stores/workflows/workflow/store.ts +++ b/apps/sim/stores/workflows/workflow/store.ts @@ -19,6 +19,7 @@ import { getDynamicHandleSubblockType, isDynamicHandleSubblock, } from '@/lib/workflows/dynamic-handle-topology' +import { rewriteBlockNameReferences } from '@/lib/workflows/references/block-rename' import { normalizeName } from '@/executor/constants' import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { @@ -711,34 +712,7 @@ export const useWorkflowStore = create()( const workflowValues = subBlockStore.workflowValues[activeWorkflowId] || {} const updatedWorkflowValues = { ...workflowValues } - // Helper function to recursively update references in any data structure - function updateReferences(value: any, regex: RegExp, replacement: string): any { - // Handle string values - if (typeof value === 'string') { - return regex.test(value) ? value.replace(regex, replacement) : value - } - - // Handle arrays - if (Array.isArray(value)) { - return value.map((item) => updateReferences(item, regex, replacement)) - } - - // Handle objects - if (value !== null && typeof value === 'object') { - const result = { ...value } - for (const key in result) { - result[key] = updateReferences(result[key], regex, replacement) - } - return result - } - - // Return unchanged for other types - return value - } - - const oldBlockName = normalizeName(oldBlock.name) - const newBlockName = normalizeName(name) - const regex = new RegExp(`<${oldBlockName}\\.`, 'g') + const nameMap = new Map([[normalizeName(oldBlock.name), normalizeName(name)]]) // Loop through blocks Object.entries(workflowValues).forEach(([blockId, blockValues]) => { @@ -749,8 +723,7 @@ export const useWorkflowStore = create()( // Loop through subblocks and update references Object.entries(blockValues).forEach(([subBlockId, value]) => { - // Use a recursive function to handle all object types - const updatedValue = updateReferences(value, regex, `<${newBlockName}.`) + const updatedValue = rewriteBlockNameReferences(value, nameMap) // Check if the value actually changed if (JSON.stringify(updatedValue) !== JSON.stringify(value)) { diff --git a/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts index ee5a49fda00..825607bb8e6 100644 --- a/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts +++ b/packages/sim-cli/src/commands/protocol/source-apply-interrupt.test.ts @@ -1,5 +1,5 @@ import { spawn } from 'node:child_process' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import type { AddressInfo } from 'node:net' import { tmpdir } from 'node:os' @@ -143,6 +143,7 @@ describe('Ctrl-C while a workflow apply is committing', () => { expect(run.stderr).toContain('the apply was already sent') expect(savedFingerprint()).toBe('sf1:b') expect(readFileSync(join(dir, 'src/workflow.ts'), 'utf8')).toContain('// stored') + expect(existsSync(join(dir, 'src/.sim/command.lock'))).toBe(false) }, 30_000) it('stops at a second Ctrl-C and says the apply may have been saved and how to recover', async () => { @@ -154,5 +155,6 @@ describe('Ctrl-C while a workflow apply is committing', () => { expect(run.stderr).toContain('Stopped before the apply answered. It may have been saved.') expect(run.stderr).toContain(`sim workflows source get ${WORKFLOW_ID}`) expect(savedFingerprint()).toBe('sf1:a') + expect(existsSync(join(dir, 'src/.sim/command.lock'))).toBe(false) }, 30_000) }) diff --git a/packages/sim-cli/src/commands/protocol/source-check.ts b/packages/sim-cli/src/commands/protocol/source-check.ts index 298816947fd..ff3c7202c2d 100644 --- a/packages/sim-cli/src/commands/protocol/source-check.ts +++ b/packages/sim-cli/src/commands/protocol/source-check.ts @@ -9,6 +9,7 @@ import { isRecordLike, toRecord } from '@sim/utils/object' import { type BlockCatalog, type CanonicalWorkflow, + catalogBlock, codeFrame, codePointLabel, compileWorkflow, @@ -20,6 +21,7 @@ import { parseWorkflowLock, raisedMessage, refuseEditedFindings, + refusesEdited, scriptName, startInputs, type Value, @@ -389,7 +391,7 @@ async function compileWorkflowDir( ...(storedEdges ? { storedUnreadEdges: storedEdges } : {}), ...(envNames ? { envNames, envDeclaredIn: KIND_DIRS.secret } : {}), }) - return { lock, files, compiled, storedBlocks, printedWith } + return { lock, files, compiled, storedBlocks, printedWith, catalog } } /** @@ -435,12 +437,9 @@ export async function checkWorkflowDir( dir: string, catalog: BlockCatalog, options: { declarations?: WorkspaceDeclarations; strict: StrictMode; prefix?: string } -): Promise<{ findings: CheckFinding[]; entry: string }> { - const { lock, files, compiled, storedBlocks, printedWith } = await compileWorkflowDir( - dir, - catalog, - options.declarations - ) +): Promise<{ findings: CheckFinding[]; entry: string; unlisted: UnlistedCounts }> { + const compiledDir = await compileWorkflowDir(dir, catalog, options.declarations) + const { lock, files, compiled, storedBlocks, printedWith } = compiledDir const entryText = files[SOURCE_ENTRY_FILE] const locate = (block: string, param: string, names: string | undefined) => { const prefix = encodeAnchor({ kind: 'param', block, param: '' }) @@ -551,7 +550,27 @@ export async function checkWorkflowDir( const head = /^(?:\$ref\.|<)([A-Za-z0-9_]+)/.exec(text)?.[1] return head !== undefined && removedKeys.has(head) } - const refusable = (finding: (typeof compiled.diagnostics)[number]) => !mayBeRenamed(finding) + /** + * A dropdown value outside options that vary with deployment settings (`optionsVary`): the + * cached catalog may have been built under other settings than the server's gate runs with, so + * the check keeps the warning and leaves the verdict to the server. + */ + const variesByDeployment = (finding: { code: string; blockKey?: string; param?: string }) => { + const { blockKey, param } = finding + if (finding.code !== 'W_UNKNOWN_OPTION' || blockKey === undefined || param === undefined) { + return false + } + return optionsVaryOn(blockKey, (id) => id === param) + } + /** Whether a field of block `blockKey` that `matches` takes options varying by deployment. */ + const optionsVaryOn = (blockKey: string, matches: (subBlockId: string) => boolean) => { + const type = compiled.workflow ? getOwn(compiled.workflow.blocks, blockKey)?.type : undefined + const block = type === undefined ? undefined : catalogBlock(compiledDir.catalog, type) + return block?.subBlocks.some((sub) => matches(sub.id) && sub.optionsVary === true) === true + } + const exempt = (finding: (typeof compiled.diagnostics)[number]) => + mayBeRenamed(finding) || variesByDeployment(finding) + const refusable = (finding: (typeof compiled.diagnostics)[number]) => !exempt(finding) const refused = [ ...refuseEditedFindings( { @@ -560,13 +579,14 @@ export async function checkWorkflowDir( }, { changed, codeEdited } ), - ...compiled.diagnostics.filter(mayBeRenamed), + ...compiled.diagnostics.filter(exempt), ] const findings = [ ...refused.filter((diagnostic) => reported(diagnostic.code, diagnostic.blockKey)), ...declared, - ].map(({ code, severity, message, hint, frame, span, blockKey }): CheckFinding => { - const raised = severity === 'warning' && fails(code, blockKey) + ].map((finding): CheckFinding => { + const { code, severity, message, hint, frame, span, blockKey } = finding + const raised = severity === 'warning' && !variesByDeployment(finding) && fails(code, blockKey) return { code, severity: raised ? 'error' : severity, @@ -576,7 +596,40 @@ export async function checkWorkflowDir( ...(span ? { file: `${prefix}${span.file}`, line: span.line, column: span.column } : {}), } }) - return { findings, entry: entryText } + // The warnings the compile's caps left out are counted as the listed ones would be: one a + // refusal raised is listed already. + const unlisted: UnlistedCounts = { errors: 0, warnings: 0 } + const edit = { changed, codeEdited } + // An omitted finding names no field, so one on a block with any deployment-shaped field stays a + // warning. + for (const { code, blockKey, count, varies } of [ + ...compiled.omitted.map((omitted) => ({ + ...omitted, + varies: + omitted.code === 'W_UNKNOWN_OPTION' && + omitted.blockKey !== undefined && + optionsVaryOn(omitted.blockKey, () => true), + })), + ...compiled.withheld + .filter((finding) => !refusable(finding) || !refusesEdited(finding, edit)) + .map((finding) => ({ + code: finding.code, + blockKey: finding.blockKey, + count: 1, + varies: variesByDeployment(finding), + })), + ]) { + if (!reported(code, blockKey)) continue + if (!varies && fails(code, blockKey)) unlisted.errors += count + else unlisted.warnings += count + } + return { findings, entry: entryText, unlisted } +} + +/** Findings a check counts but does not list: those the compile's caps left out. */ +interface UnlistedCounts { + errors: number + warnings: number } /** @@ -588,17 +641,19 @@ export function reportCheck( output: Parameters[0], findings: CheckFinding[], result: Record, - options: { verbose?: boolean; bound?: AnswerBound } = {} + options: { verbose?: boolean; bound?: AnswerBound; unlisted?: UnlistedCounts } = {} ): void { const shown = findingsOutput(output, options.verbose) const listed = boundAnswerFindings(findings, options.bound) shown.print(listed) - const errors = findings.filter((finding) => finding.severity === 'error').length + const unlisted = options.unlisted ?? { errors: 0, warnings: 0 } + const errors = findings.filter((finding) => finding.severity === 'error').length + unlisted.errors if (errors > 0) setSoftExitCode(1) printProtocolResult(output, { checked: 'compile', errors, - warnings: findings.filter((finding) => finding.severity === 'warning').length, + warnings: + findings.filter((finding) => finding.severity === 'warning').length + unlisted.warnings, ...result, ...shown.result(listed), }) diff --git a/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts b/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts index 3716ccc37b0..1550fe498dc 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/command-lock.ts @@ -189,10 +189,12 @@ export async function withSourceDirLock( await writeFile(join(lock, LOCK_OWNER_FILE), JSON.stringify(owner)) return await run() } finally { - disposeSignalRelease() + // The signal release stays installed until the lock is gone: an interrupt a deferred write + // held is raised again as `run` ends, and with no handler it would end the process mid-removal. await rm(await checkedPath(root, SOURCE_DIR_LOCK, new Set()), { recursive: true, force: true }) // Folders the lock alone created go with it; one the command filled stays. await removeCreatedFolders() + disposeSignalRelease() } } diff --git a/packages/sim-cli/src/commands/protocol/source-dir/layout.ts b/packages/sim-cli/src/commands/protocol/source-dir/layout.ts index 82966b52d6e..e9d1e637466 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/layout.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/layout.ts @@ -91,11 +91,13 @@ export function assertSendableCodeFiles(files: Record): void { /** * A path inside the source directory, normalized (`./functions/x.js` → `functions/x.js`), or - * `null` for an absolute path, a drive letter or a `..` escape. + * `null` for an absolute path, a drive letter, a `..` escape or a NUL byte (which no file name + * holds, so the compile refuses the path as `E_CODE_FILE_PATH`). */ export function sourceRelativePath(path: string): string | null { const normalized = posix.normalize(path.replace(/\\/g, '/')) if ( + normalized.includes('\0') || normalized.startsWith('/') || /^[A-Za-z]:/.test(normalized) || normalized === '.' || @@ -122,20 +124,26 @@ export function lockCodeFiles(lock: unknown): Set { } /** - * The function files a write leaves behind: named by the lock the directory held (so the CLI - * wrote them), no longer named by the new lock, and not being written now. A file the person - * added and no lock ever named is never pruned, nor is an entry that is no code file (such as - * `functions/`), whose removal would take the whole folder. + * The function files a write leaves behind: named by the lock the directory held, recorded as + * written by the CLI (`recorded`, the written record's paths relative to the same directory), no + * longer named by the new lock, and not being written now. A file the CLI never wrote is never + * pruned, even when a hand-edited lock names it (say as a trigger's `codeFile`), nor is an entry + * that is no code file (such as `functions/`), whose removal would take the whole folder. */ export function orphanedCodeFiles( previousLock: unknown, nextLock: unknown, - written: Iterable + written: Iterable, + recorded: Readonly> ): string[] { const kept = lockCodeFiles(nextLock) for (const path of written) kept.add(path) return [...lockCodeFiles(previousLock)].filter( - (path) => path.startsWith(`${SOURCE_FUNCTIONS_DIR}/`) && isCodeFilePath(path) && !kept.has(path) + (path) => + path.startsWith(`${SOURCE_FUNCTIONS_DIR}/`) && + isCodeFilePath(path) && + Object.hasOwn(recorded, path) && + !kept.has(path) ) } diff --git a/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts b/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts index 53c21b97b4b..bba48bcac50 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/write-back.ts @@ -27,6 +27,7 @@ import { digest, inWorkspace, type PrintedSourceAnswer, + readFolderWritten, readWritten, writtenFiles, writtenRecord, @@ -94,7 +95,12 @@ export async function writeBack( [SOURCE_BASE_FILE, `${JSON.stringify(answer.base, null, 2)}\n`], ] const write = [...printed, ...(folder ? [] : writtenFiles(writtenRecord(files))), ...extra.write] - const orphaned = orphanedCodeFiles(previousLock, answer.lock, Object.keys(answer.files)) + const orphaned = orphanedCodeFiles( + previousLock, + answer.lock, + Object.keys(answer.files), + (await readFolderWritten(dir)).digests + ) const unnamed = await unnamedCodeFiles(dir, previousLock, answer, orphaned) await updateSourceDir(dir, { write, diff --git a/packages/sim-cli/src/commands/protocol/source-dir/written.ts b/packages/sim-cli/src/commands/protocol/source-dir/written.ts index 76b8de80b0a..4823a24b9cc 100644 --- a/packages/sim-cli/src/commands/protocol/source-dir/written.ts +++ b/packages/sim-cli/src/commands/protocol/source-dir/written.ts @@ -214,10 +214,11 @@ export async function locallyEditedFiles( previousLock: unknown ): Promise { const touched = new Map(Object.entries(answer.files)) - for (const path of orphanedCodeFiles(previousLock, answer.lock, touched.keys())) { + const { digests } = await readFolderWritten(dir) + for (const path of orphanedCodeFiles(previousLock, answer.lock, touched.keys(), digests)) { touched.set(path, null) } - return editedFiles(dir, touched, (await readFolderWritten(dir)).digests) + return editedFiles(dir, touched, digests) } /** `path` of the workflow folder `key`, relative to its workspace directory. */ diff --git a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts index 0f7643a9edb..c991a51918f 100644 --- a/packages/sim-cli/src/commands/protocol/source-exit-codes.ts +++ b/packages/sim-cli/src/commands/protocol/source-exit-codes.ts @@ -70,7 +70,9 @@ function sourceExitCodeRows(prefix: SourceCommandGroup): SourceExitCodeRow[] { { code: 1, meaning: - 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed', + prefix === 'sim workspaces source' + ? 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another resource holds (`E_NAME_TAKEN`), a cancelled prompt, or the command failed' + : 'The source has errors (each printed as `file:line:column` with a hint), a locked block (`E_APPLY_LOCKED_BLOCK`), a name another workflow holds (`SOURCE_NAME_TAKEN`), a cancelled prompt, or the command failed', next: 'Fix what it printed and run it again', help: { 'workflows source apply': diff --git a/packages/sim-cli/src/commands/protocol/source-status.ts b/packages/sim-cli/src/commands/protocol/source-status.ts index efc6083e713..6a876e1efec 100644 --- a/packages/sim-cli/src/commands/protocol/source-status.ts +++ b/packages/sim-cli/src/commands/protocol/source-status.ts @@ -61,6 +61,8 @@ export interface StatusReport extends SourceStatus { server: ServerCheck /** Why the server was not checked, or what it left unchecked. */ note?: string + /** Files in the directory no source names, which applying never sends. */ + unnamed?: string[] plan: Plan | null next: NextStep | null } @@ -192,6 +194,13 @@ export function printStatusReport( } else { lines.push(s.yellow(`Server not checked${report.note ? `: ${sanitize(report.note)}` : '.'}`)) } + if (report.unnamed?.length) { + lines.push( + s.dim( + `Not named by any codeFile(), so applying never sends ${report.unnamed.length === 1 ? 'it' : 'them'}: ${sanitize(report.unnamed.join(', '))}` + ) + ) + } if (report.server === 'checked' && report.note) lines.push(s.dim(sanitize(report.note))) if (report.plan) lines.push(planLine(report.plan)) if (report.next) { diff --git a/packages/sim-cli/src/commands/protocol/source-ui.ts b/packages/sim-cli/src/commands/protocol/source-ui.ts index 11f0b60d55c..d0b0005bc1e 100644 --- a/packages/sim-cli/src/commands/protocol/source-ui.ts +++ b/packages/sim-cli/src/commands/protocol/source-ui.ts @@ -154,13 +154,21 @@ function resultDiagnostics( verbose = false ): { diagnostics?: ResultDiagnostic[] } { if (!isMachine(format)) return {} + const hidden = (finding: Diagnostic) => + !verbose && finding.severity === 'info' && finding.code !== 'I_FINDINGS_TRUNCATED' + const hiddenCount = diagnostics.filter(hidden).length + // The marker counts what the answer left out; the infos hidden here are left out on top of it. + const marked = (finding: Diagnostic): Diagnostic => + finding.code === 'I_FINDINGS_TRUNCATED' && hiddenCount > 0 + ? { + ...finding, + message: `${finding.message} ${hiddenCount} info finding${hiddenCount === 1 ? '' : 's'} listed ${hiddenCount === 1 ? 'is' : 'are'} hidden too; pass --verbose to see ${hiddenCount === 1 ? 'it' : 'them'}.`, + } + : finding return { diagnostics: diagnostics - .filter( - (finding) => - verbose || finding.severity !== 'info' || finding.code === 'I_FINDINGS_TRUNCATED' - ) - .map(({ frame: _frame, ...finding }) => finding), + .filter((finding) => !hidden(finding)) + .map(({ frame: _frame, ...finding }) => marked(finding)), } } diff --git a/packages/sim-cli/src/commands/protocol/workflow-dir.ts b/packages/sim-cli/src/commands/protocol/workflow-dir.ts index edfe121dbe4..c983e1ef687 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-dir.ts @@ -93,9 +93,9 @@ async function readJson(directory: string, path: string, hint: string): Promise< * reference to any other file never uploads it; the server reports a referenced file that is * missing or of another kind. */ -function codeFilePaths(entry: string, lock: WorkflowLockV1 | null): string[] { +export function codeFilePaths(entry: string | null, lock: unknown): string[] { const candidates = lockCodeFiles(lock) - for (const literal of codeFileReferences(entry)) { + for (const literal of entry === null ? [] : codeFileReferences(entry)) { const path = sourceRelativePath(literal) if (path) candidates.add(path) } @@ -212,7 +212,7 @@ The primer above is enough to edit; read a section here when a step needs it. 2. \`sim workflows source check --dir .\`: compiles offline with the server's compiler, in milliseconds. It catches what \`tsc\` cannot (duplicate names, cycles, literal secrets), and fails (exit 1) on a missing required field, an unknown option or output path, a wrong value kind, an out-of-range value, an unwired condition or router branch or a literal secret in a block you added (\`--strict\`: in every block). It also adds the blocks the source now uses to \`tsconfig.json\`. 3. \`${typecheckCommand()}\`: about a second; catches what step 2 and the dry run do not (such as a reference to a start input the trigger does not declare). It runs the TypeScript release the declarations are built for (the directory has no \`package.json\`; plain \`bunx tsc\` would run whatever version it finds). \`tsconfig.json\` includes only the blocks the source used at the last check, get or apply: a builder from another block fails with \`has no exported member\` until you run step 2 (or use \`${typecheckCommand(TSCONFIG_FULL_FILE)}\`, every current block). The dry run is the authority: a printed line you did not edit that \`tsc\` flags and the dry run accepts is fine. 4. \`sim workflows source status --dir .\`: the files you changed (\`sim workflows source diff --dir .\` shows the edits), whether the workflow changed on the server since the get, and the command to run next (\`--offline\`: skip the server). Then \`sim workflows source apply --dir . --dry-run\` until it exits 0. -5. \`sim workflows source apply --dir . --yes\`: saves the draft, refusing errors, block deletions without \`--allow-delete\` and a stale directory, so it needs no separate dry run (without \`--yes\` and a terminal it prints the plan and exits ${SOURCE_PLANNED_EXIT_CODE}). \`--deploy\` also deploys it; without it, a deployed workflow's next \`sim workflows deploy \` is printed. It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out). +5. \`sim workflows source apply --dir . --yes\`: saves the draft, refusing errors, block deletions without \`--allow-delete\` and a stale directory, so it needs no separate dry run (without \`--yes\` and a terminal it prints the plan and exits ${SOURCE_PLANNED_EXIT_CODE}). \`--deploy\` also deploys it; without it, a deployed workflow's next \`sim workflows deploy --expected-draft-fingerprint \` is printed, which deploys only the draft this apply saved. It then rewrites \`${SOURCE_ENTRY_FILE}\` in canonical form (order, quoting, defaults left out). 6. Exit ${SOURCE_STALE_EXIT_CODE} (\`SOURCE_STALE\`, HTTP 409, or \`E_APPLY_STALE\` on a dry run): the workflow changed and nothing was saved. Run \`sim workflows source get --dir \`, reapply your edit there, dry-run, apply. A get into this directory refuses to overwrite your unapplied edits unless you pass \`--force\`. Exit ${SOURCE_STALE_EXIT_CODE} right after an apply that ended without an answer (a timeout, a dropped connection) means the draft changed, by that apply or by someone else: get into a fresh directory and compare. Only when it holds your edit, run \`sim workflows source get --dir . --force\`; otherwise reapply your edit in the fresh directory. Editing the \`workflow('…')\` name or description renames or redescribes the workflow; a name another workflow in its folder holds is refused with \`E_NAME_TAKEN\`. Lines starting \`note:\` explain something the write did differently than the source says; read them. diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts index c34537eecdd..8ddfe50f8f8 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.test.ts @@ -780,6 +780,28 @@ describe('sim workflows source apply', () => { ]) }) + it('without --verbose, says the truncation marker leaves out the infos it hides too', async () => { + setTerminal(false) + const truncated = { + code: 'I_FINDINGS_TRUNCATED', + severity: 'info', + message: 'There are 140 findings; 40 more are left out.', + } + request.mockResolvedValue( + outcome({ dryRun: true, diagnostics: [layout, { ...layout, line: 4 }, truncated] }) + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--dry-run') + + expect(stdoutJson().diagnostics).toEqual([ + { + ...truncated, + message: + 'There are 140 findings; 40 more are left out. 2 info findings listed are hidden too; pass --verbose to see them.', + }, + ]) + }) + it('in a display format, prints the first warning of a code whole and the rest on one line', async () => { setTerminal(false) contextMockState.output = 'text' @@ -922,6 +944,49 @@ describe('sim workflows source apply', () => { }) }) + it('deploys no new version when a no-op apply leaves the live version matching the draft', async () => { + setTerminal(false) + const STATUS_PATH = `/api/v2/workflows/${WORKFLOW_ID}/deployment` + const noop = outcome() + noop.data.applied = false + noop.data.diff.noop = true + noop.data.needsRedeployment = false + const status = (isDeployed: boolean) => ({ + data: { + ...deployed.data, + isDeployed, + needsRedeployment: false, + isPublicApi: false, + webhooks: [], + activeDeployment: isDeployed + ? { deploymentVersionId: 'dv-7', version: 7, deployedAt: '2026-11-20T00:00:00Z' } + : null, + }, + }) + let live = true + request.mockImplementation(async (path: string) => + path === STATUS_PATH ? status(live) : path === DEPLOY_PATH ? deployed : noop + ) + + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + + expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH, STATUS_PATH]) + expect(stdoutJson()).toMatchObject({ + deployed: { version: 7, isDeployed: true }, + needsRedeployment: false, + }) + + // A workflow never deployed is deployed, no-op or not. + live = false + request.mockClear() + await apply(WORKFLOW_ID, '--dir', dir, '--yes', '--deploy') + expect(request.mock.calls.map((call) => call[0])).toEqual([ + APPLY_PATH, + STATUS_PATH, + DEPLOY_PATH, + ]) + }) + it('without --deploy, names the deploy command when the deployed workflow now differs', async () => { setTerminal(false) request.mockResolvedValue(outcome()) @@ -931,7 +996,9 @@ describe('sim workflows source apply', () => { expect(request.mock.calls.map((call) => call[0])).toEqual([APPLY_PATH]) expect(stdoutJson()).toMatchObject({ needsRedeployment: true, - next: { command: `sim workflows deploy ${WORKFLOW_ID}` }, + next: { + command: `sim workflows deploy ${WORKFLOW_ID} --expected-draft-fingerprint sf1:b`, + }, }) }) @@ -962,7 +1029,9 @@ describe('sim workflows source apply', () => { expect(error).toMatchObject({ status: 403, code: 'FORBIDDEN' }) expect((error as Error).message).toContain(`The apply was saved into ${dir}`) - expect((error as Error).message).toContain(`sim workflows deploy ${WORKFLOW_ID}`) + expect((error as Error).message).toContain( + `sim workflows deploy ${WORKFLOW_ID} --expected-draft-fingerprint sf1:b` + ) expect(readFileSync(join(dir, 'workflow.ts'), 'utf8')).toBe(`${ENTRY}// stored\n`) // The saved write's result still prints, before the deploy's failure. expect(stdoutJson()).toMatchObject({ workflowId: WORKFLOW_ID, applied: true }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts index 0e35fbabf7f..66c62c5a69c 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-apply.ts @@ -8,6 +8,7 @@ import type { ApplyWorkflowSourceResponse, CreateWorkflowFromSourceBody, DeployWorkflowResponse, + GetWorkflowDeploymentResponse, } from '../../generated/v2-api' import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' @@ -122,7 +123,7 @@ async function writeWorkflowSource(args: mode: WriteMode creating: boolean question: string - client: OperationClient + client: OperationClient operation: Operation params: Record body: ApplyWorkflowSourceBody | CreateWorkflowFromSourceBody @@ -234,11 +235,12 @@ interface DeployOutcome { * deploy says the write itself was saved. */ async function deployStep( - client: OperationClient<'deployWorkflow'>, + client: OperationClient<'deployWorkflow' | 'getWorkflowDeployment'>, data: SourceApplyData, options: { deploy?: boolean; dir: string; verb: string; machine: boolean } ): Promise { - const deployCommand = `sim workflows deploy ${data.workflowId}` + // Pinned to the draft this write stored, as `--deploy` is: an edit made since is never deployed. + const deployCommand = `sim workflows deploy ${data.workflowId} --expected-draft-fingerprint ${data.base.fingerprint}` if (!options.deploy) { if (!data.needsRedeployment) return {} return { @@ -248,6 +250,31 @@ async function deployStep( }, } } + if (!data.applied && !data.needsRedeployment) { + // A no-op write changed no draft: when the live version already matches it, a deploy would + // only publish the same version again. + const status = await client + .request('getWorkflowDeployment', { + params: { workflowId: data.workflowId }, + }) + .then( + (response) => response.data, + (error: unknown) => { + throw deployFailure(error, options, deployCommand) + } + ) + if (status.isDeployed && !status.needsRedeployment) { + if (!options.machine) + printError('The live version already matches the draft; nothing was deployed.') + return { + deployed: { + ...(status.activeDeployment ? { version: status.activeDeployment.version } : {}), + isDeployed: true, + }, + needsRedeployment: false, + } + } + } const deployed = await client .request('deployWorkflow', { params: { workflowId: data.workflowId }, @@ -316,6 +343,7 @@ export function attachWorkflowSourceApply(source: Command): void { const [command, connectApply] = apiCommand(source, 'apply', [ 'applyWorkflowSource', 'deployWorkflow', + 'getWorkflowDeployment', 'getSourceResources', ]) absoluteDirOption(command) @@ -401,6 +429,7 @@ export function attachWorkflowSourceCreate(source: Command): void { 'getSourceCatalog', 'getSourceResources', 'deployWorkflow', + 'getWorkflowDeployment', ]) absoluteDirOption(command) verboseFlag(command) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts index b26d38e116f..44ecfd54303 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.test.ts @@ -284,6 +284,22 @@ describe('sim workflows source check', () => { expect(process.exitCode).toBe(1) }) + it('refuses a codeFile() path holding a NUL byte by its code, reading no such file', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace('import { response,', 'import { codeFile, fn, response,').replace( + ' start.to(done)', + " const step = fn('Step', { code: codeFile('./functions/\\u0000x.js') })\n start.to(done)" + ) + ) + + await check() + + expect(findingLines()).toMatch(/error E_CODE_FILE_PATH/) + expect(process.exitCode).toBe(1) + }) + it('puts each finding in the JSON on stdout', async () => { cacheCatalog() writeFileSync( @@ -453,6 +469,30 @@ describe('sim workflows source check', () => { expect(codes.filter((code) => code === 'I_FINDINGS_TRUNCATED')).toHaveLength(1) }) + it('counts every literal secret it fails on, past the 20 of one code it lists', async () => { + cacheCatalog() + const bearer = `Bearer ${'sk-proj-'}${'Abc123Def456Ghi789Jkl012'}` + const calls = Array.from( + { length: 25 }, + (_, index) => + ` const call${index} = api('Call ${index}', { url: 'https://api.example.com/v1', method: 'GET', headers: [{ cells: { Key: 'Authorization', Value: '${bearer}' } }] })\n start.to(call${index})` + ).join('\n') + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace('import { response,', 'import { api, response,').replace( + ' start.to(done)', + `${calls}\n start.to(done)` + ) + ) + + await check() + + const answer = result() + expect(answer).toMatchObject({ errors: 25 }) + const codes = (answer.diagnostics as Array<{ code: string }>).map((finding) => finding.code) + expect(codes.filter((code) => code === 'W_LITERAL_SECRET')).toHaveLength(20) + }) + it('fails a row write naming a column or select option the table does not declare', async () => { cacheCatalog() writeWorkspace() @@ -481,7 +521,10 @@ describe('sim workflows source check', () => { writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) await check() - expect(findingLines()).toMatch(/error W_UNKNOWN_OPTION/) + expect(findingLines()).toMatch( + /error E_APPLY_INVALID_SUBBLOCK_FIELD: method has no option FETCH\./ + ) + expect(findingLines()).not.toContain('kept as written') expect(process.exitCode).toBe(1) process.exitCode = undefined @@ -974,12 +1017,43 @@ export default workflow('W', {}, () => { expect(process.exitCode).toBe(1) process.exitCode = undefined - writeLock([]) await checkIn(dir, '--no-strict') expect(result()).toMatchObject({ errors: 0, warnings: 1 }) expect(process.exitCode).toBeUndefined() }) + it('fails with --no-strict on a dropdown value a new block sets, as the server refuses it', async () => { + cacheCatalog() + writeFileSync(join(dir, 'workflow.ts'), UNKNOWN_METHOD) + writeLock([]) + + await checkIn(dir, '--no-strict') + + expect(result()).toMatchObject({ errors: 1, warnings: 0 }) + expect(findingLines()).toMatch( + /error E_APPLY_INVALID_SUBBLOCK_FIELD: method has no option FETCH\./ + ) + expect(process.exitCode).toBe(1) + }) + + it('keeps a value outside options that vary by deployment a warning on a new block under the default strict mode, as the cached catalog may predate the settings the server checks with', async () => { + cacheCatalog() + writeFileSync( + join(dir, 'workflow.ts'), + CLEAN.replace( + ' start.to(done)', + " const run = fn('Run', { language: 'shell', code: 'echo hi' })\n start.to(done)\n done.to(run)" + ).replace('import { response,', 'import { fn, response,') + ) + writeLock([]) + + await checkIn(dir) + + expect(findingLines()).toMatch(/warning W_UNKNOWN_OPTION: language has no option shell/) + expect(result()).toMatchObject({ errors: 0 }) + expect(process.exitCode).toBeUndefined() + }) + it('passes what a get prints from stored state the server keeps: a loop error edge, a cycle, an unwired branch', async () => { cacheCatalog() writeFileSync(join(dir, 'workflow.ts'), STORED_SHAPES) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts index df197ef165d..ba4ae9b7188 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-check.ts @@ -102,7 +102,7 @@ export function attachWorkflowSourceCheck(source: Command): void { workflowInputs: await workspaceWorkflowInputs(workspace.root, catalog), } : undefined - const { findings, entry } = await checkWorkflowDir(options.dir, catalog, { + const { findings, entry, unlisted } = await checkWorkflowDir(options.dir, catalog, { strict: strictMode(options), ...(declarations ? { declarations } : {}), }) @@ -127,6 +127,7 @@ export function attachWorkflowSourceCheck(source: Command): void { { catalogHash: catalog.hash }, { verbose: options.verbose === true, + unlisted, } ) } diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts index 1fabd5501e3..3c2e9ac19ee 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-roundtrip.test.ts @@ -617,6 +617,24 @@ describe('pruning function files', () => { expect(stderr.join('\n')).not.toContain('functions/score.js is no longer named') }) + it('never prunes a file of the author a hand-edited lock gives to a block', async () => { + await get() + writeFileSync(join(dir, 'functions/mine.js'), 'mine') + const handEdited = JSON.parse(read('.sim/lock.json')) + handEdited.blocks.start = { + id: 'b-start', + parent: null, + ident: 'start', + order: 9, + codeFile: 'functions/mine.js', + } + writeFileSync(join(dir, '.sim/lock.json'), JSON.stringify(handEdited)) + + await apply() + + expect(read('functions/mine.js')).toBe('mine') + }) + it('prunes, on a fresh get, a function file the previous get wrote', async () => { server.source = () => json(sourceResponse(twoFiles, twoLock)) await get() diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts index e491378fbfa..649fcdbb96e 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.test.ts @@ -179,7 +179,7 @@ describe('workflows source status', () => { await run(root, 'status') const { frame: _frame, ...listed } = warning - expect(printed()).toMatchObject({ plan: { warnings: 1 }, diagnostics: [listed] }) + expect(printed()).toMatchObject({ plan: null, diagnostics: [listed] }) contextMockState.output = 'text' mockProfileFrom.mockReturnValue({ @@ -217,7 +217,11 @@ describe('workflows source status', () => { }) it('reports an edit, a deleted function file and a new one, exit 11', async () => { - write(root, 'workflow.ts', `${ENTRY}// edited\n`) + write( + root, + 'workflow.ts', + ENTRY.replace('})\n})', "})\n fn('Rank', { code: codeFile('./functions/rank.js') })\n})") + ) unlinkSync(join(root, 'functions/score.js')) write(root, 'functions/rank.js', 'return 2\n') mockRequest.mockResolvedValue(dryRun([], 1)) @@ -241,6 +245,49 @@ describe('workflows source status', () => { expect(process.exitCode).toBe(11) }) + it('leaves out a function file no codeFile() names, which applying never sends, and says so', async () => { + write(root, 'functions/notes.js', 'return 3\n') + mockRequest.mockResolvedValue(dryRun()) + + await run(root, 'status') + + expect(printed()).toMatchObject({ + state: 'clean', + local: [], + unnamed: ['functions/notes.js'], + next: null, + }) + expect(process.exitCode ?? 0).toBe(0) + + vi.mocked(console.log).mockClear() + await run(root, 'diff', '--server') + expect(printed()).toMatchObject({ files: [] }) + }) + + it('does not report a recorded function file as deleted once no codeFile() names it but it is still on disk', async () => { + write( + root, + 'workflow.ts', + ENTRY.replace(" fn('Score', { code: codeFile('./functions/score.js') })\n", '') + ) + write(root, '.sim/lock.json', JSON.stringify({ ...LOCK, blocks: {} })) + mockRequest.mockResolvedValue(dryRun([], 1)) + + await run(root, 'status') + + expect(printed()).toMatchObject({ + local: [ + { + kind: 'workflow', + key: WORKFLOW_ID, + change: 'edited', + files: [{ path: 'workflow.ts', change: 'edited' }], + }, + ], + unnamed: ['functions/score.js'], + }) + }) + it('reports drift when the workflow changed on the server and nothing here, exit 12', async () => { mockRequest.mockResolvedValue(dryRun([STALE])) @@ -249,7 +296,8 @@ describe('workflows source status', () => { expect(printed()).toMatchObject({ state: 'drift', drift: [{ kind: 'workflow', key: WORKFLOW_ID, server: 'changed' }], - plan: { errors: 0 }, + // Applying is refused as stale, so no plan of what it would change is shown. + plan: null, next: { command: `sim workflows source get ${WORKFLOW_ID} --dir ${root}` }, }) expect(process.exitCode).toBe(12) diff --git a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts index 1656c0f2dba..a64e84ad222 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-source-status.ts @@ -27,6 +27,7 @@ import { digest, listSourceFiles, readFolderWritten, + readPreviousLock, readSourceText, type Written, } from './source-dir' @@ -51,7 +52,7 @@ import { } from './source-status' import { refreshWorkflowWorkspaceTypes, withSourceExitCode } from './source-tree' import type { Diagnostic } from './source-ui' -import { readBase, readLock, readSourceFiles } from './workflow-dir' +import { codeFilePaths, readBase, readLock, readSourceFiles } from './workflow-dir' type PlanData = PlanWorkflowSourceResponse['data'] @@ -62,35 +63,40 @@ interface LocalWorkflow { base: Awaited> workflowId: string written: Written - /** `workflow.ts` and every file under `functions/`, by path. */ + /** `workflow.ts` and every file under `functions/` an apply sends, by path. */ texts: Record changed: ChangedFile[] + /** Files under `functions/` that neither `workflow.ts` nor the lock names: an apply never sends them. */ + unnamed: string[] } async function readLocalWorkflow(dir: string): Promise { const base = await readBase(dir, undefined) const { workflowId } = base const written = await readFolderWritten(dir) - const paths = [ - SOURCE_ENTRY_FILE, - ...(await listSourceFiles(dir, SOURCE_CODE_DIR)).filter( - (path) => !path.split('/').some((segment) => segment.startsWith('.')) - ), - ] - const texts: Record = {} - for (const path of paths) { + const entry = await readSourceText(dir, SOURCE_ENTRY_FILE) + const sent = new Set(codeFilePaths(entry, await readPreviousLock(dir))) + const listed = (await listSourceFiles(dir, SOURCE_CODE_DIR)).filter( + (path) => !path.split('/').some((segment) => segment.startsWith('.')) + ) + const texts: Record = entry === null ? {} : { [SOURCE_ENTRY_FILE]: entry } + for (const path of listed.filter((each) => sent.has(each))) { const text = await readSourceText(dir, path) if (text !== null) texts[path] = text } const current = Object.fromEntries( Object.entries(texts).map(([path, text]) => [path, digest(text)]) ) + const unnamed = listed.filter((path) => !sent.has(path)) + // A recorded file still on disk that nothing names is left out on both sides, not deleted. + const ignored = new Set(unnamed) return { base, workflowId, written, texts, - changed: changedFiles(current, written.digests, () => true), + changed: changedFiles(current, written.digests, (path) => !ignored.has(path)), + unnamed, } } @@ -216,13 +222,15 @@ export function attachWorkflowSourceStatus(source: Command): void { } } const status = sourceStatus(locals, serverChanges) - const plan = data ? planSummary(data) : null + // A plan against a base the server moved past says nothing: applying it is refused as stale. + const plan = data && serverChanges.length === 0 ? planSummary(data) : null const report: StatusReport & { workflowId: string } = { workflowId, dir: options.dir, server, ...(note ? { note } : {}), ...status, + ...(local.unnamed.length > 0 ? { unnamed: local.unnamed } : {}), plan, next: nextStep(status, { prefix: PREFIX, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts index 9a4705c5b00..11300b73418 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.test.ts @@ -430,6 +430,33 @@ describe('workspaces source apply', () => { expect(stderr).toContain('2 change(s) delete data that applying an older directory') }) + it('prints a delete refused for want of --allow-delete as refused, not unchanged', async () => { + request.mockResolvedValue( + answer({ + dryRun: true, + plan: [ + { kind: 'skill', key: 'tone', action: 'noop' }, + { kind: 'skill', key: 'brief', action: 'noop' }, + ], + diagnostics: [ + { + code: 'E_APPLY_DELETE_NOT_ALLOWED', + severity: 'error', + message: 'skills/tone is no longer declared; deleting "tone" needs --allow-delete.', + kind: 'skill', + key: 'tone', + }, + ], + }) + ) + + await apply('--dry-run') + + const stderr = vi.mocked(console.error).mock.calls.flat().join('\n') + expect(stderr).toContain('skills: 1 delete refused, 1 unchanged') + expect(stderr).toContain(' - tone — refused, needs --allow-delete') + }) + it('prints a --to dry run with its findings in the JSON result', async () => { request.mockImplementation(async (path: string, options: { body?: unknown }) => { if (path === '/api/v2/workspaces/ws_prod/source' && options.body === undefined) { @@ -474,6 +501,7 @@ describe('workspaces source apply', () => { it('a --to dry run warns of what the target deleted since the last promotion, gives stale findings a promotion hint, and names each target-only delete', async () => { const leads = { id: 'tbl_prod', fingerprint: `rf1:${'a'.repeat(64)}` } + const dropped = { id: 'tbl_dropped', fingerprint: `rf1:${'d'.repeat(64)}` } write( '.sim/targets/ws_prod.json', JSON.stringify({ @@ -482,6 +510,7 @@ describe('workspaces source apply', () => { resources: { 'table:leads': leads, 'table:gone': { id: 'tbl_gone', fingerprint: `rf1:${'c'.repeat(64)}` }, + 'table:dropped': dropped, }, workflows: {}, }) @@ -493,12 +522,20 @@ describe('workspaces source apply', () => { data: { workspaceId: 'ws_prod', kinds: ['table'], - files: { 'tables/leads.ts': LEADS, 'tables/prod-only.ts': LEADS }, + files: { + 'tables/leads.ts': LEADS, + 'tables/prod-only.ts': LEADS, + 'tables/dropped.ts': LEADS, + }, blobs: [], base: { version: 1, workspaceId: 'ws_prod', - resources: { 'table:leads': leads, 'table:prod-only': { id: 'tbl_only' } }, + resources: { + 'table:leads': leads, + 'table:prod-only': { id: 'tbl_only' }, + 'table:dropped': dropped, + }, }, skipped: [], diagnostics: [], @@ -512,8 +549,18 @@ describe('workspaces source apply', () => { { kind: 'table', key: 'gone', action: 'noop' }, { kind: 'table', key: 'leads', action: 'noop' }, { kind: 'table', key: 'prod-only', action: 'delete' }, + { kind: 'table', key: 'dropped', action: 'delete' }, + { kind: 'table', key: 'hotfixed', action: 'noop' }, ], diagnostics: [ + { + code: 'W_SERVER_CHANGED', + severity: 'warning', + message: 'tables/hotfixed changed on the server since this directory was read.', + hint: 'Run `sim workspaces source get` into a fresh directory to see what changed.', + kind: 'table', + key: 'hotfixed', + }, { code: 'W_SERVER_DELETED', severity: 'warning', @@ -533,7 +580,15 @@ describe('workspaces source apply', () => { }) }) - await apply('--to', 'ws_prod', '--only', 'tables', '--allow-delete', '--dry-run') + await apply( + '--to', + 'ws_prod', + '--only', + 'tables', + '--allow-delete', + '--delete-target-only', + '--dry-run' + ) const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) const byCode = (code: string) => @@ -548,11 +603,103 @@ describe('workspaces source apply', () => { const [stale] = byCode('E_APPLY_STALE') expect(stale.hint).toContain('.sim/targets/ws_prod.json') expect(stale.hint).not.toContain('fresh directory') + expect(stale.hint).toContain('sim workspaces source get --workspace ws_prod') + expect(stale.hint).not.toContain('.sim/targets/ws_prod.json first') + const [drifted] = byCode('W_SERVER_CHANGED') + expect(drifted.hint).toContain('sim workspaces source get --workspace ws_prod') const stderr = vi.mocked(console.error).mock.calls.flat().join('\n') expect(stderr).toContain(' ! gone (deleted on the server; kept as the server has it)') + expect(stderr).toContain(' ! leads (changed on the server; your edit is refused as stale)') expect(stderr).not.toContain('resource(s) that only workspace ws_prod holds') }) + it('a --to with --allow-delete keeps what only the target holds, naming it, and points at --delete-target-only', async () => { + const leads = { id: 'tbl_prod', fingerprint: `rf1:${'a'.repeat(64)}` } + write( + '.sim/targets/ws_prod.json', + JSON.stringify({ + version: 1, + workspaceId: 'ws_prod', + resources: { 'table:leads': leads, 'table:dropped': { id: 'tbl_dropped' } }, + workflows: {}, + }) + ) + const sent: unknown[] = [] + request.mockImplementation(async (path: string, options: { body?: unknown }) => { + if (path === '/api/v2/workspaces/ws_prod/source' && options.body === undefined) { + return { + data: { + workspaceId: 'ws_prod', + kinds: ['table'], + files: { + 'tables/leads.ts': LEADS, + 'tables/dropped.ts': LEADS, + 'tables/audit.ts': LEADS, + }, + blobs: [], + base: { + version: 1, + workspaceId: 'ws_prod', + resources: { + 'table:leads': leads, + 'table:dropped': { id: 'tbl_dropped' }, + 'table:audit': { id: 'tbl_audit' }, + }, + }, + skipped: [], + diagnostics: [], + }, + } + } + sent.push(options.body) + return answer({ + workspaceId: 'ws_prod', + dryRun: true, + plan: [ + { kind: 'table', key: 'leads', action: 'noop' }, + { kind: 'table', key: 'dropped', action: 'delete' }, + { kind: 'table', key: 'audit', action: 'noop' }, + ], + diagnostics: [ + { + code: 'W_UNDECLARED', + severity: 'warning', + message: + '"audit" (table audit) is on the server but not in this directory; it is kept.', + kind: 'table', + key: 'audit', + }, + ], + }) + }) + + await apply('--to', 'ws_prod', '--only', 'tables', '--allow-delete', '--dry-run') + + const [body] = sent as Array<{ base: { resources: Record } }> + expect(Object.keys(body.base.resources).sort()).toEqual(['table:dropped', 'table:leads']) + const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) + const [kept] = summary.diagnostics.filter( + (finding: { code: string }) => finding.code === 'W_UNDECLARED' + ) + expect(kept).toMatchObject({ key: 'audit' }) + expect(kept.hint).toContain('--delete-target-only') + expect( + summary.diagnostics.filter( + (finding: { code: string }) => finding.code === 'W_PROMOTE_DELETES_TARGET_ONLY' + ) + ).toEqual([]) + }) + + it('refuses --delete-target-only without --to or without --allow-delete', async () => { + await expect(apply('--to', 'ws_prod', '--delete-target-only', '--yes')).rejects.toThrow( + /needs --allow-delete/ + ) + await expect(apply('--allow-delete', '--delete-target-only', '--yes')).rejects.toThrow( + /needs --to/ + ) + expect(request).not.toHaveBeenCalled() + }) + it('names only the target-only deletes a promotion that stopped part-way wrote', async () => { const target = { version: 1, @@ -585,7 +732,15 @@ describe('workspaces source apply', () => { }) }) - await apply('--to', 'ws_prod', '--only', 'tables', '--allow-delete', '--yes') + await apply( + '--to', + 'ws_prod', + '--only', + 'tables', + '--allow-delete', + '--delete-target-only', + '--yes' + ) const summary = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) expect( @@ -686,13 +841,26 @@ describe('workspaces source apply', () => { expect(Object.keys(bodies()[1].files)).not.toContain('tools/lookup.js') }) - it('removes the function files an applied workflow no longer uses, keeping files no lock named', async () => { + it('removes the function files an applied workflow no longer uses, keeping files the CLI never wrote', async () => { const lock = (codeFiles: string[]) => `${JSON.stringify({ version: 1, blocks: Object.fromEntries(codeFiles.map((file, index) => [`b${index}`, { id: `blk_${index}`, codeFile: file }])), variables: {} })}\n` - write('workflows/ingest/.sim/lock.json', lock(['functions/score.js', 'functions/old.js'])) + // A hand-edited lock names mine.js too; the CLI never wrote it, so it is never pruned. + write( + 'workflows/ingest/.sim/lock.json', + lock(['functions/score.js', 'functions/old.js', 'functions/mine.js']) + ) write('workflows/ingest/functions/score.js', 'return 1\n') write('workflows/ingest/functions/old.js', 'return 2\n') write('workflows/ingest/functions/mine.js', 'return 3\n') + write( + '.sim/written.json', + JSON.stringify({ + files: { + 'workflows/ingest/functions/score.js': digest('return 1\n'), + 'workflows/ingest/functions/old.js': digest('return 2\n'), + }, + }) + ) request.mockResolvedValue( answer({ applied: [{ kind: 'workflow', key: 'ingest', action: 'update', id: 'wf_1' }], diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts index 66ca8109d33..9c901d7fc70 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-apply.ts @@ -33,7 +33,7 @@ import { SimApiError } from '../../http/client' import { apiCommand, type OperationClient } from '../../runtime/called-operations' import { searchTasks } from '../search' import { gitProvenance } from './git-provenance' -import { SourceDirError } from './local-errors' +import { CliUsageError, SourceDirError } from './local-errors' import { printProtocolResult } from './result' import { absoluteDirOption, @@ -92,6 +92,7 @@ interface ApplyOptions { dryRun?: boolean yes?: boolean allowDelete?: boolean + deleteTargetOnly?: boolean only?: string to?: string } @@ -178,6 +179,16 @@ function printPlan( ) const driftOf = (change: Change) => change.action === 'noop' ? drifted.get(`${change.kind}:${change.key}`) : undefined + const addressesOf = (code: string) => + new Set( + diagnostics + .filter((finding) => finding.code === code) + .map((finding) => `${finding.kind}:${finding.key}`) + ) + const stale = addressesOf('E_APPLY_STALE') + const deleteRefused = addressesOf('E_APPLY_DELETE_NOT_ALLOWED') + const refusedDelete = (change: Change) => + change.action === 'noop' && deleteRefused.has(`${change.kind}:${change.key}`) for (const kind of APPLY_ORDER) { const changes = plan.filter((change) => change.kind === kind) if (changes.length === 0) continue @@ -186,9 +197,11 @@ function printPlan( const drift = driftOf(change) const label = drift ? `${drift} on the server` - : change.action === 'noop' - ? 'unchanged' - : change.action + : refusedDelete(change) + ? 'delete refused' + : change.action === 'noop' + ? 'unchanged' + : change.action counts.set(label, (counts.get(label) ?? 0) + 1) } printError( @@ -197,7 +210,14 @@ function printPlan( for (const change of changes) { const drift = driftOf(change) if (drift) { - printError(` ! ${change.key} (${drift} on the server; kept as the server has it)`) + const fate = stale.has(`${change.kind}:${change.key}`) + ? 'your edit is refused as stale' + : 'kept as the server has it' + printError(` ! ${change.key} (${drift} on the server; ${fate})`) + continue + } + if (refusedDelete(change)) { + printError(` - ${change.key} — refused, needs --allow-delete`) continue } if (change.action === 'noop') continue @@ -251,7 +271,11 @@ const WORKFLOW_LOCK_PATH = new RegExp( * a folder whose new lock does not read as one: without it nothing tells a dropped file from a * kept one. */ -async function orphanedWorkflowFiles(dir: string, files: Record) { +async function orphanedWorkflowFiles( + dir: string, + files: Record, + recorded: Written['digests'] +) { const filesByFolder = new Map() for (const path of Object.keys(files)) { if (!path.startsWith(`${KIND_DIRS.workflow}/`)) continue @@ -273,7 +297,14 @@ async function orphanedWorkflowFiles(dir: string, files: Record) } catch {} if (!isRecordLike(next) || !isRecordLike(next.blocks)) continue const previous = await readSourceJson(dir, path) - orphaned.push(...orphanedCodeFiles(previous, next, written).map((file) => `${folder}/${file}`)) + const inFolder = Object.fromEntries( + Object.entries(recorded).flatMap(([recordedPath, sha]) => + recordedPath.startsWith(`${folder}/`) ? [[recordedPath.slice(folder.length + 1), sha]] : [] + ) + ) + orphaned.push( + ...orphanedCodeFiles(previous, next, written, inFolder).map((file) => `${folder}/${file}`) + ) } return orphaned } @@ -366,7 +397,7 @@ async function saveApplied( ): Promise { assertWorkspacePaths(Object.keys(data.files)) const orphaned = [ - ...(await orphanedWorkflowFiles(dir, data.files)), + ...(await orphanedWorkflowFiles(dir, data.files, written.digests)), ...droppedResourceFiles(submission.files, data.files), ] const kept = { ...written.digests, ...appliedBlobDigests(submission, data.base) } @@ -409,6 +440,8 @@ interface Promotion { target: string /** What the target deleted since the last promotion that the directory declares (addresses). */ deletedInTarget: string[] + /** What only the target holds: undeclared, and no promotion from the directory left it there. */ + targetOnly: string[] } /** @@ -423,7 +456,7 @@ async function promoted( dir: string, submission: WorkspaceSubmission, target: string, - allowDelete: boolean + deleteTargetOnly: boolean ): Promise { if (baseWorkspaceOf(submission.base) === target) { throw new SourceDirError(`${dir} was read from workspace ${target}; apply it without --to.`, { @@ -448,7 +481,7 @@ async function promoted( kinds: submission.kinds, target: { files: data.files, base: targetBase }, record, - allowDelete, + deleteTargetOnly, }) return { submission: { ...submission, files: composed.files, base: composed.base }, @@ -456,6 +489,7 @@ async function promoted( record, target, deletedInTarget: composed.deletedInTarget, + targetOnly: composed.targetOnly, } } @@ -480,6 +514,7 @@ async function savePromotion( submission: { files: promotion.submission.files, base: parseWorkspaceBase(promotion.submission.base).base ?? base, + targetOnly: promotion.targetOnly, }, applied: { base, files: data.files, plan: data.plan }, }) @@ -497,10 +532,19 @@ function resourceWhere(kind: WorkspaceKind, key: string): string { * The hint a promotion to `target` gives a refusal the server words for an apply to the * directory's own workspace ("get it into a fresh directory"), which a promotion cannot follow. */ -function promotionHint(code: string, target: string): string | undefined { +function promotionHint(code: string, target: string, targetOnly: boolean): string | undefined { const record = promotionRecordPath(target) ?? '.sim/targets/.json' if (code === 'E_APPLY_STALE') { - return `Workspace ${target} changed this resource since the last promotion from this directory. Carry that change into the directory, or undo it in ${target}, then promote again; to take ${target} as it stands now, delete ${record} first.` + return `Workspace ${target} changed this resource since the last promotion from this directory. Carry that change into the directory (\`sim workspaces source get --workspace ${target} --dir \` prints it), or undo it in ${target}, then promote again. Don't delete ${record} to get past this: the next promotion would then overwrite every change made in ${target} since, rows its workflows wrote included.` + } + if (code === 'W_SERVER_CHANGED') { + return `Workspace ${target} changed this resource since the last promotion from this directory, and this promotion keeps that change. To carry it into the directory, \`sim workspaces source get --workspace ${target} --dir \` prints it.` + } + if (code === 'E_APPLY_DELETE_NOT_ALLOWED') { + return `This directory no longer declares it. Pass --allow-delete to delete it in ${target} too: that deletes only what this directory dropped, and resources only ${target} holds are kept. Or restore it to keep it.` + } + if (code === 'W_UNDECLARED' && targetOnly) { + return `Only workspace ${target} holds it: this directory does not declare it and no promotion from it left it there, so the promotion keeps it, with or without --allow-delete. To delete it there, add --delete-target-only beside --allow-delete; to manage it from this directory, declare it here.` } if (code === 'E_APPLY_NO_BASE') { return `Workspace ${target} holds this resource, but no promotion from this directory recorded it. Rename the declaration, or rename or delete the resource in ${target}, then promote again.` @@ -508,13 +552,14 @@ function promotionHint(code: string, target: string): string | undefined { return undefined } -/** `finding` with the hint a promotion to `target` gives its code, when it gives one. */ -function withPromotionHint( +/** `finding` with the hint `promotion` gives its code, when it gives one. */ +function withPromotionHint( finding: F, code: string, - target: string + promotion: Promotion ): F { - const hint = promotionHint(code, target) + const targetOnly = promotion.targetOnly.includes(`${finding.kind}:${finding.key}`) + const hint = promotionHint(code, promotion.target, targetOnly) return hint === undefined ? finding : { ...finding, hint } } @@ -528,14 +573,15 @@ function promotionDiagnostics(diagnostics: readonly Finding[], promotion: Promot return diagnostics.flatMap((finding) => finding.code === 'W_SERVER_DELETED' && warned.has(`${finding.kind}:${finding.key}`) ? [] - : [withPromotionHint(finding, finding.code, promotion.target)] + : [withPromotionHint(finding, finding.code, promotion)] ) } /** * The warnings only a promotion knows to give: each declared resource the target deleted since the - * last promotion (kept deleted), and each resource only the target holds that `--allow-delete` - * deletes there (of an apply that stopped part-way, each it deleted). + * last promotion (kept deleted), and each resource only the target holds that + * `--delete-target-only` deletes there (of an apply that stopped part-way, each it deleted). One + * it keeps is the server's `W_UNDECLARED`, with the promotion's hint. */ function promotionWarnings( promotion: Promotion, @@ -556,16 +602,20 @@ function promotionWarnings( }, ] }) + const onlyInTarget = new Set(promotion.targetOnly) const targetOnly = outcome.plan .filter( - (change) => change.action === 'delete' && (!outcome.failed || wasWritten(change, outcome)) + (change) => + change.action === 'delete' && + onlyInTarget.has(`${change.kind}:${change.key}`) && + (!outcome.failed || wasWritten(change, outcome)) ) .map( (change): Finding => ({ code: 'W_PROMOTE_DELETES_TARGET_ONLY', severity: 'warning', - message: `${resourceWhere(change.kind, change.key)} is only in workspace ${target}, not in this directory, so --allow-delete deletes it there.`, - hint: 'Promote without --allow-delete to keep it, or declare it in this directory.', + message: `${resourceWhere(change.kind, change.key)} is only in workspace ${target}, not in this directory, so --delete-target-only deletes it there.`, + hint: 'Promote without --delete-target-only to keep it, or declare it in this directory.', kind: change.kind, key: change.key, }) @@ -585,7 +635,7 @@ function rehinted(refusal: unknown, promotion: Promotion): unknown { ...refusal.details, diagnostics: diagnostics.map((finding) => isRecordLike(finding) && typeof finding.code === 'string' - ? withPromotionHint(finding, finding.code, promotion.target) + ? withPromotionHint(finding, finding.code, promotion) : finding ), } @@ -627,7 +677,14 @@ export function attachWorkspaceSourceApply(source: Command): void { .option('--dir ', 'Workspace source directory written by `workspaces source get`', '.') .option('--dry-run', 'Plan and check without writing anything') .option('-y, --yes', 'Apply without asking for confirmation') - .option('--allow-delete', 'Delete resources, columns and data the directory no longer declares') + .option( + '--allow-delete', + "Delete resources, columns and data the directory no longer declares (with --to, only what it deleted since the last promotion: the target's own resources are kept)" + ) + .option( + '--delete-target-only', + "With --to and --allow-delete, also delete the target's resources this directory never declared and no promotion from it left there (prod-only tables and their rows, skills made in prod). Without it a promotion keeps them, naming each with a warning" + ) .option('--only ', `Apply only these kinds (${ONLY_KINDS_HELP})`) .option( '--to ', @@ -640,6 +697,7 @@ export function attachWorkspaceSourceApply(source: Command): void { `${sourceExitCodesHelp('workspaces source apply')}${SOURCE_WRITE_EXIT_CODES_HELP}` ) .action(async (options: ApplyOptions) => { + assertTargetOnlyDeletable(options) const mode = writeMode(options) const only = parseKinds(options.only) const { client, profile } = connect() @@ -730,7 +788,13 @@ export function attachWorkspaceSourceApply(source: Command): void { } assertBlobsFetched(options.dir, read, written.digests) promotion = options.to - ? await promoted(client, options.dir, read, options.to, options.allowDelete === true) + ? await promoted( + client, + options.dir, + read, + options.to, + options.deleteTargetOnly === true + ) : null const submission = promotion?.submission ?? read /** `data` as this apply reports it: a promotion's own wording and warnings added. */ @@ -829,9 +893,29 @@ function applyCommand(options: ApplyOptions): string { ...(options.only ? [`--only ${shellWord(options.only)}`] : []), ...(options.to ? [`--to ${shellWord(options.to)}`] : []), ...(options.allowDelete ? ['--allow-delete'] : []), + ...(options.deleteTargetOnly ? ['--delete-target-only'] : []), ].join(' ') } +/** + * Refuses `--delete-target-only` without `--to` (only a promotion has a target of its own) or + * without `--allow-delete` (the server deletes nothing without it): the flag never stands in for + * consent to the directory's own deletes. + */ +function assertTargetOnlyDeletable(options: ApplyOptions): void { + if (!options.deleteTargetOnly) return + if (!options.to) { + throw new CliUsageError( + "--delete-target-only deletes a promotion target's own resources: it needs --to ." + ) + } + if (!options.allowDelete) { + throw new CliUsageError( + '--delete-target-only needs --allow-delete beside it: add it, and the promotion also deletes what this directory deleted since the last promotion.' + ) + } +} + /** * A `W_PATH_NOT_SENT` warning for each file under a kind folder the CLI does not send: a hidden * name, or a character no workspace path holds. diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts index a07d8deae35..33c0f9458d3 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-check.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-check.ts @@ -132,6 +132,7 @@ export function attachWorkspaceSourceCheck(source: Command): void { } const keys = await workspaceWorkflowKeys(options.dir) const findings: CheckFinding[] = [] + const unlisted = { errors: 0, warnings: 0 } for (const key of keys) { const checked = await checkWorkflowDir( join(options.dir, KIND_DIRS.workflow, key), @@ -143,6 +144,8 @@ export function attachWorkspaceSourceCheck(source: Command): void { } ) findings.push(...checked.findings) + unlisted.errors += checked.unlisted.errors + unlisted.warnings += checked.unlisted.warnings } for (const { code, @@ -172,7 +175,7 @@ export function attachWorkspaceSourceCheck(source: Command): void { profile.output, findings, { workflows: keys.length, catalogHash: catalog.hash }, - { verbose: options.verbose === true, bound: WORKSPACE_ANSWER_BOUND } + { verbose: options.verbose === true, bound: WORKSPACE_ANSWER_BOUND, unlisted } ) }) } diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts index 5c44370f2d8..2a9551628ad 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-dir.ts @@ -6,7 +6,9 @@ */ import { realpath, stat } from 'node:fs/promises' import { mapWithConcurrency } from '@sim/utils/concurrency' +import { compareStrings } from '@sim/utils/string' import type { WorkflowInput } from '@sim/workflow-code' +import { SOURCE_CODE_DIR, SOURCE_ENTRY_FILE, SOURCE_LOCK_FILE } from '@sim/workflow-code/limits' import { SOURCE_COMPILER_OPTIONS, typecheckCommand, @@ -77,6 +79,7 @@ import { TSCONFIG_FULL_FILE, TSCONFIG_MARKER, } from './source-tree' +import { codeFilePaths } from './workflow-dir' import { statDigest } from './workspace-source-blobs' /** Where the workspace declaration SDK is written: `@sim/workflow-sdk/workspace` resolves here. */ @@ -186,7 +189,7 @@ const ADVANCED_GUIDE = `# Sim workspace source: advanced ## Promote to another workspace -\`sim workspaces source apply --dir . --to --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name). Each promote records what it left in the target in \`.sim/targets/.json\` (commit it), and the next promote compares with that: a resource changed in the target since is kept when this directory left it alone, and refused as stale (exit ${SOURCE_STALE_EXIT_CODE}) when this directory changed it too. A resource with no record (the first promote, or one created in the target since) is updated to match this directory; delete the record to take the target as it stands again. Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory does not declare are kept with a warning, or deleted with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w \`) and connect its credentials, or the workflows using them are held back. This directory stays bound to the workspace it was read from. Without \`--to\`, \`check\`, \`status\`, \`diff\` and \`apply\` act on the workspace this directory was read from, whatever the profile's default; a \`--workspace\` naming another is refused (\`SOURCE_OTHER_WORKSPACE\`). +\`sim workspaces source apply --dir . --to --dry-run\`, then \`--yes\`: applies this directory to another workspace (dev to prod). Each resource binds by key to the target's own: one the target holds under the same key is updated in place (a workflow keeps its id and block ids, so its webhook URLs), a key it lacks is created (or adopts a resource of the same name). Each promote records what it left in the target in \`.sim/targets/.json\` (commit it), and the next promote compares with that: a resource changed in the target since is kept when this directory left it alone, and refused as stale (exit ${SOURCE_STALE_EXIT_CODE}) when this directory changed it too. A resource with no record (the first promote, or one created in the target since) is updated to match this directory; delete the record to take the target as it stands again. Secrets and credentials bind by name and key to the target's own; no value is copied. The target's resources this directory never declared (and no promote left there) are kept with a warning, even with \`--allow-delete\`; deleting them too takes \`--delete-target-only\` beside \`--allow-delete\`. A resource this directory deleted since the last promote is deleted in the target only with \`--allow-delete\`. Before the first promote, set the target's secrets (\`sim secrets set NAME --scope workspace -w \`) and connect its credentials, or the workflows using them are held back. This directory stays bound to the workspace it was read from. Without \`--to\`, \`check\`, \`status\`, \`diff\` and \`apply\` act on the workspace this directory was read from, whatever the profile's default; a \`--workspace\` naming another is refused (\`SOURCE_OTHER_WORKSPACE\`). ` /** The workspace directory's tree beside the printed source. */ @@ -552,10 +555,46 @@ export interface WorkspaceSubmission { base?: ApplyWorkspaceSourceBody['base'] /** Files under a kind folder that are not sent: hidden names and unsupported characters. */ ignored: string[] + /** + * Files under a workflow folder's `functions/` that neither its `workflow.ts` nor its lock names: + * never sent, as a workflow directory's apply never sends them. + */ + unnamed: string[] /** The blobs this read hashed or reused a recorded digest for, with their stats. */ hashed: ReadonlyMap } +/** + * Drops from `files` each file under a workflow folder's `functions/` that neither the folder's + * `workflow.ts` nor its lock names ({@link codeFilePaths}, the rule a workflow directory's apply + * and status read), returning their paths: an apply never sends one, so a status never counts it + * as a change and the folder is not sent as edited for it. + */ +function dropUnnamedCodeFiles(files: Record): string[] { + const named = new Map>() + const unnamed: string[] = [] + for (const path of Object.keys(files)) { + if (kindOfPath(path) !== 'workflow') continue + const segments = path.split('/') + if (segments[2] !== SOURCE_CODE_DIR) continue + const folder = segments.slice(0, 2).join('/') + let sent = named.get(folder) + if (!sent) { + let lock: unknown = null + try { + lock = JSON.parse(files[`${folder}/${SOURCE_LOCK_FILE}`] ?? 'null') + } catch {} + sent = new Set(codeFilePaths(files[`${folder}/${SOURCE_ENTRY_FILE}`] ?? null, lock)) + named.set(folder, sent) + } + if (!sent.has(segments.slice(2).join('/'))) { + unnamed.push(path) + delete files[path] + } + } + return unnamed.sort(compareStrings) +} + /** * Drops from `files` the workflow folders whose every file is as `.sim/written.json` records it, * keeping what `unchangedFiles` sends for one (its base alone when it was read from @@ -647,6 +686,7 @@ export async function readWorkspaceDir( const text = texts[index] if (text !== null) files[path] = text }) + const unnamed = dropUnnamedCodeFiles(files) if (!options.whole) pruneUnchangedWorkflows(files, written, workspaceId) const root = await realpath(dir) const verified = new Set() @@ -681,7 +721,7 @@ export async function readWorkspaceDir( ) } } - return { kinds, files, blobs, ignored, hashed, ...(base ? { base } : {}) } + return { kinds, files, blobs, ignored, unnamed, hashed, ...(base ? { base } : {}) } } /** `.sim/skipped.json`, or `null` when nothing was skipped. */ diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts index 431278db1aa..fae2e3b2b13 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.test.ts @@ -220,6 +220,9 @@ describe('workspaces source get', () => { }, })) await get() + expect(vi.mocked(console.error).mock.calls.flat().join('\n')).toContain( + 'Skipped file big.bin: over the size limit' + ) serve({ 'tables/leads.ts': LEADS }) await get('--only', 'table') diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts index 554edda745a..0a25b1fb408 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-get.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-get.ts @@ -374,6 +374,11 @@ export function attachWorkspaceSourceGet(source: Command): void { const result = await writeWorkspaceSource(client, options.dir, data, options.force === true) const findings = findingsOutput(profile.output) findings.print(data.diagnostics) + for (const { kind, key, reason } of data.skipped) { + printError( + `Skipped ${kind} ${key}: ${reason} It is not written here, and an apply keeps it as the server has it (${WORKSPACE_SKIPPED_FILE} lists it).` + ) + } printTreeResult(profile.output, { workspaceId: data.workspaceId, dir: options.dir, diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts index 8088ed0bd14..c68d63d979a 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.test.ts @@ -390,6 +390,60 @@ describe('workspaces source status', () => { }) }) + it('leaves out a function file no codeFile() names, as an apply never sends it: clean, exit 0, naming it', async () => { + write('workflows/ingest/functions/scratch.js', 'return 1\n') + mockRequest.mockResolvedValue(dryRun()) + + await run('status') + + expect(printed()).toMatchObject({ + state: 'clean', + local: [], + unnamed: ['workflows/ingest/functions/scratch.js'], + }) + expect(sentFiles()[0]['workflows/ingest/functions/scratch.js']).toBeUndefined() + expect(process.exitCode ?? 0).toBe(0) + }) + + it('does not report a recorded function file as deleted once no codeFile() names it but it is still on disk', async () => { + write( + '.sim/written.json', + JSON.stringify({ + version: 1, + files: { + ...Object.fromEntries(Object.entries(GOT).map(([path, text]) => [path, digest(text)])), + 'workflows/ingest/functions/scratch.js': digest('return 1\n'), + }, + }) + ) + write('workflows/ingest/functions/scratch.js', 'return 1\n') + mockRequest.mockResolvedValue(dryRun()) + + await run('status') + + expect(printed()).toMatchObject({ + state: 'clean', + local: [], + unnamed: ['workflows/ingest/functions/scratch.js'], + }) + expect(process.exitCode ?? 0).toBe(0) + }) + + it('still counts a new function file the lock names as a local change', async () => { + write( + 'workflows/ingest/.sim/lock.json', + JSON.stringify({ version: 1, blocks: { run: { id: 'b1', codeFile: 'functions/run.js' } } }) + ) + write('workflows/ingest/functions/run.js', 'return 1\n') + mockRequest.mockResolvedValue(dryRun()) + + await run('status') + + expect(printed()).not.toHaveProperty('unnamed') + expect(sentFiles()[0]['workflows/ingest/functions/run.js']).toBe('return 1\n') + expect(process.exitCode).toBe(11) + }) + it('sends every workflow folder whole to the plan read, so it compares each with the stored workflow', async () => { mockRequest.mockResolvedValue(dryRun()) diff --git a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts index 37067e8a0e8..99e91ec8614 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-source-status.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-source-status.ts @@ -116,9 +116,11 @@ async function readLocalWorkspace( const current: Record = {} for (const [path, text] of Object.entries(submission.files)) current[path] = digest(text) for (const blob of submission.blobs) current[blob.path] = blob.sha256 + // A recorded function file still on disk that no codeFile() names is left out, not deleted. + const unnamed = new Set(submission.unnamed) const changed = changedFiles(current, written.digests, (path) => { const kind = kindOfPath(path) - return kind !== null && (!only || only.includes(kind)) + return kind !== null && (!only || only.includes(kind)) && !unnamed.has(path) }) return { submission, written, changed, readFrom } } @@ -397,6 +399,7 @@ export function attachWorkspaceSourceStatus(source: Command): void { server, ...(note ? { note } : {}), ...status, + ...(local.submission.unnamed.length > 0 ? { unnamed: local.submission.unnamed } : {}), plan, next: (login diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 74a90c48863..47f56e72e70 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -9110,6 +9110,7 @@ export type GetSourceCatalogResponse = { | { opaque: true } + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean @@ -9347,6 +9348,7 @@ export type GetSourceCatalogResponse = { | { opaque: true } + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean @@ -9537,6 +9539,7 @@ export type GetSourceCatalogResponse = { | { opaque: true } + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index c4d1e892b43..df8378eeb68 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -344,7 +344,13 @@ Compile also refuses or flags source the runtime would silently mishandle: input the unified Start passes through); a format with references inside is unread; - a dropdown value outside its static options is `W_UNKNOWN_OPTION`, a `model` the catalog does not list `W_UNKNOWN_MODEL` (free text stays valid), and an unknown `operation` - `E_UNKNOWN_OPERATION` unless the stored block already holds it; + `E_UNKNOWN_OPERATION` unless the stored block already holds it. An apply refuses + `W_UNKNOWN_OPTION` (`E_APPLY_INVALID_SUBBLOCK_FIELD`, `diagnostics/refusals.ts`) only on a field + the edit sets, the fields the server's value gate judges; the CLI's offline check, which holds no + stored values, counts every field of a changed block. Options a registry function derives from + deployment settings are marked `optionsVary` in the catalog: the offline check keeps a value + outside them a warning, since its cached catalog may have been built under other settings, and + leaves the verdict to the server, whose catalog and gate run in one process; - a new block of a legacy or deprecated type is `W_LEGACY_BLOCK_TYPE`, naming its replacement. ## Materialize diff --git a/packages/workflow-code/src/catalog/types.ts b/packages/workflow-code/src/catalog/types.ts index e4773d090d6..e45f7c1c2fe 100644 --- a/packages/workflow-code/src/catalog/types.ts +++ b/packages/workflow-code/src/catalog/types.ts @@ -143,6 +143,12 @@ export interface CatalogSubBlock { * function the projection could not evaluate. */ options?: CatalogOption[] | { dynamic: true } | OpaqueMarker + /** + * The static `options` are a registry function of deployment settings (a feature or a flag), + * evaluated where this catalog was built: the server's field gate evaluates it again at write + * time, so a catalog built elsewhere, or before the settings changed, may list other options. + */ + optionsVary?: true hidden?: boolean readOnly?: boolean password?: boolean diff --git a/packages/workflow-code/src/compile/SYNTAX.md b/packages/workflow-code/src/compile/SYNTAX.md index 56ed953e179..e4e68d66e3e 100644 --- a/packages/workflow-code/src/compile/SYNTAX.md +++ b/packages/workflow-code/src/compile/SYNTAX.md @@ -495,8 +495,8 @@ Errors refuse the whole apply. Warnings are saved as written; info is informatio | `W_MAT_DELETED_SINCE_PULL` | A lock entry whose block was deleted since the read; re-created. | | `W_MAT_INFERRED_RENAME` | A rename inferred from type, parent and edges (the key and the ident both changed). | | `W_MAT_RETYPE_RECREATED` | A key kept with a new type: delete plus create, so webhook URLs and schedules change. | -| `W_MAT_LOCK_ID_CLAIMED` | A lock id another workflow holds; a new id was minted. | -| `I_MAT_NEEDS_LAYOUT` | A new block with no position; placed by autolayout. | +| `W_MAT_LOCK_ID_CLAIMED` | A lock id another workflow holds, ignored: the block binds to the stored block it matches, or gets a new id. | +| `I_MAT_NEEDS_LAYOUT` | A new block with no position, or one moved into or out of a loop or parallel; placed by autolayout. | **Apply** diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 1d43d1f8a17..0e486dbb5c2 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -415,8 +415,11 @@ function parseCodeFile( decl: BlockDecl ): string | null { const { limits } = ctx + const before = ctx.codeFiles.bytes ctx.codeFiles.bytes += utf8Bytes(text, limits.maxTotalBytes) if (ctx.codeFiles.bytes > limits.maxTotalBytes) { + // Reported once, on the file that crosses the cap; each file after it is skipped silently. + if (before > limits.maxTotalBytes) return null ctx.reporter.report( 'E_LIMIT_EXCEEDED', `The workflow and its code files total more than ${limits.maxTotalBytes} bytes.`, diff --git a/packages/workflow-code/src/compile/grammar.test.ts b/packages/workflow-code/src/compile/grammar.test.ts index 10fa4003e98..488b90128db 100644 --- a/packages/workflow-code/src/compile/grammar.test.ts +++ b/packages/workflow-code/src/compile/grammar.test.ts @@ -1159,7 +1159,7 @@ describe('errors, each with its code, location and hint', () => { const finding = result.diagnostics.find((d) => d.code === 'W_UNRESOLVED_REF') expect(finding?.hint).toBe( - '$ref.lookup is a block this workflow had that workflow.ts no longer declares under that key: if Fetch Page is it renamed, write $ref.fetchpage.data.' + '$ref.lookup is a block this workflow had that workflow.ts no longer declares under that key: if it was renamed to Fetch Page, an apply that keeps its id rewrites this reference to $ref.fetchpage.data, so no edit is needed.' ) }) diff --git a/packages/workflow-code/src/compile/hostile.test.ts b/packages/workflow-code/src/compile/hostile.test.ts index 80ed340b098..ee8e63b80be 100644 --- a/packages/workflow-code/src/compile/hostile.test.ts +++ b/packages/workflow-code/src/compile/hostile.test.ts @@ -307,6 +307,20 @@ describe('hostile sources produce diagnostics only', () => { expect(computed).toBe(FINDING_LIMITS.warningsPerCode + 1) }) + it('keeps one omitted entry per code and block, counting every warning past the caps', () => { + const reporter = new Reporter() + reporter.addFile('w.ts', 'x') + const at = { file: 'w.ts', start: 0, end: 1 } + const total = FINDING_LIMITS.perSeverity + 5_000 + for (let i = 0; i < total; i++) { + reporter.report('W_LITERAL_SECRET', `x${i}`, at, { blockKey: i % 2 === 0 ? 'a' : 'b' }) + } + + expect(reporter.omitted).toHaveLength(2) + const listed = reporter.diagnostics.filter((d) => d.code === 'W_LITERAL_SECRET').length + expect(reporter.omitted.reduce((sum, entry) => sum + entry.count, 0)).toBe(total - listed) + }) + it('keeps infos from crowding out a warning', () => { const reporter = new Reporter() reporter.addFile('w.ts', 'x') @@ -379,7 +393,7 @@ describe('hostile sources produce diagnostics only', () => { expect(errorCodes(result)).toEqual(['E_INVALID_CHARACTER']) }) - it('caps the total bytes of distinct code files', () => { + it('caps the total bytes of distinct code files with one diagnostic, on the file that crosses it', () => { const files = Object.fromEntries( Array.from({ length: 4 }, (_, i) => [ `workflows/functions/f${i}.js`, @@ -391,7 +405,8 @@ describe('hostile sources produce diagnostics only', () => { { maxTotalBytes: 2_000 }, files ) - expect(codes(result)).toContain('E_LIMIT_EXCEEDED') + const limits = result.diagnostics.filter((d) => d.code === 'E_LIMIT_EXCEEDED') + expect(limits.map((d) => d.blockKey)).toEqual(['f2']) expect(result.workflow).toBeNull() }) diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 8744c7f4a17..dce5a31729c 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -36,7 +36,7 @@ import { EdgeCollector } from './edges' import { referencedResultKind } from './function-returns' import { didYouMean, HINTS, hintForNode, nodeLabel } from './hints' import { type CompileLimits, DEFAULT_LIMITS, parseWorkflowSource, sdkHeader } from './parse' -import { ErrorLimitReached, type Range, Reporter } from './report' +import { ErrorLimitReached, type OmittedFinding, type Range, Reporter } from './report' import { checkSuspiciousName, collectBody, collisionSuffix } from './scope' import { checkCodeFileRefs, @@ -115,6 +115,11 @@ export interface CompileResult { * what `refuseEditedFindings` reads beside `diagnostics`, so a cap never hides a refusal. */ withheld: LocatedDiagnostic[] + /** + * Every other warning the caps left out, counted per code and block, so a caller counts every + * finding. + */ + omitted: OmittedFinding[] /** Where each anchor sits, to map later server findings back to lines. */ spans: Partial> /** @@ -428,6 +433,7 @@ function emptyResult(reporter: Reporter, sdkMajor: number | null): CompileResult workflow: null, diagnostics: reporter.diagnostics, withheld: reporter.withheld, + omitted: reporter.omitted, spans: {}, metaSpans: {}, hints: { blocks: {} }, @@ -793,6 +799,7 @@ function compileEntry( workflow, diagnostics: reporter.diagnostics, withheld: reporter.withheld, + omitted: reporter.omitted, spans, metaSpans: ctx.metaSpans, hints, diff --git a/packages/workflow-code/src/compile/report.ts b/packages/workflow-code/src/compile/report.ts index 30e33e930ad..69b711eb267 100644 --- a/packages/workflow-code/src/compile/report.ts +++ b/packages/workflow-code/src/compile/report.ts @@ -108,6 +108,15 @@ export class Reporter { * `refuseEditedFindings`) while the listing stays capped. */ readonly withheld: LocatedDiagnostic[] = [] + /** + * Every other warning a cap left out, counted by code and block (no text, span or frame is built + * for one), so a caller can still count every finding while what is kept stays one entry per + * code and block however many warnings come. + */ + get omitted(): OmittedFinding[] { + return [...this.omittedCounts.values()] + } + private readonly omittedCounts = new Map() /** The code, block and file of each refusable warning listed or withheld. */ private readonly refusableHeld = new Set() private readonly indexes = new Map() @@ -170,7 +179,7 @@ export class Reporter { } else if (severity === 'info') { if (this.infoCount >= MAX_LISTED) return } else if (this.warningCount >= MAX_LISTED) { - this.withhold(code, message, at, extras) + this.omit(code, message, at, extras) if (this.warningCount === MAX_LISTED) { this.truncated( `Listed ${MAX_LISTED} warnings; the rest are omitted. The next: ${message}`, @@ -183,7 +192,7 @@ export class Reporter { const ofCode = this.warningsByCode.get(code) ?? 0 this.warningsByCode.set(code, ofCode + 1) if (ofCode >= MAX_WARNINGS_PER_CODE) { - this.withhold(code, message, at, extras) + this.omit(code, message, at, extras) if (ofCode === MAX_WARNINGS_PER_CODE) { this.truncated( `Listed ${MAX_WARNINGS_PER_CODE} ${code} warnings; the rest of them are omitted. The next: ${message}`, @@ -200,17 +209,30 @@ export class Reporter { } /** - * Keeps a refusable warning a cap left out in {@link withheld}, unless one of its code, block - * and file is already listed or withheld. + * Keeps a warning a cap left out: in {@link withheld} when it is refusable and none of its code, + * block and file is listed or withheld yet, else in {@link omitted}. */ - private withhold( + private omit( code: CompileDiagnosticCode, message: string, at: Range, extras: ReportExtras ): void { const held = refusableKey(code, at, extras) - if (held === undefined || this.refusableHeld.has(held)) return + if (held === undefined || this.refusableHeld.has(held)) { + const key = `${code}\u0000${extras.blockKey ?? ''}` + const counted = this.omittedCounts.get(key) + if (counted) counted.count++ + else { + this.omittedCounts.set( + key, + extras.blockKey === undefined + ? { code, count: 1 } + : { code, blockKey: extras.blockKey, count: 1 } + ) + } + return + } this.refusableHeld.add(held) const { hint, ...listed } = extras const diagnostic = this.build(code, message, at, listed, framesSource(code)) @@ -286,6 +308,14 @@ export class Reporter { } } +/** The warnings of one code and block a compile's caps left out of every list ({@link Reporter.omitted}). */ +export interface OmittedFinding { + code: CompileDiagnosticCode + blockKey?: BlockKey + /** How many warnings of this code and block were left out. */ + count: number +} + /** The code, block and file a refusable warning is withheld by; none for any other finding. */ function refusableKey( code: CompileDiagnosticCode, diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index bda93e592c5..86c06ecc451 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -1062,7 +1062,7 @@ function formerBlockHint( const lead = `${spell([{ kind: 'name', name: formerKey }])} is a block this workflow had that ${SOURCE_ENTRY_FILE} no longer declares under that key` const [only] = unbound return unbound.length === 1 && only - ? `${lead}: if ${only.name} is it renamed, write ${rewritten(only.key)}.` + ? `${lead}: if it was renamed to ${only.name}, an apply that keeps its id rewrites this reference to ${rewritten(only.key)}, so no edit is needed.` : `${lead}: if you renamed it, write ${spell([{ kind: 'name', name: 'newkey' }])}, where newkey is its display name lowercased with whitespace and dots removed.` } diff --git a/packages/workflow-code/src/diagnostics/index.ts b/packages/workflow-code/src/diagnostics/index.ts index 148943b79d8..f93b31ccd74 100644 --- a/packages/workflow-code/src/diagnostics/index.ts +++ b/packages/workflow-code/src/diagnostics/index.ts @@ -20,5 +20,6 @@ export { raisedMessage, refusalClassifier, refuseEditedFindings, + refusesEdited, } from './refusals' export type { DiagnosticSeverity, SeverityOf } from './severity' diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index 90c6256d488..2e75eb56aef 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -65,6 +65,7 @@ export type EditRefusalCode = | 'E_APPLY_UNRESOLVED_REF' | 'E_APPLY_REF_OUT_OF_SCOPE' | 'E_APPLY_CODE_FILE_REF_SYNTAX' + | 'E_APPLY_INVALID_SUBBLOCK_FIELD' /** What an edit changed, as the caller can tell it. */ export interface EditedBlocks { @@ -76,6 +77,12 @@ export interface EditedBlocks { * code before and after may compare that alone. */ codeEdited(blockKey: string, file: string): boolean + /** + * Whether the edit sets field `param` of the changed block `blockKey`: the server's field gate + * judges only those. A caller that cannot tell (an offline check holds no stored values) leaves + * it out, and every field of a changed block counts as set. + */ + paramChanged?(blockKey: string, param: string): boolean } /** @@ -86,6 +93,18 @@ const CHANGED_BLOCK_REFUSALS: ReadonlyMap = new Map([ ['W_REF_OUT_OF_SCOPE', 'E_APPLY_REF_OUT_OF_SCOPE'], ]) +/** + * In `workflow.ts` on a field the edit set ({@link EditedBlocks.paramChanged}) of a block it + * changed: a dropdown value none of its options names, which the server's field gate refuses. A + * value the stored block already holds outside its options prints as `raw()`, which the compile + * does not check, and one written bare again is not a field the edit set, so only a value the + * author changed is refused. `operation` is left out: an unknown one is already an error, and its warning means the + * stored block holds a retired operation, which the gate keeps. + */ +const PARAM_REFUSALS: ReadonlyMap = new Map([ + ['W_UNKNOWN_OPTION', 'E_APPLY_INVALID_SUBBLOCK_FIELD'], +]) + /** In `workflow.ts` on a block the edit changed: a path no output of the block has. */ const ENTRY_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OUTPUT_PATH', 'E_APPLY_UNRESOLVED_REF'], @@ -108,6 +127,7 @@ const CODE_FILE_REFUSALS: ReadonlyMap = new Map([ * refusal. */ export const REFUSABLE_WARNING_CODES: ReadonlySet = new Set([ + 'W_UNKNOWN_OPTION', 'W_REF_OUT_OF_SCOPE', 'W_UNKNOWN_OUTPUT_PATH', 'W_UNRESOLVED_REF', @@ -125,6 +145,11 @@ export const EDIT_REFUSALS: ReadonlyArray<{ refusal, where: 'changed block' as const, })), + ...[...PARAM_REFUSALS].map(([warning, refusal]) => ({ + warning, + refusal, + where: 'workflow.ts' as const, + })), ...[...ENTRY_REFUSALS].map(([warning, refusal]) => ({ warning, refusal, @@ -158,6 +183,13 @@ function refusalOf(finding: LocatedDiagnostic, edit: EditedBlocks): EditRefusalC const edited = edit.changed(blockKey) || (!inEntryFile && edit.codeEdited(blockKey, span.file)) return edited ? onChanged : undefined } + const onParam = PARAM_REFUSALS.get(finding.code) + if (onParam) { + const { param } = finding + if (!inEntryFile || param === undefined || param === 'operation') return undefined + if (!edit.changed(blockKey)) return undefined + return (edit.paramChanged?.(blockKey, param) ?? true) ? onParam : undefined + } if (inEntryFile) { const inEntry = ENTRY_REFUSALS.get(finding.code) return inEntry && edit.changed(blockKey) ? inEntry : undefined @@ -166,6 +198,11 @@ function refusalOf(finding: LocatedDiagnostic, edit: EditedBlocks): EditRefusalC return inCode && edit.codeEdited(blockKey, span.file) ? inCode : undefined } +/** Whether the apply refuses `finding` (a compile warning), given what the edit changed. */ +export function refusesEdited(finding: LocatedDiagnostic, edit: EditedBlocks): boolean { + return refusalOf(finding, edit) !== undefined +} + /** * A compile's findings with every warning the apply refuses on an edited block raised to its * error code (see the module doc). `withheld`, the refusable warnings the compile's caps left diff --git a/packages/workflow-code/src/index.ts b/packages/workflow-code/src/index.ts index 275d1a751bc..8ac917625a4 100644 --- a/packages/workflow-code/src/index.ts +++ b/packages/workflow-code/src/index.ts @@ -57,6 +57,7 @@ export { type LocatedDiagnostic, raisedMessage, refuseEditedFindings, + refusesEdited, type SourceSpan, splitLines, } from './diagnostics' diff --git a/packages/workflow-code/src/materialize/block.ts b/packages/workflow-code/src/materialize/block.ts index c07f386f662..951de5e8288 100644 --- a/packages/workflow-code/src/materialize/block.ts +++ b/packages/workflow-code/src/materialize/block.ts @@ -274,7 +274,8 @@ export function buildBlock(inputs: BlockInputs, context: MaterializeContext): Bu } } - if (isNew || inputs.parentId !== prev?.parentId) { + const reparented = isNew || inputs.parentId !== prev?.parentId + if (reparented) { if (inputs.parentId !== null) { data.parentId = inputs.parentId data.extent = 'parent' @@ -304,9 +305,12 @@ export function buildBlock(inputs: BlockInputs, context: MaterializeContext): Bu let built = block let needsLayout = false - if (isNew) { - if (inputs.layout) built = applyLayout(block, inputs.layout) - else needsLayout = true + if (isNew && inputs.layout) built = applyLayout(block, inputs.layout) + else if (reparented) { + // A child's position is relative to its container, so a move into or out of one keeps a + // position from the wrong frame: send it to autolayout, as the server does an unplaced block. + if (!isNew) built = { ...block, position: { x: 0, y: 0 } } + needsLayout = true } const items: Record = {} diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index b8a9528b49f..7d325c35052 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -8,7 +8,13 @@ import type { MaterializeDiagnosticCode } from '../diagnostics/codes' import { diagnostic } from '../diagnostics/factory' import type { Diagnostic } from '../diagnostics/frame' import { pointer, type WorkflowLockBlock, type WorkflowLockV1 } from '../lock' -import type { BlockKey, CanonicalBlock, CanonicalWorkflow } from '../model' +import { + type BlockKey, + type CanonicalBlock, + type CanonicalWorkflow, + isSimNode, + type VarKey, +} from '../model' import { deepEqual, getOwn } from '../records' import type { MaterializeHooks } from './context' @@ -427,6 +433,32 @@ function referencesKey(block: CanonicalBlock, key: BlockKey): boolean { return false } +/** + * Whether a block reads variable `key`: a `` reference in any field a reference can + * sit in, or a variable node (a Variables block's assignment target). + */ +export function referencesVariable(block: CanonicalBlock, key: VarKey): boolean { + const reads = (value: unknown): boolean => { + if (typeof value === 'string') { + return findWorkflowReferenceTokens(value).some((token) => { + if (token.kind !== 'workflow') return false + const [head, name] = parseStoredPath(token.value.slice(1, -1)) ?? [] + return ( + head?.kind === 'name' && + head.name === 'variable' && + name?.kind === 'name' && + normalizeWorkflowBlockName(name.name) === key + ) + }) + } + if (Array.isArray(value)) return value.some(reads) + if (value === null || typeof value !== 'object') return false + if (isSimNode(value)) return value.$sim === 'var' && value.key === key + return Object.values(value).some(reads) + } + return reads([block.params, block.branches, block.subflow]) +} + /** * The strong identity signal that ties a new block to a vacated stored one, if any: `settings`, * equal settings (references to the old key read as the new); `connections`, most of their diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 7828aa36e89..519285156ef 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -729,6 +729,35 @@ describe('materialize: identity', () => { expect(codes(result)).toContain('I_MAT_NEEDS_LAYOUT') }) + it('sends a block moved into or out of a container to autolayout: its position was relative to the old one', () => { + const prev = workflowState( + [ + blockState('l1', 'loop', 'Each', {}, { data: { loopType: 'for', count: 2 } }), + blockState( + 'f1', + 'function', + 'Inner', + { code: 'x' }, + { data: { parentId: 'l1', extent: 'parent' } } + ), + blockState('f2', 'function', 'Outer', { code: 'y' }, { position: { x: 900, y: 400 } }), + ], + { loops: { l1: { id: 'l1', nodes: ['f1'], iterations: 2, loopType: 'for', enabled: true } } } + ) + const lock = lockFor(prev) + const target = edit(canonical(prev, lock).workflow, (draft) => { + draft.blocks.inner.parent = null + draft.blocks.outer.parent = 'each' + }) + + const result = run(target, lock, { prev }) + + expect(result.state.blocks.f1.position).toEqual({ x: 0, y: 0 }) + expect(result.state.blocks.f2.position).toEqual({ x: 0, y: 0 }) + expect([...result.needsLayout].sort()).toEqual(['inner', 'outer']) + expect(result.identity.blocks.inner.id).toBe('f1') + }) + it('recreates a block whose type changed under the same key, with a warning', () => { const prev = base() const target = edit(canonical(prev).workflow, (draft) => { @@ -975,6 +1004,73 @@ describe('materialize: identity', () => { } }) + it.each([ + ['number', 0], + ['boolean', false], + ['plain', ''], + ] as const)( + 'keeps the id of a variable holding %s %j renamed with every reference to it', + (type, value) => { + const pulled = workflowState( + [ + blockState('a1', 'function', 'Read', { code: 'return + 1' }), + blockState('a2', 'function', 'Other', { code: 'return 2' }), + ], + { variables: { 'var-1': { id: 'var-1', name: 'Retries', type, value } } } + ) + const rename = (code: string, renamedValue: number | boolean | string = value) => + edit(canonical(pulled).workflow, (draft) => { + draft.variables = { + attempts: { key: 'attempts', name: 'Attempts', type, value: renamedValue }, + } + draft.blocks.read.params.code = { kind: 'value', value: code } + }) + const ids = (workflow: CanonicalWorkflow) => + Object.keys(run(workflow, null, { prev: pulled }).state.variables ?? {}) + + expect(ids(rename('return + 1'))).toEqual(['var-1']) + // A reference left behind, or a changed value, ties nothing. + expect(ids(rename('return 1'))).toEqual(['new-1']) + expect(ids(rename('return + 1', type === 'number' ? 5 : 'x'))).toEqual([ + 'new-1', + ]) + } + ) + + it('keeps the id of a falsy-valued variable renamed in the same edit as the block that reads it', () => { + const pulled = workflowState( + [ + blockState('a1', 'function', 'Read', { code: 'return + 1' }), + blockState('a2', 'function', 'Other', { code: 'return 2' }), + ], + { variables: { 'var-1': { id: 'var-1', name: 'Retries', type: 'number', value: 0 } } } + ) + const lock = lockFor(pulled) + const target = edit(canonical(pulled, lock).workflow, (draft) => { + draft.variables = { + attempts: { key: 'attempts', name: 'Attempts', type: 'number', value: 0 }, + } + draft.blocks.reader = { + ...draft.blocks.read, + key: 'reader', + name: 'Reader', + params: { + ...draft.blocks.read.params, + code: { kind: 'value', value: 'return + 1' }, + }, + } + draft.blocks = omit(draft.blocks, ['read']) + }) + + const result = run(target, lock, { + prev: pulled, + compiledIdents: { reader: lock.blocks.read.ident }, + }) + + expect(result.identity.blocks.reader.id).toBe('a1') + expect(Object.keys(result.state.variables ?? {})).toEqual(['var-1']) + }) + it('leaves an inputFormat id two fields share out of the identity items', () => { const prev = workflowState([ blockState('start', 'start_trigger', 'Start', { diff --git a/packages/workflow-code/src/materialize/index.ts b/packages/workflow-code/src/materialize/index.ts index 8f1d8884a6a..f353768b423 100644 --- a/packages/workflow-code/src/materialize/index.ts +++ b/packages/workflow-code/src/materialize/index.ts @@ -25,7 +25,12 @@ import { deepEqual, getOwn, invertRecord, setOwn } from '../records' import { type BuiltBlock, buildBlock } from './block' import type { MaterializeContext, MaterializeHooks } from './context' import { type BranchHandles, writeEdges } from './edges' -import { type IdentityEntry, type LockFreshness, resolveIdentity } from './identity' +import { + type IdentityEntry, + type LockFreshness, + referencesVariable, + resolveIdentity, +} from './identity' import { restorePreserved } from './preserve' import { sharesEvidence, soleRename } from './rename' import { loopEntry, parallelEntry } from './subflows' @@ -42,7 +47,8 @@ function writeVariables( lock: WorkflowLockV1 | null, prev: WorkflowState | undefined, prevCanonical: CanonicalizeResult | null, - hooks: MaterializeHooks + hooks: MaterializeHooks, + keyByPrevKey: ReadonlyMap ): { variables: Record; idByKey: Map } { const prevVariables = prev?.variables ?? {} const prevKeyById = invertRecord(prevCanonical?.identity.variables ?? {}) @@ -68,7 +74,7 @@ function writeVariables( used.add(known) knownIds.set(key, known) } - const renamed = renamedVariable(workflow, knownIds, prevCanonical, used) + const renamed = renamedVariable(workflow, knownIds, prevCanonical, used, keyByPrevKey) if (renamed) knownIds.set(renamed.key, renamed.id) const variables: Record = {} @@ -105,7 +111,8 @@ function renamedVariable( workflow: CanonicalWorkflow, knownIds: ReadonlyMap, prevCanonical: CanonicalizeResult | null, - used: ReadonlySet + used: ReadonlySet, + keyByPrevKey: ReadonlyMap ): { key: VarKey; id: string } | null { if (!prevCanonical) return null const renamed = soleRename( @@ -120,13 +127,39 @@ function renamedVariable( former !== undefined && variable !== undefined && former.type === variable.type && - sharesEvidence(former, variable, ['value']) + (sharesEvidence(former, variable, ['value']) || + (deepEqual(former.value, variable.value) && + referencesFollow(prevCanonical.workflow, workflow, formerKey, key, keyByPrevKey))) ) } ) return renamed && { key: renamed.item, id: renamed.id } } +/** + * Whether every block that read variable `formerKey` now reads `key` instead: the evidence a + * rename carries when the value itself says nothing (`0`, `false`, `''`), as it does for counters + * and flags. Each reader is found by the block it resolved to (`keyByPrevKey`), so a reader + * renamed in the same edit still counts. + */ +function referencesFollow( + prev: CanonicalWorkflow, + workflow: CanonicalWorkflow, + formerKey: VarKey, + key: VarKey, + keyByPrevKey: ReadonlyMap +): boolean { + const readers = Object.values(prev.blocks).filter((block) => referencesVariable(block, formerKey)) + return ( + readers.length > 0 && + readers.every((block) => { + const current = keyByPrevKey.get(block.key) + const now = current === undefined ? undefined : getOwn(workflow.blocks, current) + return now !== undefined && referencesVariable(now, key) + }) + ) +} + /** * Each matched block with the preserved UI state of the stored block it continues * (`restorePreserved`): source holds what executes, and the rest comes from prev. @@ -250,12 +283,17 @@ export function materialize( ) const idByKey = new Map([...resolution.blocks].map(([key, block]) => [key, block.id])) + const keyByPrevKey = new Map() + for (const [key, block] of resolution.blocks) { + if (block.prevKey !== null) keyByPrevKey.set(block.prevKey, key) + } const { variables, idByKey: variableIdByKey } = writeVariables( workflow, lock, prev, prevCanonical, - hooks + hooks, + keyByPrevKey ) const context: MaterializeContext = { catalog, @@ -312,7 +350,7 @@ export function materialize( context.diagnostics.push( diagnostic( 'I_MAT_NEEDS_LAYOUT', - `Block "${key}" has no stored or lock position; it is placed by autolayout`, + `Block "${key}" has no position in its container yet; it is placed by autolayout`, { blockKey: key, } diff --git a/packages/workflow-sdk/test/fixtures/catalog.json b/packages/workflow-sdk/test/fixtures/catalog.json index d486709ef08..79f16b768d0 100644 --- a/packages/workflow-sdk/test/fixtures/catalog.json +++ b/packages/workflow-sdk/test/fixtures/catalog.json @@ -1096,7 +1096,8 @@ "id": "bedrock/cohere.command-r-v1:0", "label": "bedrock/cohere.command-r-v1:0" } - ] + ], + "optionsVary": true }, { "id": "vertexCredential", @@ -6890,7 +6891,8 @@ "id": "python", "label": "Python" } - ] + ], + "optionsVary": true }, { "id": "code", @@ -12142,7 +12144,8 @@ "id": "bedrock/cohere.command-r-v1:0", "label": "bedrock/cohere.command-r-v1:0" } - ] + ], + "optionsVary": true }, { "id": "vertexCredential", diff --git a/packages/workspace-code/DESIGN.md b/packages/workspace-code/DESIGN.md index 7e40ef9d54d..d462e7d66b1 100644 --- a/packages/workspace-code/DESIGN.md +++ b/packages/workspace-code/DESIGN.md @@ -332,8 +332,16 @@ to another (prod) without rebinding the directory. The CLI reads the target as a (`GET /source` for the managed kinds, no blob downloads) and composes the submission with `promotionSubmission` (`src/promote.ts`): the directory's text with every workflow folder whole, each folder's `.sim/lock.json` and `.sim/base.json` replaced by the target's folder of the same key -(or dropped, so the key is created), and the target's root base, narrowed to the keys the -directory declares unless `--allow-delete`. The server sees an ordinary apply against its own base, +(or dropped, so the key is created), and the target's root base. A base entry the directory does +not declare stays in it when the promotion record holds it for the same id (the directory deleted +it since, so the apply refuses it without `--allow-delete` as it would at home); one only the target +ever held (`targetOnly`) leaves it, so the apply keeps it (`W_UNDECLARED`) whether or not it allows +deletes. Deleting those takes `--delete-target-only` beside `--allow-delete` (the CLI refuses it +alone): `--allow-delete` is consent to the directory's own deletes, and must not also reach the +target's prod-only tables, their rows, or skills made there. The CLI names each such delete +`W_PROMOTE_DELETES_TARGET_ONLY`, and `promotionRecord` never records a target-only entry (a held +delete keeps its submitted base entry, which would otherwise read as the directory's delete on +the next promotion). The server sees an ordinary apply against its own base, so every rule holds unchanged: a resource bound under the key is updated in place (a workflow keeps its id and block ids through the target's lock), an unbound one of the same name is adopted, a missing one created, and secrets and credentials bind by name and key to the target's own. The @@ -346,8 +354,9 @@ with what the last promotion left (a target edit since is kept when the director alone, `E_APPLY_STALE` when it changed it too); a workflow folder whose digest is unchanged is sent as its base alone (a no-op), and an edited one whole with the recorded fingerprint, so a target edit since is the folder's 409. Without a record entry (the first promotion, or a resource created in -the target since) the live entry is the base and the target is updated to match the directory; -deleting the record re-takes the target as it stands. Read alone, the target's live base could +the target since) the live entry is the base and the target is updated to match the directory, so +deleting the record makes the next promotion overwrite every change made in the target since, +rows its runs wrote included: the stale hint never suggests it. Read alone, the target's live base could never look changed. Without `--to`, the CLI refuses a directory read from another workspace (`SOURCE_OTHER_WORKSPACE`); `--workspace ` instead applies it as a copy whose base the server ignores (`W_BASE_OTHER_WORKSPACE`), which only a first diff --git a/packages/workspace-code/src/promote.test.ts b/packages/workspace-code/src/promote.test.ts index ac47d5c83ac..5a1976a1b7f 100644 --- a/packages/workspace-code/src/promote.test.ts +++ b/packages/workspace-code/src/promote.test.ts @@ -34,7 +34,7 @@ function firstRecord(plan: Array<{ kind: string; key: string; action: string; he blobPaths: [], kinds: ['workflow'], target: TARGET, - allowDelete: false, + deleteTargetOnly: false, }) return promotionRecord({ workspaceId: 'ws_prod', @@ -57,7 +57,7 @@ describe('promotion records', () => { kinds: ['workflow'], target: TARGET, record, - allowDelete: false, + deleteTargetOnly: false, }) expect(next.files['workflows/ingest/workflow.ts']).toBe(DIR['workflows/ingest/workflow.ts']) }) @@ -77,7 +77,7 @@ describe('promotion records', () => { }, }, record, - allowDelete: false, + deleteTargetOnly: false, }) expect(Object.keys(promote(DIR).files)).toEqual(['workflows/ingest/.sim/base.json']) @@ -87,6 +87,47 @@ describe('promotion records', () => { ) }) + it('keeps what only the target holds out of the base, even with deletes allowed, unless its deletion is asked for, and never records it', () => { + const target = { + files: TARGET.files, + base: { + ...TARGET.base, + resources: { ...TARGET.base.resources, 'skill:audit': { id: 'sk_audit' } }, + }, + } + const promote = (deleteTargetOnly: boolean, record?: PromotionRecord) => + promotionSubmission({ + files: DIR, + blobPaths: [], + kinds: ['workflow', 'skill'], + target, + record, + deleteTargetOnly, + }) + + expect(promote(false).targetOnly).toEqual(['skill:audit']) + expect(promote(false).base.resources['skill:audit']).toBeUndefined() + const deleting = promote(true) + expect(deleting.base.resources['skill:audit']).toEqual({ id: 'sk_audit' }) + + // The delete was held back (a target workflow uses it): the answer's base keeps the entry. + const record = promotionRecord({ + workspaceId: 'ws_prod', + previous: undefined, + kinds: ['workflow', 'skill'], + files: DIR, + submission: deleting, + applied: { + base: deleting.base, + files: {}, + plan: [{ kind: 'skill', key: 'audit', action: 'delete', held: true }], + }, + }) + expect(record.resources['skill:audit']).toBeUndefined() + expect(promote(false, record).targetOnly).toEqual(['skill:audit']) + expect(promote(false, record).base.resources['skill:audit']).toBeUndefined() + }) + it('ignores a record for another target, or one this build cannot read', () => { const record: PromotionRecord = { version: 1, diff --git a/packages/workspace-code/src/promote.ts b/packages/workspace-code/src/promote.ts index a122ee238df..987f1def997 100644 --- a/packages/workspace-code/src/promote.ts +++ b/packages/workspace-code/src/promote.ts @@ -36,13 +36,22 @@ import { kindOfPath } from './layout' export interface PromotionSubmission { /** The directory's text files, each workflow folder whole, with the target's record files. */ files: Record - /** The target's base, narrowed to what the directory declares unless deletes are allowed. */ + /** + * The target's base, without what only the target holds unless its deletion is asked for (see + * {@link PromotionInput.deleteTargetOnly}). + */ base: WorkspaceBaseV1 /** * The resources (by address) the last promotion left in the target that the target deleted * since and the directory still declares: the apply keeps them deleted, so the promotion warns. */ deletedInTarget: string[] + /** + * The resources (by address) the target holds that the directory does not declare and no + * promotion from it left there: only the target ever held them. They are kept, and the server + * names each (`W_UNDECLARED`), unless {@link PromotionInput.deleteTargetOnly}. + */ + targetOnly: string[] } export interface PromotionInput { @@ -57,11 +66,13 @@ export interface PromotionInput { /** What the last promotion from this directory left in the target, when one is recorded. */ record?: PromotionRecord /** - * Whether the promotion may delete what the target holds and the directory does not declare. - * Without it, those resources leave the base, so the apply keeps them with a warning instead of - * refusing for want of `--allow-delete`. + * Whether the promotion deletes what only the target holds (`--delete-target-only`, which the + * apply's `--allow-delete` must accompany). Without it, those resources leave the base, so the + * apply keeps them with a warning whether or not it allows deletes. A resource the last + * promotion left that the directory no longer declares stays in the base either way: the + * directory deleted it, which needs `--allow-delete` as an apply to its own workspace does. */ - allowDelete: boolean + deleteTargetOnly: boolean } /** A workflow folder's record files: what binds it to one workspace's workflow and block ids. */ @@ -201,7 +212,7 @@ export function parsePromotionRecord( * The submission that promotes a directory to the target its get answer describes: the * directory's source with every workflow folder's lock and base replaced by the target's folder * of the same key (or dropped, so a key the target lacks creates a workflow), and the target's - * base, narrowed to what the directory declares unless deletes are allowed. With a + * base, without what only the target holds unless its deletion is asked for. With a * {@link PromotionRecord}, each entry the record holds for the same id stands in for the target's * own, and a workflow folder records the fingerprint the last promotion left: sent as its base * alone (a no-op) when the directory has not changed it since, else whole, so a target edit since @@ -256,15 +267,21 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission const lock = input.target.files[lockPath] if (lock !== undefined) files[lockPath] = lock } + const targetOnly: string[] = [] const resources = Object.fromEntries( Object.entries(input.target.base.resources).flatMap(([address, live]) => { const parsed = parseResourceAddress(address) if (!parsed) return [] const keys = declared.get(parsed.kind) - if (keys === undefined || !(input.allowDelete || keys.has(parsed.key))) return [] + if (keys === undefined) return [] const recorded: BaseResource | undefined = input.record?.resources[resourceAddress(parsed.kind, parsed.key)] - return [[address, recorded?.id === live.id ? recorded : live] as const] + const promoted = recorded?.id === live.id ? recorded : undefined + if (!keys.has(parsed.key) && !promoted) { + targetOnly.push(resourceAddress(parsed.kind, parsed.key)) + if (!input.deleteTargetOnly) return [] + } + return [[address, promoted ?? live] as const] }) ) // A resource the last promotion left in the target that the target no longer holds was deleted @@ -282,6 +299,7 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission files, base: { ...input.target.base, resources }, deletedInTarget: [...new Set(deletedInTarget)].sort(compareStrings), + targetOnly: targetOnly.sort(compareStrings), } } @@ -291,7 +309,9 @@ export function promotionSubmission(input: PromotionInput): PromotionSubmission * plan). A resource's entry is the answer's base entry; a workflow folder's is what the apply * wrote, or what was sent whole and planned as a no-op, else the `previous` entry (a folder sent * as its base alone, or held back, or never reached). The record of a kind the promotion did not - * manage is kept. + * manage is kept. A resource only the target held (`targetOnly`) is never recorded: a delete of it + * the apply held back keeps its submitted base entry, and recording that would make the next + * promotion read it as one the directory deleted. */ export function promotionRecord(input: { workspaceId: string @@ -299,7 +319,7 @@ export function promotionRecord(input: { kinds: readonly WorkspaceKind[] /** The directory's text files, as {@link PromotionInput} took them. */ files: Readonly> - submission: Pick + submission: Pick applied: { base: WorkspaceBaseV1 files: Readonly> @@ -314,7 +334,13 @@ export function promotionRecord(input: { resources[resourceAddress(parsed.kind, parsed.key)] = entry } } - Object.assign(resources, input.applied.base.resources) + const targetOnly = new Set(input.submission.targetOnly) + for (const [address, entry] of Object.entries(input.applied.base.resources)) { + const parsed = parseResourceAddress(address) + if (parsed && !targetOnly.has(address)) { + resources[resourceAddress(parsed.kind, parsed.key)] = entry + } + } const workflows: Record = {} if (!managed.has('workflow')) { Object.assign(workflows, input.previous?.workflows) diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 7b91a1b044a..21836fa61fb 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -326,7 +326,7 @@ "apps/sim/stores/workflow-diff/utils.ts": 2, "apps/sim/stores/workflows/subblock/store.ts": 5, "apps/sim/stores/workflows/utils.ts": 1, - "apps/sim/stores/workflows/workflow/store.ts": 6, + "apps/sim/stores/workflows/workflow/store.ts": 4, "apps/sim/stores/workflows/workflow/types.ts": 2, "apps/sim/tools/ahrefs/anchors.ts": 1, "apps/sim/tools/ahrefs/backlinks.ts": 1, @@ -1576,7 +1576,6 @@ "apps/sim/lib/slack-search/assistant-stream.ts": 2, "apps/sim/lib/table/column-keys.test.ts": 4, "apps/sim/lib/table/column-types/registry.server.ts": 7, - "apps/sim/lib/table/columns/service.ts": 1, "apps/sim/lib/table/dispatcher.ts": 2, "apps/sim/lib/table/jobs/service.test.ts": 1, "apps/sim/lib/table/query-builder/predicate.ts": 1,