diff --git a/.agents/skills/add-block/SKILL.md b/.agents/skills/add-block/SKILL.md index d063e16b485..f1f2045c4c1 100644 --- a/.agents/skills/add-block/SKILL.md +++ b/.agents/skills/add-block/SKILL.md @@ -1018,13 +1018,15 @@ When adding or changing `sunset.replacedBy`, run `bun run generate:block-success map to resolve legacy and current block IDs consistently without importing the executable registry. Verify it with `bun run check:block-successors`. +A new block type also needs `bun run generate:sdk-names` (commit the new `apps/sim/blocks/sdk-names/.json` and the regenerated `apps/sim/blocks/sdk-names.generated.ts`; names are add-only, and a merge conflict in the generated module is resolved by rerunning the generator) so workflow source gets a typed builder for it; `check:sdk-names` fails CI until it runs. + Adding a block on its own needs no **tool metadata** regeneration — a block references existing tool IDs through `tools.access` and does not change any tool's shape. But if the same change also adds, edits **or removes** a tool, run `bun run generate:tool-metadata` and commit the result, or CI fails on stale artifacts. That matters here because a block's `outputs` are authored to match its tools' outputs, and the UI reads those from the generated metadata, not the executable registry — an unregenerated tool change makes the block's outputs disagree with what the panel renders. See `.agents/skills/tool-registry-boundary/SKILL.md`. A visible integration block does require the generated integration catalog and docs to be refreshed: -`bun run generate:tool-metadata` (only when a tool changed), `bun run scripts/generate-docs.ts`, +`bun run generate:tool-metadata` (only when a tool changed), `bun run generate:docs`, `bun run generate:deployment-config`, then `bun run check:audits`. Also run `bun run apps/sim/scripts/check-block-registry.ts origin/staging` (CI runs it outside `check:audits`). Commit the full generator output. For what each check verifies, see the `validate-integration` skill → @@ -1049,7 +1051,7 @@ Regenerate Derived Artifacts. - [ ] Block + meta registered in registry-maps.ts (`BLOCK_REGISTRY` / `BLOCK_META_REGISTRY`) - [ ] If `sunset.replacedBy` changed: regenerated and committed the block successor map; `bun run check:block-successors` passes - [ ] If any tool was added, changed or removed alongside the block: ran `bun run generate:tool-metadata` and committed the artifacts -- [ ] Ran `bun run scripts/generate-docs.ts`, reviewed the generated diff, and committed the integration catalog changes +- [ ] Ran `bun run generate:docs`, reviewed the generated diff, and committed the integration catalog changes - [ ] `bun run check:integration-catalog` passes - [ ] `bun run check:docs` passes (CI gate — fails on any stale generated docs page) - [ ] If icon missing: asked user to provide SVG diff --git a/.agents/skills/add-column-type/SKILL.md b/.agents/skills/add-column-type/SKILL.md index d46f1d0ec7d..debc783f044 100644 --- a/.agents/skills/add-column-type/SKILL.md +++ b/.agents/skills/add-column-type/SKILL.md @@ -141,6 +141,15 @@ Registering the *type* is compiler-enforced. Registering its *metadata* is not, Copy `currency`'s service function and orchestration branch. +## Workspace source and the workflow SDK + +The workspace source table codec reads the registry, but the SDK restates each type: add the +`column.()` helper to `packages/workflow-sdk/src/workspace/table.d.ts` (the drift test in +`apps/sim/lib/workspace-source/application/kinds/table/codec.test.ts` fails until you do), map the +type's row value in `valueType()` (`packages/workflow-sdk/src/codegen/workspace.ts`, which defaults to +`string`), and regenerate the embedded text (`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`). +See `.claude/rules/sim-workflow-code.md`. + ## Checklist Before Finishing - [ ] Id appended to `COLUMN_TYPES` in `column-types/types.ts` diff --git a/.agents/skills/add-integration/SKILL.md b/.agents/skills/add-integration/SKILL.md index ec33d091e2e..0b5dcadd9d7 100644 --- a/.agents/skills/add-integration/SKILL.md +++ b/.agents/skills/add-integration/SKILL.md @@ -111,8 +111,8 @@ service id: `mode: 'advanced'`. An existing block that already takes the secret inline is never edited in place: add a credential-only successor (`add-block` → Credential-only successor). -Then run `generate-docs`, `generate:block-successors`, `generate:deployment-config`, and -`generate:tool-metadata`, and `check-block-registry.ts origin/staging`. This pattern does not fit +Then run `bun run generate` (every generated artifact, docs included) and +`check-block-registry.ts origin/staging`. This pattern does not fit hosted-key tools, model-provider keys, two-secret or username/password logins, or a host chosen per call. @@ -371,6 +371,8 @@ export const BLOCK_META_REGISTRY: Record = { } ``` +Then run `bun run generate:sdk-names` and commit the new `apps/sim/blocks/sdk-names/.json` and the regenerated `apps/sim/blocks/sdk-names.generated.ts` (names are add-only: never edit or delete an existing entry; on a merge conflict in the generated module, rerun the generator), so workflow source gets a typed builder for the new block (`check:sdk-names` fails CI until it runs). + ### Trigger Registry (`apps/sim/triggers/registry.ts`) - If triggers exist ```typescript @@ -436,7 +438,7 @@ a resolvable capability must fail validation; an independent service account use ## Step 8: Generate and Validate the Catalog -Run `bun run generate:tool-metadata`, `bun run scripts/generate-docs.ts`, +Run `bun run generate:tool-metadata`, `bun run generate:docs`, `bun run generate:deployment-config`, then `bun run check:audits` (see the `validate-integration` skill → Regenerate Derived Artifacts for the full list and what each check verifies). @@ -462,7 +464,7 @@ In Sim, the {Service} block lets your agents {concrete workflow uses}. Group the bullets by what the user gets done, not one bullet per tool. Only describe operations the block actually ships. Follow `.claude/rules/constitution.md` for voice. Re-run -`bun run scripts/generate-docs.ts` afterwards and confirm the section survived unchanged. +`bun run generate:docs` afterwards and confirm the section survived unchanged. The docs generator refreshes `packages/deployment-config/src/integrations.json`, and the deployment config generator projects service-account provider IDs from that catalog plus the canonical OAuth @@ -580,7 +582,7 @@ If creating V2 versions (API-aligned outputs): - [ ] Registered all triggers in `triggers/registry.ts` ### Docs and deployment metadata -- [ ] Ran `bun run scripts/generate-docs.ts` +- [ ] Ran `bun run generate:docs` - [ ] Ran `bun run generate:deployment-config` for OAuth or service-account changes - [ ] Verified docs file created - [ ] Wrote the `{/* MANUAL-CONTENT-START:intro */}` section under `` and confirmed it survives a regenerate diff --git a/.agents/skills/add-tools/SKILL.md b/.agents/skills/add-tools/SKILL.md index 5261ea8169d..f2b1972c215 100644 --- a/.agents/skills/add-tools/SKILL.md +++ b/.agents/skills/add-tools/SKILL.md @@ -620,7 +620,7 @@ If creating V2 tools (API-aligned outputs), use `_v2` suffix: - [ ] Index.ts exports all tools and re-exports types (`export * from './types'`) - [ ] Tools registered in `tools/registry.ts` - [ ] `bun run generate:tool-metadata` run and the regenerated artifacts committed -- [ ] `bun run scripts/generate-docs.ts` run and the refreshed docs committed — the integration's +- [ ] `bun run generate:docs` run and the refreshed docs committed — the integration's docs page is rendered from each tool's description, params, and outputs, and CI's `bun run check:docs` fails on stale pages - [ ] Block wired: `tools.access`, dropdown options, subBlocks, `tools.config`, outputs, inputs diff --git a/.agents/skills/add-trigger/SKILL.md b/.agents/skills/add-trigger/SKILL.md index 7f8a3c7f155..400cc8ba1a6 100644 --- a/.agents/skills/add-trigger/SKILL.md +++ b/.agents/skills/add-trigger/SKILL.md @@ -543,6 +543,6 @@ registered `InternalToolConfig.operation`; a `directExecution` property fails - [ ] `bun run type-check` passes - [ ] Manually verify output keys match trigger `outputs` keys - [ ] Trigger UI shows correctly in the block -- [ ] Ran `bun run scripts/generate-docs.ts` and committed the refreshed pages — trigger sections +- [ ] Ran `bun run generate:docs` and committed the refreshed pages — trigger sections render into the owning integration's docs page, and CI's `bun run check:docs` fails on stale pages diff --git a/.agents/skills/db-migrate/SKILL.md b/.agents/skills/db-migrate/SKILL.md index e39dd005cf0..3628a980a75 100644 --- a/.agents/skills/db-migrate/SKILL.md +++ b/.agents/skills/db-migrate/SKILL.md @@ -83,7 +83,7 @@ The lint flags risky *shapes*; it cannot know whether a given drop is *safe righ ``` The reason must be specific and name the PR/version that removed the dependency. An empty reason fails the lint. - **Warnings** (`data-backfill`): non-blocking, but confirm the batching/idempotency before merging. -4. Regenerate the test schema mock: `bun run scripts/generate-schema-mock.ts` (`check:schema-mock` in `check:audits` fails after any `schema.ts` change until you do). +4. Regenerate the test schema mock: `bun run generate:schema-mock` (`check:schema-mock` in `check:audits` fails after any `schema.ts` change until you do). 5. Re-run `(cd packages/db && bunx drizzle-kit generate)` once your migration is written: it must report no schema changes and write no new file, or CI fails on the schema and migrations disagreeing. 6. Verify locally: `cd packages/db && bun run db:migrate` against a dev DB. diff --git a/.agents/skills/ship/SKILL.md b/.agents/skills/ship/SKILL.md index 342774b7858..119a5ee218b 100644 --- a/.agents/skills/ship/SKILL.md +++ b/.agents/skills/ship/SKILL.md @@ -47,24 +47,19 @@ When the user runs `/ship`: - Run `/db-migrate` to review the migration for zero-downtime safety (expand/contract phasing, backward-compatibility with the deployed app version). - `(cd packages/db && bunx drizzle-kit generate && git status --porcelain ./migrations)` must print nothing (CI's schema/migration sync step). - `bun run check:migrations origin/staging` must pass (staging is the PR base). Do not silence a flagged statement with a `-- migration-safe:` annotation unless `/db-migrate` confirmed the old code no longer depends on it; otherwise split the destructive change into a later deploy. -6. **Run pre-ship checks** from the repo root before staging. This has two phases: first **regenerate** every committed artifact so generated files never drift into a CI failure (this is what catches things like `agent-stream-docs` going stale after a `models.ts` edit), then run the **full audit suite** CI's `lint` job enforces. Both phases parallelize — but only across commands that write **disjoint** outputs — and a bare `wait` swallows child exit codes, so both phases below explicitly collect each job's status and abort ship if any failed. +6. **Run pre-ship checks** from the repo root before staging. This has two phases: first **regenerate** every committed artifact so generated files never drift into a CI failure (this is what catches things like `agent-stream-docs` going stale after a `models.ts` edit), then run the **full audit suite** CI's `lint` job enforces. Both phases abort ship if any generator or audit fails: their runners (`scripts/generate-all.ts`, `scripts/run-audits.ts`) collect each job's exit code and exit non-zero, so never replace them with a backgrounded loop and a bare `wait`, which swallows child exit codes. - **Phase A — regenerate the always-in-repo committed artifacts (parallel), then let step 7 stage whatever changed.** Regenerate only the generators whose inputs live entirely in this repo and that any ordinary code change can drift — `generate:agent-stream-docs` (derives from the provider model registry), `generate:docs-manifest` (derives from docs page paths), `generate:tool-metadata` (derives from the tool registry), and `skills:sync` (derives from `.agents/skills/**`). They write disjoint outputs (`apps/docs/…/agent.mdx`, `apps/sim/lib/mothership/generated/docs-manifest.ts`, `apps/sim/tools/generated/*`, and `.claude/skills` links), so they parallelize safely, and each is idempotent (a no-op when already in sync): + **Phase A — regenerate the always-in-repo committed artifacts, then let step 7 stage whatever changed.** Run the umbrella generator. It derives its list from package.json (every `generate:` that has a matching `check:`, plus `skills:sync`; see the header of `scripts/generate-all.ts`), so it covers every repo-local generator an ordinary code change can drift (a `models.ts` edit drifts `agent-stream-docs` and the SDK fixture catalog, a tool change drifts `tool-metadata`), runs them in the order their inputs require, and each is idempotent (a no-op when already in sync): ```bash - rm -f /tmp/ship-gen-results - for g in generate:agent-stream-docs generate:docs-manifest generate:tool-metadata skills:sync; do - ( bun run "$g" >"/tmp/ship-gen-${g//:/-}.log" 2>&1; echo "$? $g" >>/tmp/ship-gen-results ) & - done - wait - # any non-zero line is a FAILED generator — read /tmp/ship-gen-.log and fix before shipping; - # a silently-failed generate leaves a stale artifact that Phase B / CI then rejects. Keep the - # `exit 1`: it is what makes the block's own status non-zero so a caller actually stops. - if grep -vE '^0 ' /tmp/ship-gen-results; then echo "❌ generator(s) failed — do not ship"; exit 1; fi + # A non-zero exit means a generator FAILED: the summary names it and its output is above. Fix it + # before shipping; a silently-failed generate leaves a stale artifact that Phase B / CI then + # rejects. Keep the `exit 1`: it makes the block's own status non-zero so a caller actually stops. + bun run generate || { echo "❌ generator(s) failed — do not ship"; exit 1; } echo "✅ artifacts regenerated" ``` Then `git status --short` to see what regenerated — those files must be staged in step 7 alongside your own changes. - **Do NOT blanket-run the domain generators here.** `mship:generate` (`generate-mship-contracts.ts`) is an **umbrella** that drives all nine mothership contract generators (`mship-contracts`, `billing-protocol-contract`, `mship-tools`, the four `trace-*`, `metrics-contract`, `vfs-snapshot-contract`) and biome-formats `apps/sim/lib/mothership/generated/` — never run it *and* its constituents (they write the same files and corrupt each other in parallel), and never run it on an ordinary ship: it reads an **external** copilot-contract source that isn't checked out in most worktrees, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. `scripts/generate-docs.ts` rewrites the integration docs and client-safe catalog; run it when this PR changes their block/icon/landing-content inputs or when `check:integration-catalog` reports drift, then review its broad generated diff. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (`bun run mship:check` / the individual `*:check`) — with the external source present. + **Do NOT blanket-run the domain generators here.** `mship-tools:generate` merges Sim's tool contracts into the base tool catalog in the **external** Mothership worker checkout (`/worker/packages/contracts/tool-catalog-v1.json`), which most worktrees lack, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. The other mothership contracts in `apps/sim/lib/mothership/generated/` are copied in by the worker's `contracts:sync`, never generated here. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. `scripts/generate-docs.ts` rewrites the integration docs and client-safe catalog; run it when this PR changes their block/icon/landing-content inputs or when `check:integration-catalog` reports drift, then review its broad generated diff. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (e.g. `bun run mship-tools:check`) — with the external source present. **Phase B — run lint + every audit CI enforces, in parallel, and abort ship if any fails.** Before running the commands, compare this list with `.github/workflows/checks.yml`; when CI adds an audit, run it and update this skill instead of trusting a stale snapshot. The env-flag audit is currently an inline workflow block rather than a package script: when `apps/sim/lib/core/config/env-flags.ts` changed, run that current workflow block verbatim instead of copying a second version into this skill. Run `bun run lint` first (it autofixes formatting and mutates files, so don't parallelize it with the read-only audits), then run the base-sensitive block-registry check, then fan the independent audits out and collect exit codes: ```bash diff --git a/.agents/skills/v2-api-conventions/SKILL.md b/.agents/skills/v2-api-conventions/SKILL.md index 9438c8434cd..68bee751466 100644 --- a/.agents/skills/v2-api-conventions/SKILL.md +++ b/.agents/skills/v2-api-conventions/SKILL.md @@ -182,7 +182,7 @@ Audited against the primary specs and against Stripe, GitHub, and Google's AIPs. | **`X-RateLimit-Reset` as delta-seconds** | No | It is an absolute ISO 8601 timestamp, so it is clock-skew sensitive — but the response where timing actually decides behaviour (429) also carries `Retry-After`, which is skew-free. The absolute value stays useful for scheduling. | | **422 for semantic validation** | No | RFC 9110 §15.5.21 defines 422, but Appendix B.3 records that 9110 **deleted** RFC 4918's clause saying 400 was inappropriate. 400 covers "cannot or will not process… perceived to be a client error". The split is convention, not requirement — GitHub splits, Stripe and Google do not. Our machine-readable `error.code` already carries the distinction, and restatusing now breaks clients. | | **`Location` on 201** | No | §9.3.3 makes this a `SHOULD` **for POST**; the status code itself (§15.3.2) requires nothing and defines the fallback — absent `Location`, the target URI identifies the resource. Declined knowingly: several 201 responses (signed upload sessions, table exports, knowledge folders) have no canonical single-resource GET, so a `Location` would 404, and adopting it on some of the 19 is worse for a client than on none. Every 201 returns the full representation including its `id`. Revisit per-route if one gains a canonical GET. | -| **ETag / `If-None-Match` / `If-Match`** | No | Every v2 response is `Cache-Control: private, no-store` per-caller data, so `If-None-Match` buys nothing. For writes, `If-Match` needs a **strong** validator: §8.8.3.2's strong comparison fails if *either* tag is weak, so a weak ETag silently makes every `If-Match` fail. None of the three surveyed APIs does HTTP optimistic concurrency — Google does the semantics via a resource `etag` **field** (AIP-154), deliberately not the header. If Sim needs optimistic concurrency, do it that way. | +| **ETag / `If-None-Match` / `If-Match`** | No | Every v2 response is `Cache-Control: private, no-store` per-caller data, so `If-None-Match` buys nothing. The exception is a content-addressed read (the source catalog) that sets the route builder's `entityTag`: its 200 and 304 say `private, no-cache`, so the client keeps the body and revalidates it. For writes, `If-Match` needs a **strong** validator: §8.8.3.2's strong comparison fails if *either* tag is weak, so a weak ETag silently makes every `If-Match` fail. None of the three surveyed APIs does HTTP optimistic concurrency — Google does the semantics via a resource `etag` **field** (AIP-154), deliberately not the header. If Sim needs optimistic concurrency, do it that way. | | **`Deprecation` / `Sunset` on v1** | Not yet | RFC 9745 (Standards Track) and RFC 8594 (Informational) both apply, and GitHub emits both. But `Sunset` is a timestamp and 9745 §4 makes `Sunset >= Deprecation` a `MUST`, so emitting either commits Sim to a v1 retirement date — a product decision, not an engineering one. When that date exists: `Deprecation` is an RFC 9651 Structured Field **Date** (`@1688169599`); `Sunset` is an **HTTP-date** (`Sat, 31 Dec 2033 23:59:59 GMT`). Two encodings in one response — the most common implementation error here. | | **`application/merge-patch+json`** | No | v2 PATCH bodies are merge-patch *shaped* — absent means unchanged, `null` clears — but they are `.strict()`, so unknown members are rejected where RFC 7396 §2 would merge them, and nested objects are replaced wholesale rather than merged. Advertising the media type would over-claim. Document the semantics per contract instead. | diff --git a/.agents/skills/validate-integration/SKILL.md b/.agents/skills/validate-integration/SKILL.md index ee0b092cddf..40b2472510a 100644 --- a/.agents/skills/validate-integration/SKILL.md +++ b/.agents/skills/validate-integration/SKILL.md @@ -443,12 +443,10 @@ After reporting, fix every **critical** and **warning** issue. Apply **suggestio ### Regenerate Derived Artifacts -Several files are generated from tool and block definitions. Editing a tool or block WITHOUT regenerating them fails CI, so run these before pushing: +Several files are generated from tool and block definitions. Editing a tool or block WITHOUT regenerating them fails CI, so run this before pushing: ```bash -bun run generate:tool-metadata # repo root — apps/sim/tools/generated/* -bun run scripts/generate-docs.ts # docs .mdx + deployment-config/integrations.json + docs icons -bun run generate:deployment-config # canonical OAuth registry + catalog → provider-ID facts +bun run generate # every generator below (and any other generate:/check: pair) bun run check:integration-catalog # registry ↔ committed deployment metadata drift bun run check:docs # committed docs ↔ what the generator renders today bun run check:deployment-config # OAuth registry/catalog ↔ provider-ID fact drift @@ -457,12 +455,12 @@ bun run apps/sim/scripts/check-block-registry.ts origin/staging # block ↔ too ``` - **`generate:tool-metadata`** — required whenever a tool's `outputs`, `params`, or descriptions change. CI enforces this with `bun run check:tool-metadata`, which fails with *"Generated tool metadata is stale"*. This is the easiest gate to miss, because nothing in the tool file hints that a generated artifact mirrors it. -- **`generate-docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `packages/deployment-config/src/integrations.json`, and the docs copy of `components/icons.tsx`. +- **`generate:docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `packages/deployment-config/src/integrations.json`, and the docs copy of `components/icons.tsx`. - **`generate:deployment-config`** — required for OAuth or service-account changes. Regenerates provider-ID facts from the canonical OAuth registry and integration catalog; special deployment requirements remain handwritten policy. - **`check:integration-catalog`** — loads the executable block registry, derives visible integration deployment fields, and compares them with the committed catalog. It catches missing/unexpected entries and stale auth/service IDs without loading the executable registry in client code. -- **`check:docs`** — check mode of `generate-docs.ts`: renders every generated docs artifact in +- **`check:docs`** — check mode of `generate:docs`: renders every generated docs artifact in memory and fails listing any committed file that differs. Runs in CI via `check:audits`. **Always diff the regen output before committing — but commit all of it.** These generators rewrite @@ -525,7 +523,7 @@ After fixing, confirm: - [ ] Reported all issues grouped by severity - [ ] Fixed all critical and warning issues - [ ] Ran `bun run generate:tool-metadata` if any tool outputs/params changed, and confirmed `bun run check:tool-metadata` passes -- [ ] Ran `bun run scripts/generate-docs.ts` if any block metadata changed, and committed the full generated diff — including stale-page catch-up for other integrations (`bun run check:docs` fails CI on reverted generator output) +- [ ] Ran `bun run generate:docs` if any block metadata changed, and committed the full generated diff — including stale-page catch-up for other integrations (`bun run check:docs` fails CI on reverted generator output) - [ ] Validated the docs page has an accurate `MANUAL-CONTENT-START:intro` section - [ ] Ran `bun run lint` after fixes - [ ] Verified TypeScript compiles clean diff --git a/.claude/rules/sim-api-contracts.md b/.claude/rules/sim-api-contracts.md index 643f31c65aa..9e8e90dc7c1 100644 --- a/.claude/rules/sim-api-contracts.md +++ b/.claude/rules/sim-api-contracts.md @@ -35,7 +35,7 @@ Placement: the annotation must immediately precede the call or cast; up to three ```ts // boundary-raw-fetch: streaming SSE chunks must be processed as they arrive -const response = await fetch(`/api/copilot/chat/stream?chatId=${chatId}`, { signal }) +const response = await fetch(`/api/mothership/chat/stream?streamId=${streamId}`, { signal }) ``` ```ts diff --git a/.claude/rules/sim-styling.md b/.claude/rules/sim-styling.md index 2c5e16be354..3db6e6c0fc9 100644 --- a/.claude/rules/sim-styling.md +++ b/.claude/rules/sim-styling.md @@ -99,7 +99,7 @@ The chip family is the canonical chrome: `ChipInput`, `ChipTextarea`, `ChipModal - **Errors** → `error` prop. Never `className={cn(err && 'border-[var(--text-error)]')}`. - **Leading icon** → `icon` prop (rendered 14px in `--text-icon`). - **Trailing buttons** (reveal/copy/fetch) → `endAdornment`. -- **Inner-input styling** (e.g. `font-mono`, number-spinner reset) → `inputClassName` (ChipInput only). See `ee/whitelabeling/components/whitelabeling-settings.tsx`. +- **Inner-input styling** (e.g. `font-mono`, number-spinner reset) → `inputClassName` (ChipInput only). - **`ChipModalField` controls take NO className.** Pass `title`/`value`/`onChange`/`error`/`hint`/`required`/`flush`. The field owns label, control, and error/hint rendering. See `app/workspace/[workspaceId]/skills/components/skill-modal/skill-modal.tsx`. ### What className MAY carry diff --git a/.claude/rules/sim-testing.md b/.claude/rules/sim-testing.md index 4250aa87088..68453963bc0 100644 --- a/.claude/rules/sim-testing.md +++ b/.claude/rules/sim-testing.md @@ -34,7 +34,7 @@ contracts, and demonstrated regressions. | Suffix | Needs | Run with | In CI | |--------|-------|----------|-------| | `*.test.ts(x)` | nothing; global mocks from `vitest.setup.ts` | `bun run --cwd test` | `test` jobs | -| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`; optional locally, required in CI) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | +| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`) and Redis (`TEST_REDIS_URL`: required by `apps/sim`, `apps/realtime`, and `packages/db`; `apps/sim` needs a non-zero database index) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | | `*.live.test.ts` | provider APIs, hosted sandboxes, local runtimes, or sibling checkouts | `bun run --cwd apps/sim test --mode live ` | never | | `apps/desktop/e2e/*.spec.ts` | the packaged Electron app | Playwright | desktop E2E workflow | | `apps/sim/scripts/test-*-e2e.ts` | a running app over HTTP | its `package.json` script when one exists (`bun run test:scim:e2e`; `test:workflow-version-compare:e2e` adds `--no-env-file`), else `bun scripts/test--e2e.ts` from apps/sim | `e2e` jobs (`.github/scripts/http-e2e.sh`) | @@ -46,7 +46,10 @@ contracts, and demonstrated regressions. `vi.mock` in the file. - `TEST_DATABASE_URL` must be loopback and its database name must contain a `test` segment (`sim_test`); `TEST_REDIS_URL` must be loopback. `packages/db/testing/test-infrastructure.ts` owns - those checks. Isolate with a unique schema or generated IDs, and clean up in `afterAll`. + those checks. `apps/sim` integration setup runs `FLUSHDB` before every file, so its + `TEST_REDIS_URL` must also select a non-zero database (`redis://127.0.0.1:6379/15`); never + point it at a Redis holding data you want to keep. Isolate with a unique schema or generated + IDs, and clean up in `afterAll`. - Integration files run one at a time against one shared database. A new `*.integration.ts` is picked up by CI with no workflow change, and the run writes the JSON report configured in `vitest.shared.ts`, which CI uploads. Never exclude a failing suite from the run in @@ -124,7 +127,7 @@ Global (installed by `vitest.setup.ts` — knobs only, no `vi.mock`): | `@/lib/auth` | `authMockFns.mockGetSession` (+ `auth.api.*`: `mockSignOut`, `mockSignInSSO`, `mockCreateUser`, …; `auth.$context.internalAdapter`: `mockCreateSession`, …) (`auth.mock`) | | `@/lib/auth/hybrid` | `hybridAuthMockFns` (`hybrid-auth.mock`) | | `@sim/db`, `drizzle-orm` | `dbChainMockFns.*`, `queueTableRows`, `resetDbChainMock`, `hasMockCondition`, `databaseMockFns.mockResolveDbUrl`; `@sim/db` re-exports every schema table; `sql` is a spy (`vi.mocked(sql).mock.calls`) (`database.mock`) | -| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run scripts/generate-schema-mock.ts`) (`schema.mock`) | +| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run generate:schema-mock`) (`schema.mock`) | | `@trigger.dev/sdk` | `triggerSdkMockFns` (`task()` returns its config, so call `.run`; `runs.*`, `tasks.*`, `idempotencyKeys`), `MockTriggerApiError` (`trigger-sdk.mock`) | | `@/components/icons` | stable stub per icon name — no knobs, never re-mock | | `@/lib/core/config/env` | `setEnv`, `resetEnvMock`, `envMockFns.getEnv` (`env.mock`) | diff --git a/.claude/rules/sim-workflow-code.md b/.claude/rules/sim-workflow-code.md new file mode 100644 index 00000000000..86f2404658c --- /dev/null +++ b/.claude/rules/sim-workflow-code.md @@ -0,0 +1,226 @@ +--- +description: Workflows and workspaces as code (workflow source, workspace source, the workflow SDK, the Code tab) +paths: + - "packages/workflow-code/**" + - "packages/workspace-code/**" + - "packages/workflow-sdk/**" + - "packages/table-cells/**" + - "packages/workflow-persistence/src/save.ts" + - "packages/sim-cli/src/commands/protocol/*source*" + - "packages/sim-cli/src/commands/protocol/source-dir/**" + - "packages/sim-cli/src/commands/protocol/workflow-dir.ts" + - "packages/sim-cli/src/commands/protocol/stored-edges.ts" + - "packages/sim-cli/src/commands/protocol/local-errors.ts" + - "apps/sim/lib/workflows/code/**" + - "apps/sim/lib/workflows/application/*source*" + - "apps/sim/lib/workflows/application/source-pipeline/**" + - "apps/sim/lib/workflows/api/source-*" + - "apps/sim/lib/workspace-source/**" + - "apps/sim/blocks/sdk-names/**" + - "apps/sim/app/workspace/**/panel/components/code/**" + - "apps/sim/app/api/workflows/*/source/**" + - "apps/sim/app/api/v2/**/source/**" + - "apps/sim/lib/api/contracts/workflow-source.ts" + - "apps/sim/lib/api/contracts/v2/*source*" + - "apps/sim/app/api/cron/cleanup-source-staging/**" +--- + +# Workflows and Workspaces as Code + +A workflow prints to TypeScript and compiles back; a workspace prints to a directory of +declarations and applies back. Read the design docs before changing an engine or a format: +`packages/workflow-code/DESIGN.md` (one workflow) and `packages/workspace-code/DESIGN.md` (the +workspace directory). + +## Where things live + +| Where | What | +|---|---| +| `packages/workflow-types` | Shared workflow, block and resource-kind types (`WORKFLOW_RESOURCE_KINDS`) | +| `packages/workflow-code` | Pure engines for one workflow: catalog types, canonicalize, print, compile, materialize, lock, diagnostics | +| `packages/workspace-code` | Pure workspace half: `KIND_DIRS`, `APPLY_ORDER`, one codec per kind (`src/kinds/`), `.sim/base.json`, `rf1:` fingerprints, limits | +| `packages/table-cells` | Pure table cell rules (coerce, validate, currency/date/ttl parsing) shared by the table codec's offline row check and the server's writes | +| `packages/workflow-sdk` | Authoring declarations (`src/core.d.ts`, `src/workspace/*.d.ts`), type codegen from the catalog (`src/codegen/`), the cookbook | +| `packages/sim-cli/src/commands/protocol/` | `workflow-source-*` and `workspace-source-*` commands, which never import each other's command modules; `source-dir/`, `workflow-dir.ts`, `source-check.ts`, `source-write.ts`, `source-tree.ts`, `source-ui.ts` and `workspace-source-dir.ts` (the workspace directory reader a workflow check also reads declarations through) are shared | +| `apps/sim/lib/workflows/code/` | Server glue: catalog projection from the registry (`catalog/`), source-apply gates (`apply/`), stored canonical form, the feature flag, the proof corpus (`corpus/`) | +| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources; `source-pipeline/` holds the one plan → preview/commit flow an apply and a create share | +| `apps/sim/lib/workspace-source/` | Workspace source use cases (`application/`), one adapter per kind (`application/kinds//`), route policies (`api/`) | +| `apps/sim/blocks/sdk-names/.json` | Frozen SDK builder names, one file per block type | +| `apps/sim/app/workspace/**/panel/components/code/` | The read-only Code tab | + +Dependencies point one way: `packages/workflow-types` → `packages/workflow-code` → +`packages/workspace-code` and `packages/workflow-sdk` → the CLI and `apps/sim`. `check:boundaries` keeps both engines and `packages/table-cells` pure (no +Node built-ins, `eval`, `globalThis`, dynamic import): they run in the browser, the CLI and the +server unchanged. + +## Naming: "code" and "source" + +Both words name the same feature, split by role; follow the split rather than rename: + +- **code** names the feature and its engines: the packages (`workflow-code`, `workspace-code`), the + server glue folder `lib/workflows/code/`, the `workflow-code` flag, the Code tab. +- **source** names the artifact and the operations on it: the files themselves, the use cases + (`readWorkflowSource`, `applyWorkspaceSource`), the routes (`/source`), the CLI + (`sim workflows source get`), `lib/workspace-source/`. + +A fingerprint of what the server stores now is a **stored fingerprint**: `storedFingerprint` +(`rf1:`, a resource) and `storedSourceFingerprint` (`sf1:`, a workflow); a plan records it as +`PlannedChange.storedFingerprint` and the apply re-reads it with `storedFingerprintNow`. The base +records the fingerprint the directory last read (`base.fingerprint`). + +A new module goes where its neighbours are: an engine or engine glue under a `code` folder, a use +case or surface under a `source` name. + +## The canonical model + +Every engine meets at the canonical workflow model (CWM): id-free, keyed by normalized block name, +with the executable view separated from preserved state. Print and compile are inverses over it; +canonicalize and materialize convert between it and stored state, with identity carried by the +lock. The catalog (built from the block registry, `lib/workflows/code/catalog/`) is the only way an +engine learns about blocks; engines never import the registry. See DESIGN.md "The canonical model" +and "The catalog". + +## Feature flag + +One global flag, `workflow-code` (`lib/core/config/feature-flags.ts`, env fallback +`WORKFLOW_CODE`), gates workflow source, workspace source, the source catalog and resources, and +the Code tab. Every source operation (the workflow source ones in +`lib/workflows/application/operations.ts` and the workspace source ones in +`lib/workspace-source/application/operations.ts`) declares `feature: WORKFLOW_CODE_FEATURE`, which +the shared use-case wrapper checks after authorization (`FEATURE_NOT_ENABLED`); a new source +operation declares the same feature, never its own check. A surface outside an operation (the Code +tab, Copilot entitlements) calls `isWorkflowCodeEnabled()` (`lib/workflows/code/feature-flag.ts`); +the UI reads it from the workspace layout's flags. + +## Adding things + +### A block type + +Normal integration work, then `bun run generate:sdk-names`, which adds +`apps/sim/blocks/sdk-names/.json` and regenerates `sdk-names.generated.ts`. Nothing else: the +catalog projection, SDK codegen, the corpus and default parity derive from the registry. +`check:sdk-names` fails until the generator runs. Names are add-only (`check:sdk-names-frozen`): +never edit or delete an existing entry; a collision gets a hand-picked name for the new entry. +Goldens (`packages/workflow-code/src/print/__golden__`) print real blocks, so an edit to one of +those blocks can change a golden: refresh with +`bun run --cwd apps/sim test lib/workflows/code/goldens.test.ts -u`, never by hand. + +A block whose `value()` or condition the probe cannot evaluate (`catalog/probe.ts`) degrades to an +opaque type: it works, with weaker SDK typing. + +### A subBlock type + +1. `SubBlockType` (`apps/sim/blocks/types.ts` and `@sim/workflow-types/blocks`). +2. `TYPE_RULES` in `lib/workflows/code/catalog/value-kind.ts` (a total `Record`, so type-checked). +3. `mapSubBlock` in `packages/workflow-sdk/src/codegen/map-subblock.ts` (a `never` arm). +4. `getSubBlockValueShape` in `apps/sim/tools/param-shape.ts`, which defaults to `string`. +5. `packages/workflow-code/src/compile/catalog.test-helpers.ts` restates the rule flags for package tests. +6. A **structured** value (`CatalogStructuredKind`): add an entry to `STRUCTURED_CODECS` + (`packages/workflow-code/src/structured/index.ts`); the record is total over the kinds, so tsc + fails until the kind has a codec, and `StructuredCodec` (`structured/codec.ts`) lists the hook + each stage calls. Add corpus coverage. Only a kind lifted out of the param value into its own + model shape (`tools`, `branches`, the `LIFTED_STRUCTURED_KINDS`) skips the record and has its + own stage modules. +7. A **selector to a workspace resource**: the resource kind (`WORKFLOW_RESOURCE_KINDS`), its live + loader in `lib/workspaces/resource-keys.ts`, the SDK `*Ref` type in `core.d.ts`, and the + manifest (`lib/workflows/application/read-source-resources.ts`, + `packages/workflow-sdk/src/codegen/workspace.ts`). + +### A table column type + +Follow `/add-column-type`. The workspace codec reads the column registry +(`lib/workspace-source/application/kinds/table/codec.ts`), but three places restate it: + +- the `column.()` helper in `packages/workflow-sdk/src/workspace/table.d.ts` + (`kinds/table/codec.test.ts` fails until it exists), then regenerate the embedded text + (`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`); +- `valueType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')` + gives, which defaults to `string`; +- a type that owns metadata: `TABLE_COLUMN_METADATA_KEYS` in + `packages/workspace-code/src/kinds/table.ts` plus the reader and printer there. + +### A workspace resource kind + +The kinds: secrets (`secrets.ts`), credentials (`credentials.ts`), custom tools, skills, MCP servers +(`mcp/.ts`), tables (with opt-in rows in `tables/.rows.jsonl`, see DESIGN.md "Table +rows"), knowledge bases, files, workflows and deployments +(`deployments/.ts`, applied after workflows; see DESIGN.md "The deployment kind"). +Secrets and credentials are reference kinds declared whole in one root file (`FILE_KINDS`): see +DESIGN.md "Reference kinds" for holding back the workflows that use an entry the workspace cannot +satisfy. + +1. `APPLY_ORDER` and `KIND_DIRS` (`packages/workspace-code/src/kind.ts`) and its + `WORKSPACE_KIND_SPECS` entry (`src/specs.ts`: `resourceKind` is the key-table kind a workflow + names it by and a write claims, `null` for a kind with no key row of its own: a secret, a file + keyed by path, a deployment keyed by its workflow's key); every `Record` then + fails to type-check until it has the kind. +2. A codec `packages/workspace-code/src/kinds/.ts` with tests; `KIND_LAYOUTS`, and + `BLOB_KIND_CODECS` if it holds blobs (`src/kinds/registry.ts`); its diagnostic codes. +3. An adapter `apps/sim/lib/workspace-source/application/kinds//adapter.ts` calling the + kind's own use cases; `KIND_ADAPTERS` (`application/registry.ts`) and `NOUNS` + (`application/findings.ts`); an integration test. +4. If workflows name it by key: `WORKFLOW_RESOURCE_KINDS`, the live loader in + `lib/workspaces/resource-keys.ts`, the manifest in `read-source-resources.ts` and + `codegen/workspace.ts`, and `PENDABLE_REFERENCES` in + `lib/workflows/application/source-pipeline/evaluate/finalize.ts`. +5. SDK: `packages/workflow-sdk/src/workspace/.d.ts` plus `index.d.ts`, a cookbook example + under `cookbook/workspace/`, then regenerate the embedded text. +6. CLI (`workspace-source-dir.ts`): `workspaceTsconfig` includes, the `AGENTS_GUIDE` section and + `localDeclarations`. +7. Contracts: regenerate the OpenAPI specs and `packages/sim-cli/src/generated/v2-api.ts`. + +Steps 4 to 6 have no compile-time guard: a miss ships a kind that the CLI never type-checks or +that a workflow cannot reference. + +A kind whose resources workflows name a member of (an MCP server's +tools) carries the member on the workflow change's `refs` and checks it in its adapter's +`checkRefs`, which runs once every kind is planned. A secret-bearing field prints `stored()` or an +env reference, never a value, and a literal secret in source is refused (`E_MCP_LITERAL_SECRET`). + +### A field on an existing kind + +Add it to the declaration type and to the kind's field table (`DeclFields`, +`declaration/field-table.ts`: `KNOWLEDGE_FIELDS`, `MCP_FIELDS`, `SECRET_FIELDS`, …). The table is +typed over every field of the declaration, so tsc fails until the field has an entry, and that one +entry is how the kind reads, prints and fingerprints it. Then the server's stored → declaration +mapping and diff, and the SDK `.d.ts`. Custom tools (`TOOL_FIELDS`) and skills +(`FRONTMATTER_FIELDS`) use the same tables; only a tool's `code` and a skill's `content` are read, +printed and fingerprinted by hand. A field left out of the fingerprint +makes an edit fingerprint as unchanged and silently drops it; the kind's test fixture must set the +field so the "reads back every declaration it prints" test covers it. A default mirrored from +`apps/sim` (`DEFAULT_CHUNKING`) must match the server's, or every undeclared resource reads as +changed. + +## Formats and versions + +| Artifact | Marker | Rule | +|---|---|---| +| `.sim/lock.json` (workflow) | `version: 1` | Add optional fields without a bump; validators learn a field before writers emit it; unknown data goes under `x-` keys | +| `.sim/base.json` (workspace) | `version: 1` | Same policy | +| `workflow_source_meta` | `version: 1` | Same policy; a bump needs a data migration | +| Fingerprints | `sf1:` (workflow), `rf1:` (resource) | Any change to canonicalize or a kind's `canonical()` output changes every stored fingerprint and makes every checkout stale; treat it as a format change | +| SDK surface | `// @sim-sdk 1`, `SDK_MAJOR`, `OLDEST_SDK_MAJOR` | Compile refuses a newer major and reads down to the floor | +| `apps/sim/blocks/sdk-names/` | append-only | Never rename or remove a name | + +Nothing upgrades an old version: a bump is a hard refusal until a migration exists. A lock or base +field also appears in the zod contracts (`workflowLockSchema`, `sourceBaseSchema` in +`lib/api/contracts/workflow-source.ts`), so keep both in step. + +## Proof + +Behaviour is proven by properties over the real registry, not restated unit tests: + +- `bun run --cwd apps/sim test lib/workflows/code` runs the default corpus (sampled per block type, + under 60 s): round trip (`corpus/roundtrip.corpus.test.ts`, properties P0–P7 and O1–O3 in + DESIGN.md "Proof"), inactive params, stale outputs, secrets, default parity, goldens and SDK + examples. +- `ROUNDTRIP_CORPUS_FULL=1` runs every case; `ROUNDTRIP_CORPUS_EXTRA_DIR` adds redacted production + states; `ROUNDTRIP_CORPUS_REPORT_PATH` writes a report. `corpus/typecheck.corpus.test.ts` with + `ROUNDTRIP_CORPUS_FULL=1` type-checks every printed case against the generated SDK. +- Apply behaviour is proven against Postgres: `apps/sim/lib/workflows/application/*.integration.ts` + and `apps/sim/lib/workspace-source/application/*.integration.ts`. +- The Code tab and the CLI have HTTP e2e scripts (`apps/sim/scripts/test-workflow-code-ui-e2e.ts`, + `test-workflow-source-e2e.ts`, `test-workspace-source-e2e.ts`). +- Generated artifacts and their checks: `generate:sdk-names` (`check:sdk-names`, + `check:sdk-names-frozen`), the SDK embedded text (its `*-text.test.ts`), goldens + (`check:workflow-code-golden`). diff --git a/.cursor/rules/sim-api-contracts.mdc b/.cursor/rules/sim-api-contracts.mdc index 840e3bbbc23..483379b72a1 100644 --- a/.cursor/rules/sim-api-contracts.mdc +++ b/.cursor/rules/sim-api-contracts.mdc @@ -33,7 +33,7 @@ Placement: the annotation must immediately precede the call or cast; up to three ```ts // boundary-raw-fetch: streaming SSE chunks must be processed as they arrive -const response = await fetch(`/api/copilot/chat/stream?chatId=${chatId}`, { signal }) +const response = await fetch(`/api/mothership/chat/stream?streamId=${streamId}`, { signal }) ``` ```ts diff --git a/.cursor/rules/sim-styling.mdc b/.cursor/rules/sim-styling.mdc index f016cfc5b3f..2569d96c9d1 100644 --- a/.cursor/rules/sim-styling.mdc +++ b/.cursor/rules/sim-styling.mdc @@ -98,7 +98,7 @@ The chip family is the canonical chrome: `ChipInput`, `ChipTextarea`, `ChipModal - **Errors** → `error` prop. Never `className={cn(err && 'border-[var(--text-error)]')}`. - **Leading icon** → `icon` prop (rendered 14px in `--text-icon`). - **Trailing buttons** (reveal/copy/fetch) → `endAdornment`. -- **Inner-input styling** (e.g. `font-mono`, number-spinner reset) → `inputClassName` (ChipInput only). See `ee/whitelabeling/components/whitelabeling-settings.tsx`. +- **Inner-input styling** (e.g. `font-mono`, number-spinner reset) → `inputClassName` (ChipInput only). - **`ChipModalField` controls take NO className.** Pass `title`/`value`/`onChange`/`error`/`hint`/`required`/`flush`. The field owns label, control, and error/hint rendering. See `app/workspace/[workspaceId]/skills/components/skill-modal/skill-modal.tsx`. ### What className MAY carry diff --git a/.cursor/rules/sim-testing.mdc b/.cursor/rules/sim-testing.mdc index d4906ad3d77..18378075937 100644 --- a/.cursor/rules/sim-testing.mdc +++ b/.cursor/rules/sim-testing.mdc @@ -31,7 +31,7 @@ contracts, and demonstrated regressions. | Suffix | Needs | Run with | In CI | |--------|-------|----------|-------| | `*.test.ts(x)` | nothing; global mocks from `vitest.setup.ts` | `bun run --cwd test` | `test` jobs | -| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`), Redis (`TEST_REDIS_URL`; optional locally, required in CI) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | +| `*.integration.ts` | real PostgreSQL (`TEST_DATABASE_URL`) and Redis (`TEST_REDIS_URL`: required by `apps/sim`, `apps/realtime`, and `packages/db`; `apps/sim` needs a non-zero database index) | `bun run --cwd test --mode integration` | `integration` jobs, by glob (sharded per provisioning path) | | `*.live.test.ts` | provider APIs, hosted sandboxes, local runtimes, or sibling checkouts | `bun run --cwd apps/sim test --mode live ` | never | | `apps/desktop/e2e/*.spec.ts` | the packaged Electron app | Playwright | desktop E2E workflow | | `apps/sim/scripts/test-*-e2e.ts` | a running app over HTTP | its `package.json` script when one exists (`bun run test:scim:e2e`; `test:workflow-version-compare:e2e` adds `--no-env-file`), else `bun scripts/test--e2e.ts` from apps/sim | `e2e` jobs (`.github/scripts/http-e2e.sh`) | @@ -43,7 +43,10 @@ contracts, and demonstrated regressions. `vi.mock` in the file. - `TEST_DATABASE_URL` must be loopback and its database name must contain a `test` segment (`sim_test`); `TEST_REDIS_URL` must be loopback. `packages/db/testing/test-infrastructure.ts` owns - those checks. Isolate with a unique schema or generated IDs, and clean up in `afterAll`. + those checks. `apps/sim` integration setup runs `FLUSHDB` before every file, so its + `TEST_REDIS_URL` must also select a non-zero database (`redis://127.0.0.1:6379/15`); never + point it at a Redis holding data you want to keep. Isolate with a unique schema or generated + IDs, and clean up in `afterAll`. - Integration files run one at a time against one shared database. A new `*.integration.ts` is picked up by CI with no workflow change, and the run writes the JSON report configured in `vitest.shared.ts`, which CI uploads. Never exclude a failing suite from the run in @@ -121,7 +124,7 @@ Global (installed by `vitest.setup.ts` — knobs only, no `vi.mock`): | `@/lib/auth` | `authMockFns.mockGetSession` (+ `auth.api.*`: `mockSignOut`, `mockSignInSSO`, `mockCreateUser`, …; `auth.$context.internalAdapter`: `mockCreateSession`, …) (`auth.mock`) | | `@/lib/auth/hybrid` | `hybridAuthMockFns` (`hybrid-auth.mock`) | | `@sim/db`, `drizzle-orm` | `dbChainMockFns.*`, `queueTableRows`, `resetDbChainMock`, `hasMockCondition`, `databaseMockFns.mockResolveDbUrl`; `@sim/db` re-exports every schema table; `sql` is a spy (`vi.mocked(sql).mock.calls`) (`database.mock`) | -| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run scripts/generate-schema-mock.ts`) (`schema.mock`) | +| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run generate:schema-mock`) (`schema.mock`) | | `@trigger.dev/sdk` | `triggerSdkMockFns` (`task()` returns its config, so call `.run`; `runs.*`, `tasks.*`, `idempotencyKeys`), `MockTriggerApiError` (`trigger-sdk.mock`) | | `@/components/icons` | stable stub per icon name — no knobs, never re-mock | | `@/lib/core/config/env` | `setEnv`, `resetEnvMock`, `envMockFns.getEnv` (`env.mock`) | diff --git a/.cursor/rules/sim-workflow-code.mdc b/.cursor/rules/sim-workflow-code.mdc new file mode 100644 index 00000000000..65c08f7ef36 --- /dev/null +++ b/.cursor/rules/sim-workflow-code.mdc @@ -0,0 +1,206 @@ +--- +description: "Workflows and workspaces as code (workflow source, workspace source, the workflow SDK, the Code tab)" +globs: ["packages/workflow-code/**","packages/workspace-code/**","packages/workflow-sdk/**","packages/table-cells/**","packages/workflow-persistence/src/save.ts","packages/sim-cli/src/commands/protocol/*source*","packages/sim-cli/src/commands/protocol/source-dir/**","packages/sim-cli/src/commands/protocol/workflow-dir.ts","packages/sim-cli/src/commands/protocol/stored-edges.ts","packages/sim-cli/src/commands/protocol/local-errors.ts","apps/sim/lib/workflows/code/**","apps/sim/lib/workflows/application/*source*","apps/sim/lib/workflows/application/source-pipeline/**","apps/sim/lib/workflows/api/source-*","apps/sim/lib/workspace-source/**","apps/sim/blocks/sdk-names/**","apps/sim/app/workspace/**/panel/components/code/**","apps/sim/app/api/workflows/*/source/**","apps/sim/app/api/v2/**/source/**","apps/sim/lib/api/contracts/workflow-source.ts","apps/sim/lib/api/contracts/v2/*source*","apps/sim/app/api/cron/cleanup-source-staging/**"] +--- + + + +# Workflows and Workspaces as Code + +A workflow prints to TypeScript and compiles back; a workspace prints to a directory of +declarations and applies back. Read the design docs before changing an engine or a format: +`packages/workflow-code/DESIGN.md` (one workflow) and `packages/workspace-code/DESIGN.md` (the +workspace directory). + +## Where things live + +| Where | What | +|---|---| +| `packages/workflow-types` | Shared workflow, block and resource-kind types (`WORKFLOW_RESOURCE_KINDS`) | +| `packages/workflow-code` | Pure engines for one workflow: catalog types, canonicalize, print, compile, materialize, lock, diagnostics | +| `packages/workspace-code` | Pure workspace half: `KIND_DIRS`, `APPLY_ORDER`, one codec per kind (`src/kinds/`), `.sim/base.json`, `rf1:` fingerprints, limits | +| `packages/table-cells` | Pure table cell rules (coerce, validate, currency/date/ttl parsing) shared by the table codec's offline row check and the server's writes | +| `packages/workflow-sdk` | Authoring declarations (`src/core.d.ts`, `src/workspace/*.d.ts`), type codegen from the catalog (`src/codegen/`), the cookbook | +| `packages/sim-cli/src/commands/protocol/` | `workflow-source-*` and `workspace-source-*` commands, which never import each other's command modules; `source-dir/`, `workflow-dir.ts`, `source-check.ts`, `source-write.ts`, `source-tree.ts`, `source-ui.ts` and `workspace-source-dir.ts` (the workspace directory reader a workflow check also reads declarations through) are shared | +| `apps/sim/lib/workflows/code/` | Server glue: catalog projection from the registry (`catalog/`), source-apply gates (`apply/`), stored canonical form, the feature flag, the proof corpus (`corpus/`) | +| `apps/sim/lib/workflows/application/*source*` | Workflow source use cases: read, apply, create, catalog, resources; `source-pipeline/` holds the one plan → preview/commit flow an apply and a create share | +| `apps/sim/lib/workspace-source/` | Workspace source use cases (`application/`), one adapter per kind (`application/kinds//`), route policies (`api/`) | +| `apps/sim/blocks/sdk-names/.json` | Frozen SDK builder names, one file per block type | +| `apps/sim/app/workspace/**/panel/components/code/` | The read-only Code tab | + +Dependencies point one way: `packages/workflow-types` → `packages/workflow-code` → +`packages/workspace-code` and `packages/workflow-sdk` → the CLI and `apps/sim`. `check:boundaries` keeps both engines and `packages/table-cells` pure (no +Node built-ins, `eval`, `globalThis`, dynamic import): they run in the browser, the CLI and the +server unchanged. + +## Naming: "code" and "source" + +Both words name the same feature, split by role; follow the split rather than rename: + +- **code** names the feature and its engines: the packages (`workflow-code`, `workspace-code`), the + server glue folder `lib/workflows/code/`, the `workflow-code` flag, the Code tab. +- **source** names the artifact and the operations on it: the files themselves, the use cases + (`readWorkflowSource`, `applyWorkspaceSource`), the routes (`/source`), the CLI + (`sim workflows source get`), `lib/workspace-source/`. + +A fingerprint of what the server stores now is a **stored fingerprint**: `storedFingerprint` +(`rf1:`, a resource) and `storedSourceFingerprint` (`sf1:`, a workflow); a plan records it as +`PlannedChange.storedFingerprint` and the apply re-reads it with `storedFingerprintNow`. The base +records the fingerprint the directory last read (`base.fingerprint`). + +A new module goes where its neighbours are: an engine or engine glue under a `code` folder, a use +case or surface under a `source` name. + +## The canonical model + +Every engine meets at the canonical workflow model (CWM): id-free, keyed by normalized block name, +with the executable view separated from preserved state. Print and compile are inverses over it; +canonicalize and materialize convert between it and stored state, with identity carried by the +lock. The catalog (built from the block registry, `lib/workflows/code/catalog/`) is the only way an +engine learns about blocks; engines never import the registry. See DESIGN.md "The canonical model" +and "The catalog". + +## Feature flag + +One global flag, `workflow-code` (`lib/core/config/feature-flags.ts`, env fallback +`WORKFLOW_CODE`), gates workflow source, workspace source, the source catalog and resources, and +the Code tab. Every source operation (the workflow source ones in +`lib/workflows/application/operations.ts` and the workspace source ones in +`lib/workspace-source/application/operations.ts`) declares `feature: WORKFLOW_CODE_FEATURE`, which +the shared use-case wrapper checks after authorization (`FEATURE_NOT_ENABLED`); a new source +operation declares the same feature, never its own check. A surface outside an operation (the Code +tab, Copilot entitlements) calls `isWorkflowCodeEnabled()` (`lib/workflows/code/feature-flag.ts`); +the UI reads it from the workspace layout's flags. + +## Adding things + +### A block type + +Normal integration work, then `bun run generate:sdk-names`, which adds +`apps/sim/blocks/sdk-names/.json` and regenerates `sdk-names.generated.ts`. Nothing else: the +catalog projection, SDK codegen, the corpus and default parity derive from the registry. +`check:sdk-names` fails until the generator runs. Names are add-only (`check:sdk-names-frozen`): +never edit or delete an existing entry; a collision gets a hand-picked name for the new entry. +Goldens (`packages/workflow-code/src/print/__golden__`) print real blocks, so an edit to one of +those blocks can change a golden: refresh with +`bun run --cwd apps/sim test lib/workflows/code/goldens.test.ts -u`, never by hand. + +A block whose `value()` or condition the probe cannot evaluate (`catalog/probe.ts`) degrades to an +opaque type: it works, with weaker SDK typing. + +### A subBlock type + +1. `SubBlockType` (`apps/sim/blocks/types.ts` and `@sim/workflow-types/blocks`). +2. `TYPE_RULES` in `lib/workflows/code/catalog/value-kind.ts` (a total `Record`, so type-checked). +3. `mapSubBlock` in `packages/workflow-sdk/src/codegen/map-subblock.ts` (a `never` arm). +4. `getSubBlockValueShape` in `apps/sim/tools/param-shape.ts`, which defaults to `string`. +5. `packages/workflow-code/src/compile/catalog.test-helpers.ts` restates the rule flags for package tests. +6. A **structured** value (`CatalogStructuredKind`): add an entry to `STRUCTURED_CODECS` + (`packages/workflow-code/src/structured/index.ts`); the record is total over the kinds, so tsc + fails until the kind has a codec, and `StructuredCodec` (`structured/codec.ts`) lists the hook + each stage calls. Add corpus coverage. Only a kind lifted out of the param value into its own + model shape (`tools`, `branches`, the `LIFTED_STRUCTURED_KINDS`) skips the record and has its + own stage modules. +7. A **selector to a workspace resource**: the resource kind (`WORKFLOW_RESOURCE_KINDS`), its live + loader in `lib/workspaces/resource-keys.ts`, the SDK `*Ref` type in `core.d.ts`, and the + manifest (`lib/workflows/application/read-source-resources.ts`, + `packages/workflow-sdk/src/codegen/workspace.ts`). + +### A table column type + +Follow `/add-column-type`. The workspace codec reads the column registry +(`lib/workspace-source/application/kinds/table/codec.ts`), but three places restate it: + +- the `column.()` helper in `packages/workflow-sdk/src/workspace/table.d.ts` + (`kinds/table/codec.test.ts` fails until it exists), then regenerate the embedded text + (`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`); +- `valueType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')` + gives, which defaults to `string`; +- a type that owns metadata: `TABLE_COLUMN_METADATA_KEYS` in + `packages/workspace-code/src/kinds/table.ts` plus the reader and printer there. + +### A workspace resource kind + +The kinds: secrets (`secrets.ts`), credentials (`credentials.ts`), custom tools, skills, MCP servers +(`mcp/.ts`), tables (with opt-in rows in `tables/.rows.jsonl`, see DESIGN.md "Table +rows"), knowledge bases, files, workflows and deployments +(`deployments/.ts`, applied after workflows; see DESIGN.md "The deployment kind"). +Secrets and credentials are reference kinds declared whole in one root file (`FILE_KINDS`): see +DESIGN.md "Reference kinds" for holding back the workflows that use an entry the workspace cannot +satisfy. + +1. `APPLY_ORDER` and `KIND_DIRS` (`packages/workspace-code/src/kind.ts`) and its + `WORKSPACE_KIND_SPECS` entry (`src/specs.ts`: `resourceKind` is the key-table kind a workflow + names it by and a write claims, `null` for a kind with no key row of its own: a secret, a file + keyed by path, a deployment keyed by its workflow's key); every `Record` then + fails to type-check until it has the kind. +2. A codec `packages/workspace-code/src/kinds/.ts` with tests; `KIND_LAYOUTS`, and + `BLOB_KIND_CODECS` if it holds blobs (`src/kinds/registry.ts`); its diagnostic codes. +3. An adapter `apps/sim/lib/workspace-source/application/kinds//adapter.ts` calling the + kind's own use cases; `KIND_ADAPTERS` (`application/registry.ts`) and `NOUNS` + (`application/findings.ts`); an integration test. +4. If workflows name it by key: `WORKFLOW_RESOURCE_KINDS`, the live loader in + `lib/workspaces/resource-keys.ts`, the manifest in `read-source-resources.ts` and + `codegen/workspace.ts`, and `PENDABLE_REFERENCES` in + `lib/workflows/application/source-pipeline/evaluate/finalize.ts`. +5. SDK: `packages/workflow-sdk/src/workspace/.d.ts` plus `index.d.ts`, a cookbook example + under `cookbook/workspace/`, then regenerate the embedded text. +6. CLI (`workspace-source-dir.ts`): `workspaceTsconfig` includes, the `AGENTS_GUIDE` section and + `localDeclarations`. +7. Contracts: regenerate the OpenAPI specs and `packages/sim-cli/src/generated/v2-api.ts`. + +Steps 4 to 6 have no compile-time guard: a miss ships a kind that the CLI never type-checks or +that a workflow cannot reference. + +A kind whose resources workflows name a member of (an MCP server's +tools) carries the member on the workflow change's `refs` and checks it in its adapter's +`checkRefs`, which runs once every kind is planned. A secret-bearing field prints `stored()` or an +env reference, never a value, and a literal secret in source is refused (`E_MCP_LITERAL_SECRET`). + +### A field on an existing kind + +Add it to the declaration type and to the kind's field table (`DeclFields`, +`declaration/field-table.ts`: `KNOWLEDGE_FIELDS`, `MCP_FIELDS`, `SECRET_FIELDS`, …). The table is +typed over every field of the declaration, so tsc fails until the field has an entry, and that one +entry is how the kind reads, prints and fingerprints it. Then the server's stored → declaration +mapping and diff, and the SDK `.d.ts`. Custom tools (`TOOL_FIELDS`) and skills +(`FRONTMATTER_FIELDS`) use the same tables; only a tool's `code` and a skill's `content` are read, +printed and fingerprinted by hand. A field left out of the fingerprint +makes an edit fingerprint as unchanged and silently drops it; the kind's test fixture must set the +field so the "reads back every declaration it prints" test covers it. A default mirrored from +`apps/sim` (`DEFAULT_CHUNKING`) must match the server's, or every undeclared resource reads as +changed. + +## Formats and versions + +| Artifact | Marker | Rule | +|---|---|---| +| `.sim/lock.json` (workflow) | `version: 1` | Add optional fields without a bump; validators learn a field before writers emit it; unknown data goes under `x-` keys | +| `.sim/base.json` (workspace) | `version: 1` | Same policy | +| `workflow_source_meta` | `version: 1` | Same policy; a bump needs a data migration | +| Fingerprints | `sf1:` (workflow), `rf1:` (resource) | Any change to canonicalize or a kind's `canonical()` output changes every stored fingerprint and makes every checkout stale; treat it as a format change | +| SDK surface | `// @sim-sdk 1`, `SDK_MAJOR`, `OLDEST_SDK_MAJOR` | Compile refuses a newer major and reads down to the floor | +| `apps/sim/blocks/sdk-names/` | append-only | Never rename or remove a name | + +Nothing upgrades an old version: a bump is a hard refusal until a migration exists. A lock or base +field also appears in the zod contracts (`workflowLockSchema`, `sourceBaseSchema` in +`lib/api/contracts/workflow-source.ts`), so keep both in step. + +## Proof + +Behaviour is proven by properties over the real registry, not restated unit tests: + +- `bun run --cwd apps/sim test lib/workflows/code` runs the default corpus (sampled per block type, + under 60 s): round trip (`corpus/roundtrip.corpus.test.ts`, properties P0–P7 and O1–O3 in + DESIGN.md "Proof"), inactive params, stale outputs, secrets, default parity, goldens and SDK + examples. +- `ROUNDTRIP_CORPUS_FULL=1` runs every case; `ROUNDTRIP_CORPUS_EXTRA_DIR` adds redacted production + states; `ROUNDTRIP_CORPUS_REPORT_PATH` writes a report. `corpus/typecheck.corpus.test.ts` with + `ROUNDTRIP_CORPUS_FULL=1` type-checks every printed case against the generated SDK. +- Apply behaviour is proven against Postgres: `apps/sim/lib/workflows/application/*.integration.ts` + and `apps/sim/lib/workspace-source/application/*.integration.ts`. +- The Code tab and the CLI have HTTP e2e scripts (`apps/sim/scripts/test-workflow-code-ui-e2e.ts`, + `test-workflow-source-e2e.ts`, `test-workspace-source-e2e.ts`). +- Generated artifacts and their checks: `generate:sdk-names` (`check:sdk-names`, + `check:sdk-names-frozen`), the SDK embedded text (its `*-text.test.ts`), goldens + (`check:workflow-code-golden`). diff --git a/.github/CI.md b/.github/CI.md index bda3d5c7182..6bc50a4c8fc 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -36,9 +36,7 @@ Database integration uses 4-vCPU Blacksmith runners with sticky dependency disks GitHub runners install dependencies directly rather than transferring Bun and `node_modules` archives. In a same-source paired comparison, those restores took 26.2 seconds before an 82-millisecond install, while a clean install took 12.42 seconds. This is one paired observation; retain the policy only while repeated CI timings support it. Blacksmith keeps its sticky dependency disks. GitHub Turbo output caches remain enabled for jobs that request them, with a commit-specific key and a task/event/fork-scoped restore prefix, so a rerun can reuse an exact entry without creating another archive. Push jobs never restore pull-request entries. -Trusted Blacksmith E2E builders can reuse Next's incremental compiler state when a complete Turbo build is not already cached. The compiler cache has its own mode, event, OS/architecture, resolved toolchain, lockfile and build-recipe namespace; fork PRs and GitHub fallback keep the compiler cache off. The build command always runs, and an uncertain cache probe restores state rather than skipping compilation. - -Compiler maintenance enforces a 20 GiB regular-file working-state limit before and after compilation, preserves the mounted root, and fails closed on access or ownership errors. Dev maintenance remains separate and nonblocking. Sticky disks are optional last-good compiler snapshots, not cross-job artifact handoff or validation receipts; no freshest-snapshot or physical-storage eviction guarantee is assumed. Hosted/deployment builds remain cache-off until separately qualified. A cold E2E qualification job took 259 seconds versus 225 uncached; fresh-runner warm and changed-source jobs took 148 and 159 seconds versus 227 and 229 uncached. These paired observations do not establish a universal CI speedup or justify smaller runners. Compilation and all existing acceptance coverage remain required. +E2E builds compile without Next's persistent compiler cache (`turbopackFileSystemCacheForBuild: false`; Next enables it by default). Its earlier qualification showed warm, identical-source builds at 148 seconds, but real PR compiles restoring a shared cache took 2.9-4.6 minutes plus 15-37 seconds writing the cache back, while the same builds without it compiled in 2.5 minutes with no write. A complete Turbo build output is still reused when nothing it depends on changed. `scripts/prune-turbopack-cache.mts` caps only the dev cache. Closing a same-repository pull request deletes only its GitHub merge-ref caches after verifying that it remains closed or merged. The regular `pull_request` workflow skips forks and Dependabot because their tokens cannot delete caches; those entries retain GitHub's normal cache eviction. Cleanup checks out no code and never selects a branch-wide or default-branch cache. Blacksmith sticky disks retain their separate lifecycle. @@ -50,7 +48,7 @@ HTTP and docs browser installation first asks the pinned Playwright driver to si Every HTTP and browser group consumes the same production build in an isolated job/database. Electron also uses that build and skips its development-route warm-up. Local HTTP and Electron development runs retain their dev-server behavior. Build inputs exclude test-only files so a harness edit can reuse an unchanged production build. -Desktop inbox protocol, assistant actions UI, and organization search UI run as three parallel HTTP jobs with isolated databases, apps, and reports. Each harness owns its acceptance task, so an edit selects its own job. The inbox protocol task depends on server and realtime code; the browser tasks also depend on workspace UI. Desktop inbox needs no browser. The UI jobs and browser extension install Chromium while their app starts, waiting for both before executing assertions; browser-install failures still fail the job and preserve the install log. No acceptance assertions are removed by the split. +Desktop inbox protocol, assistant actions UI, organization search UI, and the large search-permission import run as four parallel HTTP jobs with isolated databases, apps, and reports. Each harness owns its acceptance task, so an edit selects its own job. The inbox protocol task depends on server and realtime code; the browser tasks also depend on workspace UI. The permission import uploads its large import in order over HTTP and runs beside organization search rather than ahead of it; the same suite separately uploads a small mapping through the Sim CLI, so its task depends on server code and the CLI runtime. Desktop inbox and the permission import need no browser. The UI jobs and browser extension install Chromium while their app starts, waiting for both before executing assertions; browser-install failures still fail the job and preserve the install log. No acceptance assertions are removed by the split. Live Electron acceptance tracks its live-app suite, fixtures, and Playwright configuration separately from shell-only smoke tests. Changing a shell-only smoke case runs the dedicated desktop workflow without starting the app's production build and live lease checks. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f03ff816348..deb4e526c78 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,28 +5,28 @@ /apps/sim/app/api/mothership/chat/ @simstudioai/mothership /apps/sim/app/api/mothership/chats/ @simstudioai/mothership /apps/sim/app/api/mothership/execute/ @simstudioai/mothership -/apps/sim/app/api/v1/copilot/chat/ @simstudioai/mothership # Server-side stream orchestration, persistence, and protocol. -/apps/sim/lib/copilot/chat/ @simstudioai/mothership -/apps/sim/lib/copilot/async-runs/ @simstudioai/mothership -/apps/sim/lib/copilot/request/ @simstudioai/mothership -/apps/sim/lib/copilot/generated/ @simstudioai/mothership -/apps/sim/lib/copilot/constants.ts @simstudioai/mothership +/apps/sim/lib/mothership/chat/ @simstudioai/mothership +/apps/sim/lib/mothership/async-runs/ @simstudioai/mothership +/apps/sim/lib/mothership/request/ @simstudioai/mothership +/apps/sim/lib/mothership/generated/ @simstudioai/mothership +/apps/sim/lib/mothership/observability/ @simstudioai/mothership +/apps/sim/lib/mothership/constants.ts @simstudioai/mothership /apps/sim/lib/core/utils/sse.ts @simstudioai/mothership # Stream-time tool execution, confirmations, resource persistence, and handlers. -/apps/sim/lib/copilot/tool-executor/ @simstudioai/mothership -/apps/sim/lib/copilot/tools/ @simstudioai/mothership -/apps/sim/lib/copilot/persistence/ @simstudioai/mothership -/apps/sim/lib/copilot/resources/ @simstudioai/mothership +/apps/sim/lib/mothership/tool-executor/ @simstudioai/mothership +/apps/sim/lib/mothership/tools/ @simstudioai/mothership +/apps/sim/lib/mothership/persistence/ @simstudioai/mothership +/apps/sim/lib/mothership/resources/ @simstudioai/mothership # Client-side stream consumption, hydration, and reconnect. /apps/sim/app/workspace/*/home/hooks/index.ts @simstudioai/mothership /apps/sim/app/workspace/*/home/hooks/use-chat.ts @simstudioai/mothership /apps/sim/app/workspace/*/home/hooks/preview/ @simstudioai/mothership /apps/sim/app/workspace/*/home/hooks/stream/ @simstudioai/mothership -/apps/sim/hooks/queries/tasks.ts @simstudioai/mothership +/apps/sim/hooks/queries/mothership-tasks.ts @simstudioai/mothership # Dependency manifests and package-manager config. Any change here — adding, # removing, or bumping a dependency, or altering install/security settings — diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 454529f0dfd..4b8a9c59d91 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -169,6 +169,7 @@ BETTER_AUTH_SECRET=$(openssl rand -hex 32) ENCRYPTION_KEY=$(openssl rand -hex 32) INTERNAL_API_SECRET=$(openssl rand -hex 32) CRON_SECRET=$(openssl rand -hex 32) +POSTGRES_PASSWORD=$(openssl rand -hex 24) EOF # Start Sim @@ -179,31 +180,34 @@ Access the application at [http://localhost:3000/](http://localhost:3000/) #### Using Local Models -To use local models with Sim: +To use local models with Sim, either point it at an Ollama server you already run or start one alongside the stack. -1. Install Ollama and pull models: +- **Ollama on your machine**: install it, pull a model, and point the app at the host: - ```bash - # Install Ollama (if not already installed) - curl -fsSL https://ollama.ai/install.sh | sh + ```bash + curl -fsSL https://ollama.ai/install.sh | sh + ollama pull gemma3:4b - # Pull a model (e.g., gemma3:4b) - ollama pull gemma3:4b - ``` + echo 'OLLAMA_URL=http://host.docker.internal:11434' >> .env + docker compose -f docker-compose.prod.yml up -d + ``` -2. Start Sim with local model support: + Ollama must listen on an interface the container can reach (`OLLAMA_HOST=0.0.0.0:11434`). For an Ollama server elsewhere, set `OLLAMA_URL` to its address instead. - ```bash - # With NVIDIA GPU support - docker compose --profile local-gpu -f docker-compose.ollama.yml up -d +- **Bundled Ollama**: layer `docker-compose.ollama.yml` on the production file, with exactly one of the `gpu` or `cpu` profiles, then pull the starter model once: - # Without GPU (CPU only) - docker compose --profile local-cpu -f docker-compose.ollama.yml up -d + ```bash + echo 'OLLAMA_URL=http://ollama:11434' >> .env - # If hosting on a server, update the environment variables in the docker-compose.prod.yml file - # to include the server's public IP then start again (OLLAMA_URL to i.e. http://1.1.1.1:11434) - docker compose -f docker-compose.prod.yml up -d - ``` + # With NVIDIA GPU support + docker compose -f docker-compose.prod.yml -f docker-compose.ollama.yml --profile gpu up -d + + # Without GPU (CPU only) + docker compose -f docker-compose.prod.yml -f docker-compose.ollama.yml --profile cpu up -d + + # Pull gemma3:4b into the running server + docker compose -f docker-compose.prod.yml -f docker-compose.ollama.yml --profile setup run --rm model-setup + ``` ### Option 3: Manual Setup @@ -234,7 +238,7 @@ If you prefer not to use Docker. **All commands run from the repository root unl cp packages/db/.env.example packages/db/.env ``` - At minimum, each `.env` needs `DATABASE_URL`. `apps/sim/.env` and `apps/realtime/.env` additionally need matching values for `BETTER_AUTH_URL`, `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, and `NEXT_PUBLIC_APP_URL`. `apps/sim/.env` also needs `ENCRYPTION_KEY` and `API_ENCRYPTION_KEY`. Generate any 32-char secrets with `openssl rand -hex 32`. + At minimum, each `.env` needs `DATABASE_URL`. `apps/sim/.env` and `apps/realtime/.env` additionally need matching values for `BETTER_AUTH_URL`, `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, `NEXT_PUBLIC_APP_URL`, and `REDIS_URL`: both services refuse to start without Redis (for a local one, run `docker run -d -p 127.0.0.1:6379:6379 redis:7-alpine` and set `REDIS_URL=redis://localhost:6379`). `apps/sim/.env` also needs `ENCRYPTION_KEY` and `API_ENCRYPTION_KEY`. Generate any 32-char secrets with `openssl rand -hex 32`. The same `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, and `DATABASE_URL` must appear in both `apps/sim/.env` and `apps/realtime/.env` so the two services share auth and DB. After editing `apps/sim/.env`, you can mirror the shared subset into the realtime env in one shot: @@ -516,10 +520,10 @@ In addition, you will need to update the registries: Ensure that your tool functions correctly by making test requests and verifying the responses. 8. **Generate Documentation:** - Run the documentation generator (from `apps/sim`) to create docs for your new tool: + Regenerate every derived artifact, including your tool's docs page, from the repo root: ```bash - cd apps/sim && bun run generate-docs + bun run generate ``` ### Naming Conventions diff --git a/.github/ci-suites.json b/.github/ci-suites.json index d8f1e3953fd..bbd2f26d1c7 100644 --- a/.github/ci-suites.json +++ b/.github/ci-suites.json @@ -24,6 +24,10 @@ "group": "organization-search", "task": "ci:e2e-organization-search" }, + { + "group": "search-permissions", + "task": "ci:e2e-search-permissions" + }, { "group": "browser-extension-navigation", "task": "ci:e2e-browser-extension" @@ -47,6 +51,10 @@ { "group": "markdown-viewer", "task": "ci:e2e-markdown-viewer" + }, + { + "group": "workflow-code-ui", + "task": "ci:e2e-workflow-code-ui" } ], "production": [ diff --git a/.github/scripts/ci-plan.gql b/.github/scripts/ci-plan.gql index f393f1a0aed..e3551074557 100644 --- a/.github/scripts/ci-plan.gql +++ b/.github/scripts/ci-plan.gql @@ -3,10 +3,10 @@ query($base: String!, $head: String!) { affectedTasks(base: $base, head: $head, tasks: [ "test", "build", "type-check", "ci:integration", "ci:desktop-live", "ci:e2e-scim", "ci:e2e-cli", "ci:e2e-stop-after", "ci:e2e-desktop-inbox", - "ci:e2e-assistant-actions", "ci:e2e-organization-search", + "ci:e2e-assistant-actions", "ci:e2e-organization-search", "ci:e2e-search-permissions", "ci:e2e-docs-platform", "ci:e2e-browser-extension", "ci:e2e-project-files", "ci:e2e-mobile", "ci:e2e-seo", - "ci:e2e-markdown-viewer" + "ci:e2e-markdown-viewer", "ci:e2e-workflow-code-ui" ]) { items { name diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index 61b7a752793..a27b9acd6bb 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -3,9 +3,10 @@ # CI groups, which reuse the production build from `e2e-build`. # The job extracts it into apps/sim/.next before running these suites. # -# Usage: http-e2e.sh (run from apps/sim) +# Usage: http-e2e.sh (run from apps/sim) # -# The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of +# The job provides DATABASE_URL, BETTER_AUTH_SECRET, ENCRYPTION_KEY and a Redis on 127.0.0.1:6379 +# (Redis is required, so every app and realtime server gets REDIS_URL); each group sets the rest of # its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e. # # Under `next dev` the first request cold-compiles the app under Turbopack, which takes 60-185s on @@ -20,9 +21,10 @@ # telemetry flush runs detached and still writes .next/dev). set -euo pipefail -group=${1:?usage: http-e2e.sh } +group=${1:?usage: http-e2e.sh } report_dir="$RUNNER_TEMP/e2e" ready_timeout_seconds=600 +export REDIS_URL=${REDIS_URL:-redis://127.0.0.1:6379} mkdir -p "$report_dir" server_pid='' @@ -34,6 +36,9 @@ realtime_tag='' realtime_log='' browsers_pid='' local_app_url='' +extra_pids=() +extra_tags=() +extra_logs=() # `dev` or `start`; a group sets it before start_app. app_server=${E2E_APP_SERVER:-dev} @@ -56,6 +61,13 @@ finish() { tail -n 200 "$server_log" fi fi + for i in "${!extra_pids[@]}"; do + bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "${extra_pids[$i]}" "${extra_tags[$i]}" || status=1 + wait "${extra_pids[$i]}" 2>/dev/null || true + if [ "$status" -ne 0 ]; then + tail -n 100 "${extra_logs[$i]}" + fi + done if [ -n "$realtime_pid" ]; then bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$realtime_pid" "$realtime_tag" || status=1 wait "$realtime_pid" 2>/dev/null || true @@ -87,6 +99,7 @@ wait_for_browsers() { exit 1 fi browsers_pid='' + grep -E '^(Installed system dependencies|Cache install failed|Stored [0-9]+ packages|Browser install took)' "$browsers_log" || true } # start_app