diff --git a/.agents/skills/add-block/SKILL.md b/.agents/skills/add-block/SKILL.md index 091fff7ab20..f1f2045c4c1 100644 --- a/.agents/skills/add-block/SKILL.md +++ b/.agents/skills/add-block/SKILL.md @@ -1026,7 +1026,7 @@ tool IDs through `tools.access` and does not change any tool's shape. But if the same change also adds, edits **or removes** a tool, run `bun run generate:tool-metadata` and commit the result, or CI fails on stale artifacts. That matters here because a block's `outputs` are authored to match its tools' outputs, and the UI reads those from the generated metadata, not the executable registry — an unregenerated tool change makes the block's outputs disagree with what the panel renders. See `.agents/skills/tool-registry-boundary/SKILL.md`. A visible integration block does require the generated integration catalog and docs to be refreshed: -`bun run generate:tool-metadata` (only when a tool changed), `bun run scripts/generate-docs.ts`, +`bun run generate:tool-metadata` (only when a tool changed), `bun run generate:docs`, `bun run generate:deployment-config`, then `bun run check:audits`. Also run `bun run apps/sim/scripts/check-block-registry.ts origin/staging` (CI runs it outside `check:audits`). Commit the full generator output. For what each check verifies, see the `validate-integration` skill → @@ -1051,7 +1051,7 @@ Regenerate Derived Artifacts. - [ ] Block + meta registered in registry-maps.ts (`BLOCK_REGISTRY` / `BLOCK_META_REGISTRY`) - [ ] If `sunset.replacedBy` changed: regenerated and committed the block successor map; `bun run check:block-successors` passes - [ ] If any tool was added, changed or removed alongside the block: ran `bun run generate:tool-metadata` and committed the artifacts -- [ ] Ran `bun run scripts/generate-docs.ts`, reviewed the generated diff, and committed the integration catalog changes +- [ ] Ran `bun run generate:docs`, reviewed the generated diff, and committed the integration catalog changes - [ ] `bun run check:integration-catalog` passes - [ ] `bun run check:docs` passes (CI gate — fails on any stale generated docs page) - [ ] If icon missing: asked user to provide SVG diff --git a/.agents/skills/add-integration/SKILL.md b/.agents/skills/add-integration/SKILL.md index e6e6a8ba2ae..0b5dcadd9d7 100644 --- a/.agents/skills/add-integration/SKILL.md +++ b/.agents/skills/add-integration/SKILL.md @@ -111,8 +111,8 @@ service id: `mode: 'advanced'`. An existing block that already takes the secret inline is never edited in place: add a credential-only successor (`add-block` → Credential-only successor). -Then run `generate-docs`, `generate:block-successors`, `generate:deployment-config`, and -`generate:tool-metadata`, and `check-block-registry.ts origin/staging`. This pattern does not fit +Then run `bun run generate` (every generated artifact, docs included) and +`check-block-registry.ts origin/staging`. This pattern does not fit hosted-key tools, model-provider keys, two-secret or username/password logins, or a host chosen per call. @@ -438,7 +438,7 @@ a resolvable capability must fail validation; an independent service account use ## Step 8: Generate and Validate the Catalog -Run `bun run generate:tool-metadata`, `bun run scripts/generate-docs.ts`, +Run `bun run generate:tool-metadata`, `bun run generate:docs`, `bun run generate:deployment-config`, then `bun run check:audits` (see the `validate-integration` skill → Regenerate Derived Artifacts for the full list and what each check verifies). @@ -464,7 +464,7 @@ In Sim, the {Service} block lets your agents {concrete workflow uses}. Group the bullets by what the user gets done, not one bullet per tool. Only describe operations the block actually ships. Follow `.claude/rules/constitution.md` for voice. Re-run -`bun run scripts/generate-docs.ts` afterwards and confirm the section survived unchanged. +`bun run generate:docs` afterwards and confirm the section survived unchanged. The docs generator refreshes `packages/deployment-config/src/integrations.json`, and the deployment config generator projects service-account provider IDs from that catalog plus the canonical OAuth @@ -582,7 +582,7 @@ If creating V2 versions (API-aligned outputs): - [ ] Registered all triggers in `triggers/registry.ts` ### Docs and deployment metadata -- [ ] Ran `bun run scripts/generate-docs.ts` +- [ ] Ran `bun run generate:docs` - [ ] Ran `bun run generate:deployment-config` for OAuth or service-account changes - [ ] Verified docs file created - [ ] Wrote the `{/* MANUAL-CONTENT-START:intro */}` section under `` and confirmed it survives a regenerate diff --git a/.agents/skills/add-tools/SKILL.md b/.agents/skills/add-tools/SKILL.md index 5261ea8169d..f2b1972c215 100644 --- a/.agents/skills/add-tools/SKILL.md +++ b/.agents/skills/add-tools/SKILL.md @@ -620,7 +620,7 @@ If creating V2 tools (API-aligned outputs), use `_v2` suffix: - [ ] Index.ts exports all tools and re-exports types (`export * from './types'`) - [ ] Tools registered in `tools/registry.ts` - [ ] `bun run generate:tool-metadata` run and the regenerated artifacts committed -- [ ] `bun run scripts/generate-docs.ts` run and the refreshed docs committed — the integration's +- [ ] `bun run generate:docs` run and the refreshed docs committed — the integration's docs page is rendered from each tool's description, params, and outputs, and CI's `bun run check:docs` fails on stale pages - [ ] Block wired: `tools.access`, dropdown options, subBlocks, `tools.config`, outputs, inputs diff --git a/.agents/skills/add-trigger/SKILL.md b/.agents/skills/add-trigger/SKILL.md index 7f8a3c7f155..400cc8ba1a6 100644 --- a/.agents/skills/add-trigger/SKILL.md +++ b/.agents/skills/add-trigger/SKILL.md @@ -543,6 +543,6 @@ registered `InternalToolConfig.operation`; a `directExecution` property fails - [ ] `bun run type-check` passes - [ ] Manually verify output keys match trigger `outputs` keys - [ ] Trigger UI shows correctly in the block -- [ ] Ran `bun run scripts/generate-docs.ts` and committed the refreshed pages — trigger sections +- [ ] Ran `bun run generate:docs` and committed the refreshed pages — trigger sections render into the owning integration's docs page, and CI's `bun run check:docs` fails on stale pages diff --git a/.agents/skills/db-migrate/SKILL.md b/.agents/skills/db-migrate/SKILL.md index e39dd005cf0..3628a980a75 100644 --- a/.agents/skills/db-migrate/SKILL.md +++ b/.agents/skills/db-migrate/SKILL.md @@ -83,7 +83,7 @@ The lint flags risky *shapes*; it cannot know whether a given drop is *safe righ ``` The reason must be specific and name the PR/version that removed the dependency. An empty reason fails the lint. - **Warnings** (`data-backfill`): non-blocking, but confirm the batching/idempotency before merging. -4. Regenerate the test schema mock: `bun run scripts/generate-schema-mock.ts` (`check:schema-mock` in `check:audits` fails after any `schema.ts` change until you do). +4. Regenerate the test schema mock: `bun run generate:schema-mock` (`check:schema-mock` in `check:audits` fails after any `schema.ts` change until you do). 5. Re-run `(cd packages/db && bunx drizzle-kit generate)` once your migration is written: it must report no schema changes and write no new file, or CI fails on the schema and migrations disagreeing. 6. Verify locally: `cd packages/db && bun run db:migrate` against a dev DB. diff --git a/.agents/skills/validate-integration/SKILL.md b/.agents/skills/validate-integration/SKILL.md index ee0b092cddf..40b2472510a 100644 --- a/.agents/skills/validate-integration/SKILL.md +++ b/.agents/skills/validate-integration/SKILL.md @@ -443,12 +443,10 @@ After reporting, fix every **critical** and **warning** issue. Apply **suggestio ### Regenerate Derived Artifacts -Several files are generated from tool and block definitions. Editing a tool or block WITHOUT regenerating them fails CI, so run these before pushing: +Several files are generated from tool and block definitions. Editing a tool or block WITHOUT regenerating them fails CI, so run this before pushing: ```bash -bun run generate:tool-metadata # repo root — apps/sim/tools/generated/* -bun run scripts/generate-docs.ts # docs .mdx + deployment-config/integrations.json + docs icons -bun run generate:deployment-config # canonical OAuth registry + catalog → provider-ID facts +bun run generate # every generator below (and any other generate:/check: pair) bun run check:integration-catalog # registry ↔ committed deployment metadata drift bun run check:docs # committed docs ↔ what the generator renders today bun run check:deployment-config # OAuth registry/catalog ↔ provider-ID fact drift @@ -457,12 +455,12 @@ bun run apps/sim/scripts/check-block-registry.ts origin/staging # block ↔ too ``` - **`generate:tool-metadata`** — required whenever a tool's `outputs`, `params`, or descriptions change. CI enforces this with `bun run check:tool-metadata`, which fails with *"Generated tool metadata is stale"*. This is the easiest gate to miss, because nothing in the tool file hints that a generated artifact mirrors it. -- **`generate-docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `packages/deployment-config/src/integrations.json`, and the docs copy of `components/icons.tsx`. +- **`generate:docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `packages/deployment-config/src/integrations.json`, and the docs copy of `components/icons.tsx`. - **`generate:deployment-config`** — required for OAuth or service-account changes. Regenerates provider-ID facts from the canonical OAuth registry and integration catalog; special deployment requirements remain handwritten policy. - **`check:integration-catalog`** — loads the executable block registry, derives visible integration deployment fields, and compares them with the committed catalog. It catches missing/unexpected entries and stale auth/service IDs without loading the executable registry in client code. -- **`check:docs`** — check mode of `generate-docs.ts`: renders every generated docs artifact in +- **`check:docs`** — check mode of `generate:docs`: renders every generated docs artifact in memory and fails listing any committed file that differs. Runs in CI via `check:audits`. **Always diff the regen output before committing — but commit all of it.** These generators rewrite @@ -525,7 +523,7 @@ After fixing, confirm: - [ ] Reported all issues grouped by severity - [ ] Fixed all critical and warning issues - [ ] Ran `bun run generate:tool-metadata` if any tool outputs/params changed, and confirmed `bun run check:tool-metadata` passes -- [ ] Ran `bun run scripts/generate-docs.ts` if any block metadata changed, and committed the full generated diff — including stale-page catch-up for other integrations (`bun run check:docs` fails CI on reverted generator output) +- [ ] Ran `bun run generate:docs` if any block metadata changed, and committed the full generated diff — including stale-page catch-up for other integrations (`bun run check:docs` fails CI on reverted generator output) - [ ] Validated the docs page has an accurate `MANUAL-CONTENT-START:intro` section - [ ] Ran `bun run lint` after fixes - [ ] Verified TypeScript compiles clean diff --git a/.claude/rules/sim-testing.md b/.claude/rules/sim-testing.md index 4250aa87088..867f6393387 100644 --- a/.claude/rules/sim-testing.md +++ b/.claude/rules/sim-testing.md @@ -124,7 +124,7 @@ Global (installed by `vitest.setup.ts` — knobs only, no `vi.mock`): | `@/lib/auth` | `authMockFns.mockGetSession` (+ `auth.api.*`: `mockSignOut`, `mockSignInSSO`, `mockCreateUser`, …; `auth.$context.internalAdapter`: `mockCreateSession`, …) (`auth.mock`) | | `@/lib/auth/hybrid` | `hybridAuthMockFns` (`hybrid-auth.mock`) | | `@sim/db`, `drizzle-orm` | `dbChainMockFns.*`, `queueTableRows`, `resetDbChainMock`, `hasMockCondition`, `databaseMockFns.mockResolveDbUrl`; `@sim/db` re-exports every schema table; `sql` is a spy (`vi.mocked(sql).mock.calls`) (`database.mock`) | -| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run scripts/generate-schema-mock.ts`) (`schema.mock`) | +| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run generate:schema-mock`) (`schema.mock`) | | `@trigger.dev/sdk` | `triggerSdkMockFns` (`task()` returns its config, so call `.run`; `runs.*`, `tasks.*`, `idempotencyKeys`), `MockTriggerApiError` (`trigger-sdk.mock`) | | `@/components/icons` | stable stub per icon name — no knobs, never re-mock | | `@/lib/core/config/env` | `setEnv`, `resetEnvMock`, `envMockFns.getEnv` (`env.mock`) | diff --git a/.cursor/rules/sim-testing.mdc b/.cursor/rules/sim-testing.mdc index d4906ad3d77..e8ab580643a 100644 --- a/.cursor/rules/sim-testing.mdc +++ b/.cursor/rules/sim-testing.mdc @@ -121,7 +121,7 @@ Global (installed by `vitest.setup.ts` — knobs only, no `vi.mock`): | `@/lib/auth` | `authMockFns.mockGetSession` (+ `auth.api.*`: `mockSignOut`, `mockSignInSSO`, `mockCreateUser`, …; `auth.$context.internalAdapter`: `mockCreateSession`, …) (`auth.mock`) | | `@/lib/auth/hybrid` | `hybridAuthMockFns` (`hybrid-auth.mock`) | | `@sim/db`, `drizzle-orm` | `dbChainMockFns.*`, `queueTableRows`, `resetDbChainMock`, `hasMockCondition`, `databaseMockFns.mockResolveDbUrl`; `@sim/db` re-exports every schema table; `sql` is a spy (`vi.mocked(sql).mock.calls`) (`database.mock`) | -| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run scripts/generate-schema-mock.ts`) (`schema.mock`) | +| `@sim/db/schema` | `schemaMock`, generated from `packages/db/schema.ts` (`bun run generate:schema-mock`) (`schema.mock`) | | `@trigger.dev/sdk` | `triggerSdkMockFns` (`task()` returns its config, so call `.run`; `runs.*`, `tasks.*`, `idempotencyKeys`), `MockTriggerApiError` (`trigger-sdk.mock`) | | `@/components/icons` | stable stub per icon name — no knobs, never re-mock | | `@/lib/core/config/env` | `setEnv`, `resetEnvMock`, `envMockFns.getEnv` (`env.mock`) | diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 454529f0dfd..c009be91430 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -516,10 +516,10 @@ In addition, you will need to update the registries: Ensure that your tool functions correctly by making test requests and verifying the responses. 8. **Generate Documentation:** - Run the documentation generator (from `apps/sim`) to create docs for your new tool: + Regenerate every derived artifact, including your tool's docs page, from the repo root: ```bash - cd apps/sim && bun run generate-docs + bun run generate ``` ### Naming Conventions diff --git a/.husky/pre-push b/.husky/pre-push index 8d6d1f415fe..689a013dbd7 100755 --- a/.husky/pre-push +++ b/.husky/pre-push @@ -1,6 +1,7 @@ #!/bin/sh # Checks only what this push changes against its merge base with origin/staging (the PR base): -# biome format + lint on the changed files, then type-check for the affected workspaces. +# biome format + lint on the changed files, type-check for the affected workspaces, then +# `bun run generate` so no committed generated artifact is left stale. # CI runs the full lint, type-check and audit gate on every PR, so it stays the backstop. set -e @@ -42,3 +43,18 @@ if printf '%s\n' "$changed" | grep -qE '^scripts/'; then bun run type-check:scripts:esm fi TURBO_SCM_BASE="$base" bunx turbo run type-check --affected --output-logs=errors-only + +# A stale generated artifact would only fail CI's check:audits. Generators read the working tree, +# so the result describes the pushed commit only when the tree is that commit; with uncommitted or +# untracked files the check is left to CI rather than judging (or blocking) unrelated work. +if git diff --quiet HEAD && [ -z "$(git ls-files --others --exclude-standard)" ]; then + bun run generate >/dev/null + if ! git diff --quiet; then + echo "pre-push: generated artifacts were stale and have been regenerated:" + git diff --name-only + echo "pre-push: commit them and push again." + exit 1 + fi +else + echo "pre-push: uncommitted changes; leaving generated-artifact freshness to CI" +fi diff --git a/CLAUDE.md b/CLAUDE.md index 3a5b7acdf9a..299ed9635b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,6 +69,7 @@ Build order **Tools** → **Block** → **Icon** → optional **Trigger**, from Before declaring a change done, run the local gate from the repo root; CI fails on any of them: ```bash +bun run generate # rewrites every committed generated artifact (docs, metadata, fixtures, …) bun run lint # biome format + lint, autofixes (CI runs lint:check) bun run type-check # every workspace bun run check:audits # every check:* audit, generated-artifact checks included @@ -77,4 +78,6 @@ git fetch origin staging # the block-registry check diffs against it bun run apps/sim/scripts/check-block-registry.ts origin/staging ``` +A generated artifact gets a `generate:` / `check:` pair in the root package.json; `bun run generate` and `check:audits` pick it up from there (`check:generators`), so never regenerate one from a test. + Author a `packages/db` schema change or migration with the `db-migrate` skill and run `bun run check:migrations` (not in `check:audits`). Ratchet baselines (`scripts/*baseline.json`) only shrink: regenerate one with `--update` after removing violations. The one exception is `check:tool-registry-boundary`, whose module-count baseline is re-recorded with `--update-baseline` when growth is deliberate (see the `tool-registry-boundary` skill). diff --git a/apps/docs/EDITORIAL_GUIDE.md b/apps/docs/EDITORIAL_GUIDE.md index e7fb832e5ce..c13c519aaff 100644 --- a/apps/docs/EDITORIAL_GUIDE.md +++ b/apps/docs/EDITORIAL_GUIDE.md @@ -87,7 +87,7 @@ Edit generated prose at its owner, then regenerate: | MCP operation catalog | Authorized operation definitions and generated OpenAPI | `bun run generate:mcp-operations` | | v2 route table | Canonical route definitions | `bun run generate:v2-route-table` | | Marked Agent streaming table | Streaming protocol metadata and `scripts/sync-agent-stream-docs.ts` | `bun run generate:agent-stream-docs` | -| Integration references, icons, navigation inventory and public catalog | Block/tool/trigger and OAuth metadata; authored `MANUAL-CONTENT` regions; registered credential guides; `scripts/generate-docs.ts` | `bun run scripts/generate-docs.ts` | +| Integration references, icons, navigation inventory and public catalog | Block/tool/trigger and OAuth metadata; authored `MANUAL-CONTENT` regions; registered credential guides; `scripts/generate-docs.ts` | `bun run generate:docs` | Do not hand-edit generated pages, JSON specifications, or marked generated regions. Preserve all argument, option, schema, authentication, and confirmation detail. CLI descriptions also feed command discovery; run the existing search-ranking checks after changing help copy. Validate rendered request examples against their canonical input contracts; schema validity alone may still leave a semantic error such as an invalid serialized workflow. diff --git a/apps/sim/package.json b/apps/sim/package.json index 1f25af6afa7..1dbee920890 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -50,7 +50,6 @@ "format:check": "biome format .", "generate:pi-model-catalog": "bun run scripts/generate-pi-model-catalog.ts", "generate:disposable-email-domains": "bun run scripts/generate-disposable-email-domains.ts", - "generate-docs": "bun run ../../scripts/generate-docs.ts", "org:consolidate-users": "bun run scripts/consolidate-users-into-organization.ts", "verify:oracledb": "node scripts/oracledb-patch-self-test.cjs --worker lib/internal/oracledb/oracle-worker.cjs", "build:cli-runtime": "bun build ../../packages/sim-cli/src/runtime.ts --target=node --format=esm --packages=bundle --outfile=../../packages/sim-cli/dist/runtime.js", diff --git a/package.json b/package.json index 1791b665d99..724c6f04ee4 100644 --- a/package.json +++ b/package.json @@ -80,8 +80,10 @@ "check:script-tests": "bun run scripts/check-script-test-coverage.ts", "check:test-patterns": "bun run scripts/check-test-patterns.ts", "check:sealed-columns": "bun run scripts/check-sealed-columns.ts", + "generate:schema-mock": "bun run scripts/generate-schema-mock.ts", "check:schema-mock": "bun run scripts/generate-schema-mock.ts --check", "check:audits": "bun run scripts/run-audits.ts", + "check:generators": "bun run scripts/check-generators.ts", "generate": "bun run scripts/generate-all.ts", "check:skills": "bun run scripts/sync-skills.ts --check", "check:guidance-refs": "bun run scripts/check-guidance-refs.ts", @@ -106,12 +108,13 @@ "generate:sdk-fixture-catalog": "bun run apps/sim/scripts/generate-sdk-fixture-catalog.ts", "check:sdk-fixture-catalog": "bun run apps/sim/scripts/generate-sdk-fixture-catalog.ts --check", "check:sdk-names-frozen": "bun run scripts/check-sdk-names-frozen.ts", + "generate:docs": "bun run scripts/generate-docs.ts", + "check:docs": "bun run scripts/generate-docs.ts --check", "generate:deployment-config": "bun run scripts/generate-deployment-config.ts", "check:deployment-config": "bun run scripts/generate-deployment-config.ts --check", "images:generate": "bun run scripts/generate-image-manifest.ts", "images:check": "bun run scripts/generate-image-manifest.ts --check", "check:integration-catalog": "bun run scripts/check-integration-catalog.ts", - "check:docs": "bun run scripts/generate-docs.ts --check", "mship-tools:generate": "bun run scripts/sync-tool-catalog.ts", "mship-tools:check": "bun run scripts/sync-tool-catalog.ts --check", "trace-spans-contract:generate": "bun run scripts/sync-trace-spans-contract.ts", diff --git a/packages/testing/src/mocks/schema-tables.generated.ts b/packages/testing/src/mocks/schema-tables.generated.ts index a9680087445..29d85c5a3dd 100644 --- a/packages/testing/src/mocks/schema-tables.generated.ts +++ b/packages/testing/src/mocks/schema-tables.generated.ts @@ -1,6 +1,6 @@ /** * Generated by `scripts/generate-schema-mock.ts` from `packages/db/schema.ts` — do not edit. - * Regenerate with `bun run scripts/generate-schema-mock.ts`; `check:schema-mock` fails on drift. + * Regenerate with `bun run generate:schema-mock`; `check:schema-mock` fails on drift. */ /** Every `pgTable` export mapped to its column property names. */ diff --git a/packages/testing/src/mocks/schema.mock.ts b/packages/testing/src/mocks/schema.mock.ts index 0d51eb2ebab..cf18d088c76 100644 --- a/packages/testing/src/mocks/schema.mock.ts +++ b/packages/testing/src/mocks/schema.mock.ts @@ -8,7 +8,7 @@ import { * Comprehensive mock for `@sim/db/schema`, derived from `packages/db/schema.ts`. * * `schema-tables.generated.ts` lists every table's column names, every enum's values and every - * literal constant (`bun run scripts/generate-schema-mock.ts`; `check:schema-mock` fails on drift), + * literal constant (`bun run generate:schema-mock`; `check:schema-mock` fails on drift), * so a column added to the real schema is present here without a hand edit. Only names are copied — * no drizzle table object is imported at runtime. * diff --git a/scripts/README.md b/scripts/README.md index 5ad330b17f0..4e36afceb44 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -27,8 +27,8 @@ The block references actions by id in `tools.access`; the generator looks each o ## What the generator does -Run with `cd apps/sim && bun run generate-docs` (or `bun run scripts/generate-docs.ts` -from the repo root). One pass (`generateAllBlockDocs`): +Run with `bun run generate:docs` from the repo root (`bun run generate` runs it with every +other generator). One pass (`generateAllBlockDocs`): 1. **Copies icons** `apps/sim/components/icons.tsx` → `apps/docs/components/icons.tsx` and builds `apps/docs/components/ui/icon-mapping.ts`. diff --git a/scripts/check-generators.ts b/scripts/check-generators.ts new file mode 100644 index 00000000000..0d54a3cb92a --- /dev/null +++ b/scripts/check-generators.ts @@ -0,0 +1,46 @@ +#!/usr/bin/env bun +/** + * Enforces the generated-artifact contract in `scripts/generators.ts`: + * + * - every `generate:` has a `check:`, so CI fails when its artifact drifts; + * - every `check:` that runs a generator in check mode has a `generate:`, so + * `bun run generate` regenerates it; + * - no test rewrites a committed artifact behind an `UPDATE_*` environment flag, where nobody + * runs it and the drift only surfaces in a test shard after merge. + * + * Run: `bun run check:generators` + */ +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import path from 'node:path' +import { findPairingViolations, findRegeneratingTests } from './generators' + +const ROOT = path.resolve(import.meta.dir, '..') + +function trackedTestFiles(): string[] { + return execFileSync('git', ['ls-files', '-z', '--', '*.test.ts', '*.test.tsx'], { + cwd: ROOT, + encoding: 'utf8', + }) + .split('\0') + .filter(Boolean) +} + +function main(): void { + const { scripts } = JSON.parse(readFileSync(path.join(ROOT, 'package.json'), 'utf8')) as { + scripts: Record + } + const tests = trackedTestFiles().map((file) => ({ + path: file, + content: readFileSync(path.join(ROOT, file), 'utf8'), + })) + const violations = [...findPairingViolations(scripts), ...findRegeneratingTests(tests)] + + if (violations.length > 0) { + console.error(`✗ generated-artifact contract:\n- ${violations.join('\n- ')}`) + process.exit(1) + } + console.log('✓ every generated artifact has a generate/check pair') +} + +if (import.meta.main) main() diff --git a/scripts/check-integration-catalog.ts b/scripts/check-integration-catalog.ts index 546f85eba7b..169be95a1f5 100644 --- a/scripts/check-integration-catalog.ts +++ b/scripts/check-integration-catalog.ts @@ -146,7 +146,7 @@ function verifyIntegrationCatalog(): void { ) if (JSON.stringify(INTEGRATION_METADATA) !== JSON.stringify(expectedMetadata)) { throw new Error( - 'Integration metadata is stale. Run `bun run scripts/generate-docs.ts` and commit the result.' + 'Integration metadata is stale. Run `bun run generate:docs` and commit the result.' ) } const expectedByType = new Map(expected.map((block) => [block.type, expectedEntry(block)])) @@ -190,7 +190,7 @@ function verifyIntegrationCatalog(): void { if (issues.length > 0) { throw new Error( - `Generated integration catalog is stale:\n- ${issues.join('\n- ')}\nRun \`bun run scripts/generate-docs.ts\` and commit the generated catalog.` + `Generated integration catalog is stale:\n- ${issues.join('\n- ')}\nRun \`bun run generate:docs\` and commit the generated catalog.` ) } process.stdout.write( diff --git a/scripts/generate-all.ts b/scripts/generate-all.ts index 584fc702ea8..dd3efd46093 100644 --- a/scripts/generate-all.ts +++ b/scripts/generate-all.ts @@ -1,38 +1,26 @@ #!/usr/bin/env bun /** - * Runs every self-contained, repo-local generator, so a contributor (or `/ship`) can bring all - * committed artifacts back in sync with one command: `bun run generate`. + * Brings every committed generated artifact back in sync with one command: `bun run generate`. * - * The list is derived from package.json the way `run-audits.ts` derives its audits, so a new - * generator is picked up by default rather than added to a list here. A generator qualifies when - * it has a matching audit: every `generate:` that has a `check:`, plus the older names in - * {@link ALIASED_PAIRS}. That pairing is the repo's convention for "this artifact is a pure - * function of the tree" (`check:audits` runs the check), so it is also what makes a generator safe - * to run unprompted. Name a new generator `generate:` / `check:` and it runs here. + * It runs the generators `scripts/generators.ts` derives from package.json, so a new + * `generate:` / `check:` pair is picked up with no list to edit here; `check:generators` + * fails CI when a pair is incomplete. * - * Deliberately not run: the `:generate` / `:check` suffix pairs (`mship*`, `trace-*`, - * `metrics`, `vfs`, `billing-protocol`, `images`, `library:covers`), which need the sibling - * copilot repo, a Helm render, or a per-machine renderer (see `run-audits.ts`); - * `generate:pi-model-catalog` under `apps/sim`, which reads an installed package; and - * `scripts/generate-docs.ts`, whose broad integration-docs rewrite has no `generate:` script. + * Generators run one at a time, in package.json order, because some read what an earlier one + * wrote: `generate:cli-docs` reads the operation table `generate:cli-api` emits, + * `generate:deployment-config` imports the integration metadata `generate:docs` writes, + * `generate:docs-manifest` lists the pages `generate:cli-docs` and `generate:docs` write, and + * `generate:sdk-fixture-catalog` projects through `generate:tool-metadata` and `generate:sdk-names` + * output. Keep a new pair below the generators it reads. * - * Generators run one at a time, in package.json order, because they are not independent: some - * read what an earlier one wrote. `generate:cli-docs` reads the operation table - * `generate:cli-api` emits, `generate:docs-manifest` lists the pages `generate:cli-docs` writes, - * and `generate:sdk-fixture-catalog` projects through `generate:tool-metadata` and - * `generate:sdk-names` output. Keep a new pair below the generators it reads. + * Not run: the `:generate` / `:check` suffix pairs, which need the sibling copilot repo, a + * Helm render, or a per-machine renderer (see `run-audits.ts`), and workspace-local scripts such + * as `apps/sim`'s `generate:pi-model-catalog`, which reads an installed package. * * Exits non-zero when any generator fails; the rest still run so one failure does not hide another. - * - * Usage: - * bun run generate */ import path from 'node:path' - -/** Generators whose audit predates the `generate:` / `check:` naming, keyed to that audit. */ -const ALIASED_PAIRS: Record = { - 'skills:sync': 'check:skills', -} +import { listGenerators } from './generators' const ROOT = path.resolve(import.meta.dir, '..') @@ -42,14 +30,6 @@ interface GeneratorResult { durationMs: number } -function generatorScripts(scripts: Record): string[] { - return Object.keys(scripts).filter((name) => { - if (name.startsWith('generate:')) return `check:${name.slice('generate:'.length)}` in scripts - const audit = ALIASED_PAIRS[name] - return audit !== undefined && audit in scripts - }) -} - /** * Runs one generator with inherited output. * @@ -70,7 +50,7 @@ async function runGenerator(script: string, command: string): Promise -const generators = generatorScripts(commands) +const generators = listGenerators(commands) const results: GeneratorResult[] = [] for (const script of generators) { diff --git a/scripts/generate-docs.ts b/scripts/generate-docs.ts index 5dff1542010..f371dda0e1c 100755 --- a/scripts/generate-docs.ts +++ b/scripts/generate-docs.ts @@ -5702,7 +5702,7 @@ if (import.meta.main) { if (stale.length > 0) { console.error( `Generated integration docs are stale:\n- ${stale.join('\n- ')}\n` + - 'Run `bun run scripts/generate-docs.ts` and commit the result.' + 'Run `bun run generate:docs` and commit the result.' ) process.exit(1) } diff --git a/scripts/generate-schema-mock.ts b/scripts/generate-schema-mock.ts index 368eec4d903..f8afb32526c 100644 --- a/scripts/generate-schema-mock.ts +++ b/scripts/generate-schema-mock.ts @@ -11,7 +11,7 @@ * re-mocking `@sim/db/schema` locally just because a new column was missing. Only names and * literal values are copied; the mock never imports drizzle table objects at runtime. * - * Run: `bun run scripts/generate-schema-mock.ts` (write) or `--check` (fail on drift; collected by + * Run: `bun run generate:schema-mock` (write) or `--check` (fail on drift; collected by * `check:audits` as `check:schema-mock`). */ import { readFileSync, writeFileSync } from 'node:fs' @@ -121,7 +121,7 @@ function render(): string { const json = (value: unknown) => JSON.stringify(value, null, 2) return `/** * Generated by \`scripts/generate-schema-mock.ts\` from \`packages/db/schema.ts\` — do not edit. - * Regenerate with \`bun run scripts/generate-schema-mock.ts\`; \`check:schema-mock\` fails on drift. + * Regenerate with \`bun run generate:schema-mock\`; \`check:schema-mock\` fails on drift. */ /** Every \`pgTable\` export mapped to its column property names. */ @@ -156,7 +156,7 @@ if (process.argv.includes('--check')) { if (current !== output) { console.error( 'packages/testing/src/mocks/schema-tables.generated.ts is out of date with packages/db/schema.ts.\n' + - 'Run: bun run scripts/generate-schema-mock.ts' + 'Run: bun run generate:schema-mock' ) process.exit(1) } diff --git a/scripts/generators.test.ts b/scripts/generators.test.ts new file mode 100644 index 00000000000..3eab2ce37f5 --- /dev/null +++ b/scripts/generators.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' +import { findPairingViolations, findRegeneratingTests, listGenerators } from './generators' + +describe('findPairingViolations', () => { + it('accepts a generator whose audit runs it with --check', () => { + expect( + findPairingViolations({ + 'generate:docs': 'bun run scripts/generate-docs.ts', + 'check:docs': 'bun run scripts/generate-docs.ts --check', + }) + ).toEqual([]) + }) + + it('flags a generator with no audit', () => { + expect(findPairingViolations({ 'generate:docs': 'bun run scripts/generate-docs.ts' })).toEqual([ + expect.stringContaining('`generate:docs` has no `check:docs`'), + ]) + }) + + it('flags a check-mode generator that nothing regenerates', () => { + expect( + findPairingViolations({ 'check:docs': 'bun run scripts/generate-docs.ts --check' }) + ).toEqual([ + expect.stringContaining('Add `"generate:docs": "bun run scripts/generate-docs.ts"`'), + ]) + }) + + it('accepts a standalone check-* audit that takes --check', () => { + expect( + findPairingViolations({ + 'check:client-boundary': 'bun run scripts/check-client-boundary-imports.ts --check', + }) + ).toEqual([]) + }) + + it('accepts an aliased generator and flags its audit when the alias is missing', () => { + const audit = { 'check:skills': 'bun run scripts/sync-skills.ts --check' } + expect( + findPairingViolations({ ...audit, 'skills:sync': 'bun run scripts/sync-skills.ts' }) + ).toEqual([]) + expect(findPairingViolations(audit)).toHaveLength(1) + }) +}) + +describe('listGenerators', () => { + it('lists paired and aliased generators in package.json order and skips unpaired ones', () => { + expect( + listGenerators({ + 'skills:sync': 'bun run scripts/sync-skills.ts', + 'generate:a': 'bun run a.ts', + 'check:a': 'bun run a.ts --check', + 'generate:pi-model-catalog': 'bun run b.ts', + 'check:skills': 'bun run scripts/sync-skills.ts --check', + }) + ).toEqual(['skills:sync', 'generate:a']) + }) +}) + +describe('findRegeneratingTests', () => { + it('flags a test that rewrites an artifact behind an UPDATE_ flag and ignores non-tests', () => { + const flag = 'UPDATE_SDK_FIXTURE_CATALOG' + const content = `if (process.env.${flag} === '1') writeFileSync(p, text)` + expect( + findRegeneratingTests([ + { path: 'apps/sim/lib/catalog.test.ts', content }, + { path: 'scripts/generate-x.ts', content }, + ]) + ).toEqual([expect.stringContaining('apps/sim/lib/catalog.test.ts regenerates')]) + }) +}) diff --git a/scripts/generators.ts b/scripts/generators.ts new file mode 100644 index 00000000000..f4e86f8f633 --- /dev/null +++ b/scripts/generators.ts @@ -0,0 +1,91 @@ +/** + * The repo's contract for committed generated artifacts, shared by `bun run generate` + * (`generate-all.ts`) and its audit (`check-generators.ts`) so the two can never disagree. + * + * An artifact derived purely from the tree gets one script pair in the root package.json: + * `generate:` rewrites it and `check:` fails when it is stale (usually the same command plus + * `--check`; an audit may validate further, as `check:openapi` does). `check:audits` runs every + * `check:*`, so drift fails CI, and `bun run generate` runs every generator, so one command fixes + * it. Nothing else lists generators: a new pair is picked up by both sides automatically. + */ + +/** Generators whose audit predates the `generate:` / `check:` naming, keyed to that audit. */ +export const GENERATOR_ALIASES: Readonly> = { + 'skills:sync': 'check:skills', +} + +const CHECK_FLAG = ' --check' + +/** The audit that guards a generator script, or `null` when the name is not a generator. */ +function auditFor(script: string): string | null { + if (script.startsWith('generate:')) return `check:${script.slice('generate:'.length)}` + return GENERATOR_ALIASES[script] ?? null +} + +/** + * Whether a `check:*` command runs a generator in check mode rather than a standalone audit. + * Standalone audits live in `check-*.ts` files, some of which also take `--check`. + */ +function runsGeneratorInCheckMode(command: string): boolean { + if (!command.endsWith(CHECK_FLAG)) return false + const file = command.slice(0, -CHECK_FLAG.length).trim().split(/\s+/).at(-1) ?? '' + return !/(^|\/)check-[^/]*$/.test(file) +} + +/** Every generator `bun run generate` runs, in package.json order (later ones may read earlier output). */ +export function listGenerators(scripts: Readonly>): string[] { + return Object.keys(scripts).filter((script) => { + const audit = auditFor(script) + return audit !== null && audit in scripts + }) +} + +/** Ways the root scripts break the generator contract, one message per problem. */ +export function findPairingViolations(scripts: Readonly>): string[] { + const violations: string[] = [] + const generatorByAudit = new Map() + + for (const [script, command] of Object.entries(scripts)) { + const audit = auditFor(script) + if (audit === null) continue + generatorByAudit.set(audit, script) + if (!(audit in scripts)) { + violations.push( + `\`${script}\` has no \`${audit}\`, so its artifact can drift without failing CI. Add \`"${audit}": "${command}${CHECK_FLAG}"\`.` + ) + } + } + + for (const [script, command] of Object.entries(scripts)) { + if (!script.startsWith('check:') || generatorByAudit.has(script)) continue + if (!runsGeneratorInCheckMode(command)) continue + const generator = `generate:${script.slice('check:'.length)}` + violations.push( + `\`${script}\` checks a generated artifact but nothing regenerates it, so \`bun run generate\` skips it. Add \`"${generator}": "${command.slice(0, -CHECK_FLAG.length)}"\`.` + ) + } + + return violations +} + +const TEST_FILE = /\.test\.tsx?$/ +const REGENERATE_FLAG = /process\.env\.(UPDATE_[A-Z0-9_]+)/ + +/** + * Test files that rewrite a committed artifact behind an environment flag. Such a test is a + * generator nobody runs: the artifact only goes stale in CI's test shard, after merge. Move the + * projection into a `generate:` script instead. + */ +export function findRegeneratingTests( + files: ReadonlyArray<{ path: string; content: string }> +): string[] { + return files.flatMap(({ path, content }) => { + if (!TEST_FILE.test(path)) return [] + const flag = content.match(REGENERATE_FLAG)?.[1] + return flag + ? [ + `${path} regenerates a committed artifact behind \`${flag}\`. Move it into a \`generate:\` / \`check:\` script pair.`, + ] + : [] + }) +}