diff --git a/.agents/skills/add-column-type/SKILL.md b/.agents/skills/add-column-type/SKILL.md index 0f5065d8dab..debc783f044 100644 --- a/.agents/skills/add-column-type/SKILL.md +++ b/.agents/skills/add-column-type/SKILL.md @@ -145,8 +145,8 @@ Copy `currency`'s service function and orchestration branch. The workspace source table codec reads the registry, but the SDK restates each type: add the `column.()` helper to `packages/workflow-sdk/src/workspace/table.d.ts` (the drift test in -`lib/workspace-source/application/kinds/table/codec.test.ts` fails until you do), map the type's row -value in `columnType()` (`packages/workflow-sdk/src/codegen/workspace.ts`, which defaults to +`apps/sim/lib/workspace-source/application/kinds/table/codec.test.ts` fails until you do), map the +type's row value in `valueType()` (`packages/workflow-sdk/src/codegen/workspace.ts`, which defaults to `string`), and regenerate the embedded text (`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`). See `.claude/rules/sim-workflow-code.md`. diff --git a/.claude/rules/sim-workflow-code.md b/.claude/rules/sim-workflow-code.md index 8bf5712ad06..86f2404658c 100644 --- a/.claude/rules/sim-workflow-code.md +++ b/.claude/rules/sim-workflow-code.md @@ -134,7 +134,7 @@ Follow `/add-column-type`. The workspace codec reads the column registry - the `column.()` helper in `packages/workflow-sdk/src/workspace/table.d.ts` (`kinds/table/codec.test.ts` fails until it exists), then regenerate the embedded text (`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`); -- `columnType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')` +- `valueType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')` gives, which defaults to `string`; - a type that owns metadata: `TABLE_COLUMN_METADATA_KEYS` in `packages/workspace-code/src/kinds/table.ts` plus the reader and printer there. diff --git a/.cursor/rules/sim-workflow-code.mdc b/.cursor/rules/sim-workflow-code.mdc index c631f32be1b..65c08f7ef36 100644 --- a/.cursor/rules/sim-workflow-code.mdc +++ b/.cursor/rules/sim-workflow-code.mdc @@ -114,7 +114,7 @@ Follow `/add-column-type`. The workspace codec reads the column registry - the `column.()` helper in `packages/workflow-sdk/src/workspace/table.d.ts` (`kinds/table/codec.test.ts` fails until it exists), then regenerate the embedded text (`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`); -- `columnType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')` +- `valueType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')` gives, which defaults to `string`; - a type that owns metadata: `TABLE_COLUMN_METADATA_KEYS` in `packages/workspace-code/src/kinds/table.ts` plus the reader and printer there. diff --git a/.github/workflows/publish-sim-cli.yml b/.github/workflows/publish-sim-cli.yml index 4ba40ae9560..7932e49c88d 100644 --- a/.github/workflows/publish-sim-cli.yml +++ b/.github/workflows/publish-sim-cli.yml @@ -7,7 +7,9 @@ on: - 'packages/sim-cli/**' # The build bundles its workspace dependencies, so a change there # changes what ships. `bun run check:sim-cli-bundle` fails when this - # list and the bundle's workspace packages disagree. + # list and the bundle's workspace packages disagree. bun.lock is not + # watched: nearly every lockfile change leaves the bundle byte-identical, + # and the next change to a watched package ships the ones that do not. - 'packages/table-cells/**' - 'packages/utils/**' - 'packages/workflow-code/**' diff --git a/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content/route.ts b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content/route.ts index e0632adfa15..0e90c739d86 100644 --- a/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content/route.ts +++ b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content/route.ts @@ -5,6 +5,7 @@ import { v2KnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { readOriginalKnowledgeDocument } from '@/lib/knowledge/application/read-original-document' import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' export const dynamic = 'force-dynamic' export const revalidate = 0 @@ -14,8 +15,9 @@ export const revalidate = 0 * original bytes. * * What `sim workspaces source get` downloads for each document of a knowledge base. A document - * whose original is not stored (a connector-synced page) answers `NOT_FOUND`. `headSafe: false` - * because the read pulls the bytes out of object storage. + * whose original is not stored (a connector-synced page) answers `NOT_FOUND`. Nothing parses the + * bytes, so any original up to the stored-document limit downloads, not just those under the + * text-extraction cap. `headSafe: false` because the read pulls the bytes out of object storage. */ export const GET = defineV2BinaryRoute({ contract: v2DownloadKnowledgeDocumentContract, @@ -28,6 +30,7 @@ export const GET = defineV2BinaryRoute({ knowledgeBaseId: params.knowledgeBaseId, documentId: params.documentId, assertedWorkspaceId: query.workspaceId, + maxBytes: MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE, }), useCase: readOriginalKnowledgeDocument, present: (document) => { diff --git a/apps/sim/lib/knowledge/__integration__/original-download.integration.ts b/apps/sim/lib/knowledge/__integration__/original-download.integration.ts new file mode 100644 index 00000000000..04533bef68f --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/original-download.integration.ts @@ -0,0 +1,124 @@ +/** + * The v2 knowledge document download route against real PostgreSQL and local object storage, + * through its route handler with a real workspace API key. A download parses nothing, so it serves + * any stored original up to the stored-document limit, including one past the text-extraction cap + * that a parsing read refuses. + */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { apiKey, document, knowledgeBase, organization, user, workspace } from '@sim/db/schema' +import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { generateId } from '@sim/utils/id' +import { eq, inArray } from 'drizzle-orm' +import { NextRequest } from 'next/server' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) + +import { hashApiKey } from '@/lib/api-key/crypto' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { uploadFile } from '@/lib/uploads/core/storage-service' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' +import { MAX_TEXT_EXTRACTION_BYTES } from '@/lib/uploads/utils/file' +import { GET } from '@/app/api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content/route' + +const ids = createKnowledgeAclFixtureIds() +const key = `sk-sim-fixture-${generateId()}` + +async function seedStoredDocument(bytes: Buffer, fileSize = bytes.byteLength) { + const id = generateId() + const storageKey = `kb/${id}.bin` + await uploadFile({ + file: bytes, + fileName: 'original.bin', + contentType: 'application/octet-stream', + context: 'knowledge-base', + customKey: storageKey, + preserveKey: true, + persistMetadata: false, + }) + await db.insert(document).values({ + id, + knowledgeBaseId: ids.knowledgeBaseId, + filename: 'original.bin', + fileUrl: '', + storageKey, + fileSize, + mimeType: 'application/octet-stream', + processingStatus: 'completed', + acl: ['ws'], + aclRequirements: [], + aclVerifiedAt: new Date(), + }) + return id +} + +function download(documentId: string) { + const request = new NextRequest( + `http://localhost:3000/api/v2/knowledge/${ids.knowledgeBaseId}/documents/${documentId}/content?workspaceId=${ids.workspaceId}`, + { headers: { 'x-api-key': key, 'x-forwarded-for': '127.0.0.1' } } + ) + return GET(request, { + params: Promise.resolve({ knowledgeBaseId: ids.knowledgeBaseId, documentId }), + }) +} + +beforeAll(async () => { + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-knowledge-download-')) + await seedKnowledgeAclFixture(ids) + await db.insert(apiKey).values({ + id: generateId(), + userId: ids.aliceId, + workspaceId: ids.workspaceId, + createdBy: ids.aliceId, + name: 'Download fixture', + key, + keyHash: hashApiKey(key), + type: 'workspace', + access: 'read', + }) +}) + +afterAll(async () => { + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await rm(fixtureStorage.root, { recursive: true, force: true }) + await db.$client.end() +}) + +describe('GET /api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content', () => { + it('downloads an original larger than the text-extraction cap', async () => { + const bytes = Buffer.alloc(MAX_TEXT_EXTRACTION_BYTES + 1024 * 1024, 'k') + const documentId = await seedStoredDocument(bytes) + + const response = await download(documentId) + + expect(response.status).toBe(200) + expect(response.headers.get('content-length')).toBe(String(bytes.byteLength)) + expect(Buffer.from(await response.arrayBuffer()).equals(bytes)).toBe(true) + }) + + it('refuses a document recorded past the stored-document limit', async () => { + const documentId = await seedStoredDocument( + Buffer.from('small'), + MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE + 1 + ) + + const response = await download(documentId) + + expect(response.status).toBe(413) + }) +}) diff --git a/apps/sim/lib/knowledge/application/read-original-document.ts b/apps/sim/lib/knowledge/application/read-original-document.ts index 50a4e8f6ed7..c6ecf3231d9 100644 --- a/apps/sim/lib/knowledge/application/read-original-document.ts +++ b/apps/sim/lib/knowledge/application/read-original-document.ts @@ -14,12 +14,17 @@ import { } from '@/lib/knowledge/secret-provenance' import { isObjectNotFoundError } from '@/lib/uploads/core/errors' import { downloadFile } from '@/lib/uploads/core/storage-service' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' import { MAX_TEXT_EXTRACTION_BYTES } from '@/lib/uploads/utils/file' export interface ReadOriginalKnowledgeDocumentInput { knowledgeBaseId: string documentId: string assertedWorkspaceId: string + /** + * Largest original the read accepts. Defaults to the text-extraction cap, for a caller that + * parses the bytes; a raw download raises it up to the stored-document limit. + */ maxBytes?: number signal?: AbortSignal } @@ -41,7 +46,11 @@ export const readOriginalKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ }) => resolveCanonicalActiveKnowledgeDocumentContext(input, principal), async execute({ input, context, principal }) { const maxBytes = input.maxBytes ?? MAX_TEXT_EXTRACTION_BYTES - if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > MAX_TEXT_EXTRACTION_BYTES) + if ( + !Number.isSafeInteger(maxBytes) || + maxBytes < 1 || + maxBytes > MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE + ) throw new OrchestrationError('validation', 'Invalid document read byte limit') input.signal?.throwIfAborted() const doc = context.document diff --git a/bun.lock b/bun.lock index 0cb4d786ab5..52dbfeedb0a 100644 --- a/bun.lock +++ b/bun.lock @@ -769,6 +769,7 @@ "proper-lockfile": "4.1.2", }, "devDependencies": { + "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-code": "workspace:*", @@ -944,6 +945,7 @@ "@sim/workflow-types": "workspace:*", }, "devDependencies": { + "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@types/node": "24.19.2", "@xyflow/react": "12.12.1", diff --git a/packages/sim-cli/package.json b/packages/sim-cli/package.json index 3467697e29e..79417a22522 100644 --- a/packages/sim-cli/package.json +++ b/packages/sim-cli/package.json @@ -59,6 +59,7 @@ "proper-lockfile": "4.1.2" }, "devDependencies": { + "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-code": "workspace:*", diff --git a/packages/sim-cli/src/commands/protocol/logs-follow.test.ts b/packages/sim-cli/src/commands/protocol/logs-follow.test.ts index bdb383787c9..0458fd568cb 100644 --- a/packages/sim-cli/src/commands/protocol/logs-follow.test.ts +++ b/packages/sim-cli/src/commands/protocol/logs-follow.test.ts @@ -1,3 +1,4 @@ +import { utilsHelpersMock, utilsHelpersMockFns } from '@sim/testing/mocks/utils-helpers.mock' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ListLogsResponse } from '../../generated/v2-api' @@ -5,11 +6,9 @@ import { SimApiError } from '../../http/client' import { contextMockFns, contextMockState } from '../../test/context-mock' import { attachLogsFollow, type LogRow } from './logs-follow' -const { mockSleep } = vi.hoisted(() => ({ - mockSleep: vi.fn(() => Promise.resolve()), -})) +const { mockSleep } = utilsHelpersMockFns -vi.mock('@sim/utils/helpers', () => ({ sleep: mockSleep })) +vi.mock('@sim/utils/helpers', () => utilsHelpersMock) vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock) diff --git a/packages/sim-cli/src/commands/protocol/source-status.test.ts b/packages/sim-cli/src/commands/protocol/source-status.test.ts new file mode 100644 index 00000000000..2ed2708c202 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/source-status.test.ts @@ -0,0 +1,12 @@ +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' +import { describe, it } from 'vitest' +import { pathFilter } from './source-status' + +describe('pathFilter', () => { + it('reads a filter with a long run of slashes before its last segment in linear time', () => { + expectLinearGrowth((n) => { + const filter = `wf${'/'.repeat(n)}workflow.ts` + return () => pathFilter([filter], '.') + }, 4_000) + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/source-status.ts b/packages/sim-cli/src/commands/protocol/source-status.ts index 756b2b3dc73..81a70f45900 100644 --- a/packages/sim-cli/src/commands/protocol/source-status.ts +++ b/packages/sim-cli/src/commands/protocol/source-status.ts @@ -7,6 +7,7 @@ import { realpathSync } from 'node:fs' import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { stripTrailingSlashes } from '@sim/utils/string' import type { FileChange, SourceStatus } from '@sim/workspace-code/status' import { createTwoFilesPatch, FILE_HEADERS_ONLY } from 'diff' import type { OutputFormat } from '../../config/index' @@ -353,10 +354,7 @@ function realPath(path: string): string { function filterForms(filter: string, dir: string): string[] { const forms = new Set() if (!isAbsolute(filter)) { - const given = filter - .replace(/\\/g, '/') - .replace(/^(\.\/)+/, '') - .replace(/\/+$/, '') + const given = stripTrailingSlashes(filter.replace(/\\/g, '/').replace(/^(\.\/)+/, '')) forms.add(given === '' ? '.' : given) } const resolved = relative(realPath(resolve(dir)), realPath(resolve(filter))) diff --git a/packages/sim-cli/src/commands/protocol/workflow-run-follow.test.ts b/packages/sim-cli/src/commands/protocol/workflow-run-follow.test.ts index 207d32076ee..7c0987ba354 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-run-follow.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-run-follow.test.ts @@ -1,6 +1,7 @@ import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { jsonResponse } from '@sim/testing/helpers/http' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SimApiError } from '../../http/client' @@ -61,13 +62,6 @@ function _streamResponse(body: ReadableStream): Response { } as unknown as Response } -function jsonResponse(data: Record): Response { - return new Response(JSON.stringify({ data }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }) -} - function ndjsonResponse(...events: Array>): Response { return new Response(bodyOf(events.map((event) => `${JSON.stringify(event)}\n`)), { status: 200, @@ -85,11 +79,13 @@ beforeEach(() => { requestRaw.mockReset() requestRaw.mockImplementation(async () => jsonResponse({ - runId: 'run-1', - workflowId: WORKFLOW_ID, - status: 'completed', - output: {}, - error: null, + data: { + runId: 'run-1', + workflowId: WORKFLOW_ID, + status: 'completed', + output: {}, + error: null, + }, }) ) }) diff --git a/packages/sim-cli/src/commands/protocol/workflow-run-wait.test.ts b/packages/sim-cli/src/commands/protocol/workflow-run-wait.test.ts index 980ece89cbc..0a9fb14ffa3 100644 --- a/packages/sim-cli/src/commands/protocol/workflow-run-wait.test.ts +++ b/packages/sim-cli/src/commands/protocol/workflow-run-wait.test.ts @@ -1,3 +1,4 @@ +import { utilsHelpersMock, utilsHelpersMockFns } from '@sim/testing/mocks/utils-helpers.mock' import { Command } from 'commander' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { attachWorkflowRunWait } from './workflow-run-wait' @@ -22,18 +23,7 @@ vi.mock('../../context', () => ({ }), })) -/** - * Waiting is simulated rather than timed: the mock advances the same clock the - * command reads its deadline from, so a `--wait-timeout 3600` test costs - * nothing and the poll schedule is exactly what the assertions say it is. - */ -vi.mock('@sim/utils/helpers', () => ({ - sleep: (ms: number) => { - sleeps.push(ms) - clock.now += ms - return Promise.resolve() - }, -})) +vi.mock('@sim/utils/helpers', () => utilsHelpersMock) let logged: string[] let errored: string[] @@ -53,6 +43,15 @@ beforeEach(() => { process.exitCode = undefined stderr.isTTY = false + /** + * Waiting is simulated rather than timed: the mock advances the same clock the + * command reads its deadline from, so a `--wait-timeout 3600` test costs + * nothing and the poll schedule is exactly what the assertions say it is. + */ + utilsHelpersMockFns.mockSleep.mockImplementation(async (ms: number) => { + sleeps.push(ms) + clock.now += ms + }) vi.spyOn(Date, 'now').mockImplementation(() => clock.now) vi.spyOn(console, 'log').mockImplementation((line: string) => { logged.push(line) diff --git a/packages/sim-cli/src/embed.test.ts b/packages/sim-cli/src/embed.test.ts index f47df6ce2d5..6cfd42ec86d 100644 --- a/packages/sim-cli/src/embed.test.ts +++ b/packages/sim-cli/src/embed.test.ts @@ -1,3 +1,4 @@ +import { jsonResponse } from '@sim/testing/helpers/http' import { sleep } from '@sim/utils/helpers' import { describe, expect, it, vi } from 'vitest' import { resolveEmbeddedCliCommand, runEmbeddedCli } from './embed' @@ -8,13 +9,6 @@ const IDENTITY = { workspaceId: 'a2e3ab27-2f9d-4b8a-a2f2-3c47a1b0c9d1', } -function jsonResponse(body: unknown): Response { - return new Response(JSON.stringify(body), { - status: 200, - headers: { 'content-type': 'application/json' }, - }) -} - describe('runEmbeddedCli', () => { it('cancels only the selected embedded invocation and refuses new requests after Stop', async () => { const controller = new AbortController() diff --git a/packages/workflow-code/src/compile/hints.ts b/packages/workflow-code/src/compile/hints.ts index 41195095df6..f2d838d6a1c 100644 --- a/packages/workflow-code/src/compile/hints.ts +++ b/packages/workflow-code/src/compile/hints.ts @@ -205,8 +205,8 @@ const VERB_SYNONYMS: ReadonlyArray = [ /** `postMessage` → `['post', 'message']`: the leading lowercase verb and the rest, lowercased. */ function splitVerb(method: string): [string, string] { - const match = /^([a-z]+)(.*)$/.exec(method) - return match ? [match[1], (match[2] ?? '').toLowerCase()] : ['', method.toLowerCase()] + const verb = /^[a-z]*/.exec(method)?.[0] ?? '' + return [verb, method.slice(verb.length).toLowerCase()] } /** diff --git a/packages/workflow-code/src/compile/scaling.test.ts b/packages/workflow-code/src/compile/scaling.test.ts index e0b62ea33cf..0ed749861dd 100644 --- a/packages/workflow-code/src/compile/scaling.test.ts +++ b/packages/workflow-code/src/compile/scaling.test.ts @@ -7,6 +7,7 @@ import { describe, it } from 'vitest' import { SOURCE_LIMITS } from '../limits' import { TEST_CATALOG } from './catalog.test-helpers' import { stronglyConnected } from './edges' +import { suggestBuilder } from './hints' import { compileWorkflow } from './index' const ENTRY = 'workflows/test.ts' @@ -67,4 +68,12 @@ describe('compile scaling', () => { return compileReading(n, n, (i) => `$env.MISSING_${i}`, { envNames }) }, 200) }) + + it('splits a long written method that spans lines in linear time', () => { + const paths = ['slack.sendMessage', 'slack.getMessage'] + expectLinearGrowth((n) => { + const written = `slack.${'a'.repeat(n)}\n` + return () => suggestBuilder(written, paths) + }, 4_000) + }) }) diff --git a/packages/workflow-sdk/package.json b/packages/workflow-sdk/package.json index cba2f02dd0c..b92432ca1ab 100644 --- a/packages/workflow-sdk/package.json +++ b/packages/workflow-sdk/package.json @@ -38,6 +38,7 @@ "@sim/workflow-types": "workspace:*" }, "devDependencies": { + "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@types/node": "24.19.2", "@xyflow/react": "12.12.1", diff --git a/packages/workflow-sdk/src/codegen/generate.test.ts b/packages/workflow-sdk/src/codegen/generate.test.ts index 8df1a46b7c8..86498133ce9 100644 --- a/packages/workflow-sdk/src/codegen/generate.test.ts +++ b/packages/workflow-sdk/src/codegen/generate.test.ts @@ -1,6 +1,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import path from 'node:path' +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' import { describe, expect, it } from 'vitest' import { FIXTURE_WORKSPACE } from '../../test/fixtures/workspace' import { @@ -9,7 +10,12 @@ import { writeTypecheckProject, } from '../../test/typecheck-project' import { generateSdkTypes } from './generate' -import { defaultBlockFiles, fullBlockFiles, referencedBlockTypes } from './layout' +import { + defaultBlockFiles, + fullBlockFiles, + referencedBlockTypes, + referencedBuilders, +} from './layout' import { generateWorkspaceTypes } from './workspace' function reversedKeys(record: Record): Record { @@ -155,6 +161,17 @@ describe('the SDK layout', () => { expect(defaults).not.toContain('blocks/google_sheets_v2.d.ts') expect(fullBlockFiles(catalog, used)).toEqual(['index.d.ts', 'blocks/slack.d.ts']) }) + + it.each([ + ['a long identifier', (n: number) => 'a'.repeat(n)], + ['a long dotted chain', (n: number) => '$.'.repeat(n)], + ])('scans a source holding %s with no call in linear time', (_, source) => { + const catalog = loadCatalog() + expectLinearGrowth((n) => { + const text = source(n) + return () => referencedBuilders(catalog, [text]) + }, 4_000) + }) }) describe('generateWorkspaceTypes', () => { diff --git a/packages/workflow-sdk/src/codegen/layout.ts b/packages/workflow-sdk/src/codegen/layout.ts index ce65bb8d80e..200b67908ec 100644 --- a/packages/workflow-sdk/src/codegen/layout.ts +++ b/packages/workflow-sdk/src/codegen/layout.ts @@ -62,7 +62,21 @@ export function legacyOnlyBlockTypes(catalog: BlockCatalog): Set { ) } -const CALL = /([A-Za-z_$][\w$]*(?:\s*\.\s*[A-Za-z_$][\w$]*)*)\s*\(/g +/** A dotted path (`slack . sendMessage`). It always succeeds greedily, so a scan never rescans one. */ +const CHAIN = /[A-Za-z_$][\w$]*(?:\s*\.\s*[A-Za-z_$][\w$]*)*/g +const OPEN_PAREN = /\s*\(/y + +/** + * The segments of each dotted call in `text` (`slack . sendMessage (` → `['slack', 'sendMessage']`). + * The `(` is checked after the chain matches, not inside its pattern: a pattern that can fail after + * the chain retries from each of its characters, quadratic on a long identifier or `a.b.c…` run. + */ +function* calledPaths(text: string): Generator { + for (const match of text.matchAll(CHAIN)) { + OPEN_PAREN.lastIndex = match.index + match[0].length + if (OPEN_PAREN.test(text)) yield match[0].replace(/\s+/g, '').split('.') + } +} /** * The catalog builders `sources` (workflow files) call, sorted: every dotted call whose path is a @@ -73,8 +87,7 @@ const CALL = /([A-Za-z_$][\w$]*(?:\s*\.\s*[A-Za-z_$][\w$]*)*)\s*\(/g export function referencedBuilders(catalog: BlockCatalog, sources: Iterable): string[] { const keys = new Set() for (const text of sources) { - for (const match of text.matchAll(CALL)) { - const path = match[1].replace(/\s+/g, '').split('.') + for (const path of calledPaths(text)) { if (path.length > 1 && path[path.length - 2] === TOOL_NAMESPACE) path.splice(path.length - 2, 1) else if (path.length > 1 && path[path.length - 1] === TOOL_NAMESPACE) path.pop() diff --git a/packages/workflow-sdk/src/codegen/reference.ts b/packages/workflow-sdk/src/codegen/reference.ts index 8cc540022b6..7d9ccb8b497 100644 --- a/packages/workflow-sdk/src/codegen/reference.ts +++ b/packages/workflow-sdk/src/codegen/reference.ts @@ -5,7 +5,7 @@ * the generated declarations' own (`builderTypes`), so the two never disagree. */ -import { compareStrings } from '@sim/utils/string' +import { compareStrings, escapeRegExp } from '@sim/utils/string' import type { BlockCatalog } from '@sim/workflow-code' import { suggestBuilder } from '@sim/workflow-code' import { catalogBlock, catalogBuilder, catalogBuilderPaths } from '@sim/workflow-code/catalog/view' @@ -129,7 +129,7 @@ const MAX_CORE_LINES = 80 function coreDeclarationRanges(name: string): Array<{ start: number; end: number }> { const lines = CORE_DTS.split('\n') const start = new RegExp( - `^export (?:declare )?(?:const|function|namespace|interface|type) ${name}\\b` + `^export (?:declare )?(?:const|function|namespace|interface|type) ${escapeRegExp(name)}\\b` ) const declarations: Array<{ start: number; end: number }> = [] for (let index = 0; index < lines.length; index++) {