Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .agents/skills/add-column-type/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,8 @@ Copy `currency`'s service function and orchestration branch.

The workspace source table codec reads the registry, but the SDK restates each type: add the
`column.<type>()` helper to `packages/workflow-sdk/src/workspace/table.d.ts` (the drift test in
`lib/workspace-source/application/kinds/table/codec.test.ts` fails until you do), map the type's row
value in `columnType()` (`packages/workflow-sdk/src/codegen/workspace.ts`, which defaults to
`apps/sim/lib/workspace-source/application/kinds/table/codec.test.ts` fails until you do), map the
type's row value in `valueType()` (`packages/workflow-sdk/src/codegen/workspace.ts`, which defaults to
`string`), and regenerate the embedded text (`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`).
See `.claude/rules/sim-workflow-code.md`.

Expand Down
2 changes: 1 addition & 1 deletion .claude/rules/sim-workflow-code.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ Follow `/add-column-type`. The workspace codec reads the column registry
- the `column.<type>()` helper in `packages/workflow-sdk/src/workspace/table.d.ts`
(`kinds/table/codec.test.ts` fails until it exists), then regenerate the embedded text
(`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`);
- `columnType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')`
- `valueType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')`
gives, which defaults to `string`;
- a type that owns metadata: `TABLE_COLUMN_METADATA_KEYS` in
`packages/workspace-code/src/kinds/table.ts` plus the reader and printer there.
Expand Down
2 changes: 1 addition & 1 deletion .cursor/rules/sim-workflow-code.mdc
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ Follow `/add-column-type`. The workspace codec reads the column registry
- the `column.<type>()` helper in `packages/workflow-sdk/src/workspace/table.d.ts`
(`kinds/table/codec.test.ts` fails until it exists), then regenerate the embedded text
(`bun packages/workflow-sdk/scripts/generate-embedded-text.ts`);
- `columnType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')`
- `valueType()` in `packages/workflow-sdk/src/codegen/workspace.ts`, the row type `table('k')`
gives, which defaults to `string`;
- a type that owns metadata: `TABLE_COLUMN_METADATA_KEYS` in
`packages/workspace-code/src/kinds/table.ts` plus the reader and printer there.
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/publish-sim-cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ on:
- 'packages/sim-cli/**'
# The build bundles its workspace dependencies, so a change there
# changes what ships. `bun run check:sim-cli-bundle` fails when this
# list and the bundle's workspace packages disagree.
# list and the bundle's workspace packages disagree. bun.lock is not
# watched: nearly every lockfile change leaves the bundle byte-identical,
# and the next change to a watched package ships the ones that do not.
- 'packages/table-cells/**'
- 'packages/utils/**'
- 'packages/workflow-code/**'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { v2KnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies'
import { knowledgeOperations } from '@/lib/knowledge/application/operations'
import { readOriginalKnowledgeDocument } from '@/lib/knowledge/application/read-original-document'
import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery'
import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types'

export const dynamic = 'force-dynamic'
export const revalidate = 0
Expand All @@ -14,8 +15,9 @@ export const revalidate = 0
* original bytes.
*
* What `sim workspaces source get` downloads for each document of a knowledge base. A document
* whose original is not stored (a connector-synced page) answers `NOT_FOUND`. `headSafe: false`
* because the read pulls the bytes out of object storage.
* whose original is not stored (a connector-synced page) answers `NOT_FOUND`. Nothing parses the
* bytes, so any original up to the stored-document limit downloads, not just those under the
* text-extraction cap. `headSafe: false` because the read pulls the bytes out of object storage.
*/
export const GET = defineV2BinaryRoute({
contract: v2DownloadKnowledgeDocumentContract,
Expand All @@ -28,6 +30,7 @@ export const GET = defineV2BinaryRoute({
knowledgeBaseId: params.knowledgeBaseId,
documentId: params.documentId,
assertedWorkspaceId: query.workspaceId,
maxBytes: MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE,
}),
useCase: readOriginalKnowledgeDocument,
present: (document) => {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/**
* The v2 knowledge document download route against real PostgreSQL and local object storage,
* through its route handler with a real workspace API key. A download parses nothing, so it serves
* any stored original up to the stored-document limit, including one past the text-extraction cap
* that a parsing read refuses.
*/
import { mkdtempSync } from 'node:fs'
import { rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { db } from '@sim/db'
import { apiKey, document, knowledgeBase, organization, user, workspace } from '@sim/db/schema'
import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
import { generateId } from '@sim/utils/id'
import { eq, inArray } from 'drizzle-orm'
import { NextRequest } from 'next/server'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'

const fixtureStorage = vi.hoisted(() => ({ root: '' }))
vi.mock('@/lib/uploads/core/setup.server', () => ({
get UPLOAD_DIR_SERVER() {
return fixtureStorage.root
},
}))

import { hashApiKey } from '@/lib/api-key/crypto'
import {
createKnowledgeAclFixtureIds,
seedKnowledgeAclFixture,
} from '@/lib/knowledge/__integration__/seed-source-access-fixture'
import { uploadFile } from '@/lib/uploads/core/storage-service'
import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types'
import { MAX_TEXT_EXTRACTION_BYTES } from '@/lib/uploads/utils/file'
import { GET } from '@/app/api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content/route'

const ids = createKnowledgeAclFixtureIds()
const key = `sk-sim-fixture-${generateId()}`

async function seedStoredDocument(bytes: Buffer, fileSize = bytes.byteLength) {
const id = generateId()
const storageKey = `kb/${id}.bin`
await uploadFile({
file: bytes,
fileName: 'original.bin',
contentType: 'application/octet-stream',
context: 'knowledge-base',
customKey: storageKey,
preserveKey: true,
persistMetadata: false,
})
await db.insert(document).values({
id,
knowledgeBaseId: ids.knowledgeBaseId,
filename: 'original.bin',
fileUrl: '',
storageKey,
fileSize,
mimeType: 'application/octet-stream',
processingStatus: 'completed',
acl: ['ws'],
aclRequirements: [],
aclVerifiedAt: new Date(),
})
return id
}

function download(documentId: string) {
const request = new NextRequest(
`http://localhost:3000/api/v2/knowledge/${ids.knowledgeBaseId}/documents/${documentId}/content?workspaceId=${ids.workspaceId}`,
{ headers: { 'x-api-key': key, 'x-forwarded-for': '127.0.0.1' } }
)
return GET(request, {
params: Promise.resolve({ knowledgeBaseId: ids.knowledgeBaseId, documentId }),
})
}

beforeAll(async () => {
fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-knowledge-download-'))
await seedKnowledgeAclFixture(ids)
await db.insert(apiKey).values({
id: generateId(),
userId: ids.aliceId,
workspaceId: ids.workspaceId,
createdBy: ids.aliceId,
name: 'Download fixture',
key,
keyHash: hashApiKey(key),
type: 'workspace',
access: 'read',
})
})

afterAll(async () => {
await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId))
await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId))
await db.delete(organization).where(eq(organization.id, ids.organizationId))
await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId]))
await rm(fixtureStorage.root, { recursive: true, force: true })
await db.$client.end()
})

describe('GET /api/v2/knowledge/[knowledgeBaseId]/documents/[documentId]/content', () => {
it('downloads an original larger than the text-extraction cap', async () => {
const bytes = Buffer.alloc(MAX_TEXT_EXTRACTION_BYTES + 1024 * 1024, 'k')
const documentId = await seedStoredDocument(bytes)

const response = await download(documentId)

expect(response.status).toBe(200)
expect(response.headers.get('content-length')).toBe(String(bytes.byteLength))
expect(Buffer.from(await response.arrayBuffer()).equals(bytes)).toBe(true)
})

it('refuses a document recorded past the stored-document limit', async () => {
const documentId = await seedStoredDocument(
Buffer.from('small'),
MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE + 1
)

const response = await download(documentId)

expect(response.status).toBe(413)
})
})
11 changes: 10 additions & 1 deletion apps/sim/lib/knowledge/application/read-original-document.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,17 @@ import {
} from '@/lib/knowledge/secret-provenance'
import { isObjectNotFoundError } from '@/lib/uploads/core/errors'
import { downloadFile } from '@/lib/uploads/core/storage-service'
import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types'
import { MAX_TEXT_EXTRACTION_BYTES } from '@/lib/uploads/utils/file'

export interface ReadOriginalKnowledgeDocumentInput {
knowledgeBaseId: string
documentId: string
assertedWorkspaceId: string
/**
* Largest original the read accepts. Defaults to the text-extraction cap, for a caller that
* parses the bytes; a raw download raises it up to the stored-document limit.
*/
maxBytes?: number
signal?: AbortSignal
}
Expand All @@ -41,7 +46,11 @@ export const readOriginalKnowledgeDocument = defineAuthorizedKnowledgeUseCase({
}) => resolveCanonicalActiveKnowledgeDocumentContext(input, principal),
async execute({ input, context, principal }) {
const maxBytes = input.maxBytes ?? MAX_TEXT_EXTRACTION_BYTES
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > MAX_TEXT_EXTRACTION_BYTES)
if (
!Number.isSafeInteger(maxBytes) ||
maxBytes < 1 ||
maxBytes > MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE
)
throw new OrchestrationError('validation', 'Invalid document read byte limit')
input.signal?.throwIfAborted()
const doc = context.document
Expand Down
2 changes: 2 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions packages/sim-cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"proper-lockfile": "4.1.2"
},
"devDependencies": {
"@sim/testing": "workspace:*",
"@sim/tsconfig": "workspace:*",
"@sim/utils": "workspace:*",
"@sim/workflow-code": "workspace:*",
Expand Down
7 changes: 3 additions & 4 deletions packages/sim-cli/src/commands/protocol/logs-follow.test.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
import { utilsHelpersMock, utilsHelpersMockFns } from '@sim/testing/mocks/utils-helpers.mock'
import { Command } from 'commander'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { ListLogsResponse } from '../../generated/v2-api'
import { SimApiError } from '../../http/client'
import { contextMockFns, contextMockState } from '../../test/context-mock'
import { attachLogsFollow, type LogRow } from './logs-follow'

const { mockSleep } = vi.hoisted(() => ({
mockSleep: vi.fn(() => Promise.resolve()),
}))
const { mockSleep } = utilsHelpersMockFns

vi.mock('@sim/utils/helpers', () => ({ sleep: mockSleep }))
vi.mock('@sim/utils/helpers', () => utilsHelpersMock)

vi.mock('../../context', async () => (await import('../../test/context-mock')).contextMock)

Expand Down
12 changes: 12 additions & 0 deletions packages/sim-cli/src/commands/protocol/source-status.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import { expectLinearGrowth } from '@sim/testing/helpers/scaling'
import { describe, it } from 'vitest'
import { pathFilter } from './source-status'

describe('pathFilter', () => {
it('reads a filter with a long run of slashes before its last segment in linear time', () => {
expectLinearGrowth((n) => {
const filter = `wf${'/'.repeat(n)}workflow.ts`
return () => pathFilter([filter], '.')
}, 4_000)
})
})
6 changes: 2 additions & 4 deletions packages/sim-cli/src/commands/protocol/source-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import { realpathSync } from 'node:fs'
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
import { stripTrailingSlashes } from '@sim/utils/string'
import type { FileChange, SourceStatus } from '@sim/workspace-code/status'
import { createTwoFilesPatch, FILE_HEADERS_ONLY } from 'diff'
import type { OutputFormat } from '../../config/index'
Expand Down Expand Up @@ -353,10 +354,7 @@ function realPath(path: string): string {
function filterForms(filter: string, dir: string): string[] {
const forms = new Set<string>()
if (!isAbsolute(filter)) {
const given = filter
.replace(/\\/g, '/')
.replace(/^(\.\/)+/, '')
.replace(/\/+$/, '')
const given = stripTrailingSlashes(filter.replace(/\\/g, '/').replace(/^(\.\/)+/, ''))
forms.add(given === '' ? '.' : given)
}
const resolved = relative(realPath(resolve(dir)), realPath(resolve(filter)))
Expand Down
20 changes: 8 additions & 12 deletions packages/sim-cli/src/commands/protocol/workflow-run-follow.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { jsonResponse } from '@sim/testing/helpers/http'
import { Command } from 'commander'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { SimApiError } from '../../http/client'
Expand Down Expand Up @@ -61,13 +62,6 @@ function _streamResponse(body: ReadableStream<Uint8Array>): Response {
} as unknown as Response
}

function jsonResponse(data: Record<string, unknown>): Response {
return new Response(JSON.stringify({ data }), {
status: 200,
headers: { 'content-type': 'application/json' },
})
}

function ndjsonResponse(...events: Array<Record<string, unknown>>): Response {
return new Response(bodyOf(events.map((event) => `${JSON.stringify(event)}\n`)), {
status: 200,
Expand All @@ -85,11 +79,13 @@ beforeEach(() => {
requestRaw.mockReset()
requestRaw.mockImplementation(async () =>
jsonResponse({
runId: 'run-1',
workflowId: WORKFLOW_ID,
status: 'completed',
output: {},
error: null,
data: {
runId: 'run-1',
workflowId: WORKFLOW_ID,
status: 'completed',
output: {},
error: null,
},
})
)
})
Expand Down
23 changes: 11 additions & 12 deletions packages/sim-cli/src/commands/protocol/workflow-run-wait.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { utilsHelpersMock, utilsHelpersMockFns } from '@sim/testing/mocks/utils-helpers.mock'
import { Command } from 'commander'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { attachWorkflowRunWait } from './workflow-run-wait'
Expand All @@ -22,18 +23,7 @@ vi.mock('../../context', () => ({
}),
}))

/**
* Waiting is simulated rather than timed: the mock advances the same clock the
* command reads its deadline from, so a `--wait-timeout 3600` test costs
* nothing and the poll schedule is exactly what the assertions say it is.
*/
vi.mock('@sim/utils/helpers', () => ({
sleep: (ms: number) => {
sleeps.push(ms)
clock.now += ms
return Promise.resolve()
},
}))
vi.mock('@sim/utils/helpers', () => utilsHelpersMock)

let logged: string[]
let errored: string[]
Expand All @@ -53,6 +43,15 @@ beforeEach(() => {
process.exitCode = undefined
stderr.isTTY = false

/**
* Waiting is simulated rather than timed: the mock advances the same clock the
* command reads its deadline from, so a `--wait-timeout 3600` test costs
* nothing and the poll schedule is exactly what the assertions say it is.
*/
utilsHelpersMockFns.mockSleep.mockImplementation(async (ms: number) => {
sleeps.push(ms)
clock.now += ms
})
vi.spyOn(Date, 'now').mockImplementation(() => clock.now)
vi.spyOn(console, 'log').mockImplementation((line: string) => {
logged.push(line)
Expand Down
Loading
Loading