Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 31 additions & 4 deletions apps/sim/lib/knowledge/__integration__/gitlab-group.integration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ describe('GitLab group Search permission boundary', () => {
let moved = false
let failedDiscovery = false
let added = false
let aliceLegacyResult: { documentId: string } | undefined
let confidential = false
let providerState = 'healthy'
const checks: string[] = []
Expand Down Expand Up @@ -259,7 +260,7 @@ describe('GitLab group Search permission boundary', () => {
boundary:
'Application use cases, real PostgreSQL and HTTP GitLab API fixture; TLS/SSRF transport substituted',
checks,
passed: checks.length === 10,
passed: checks.length === 11,
},
null,
2
Expand Down Expand Up @@ -433,9 +434,8 @@ describe('GitLab group Search permission boundary', () => {
input: { organizationId: ids.organizationId, sourceId: legacyId },
})
expect(legacyRevision).toEqual({ revision: 1 })
expect(
(await search(ids.aliceId, '11', undefined, 20, legacyId)).results.length
).toBeGreaterThan(0)
aliceLegacyResult = (await search(ids.aliceId, '11', undefined, 20, legacyId)).results[0]
expect(aliceLegacyResult).toBeDefined()
await replaceSearchSourcePermissions.execute({
principal: principal(ids.aliceId),
input: {
Expand All @@ -461,6 +461,33 @@ describe('GitLab group Search permission boundary', () => {
checks.push('Atomic group refresh, revocation and existing source preservation')
})

it('leaves a source out of the searches of a reader its uploaded permissions no longer grant', async () => {
const accounts = async (userId: string) =>
(
await searchLiveKnowledge.execute({
principal: principal(userId),
input: {
organizationId: ids.organizationId,
query: 'Atlas',
topK: 20,
filters: { source: 'gitlab' },
resultSecretRegistry: new ResolvedSecretTraceRegistry(),
},
})
).live?.accounts ?? []
const alice = await accounts(ids.aliceId)
expect(alice.map((account) => account.accountId)).toEqual([`gitlab-source:${sourceId}`])
expect(alice.every((account) => account.status === 'ok')).toBe(true)
expect(new Set((await accounts(ids.bobId)).map((account) => account.accountId))).toEqual(
new Set([`gitlab-source:${sourceId}`, `gitlab-source:${legacyId}`])
)
if (!aliceLegacyResult) throw new Error('Missing result found before revocation')
await expect(read(ids.aliceId, aliceLegacyResult.documentId)).rejects.toThrow(
'This GitLab source is not available to your account.'
)
checks.push('Sources granting the reader nothing are left out of their searches')
})

it('preserves all grants on discovery failure and admits only one concurrent revision', async () => {
failedDiscovery = true
await expect(replace(2)).rejects.toMatchObject({ code: 'validation' })
Expand Down
22 changes: 22 additions & 0 deletions apps/sim/lib/knowledge/connectors/permission-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,28 @@ export async function hasConnectorPermissionGrant(
return Boolean(grant)
}

/** The connectors among `connectorIds` that grant the reader's subject at least one collection. */
export async function connectorsGrantingSubject(
connectorIds: readonly string[],
subjectToken: string
): Promise<Set<string>> {
const table = knowledgeConnectorPermissionGrant
const granted = new Set<string>()
for (let offset = 0; offset < connectorIds.length; offset += 1000) {
const rows = await db
.selectDistinct({ connectorId: table.connectorId })
.from(table)
.where(
and(
eq(table.subjectToken, subjectToken),
inArray(table.connectorId, connectorIds.slice(offset, offset + 1000))
)
)
for (const row of rows) granted.add(row.connectorId)
}
return granted
}

/** Returns grants only for the bounded, canonical collections requested by the authorized caller. */
export async function readConnectorPermissionGrants(
connectorId: string,
Expand Down
4 changes: 2 additions & 2 deletions apps/sim/lib/sim-search/live/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ This describes the live enterprise-search path. Credential Groups and ordinary k
- **Sources** controls integration availability and account mode per integration.
- **Member accounts** has no resource configuration. A member connects their own account under Integrations, and searches content that account can access through the provider API. A member can add several accounts of one provider, such as Google Drive in two tenants; each is searched and cited separately, and every search counts them toward its account limit. Old member resource filters are ignored and cleared when settings are saved.
- **Service account** selects an admin-managed source. Its connection owns the resource settings; Sources does not duplicate them. GitHub uses one GitHub App source per repository instead of a single selected source. A member still connects their own account, except for GitLab. Search returns the intersection of the member's provider permissions and the source's current resource boundary.
- GitLab is always service-account mode. Slack and Jira currently support member mode only.
- GitLab is always service-account mode. A source whose uploaded CSV permissions grant the member no project is left out of their searches rather than reported unavailable on each one; a source checked against GitLab itself stays listed and its session decides. Slack and Jira currently support member mode only.
- Removing a source integration prevents subsequent account resolution, searches, and reads. Paused, archived, deleted, wrong-organization, wrong-provider, and unapproved service sources cannot authorize search.

The Search assistant exposes the dedicated search/read tools. It cannot discover or execute integration tools through `search_integration_tools`, `call_integration_tool`, or direct provider-tool names. The broader assistant outside Search retains its integration tools.
Expand Down Expand Up @@ -58,7 +58,7 @@ Each event must exist under the source's delegated token, be in an allowed calen

### Slack

Member mode only. Search uses the connected user's Slack real-time search grant and provider search syntax. Date bounds are sent as numeric provider timestamps; date ordering selects keyword retrieval and timestamp sort, because Slack semantic retrieval uses relevance ordering. Channels, messages, direct messages, and files are limited by that grant and provider scopes; there is no admin channel filter in member mode. Missing required search scopes produce a reconnect status. Thread/file reads use the same connected identity. A file read downloads the full text of a canvas (HTML), text snippet, PDF or DOCX from `files.slack.com` only, using `url_private_download` when present (Slack redirects a PDF's `url_private` to a viewer on another domain), never a remote file's own URL (redirects are followed only within that host; a hop elsewhere, such as a sign-in page, keeps the preview), within the shared response-size limit; when Slack will not serve it, the read returns the `files.info` preview. Real-time Search returns at most 20 results a call and counts every page against the member's rate limit, so a search across every source reads one page; a Slack-scoped or native Slack search follows the cursor for up to three pages to fill its limit, and a later page that fails keeps the earlier pages and returns the unread cursor.
Member mode only. Search uses the connected user's Slack real-time search grant and provider search syntax. Date bounds are sent as numeric provider timestamps; date ordering selects keyword retrieval and timestamp sort, because Slack semantic retrieval uses relevance ordering. Channels, messages, direct messages, and files are limited by that grant and provider scopes; there is no admin channel filter in member mode. Missing required search scopes produce a reconnect status. Thread/file reads use the same connected identity. Real-time Search returns only a file's title for a file hit, so the hit names the file's type and size and whether a read returns its text. A file read downloads the full text of a canvas (HTML), text snippet, PDF or DOCX from `files.slack.com` only, using `url_private_download` when present (Slack redirects a PDF's `url_private` to a viewer on another domain), never a remote file's own URL (redirects are followed only within that host; a hop elsewhere, such as a sign-in page, keeps the preview), within the shared response-size limit; when Slack will not serve it, the read returns the `files.info` preview. Real-time Search returns at most 20 results a call and counts every page against the member's rate limit, so a search across every source reads one page; a Slack-scoped or native Slack search follows the cursor for up to three pages to fill its limit, and a later page that fails keeps the earlier pages and returns the unread cursor.

### GitHub

Expand Down
1 change: 1 addition & 0 deletions apps/sim/lib/sim-search/live/accounts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const hoisted = vi.hoisted(() => ({
}))
vi.mock('@/lib/knowledge/search/integration-policy', () => knowledgeSearchIntegrationPolicyMock)
vi.mock('@/lib/sim-search/live/gitlab-admin', () => ({
isAdminGitLabAccountId: (accountId: string) => accountId.startsWith('gitlab-source:'),
listAdminGitLabAccounts: hoisted.admin,
resolveAdminGitLabAccount: hoisted.resolveAdmin,
}))
Expand Down
7 changes: 5 additions & 2 deletions apps/sim/lib/sim-search/live/accounts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { resolveKnowledgeWorkspaceContext } from '@/lib/knowledge/application/co
import { listOrganizationSearchApprovals } from '@/lib/knowledge/search/integration-policy'
import { resolveCredentialTokenBundle } from '@/lib/oauth/credential-service'
import {
isAdminGitLabAccountId,
listAdminGitLabAccounts,
resolveAdminGitLabAccount,
} from '@/lib/sim-search/live/gitlab-admin'
Expand Down Expand Up @@ -114,7 +115,7 @@ export async function listLiveAccounts(
const [visible, mcp, admin] = await Promise.all([
workspaceContext ? filterWorkspaceAccountCredentials(workspaceContext, candidates) : candidates,
listManagedMcpSearchAccounts(owner, userId, denied),
denied.has('gitlab') ? [] : listAdminGitLabAccounts(owner),
denied.has('gitlab') ? [] : listAdminGitLabAccounts(owner, userId),
])
if (visible.length === 0) return [...mcp, ...admin]
// Metadata is fetched in one batch; a fresh binding check still precedes token resolution.
Expand Down Expand Up @@ -162,7 +163,9 @@ export async function resolveLiveAccount(
) {
const current = (await listLiveAccounts(owner, userId)).find((row) => row.id === accountId)
if (!current)
throw new NativeSearchError('reconnect', 'This search connection is no longer available.')
throw isAdminGitLabAccountId(accountId)
? new NativeSearchError('unavailable', 'This GitLab source is not available to your account.')
: new NativeSearchError('reconnect', 'This search connection is no longer available.')
return resolveListedLiveAccount(owner, userId, current, signal)
}

Expand Down
21 changes: 20 additions & 1 deletion apps/sim/lib/sim-search/live/gitlab-admin.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import { queueTableRows, schemaMock } from '@sim/testing'
import {
knowledgeAccessScopeMock,
knowledgeAccessScopeMockFns,
} from '@sim/testing/mocks/knowledge-access-scope.mock'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { groupToken } from '@/lib/knowledge/access/tokens'
import { createAdminGitLabSession } from '@/lib/sim-search/live/gitlab-admin'
import {
createAdminGitLabSession,
listAdminGitLabAccounts,
} from '@/lib/sim-search/live/gitlab-admin'
import {
gitLabCsvGroupToken,
setGitLabCsvContext,
Expand All @@ -19,13 +23,15 @@ const mocks = vi.hoisted(() => ({
search: vi.fn(),
read: vi.fn(),
grant: vi.fn(),
metadata: vi.fn(),
}))
vi.mock('@/lib/knowledge/access/scope', () => knowledgeAccessScopeMock)
vi.mock('@/connectors/gitlab/permission-config/repository', () => ({
seedGitLabPermissionContext: mocks.seed,
}))
vi.mock('@/lib/knowledge/connectors/permission-store', () => ({
hasConnectorPermissionGrant: mocks.grant,
readConnectorPermissionMetadata: mocks.metadata,
}))
vi.mock('@/connectors/gitlab/permissions', () => ({
openGitLabDirectory: mocks.directory,
Expand Down Expand Up @@ -236,3 +242,16 @@ describe('non-admin token and CSV permissions', () => {
await expect(session()).rejects.toThrow('not available')
})
})

describe('administrator-managed GitLab source listing', () => {
it('lists every source when its grants cannot be checked, leaving the session to refuse it', async () => {
queueTableRows(schemaMock.knowledgeConnector, [
{ id: 'source-a', config: source.config, encryptedApiKey: 'sealed' },
])
mocks.metadata.mockRejectedValue(new Error('connection reset'))

const accounts = await listAdminGitLabAccounts({ organizationId: 'org' }, 'reader')

expect(accounts.map((account) => account.id)).toEqual(['gitlab-source:source-a'])
})
})
96 changes: 75 additions & 21 deletions apps/sim/lib/sim-search/live/gitlab-admin.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import { db } from '@sim/db'
import { knowledgeBase, knowledgeConnector } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { describeError } from '@sim/utils/errors'
import { and, eq, inArray, isNull } from 'drizzle-orm'
import { decryptApiKey } from '@/lib/api-key/crypto'
import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope'
import { resourceScopeCondition } from '@/lib/core/resource-scope.server'
import { createUserKnowledgeAccessProvider } from '@/lib/knowledge/access/scope'
import { groupToken } from '@/lib/knowledge/access/tokens'
import {
connectorsGrantingSubject,
hasConnectorPermissionGrant,
readConnectorPermissionGrants,
readConnectorPermissionMetadata,
Expand Down Expand Up @@ -44,6 +47,7 @@ import {
} from '@/connectors/gitlab/permissions'
import { normalizeGitLabHost } from '@/tools/gitlab/utils'

const logger = createLogger('SimSearchGitLabSources')
const PREFIX = 'gitlab-source:'

/** Canonical admin-managed sources only. Neither credentials nor indexed documents leave this module. */
Expand Down Expand Up @@ -72,20 +76,80 @@ async function sources(owner: ResourceOwner, connectorId?: string) {
)
}

export async function listAdminGitLabAccounts(owner: ResourceOwner): Promise<LiveAccount[]> {
return (await sources(owner)).map((source) => ({
id: `${PREFIX}${source.id}`,
provider: 'gitlab',
providerId: 'gitlab',
displayName: 'GitLab',
type: 'admin_source',
scopes: [],
}))
/** The viewer's own identity token, the subject every GitLab grant is checked against. */
async function viewerToken(owner: ResourceOwner, userId: string): Promise<string | undefined> {
const access = await createUserKnowledgeAccessProvider(userId, {
workspaceId: owner.workspaceId ?? undefined,
organizationId: owner.organizationId ?? undefined,
}).get()
return access.kind === 'user' ? access.tokens.find((value) => value.startsWith('u:')) : undefined
}

function hasCsvPermissions(metadata: unknown): boolean {
try {
return readGitLabPermissionMetadata(metadata).mode === 'csv'
} catch {
return false
}
}

/** The IDs of CSV-permission sources that grant the viewer no project. */
async function ungrantedSources(
owner: ResourceOwner,
userId: string,
ids: string[]
): Promise<Set<string>> {
const csv = (await readConnectorPermissionMetadata(ids))
.filter((row) => hasCsvPermissions(row.metadata))
.map((row) => row.connectorId)
const own = csv.length ? await viewerToken(owner, userId) : undefined
if (!own) return new Set()
const granted = await connectorsGrantingSubject(csv, own)
return new Set(csv.filter((id) => !granted.has(id)))
}

/**
* A source whose uploaded permissions grant the viewer nothing could never return a result, so it
* is left out rather than reported unavailable on every search. Sources that check permissions
* with GitLab, any whose stored permissions cannot be read, and every source when this check
* itself fails stay listed: their session decides.
*/
export async function listAdminGitLabAccounts(
owner: ResourceOwner,
userId: string
): Promise<LiveAccount[]> {
const rows = await sources(owner)
if (!rows.length) return []
const hidden = await ungrantedSources(
owner,
userId,
rows.map((source) => source.id)
).catch((error: unknown) => {
logger.warn('GitLab source grants could not be checked; listing every source', {
error: describeError(error),
})
return new Set<string>()
})
return rows
.filter((source) => !hidden.has(source.id))
.map((source) => ({
id: `${PREFIX}${source.id}`,
provider: 'gitlab',
providerId: 'gitlab',
displayName: 'GitLab',
type: 'admin_source',
scopes: [],
}))
}

/** Whether a live account ID names an administrator-managed GitLab source. */
export function isAdminGitLabAccountId(accountId: string): boolean {
return accountId.startsWith(PREFIX)
}

export async function resolveAdminGitLabAccount(owner: ResourceOwner, account: LiveAccount) {
const [source] = await sources(owner, account.id.slice(PREFIX.length))
if (!account.id.startsWith(PREFIX) || !source?.encryptedApiKey)
if (!isAdminGitLabAccountId(account.id) || !source?.encryptedApiKey)
throw new NativeSearchError(
'unavailable',
'Ask an organization admin to check the GitLab connection.'
Expand Down Expand Up @@ -129,17 +193,7 @@ interface GitLabSessionInput {
}

export async function createAdminGitLabSession(input: GitLabSessionInput) {
const { owner, userId } = input
const access = await createUserKnowledgeAccessProvider(userId, {
workspaceId: owner.workspaceId ?? undefined,
organizationId: owner.organizationId ?? undefined,
}).get()
if (access.kind !== 'user')
throw new NativeSearchError(
'unavailable',
'GitLab search requires a verified organization identity.'
)
const own = access.tokens.find((value) => value.startsWith('u:'))
const own = await viewerToken(input.owner, input.userId)
if (!own)
throw new NativeSearchError(
'unavailable',
Expand Down
Loading
Loading