Skip to content

Commit 08d75c6

Browse files
committed
fix(integrations): drop the row cap; stream mongodb and neo4j reads and fail at the response size limit
1 parent a852401 commit 08d75c6

35 files changed

Lines changed: 215 additions & 497 deletions

‎apps/docs/content/docs/integrations/mongodb.mdx‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,6 @@ Execute find operation on MongoDB collection
5050
| `message` | string | Operation status message |
5151
| `documents` | array | Array of documents returned from the query |
5252
| `documentCount` | number | Number of documents returned |
53-
| `truncated` | boolean | Present and true only when documents were dropped to stay inside the response ceilings \(10,000 documents or 10 MiB\). Absent means the result is complete |
54-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
5553

5654
</ReferenceDetails>
5755

@@ -179,8 +177,6 @@ Execute MongoDB aggregation pipeline
179177
| `message` | string | Operation status message |
180178
| `documents` | array | Array of documents returned from aggregation |
181179
| `documentCount` | number | Number of documents returned |
182-
| `truncated` | boolean | Present and true only when documents were dropped to stay inside the response ceilings \(10,000 documents or 10 MiB\). Absent means the result is complete |
183-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
184180

185181
</ReferenceDetails>
186182

‎apps/docs/content/docs/integrations/mysql.mdx‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,6 @@ Execute SELECT query on MySQL database
5151
| `message` | string | Operation status message |
5252
| `rows` | array | Array of rows returned from the query |
5353
| `rowCount` | number | Number of rows returned |
54-
| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete |
55-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
5654

5755
</ReferenceDetails>
5856

@@ -145,8 +143,6 @@ Execute raw SQL query on MySQL database
145143
| `message` | string | Operation status message |
146144
| `rows` | array | Array of rows returned from the query |
147145
| `rowCount` | number | Number of rows affected |
148-
| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete |
149-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
150146

151147
</ReferenceDetails>
152148

‎apps/docs/content/docs/integrations/neo4j.mdx‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,6 @@ Execute MATCH queries to read nodes and relationships from Neo4j graph database.
4646
| `records` | array | Array of records returned from the query |
4747
| `recordCount` | number | Number of records returned |
4848
| `summary` | json | Query execution summary with timing and counters |
49-
| `truncated` | boolean | Present and true only when records were dropped to stay inside the response ceilings \(10,000 records or 10 MiB\). Absent means the result is complete |
50-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
5149

5250
</ReferenceDetails>
5351

@@ -191,8 +189,6 @@ Execute arbitrary Cypher queries on Neo4j graph database for complex operations
191189
| `records` | array | Array of records returned from the query |
192190
| `recordCount` | number | Number of records returned |
193191
| `summary` | json | Execution summary with timing and counters |
194-
| `truncated` | boolean | Present and true only when records were dropped to stay inside the response ceilings \(10,000 records or 10 MiB\). Absent means the result is complete |
195-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
196192

197193
</ReferenceDetails>
198194

‎apps/docs/content/docs/integrations/postgresql.mdx‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ Execute a SELECT query on PostgreSQL database
4848
| `message` | string | Operation status message |
4949
| `rows` | array | Array of rows returned from the query |
5050
| `rowCount` | number | Number of rows returned |
51-
| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete |
52-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
5351

5452
</ReferenceDetails>
5553

@@ -73,8 +71,6 @@ Insert data into PostgreSQL database
7371
| `message` | string | Operation status message |
7472
| `rows` | array | Inserted data \(if RETURNING clause used\) |
7573
| `rowCount` | number | Number of rows inserted |
76-
| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete |
77-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
7874

7975
</ReferenceDetails>
8076

@@ -99,8 +95,6 @@ Update data in PostgreSQL database
9995
| `message` | string | Operation status message |
10096
| `rows` | array | Updated data \(if RETURNING clause used\) |
10197
| `rowCount` | number | Number of rows updated |
102-
| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete |
103-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
10498

10599
</ReferenceDetails>
106100

@@ -124,8 +118,6 @@ Delete data from PostgreSQL database
124118
| `message` | string | Operation status message |
125119
| `rows` | array | Deleted data \(if RETURNING clause used\) |
126120
| `rowCount` | number | Number of rows deleted |
127-
| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete |
128-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
129121

130122
</ReferenceDetails>
131123

@@ -148,8 +140,6 @@ Execute raw SQL query on PostgreSQL database
148140
| `message` | string | Operation status message |
149141
| `rows` | array | Array of rows returned from the query |
150142
| `rowCount` | number | Number of rows affected |
151-
| `truncated` | boolean | Present and true only when rows were dropped to stay inside the response ceilings \(10,000 rows or 10 MiB\). Absent means the result is complete |
152-
| `truncationReason` | string | Which ceiling was hit and how to read the rest |
153143

154144
</ReferenceDetails>
155145

‎apps/sim/blocks/blocks/mongodb.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -959,14 +959,6 @@ Return ONLY the MongoDB query filter as valid JSON - no explanations, no markdow
959959
type: 'number',
960960
description: 'Number of documents affected by the operation',
961961
},
962-
truncated: {
963-
type: 'boolean',
964-
description: 'True when the result hit a row or byte ceiling and documents were dropped',
965-
},
966-
truncationReason: {
967-
type: 'string',
968-
description: 'Explanation of the ceiling that truncated the result',
969-
},
970962
insertedId: {
971963
type: 'string',
972964
description: 'ID of the inserted document (single insert)',

‎apps/sim/blocks/blocks/mysql.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -412,14 +412,6 @@ Return ONLY the SQL query - no explanations, no markdown, no extra text.`,
412412
type: 'number',
413413
description: 'Number of rows affected by the operation',
414414
},
415-
truncated: {
416-
type: 'boolean',
417-
description: 'True when the result hit a row or byte ceiling and rows were dropped',
418-
},
419-
truncationReason: {
420-
type: 'string',
421-
description: 'Explanation of the ceiling that truncated the result',
422-
},
423415
},
424416
}
425417

‎apps/sim/blocks/blocks/neo4j.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -706,14 +706,6 @@ Return ONLY valid JSON.`,
706706
type: 'number',
707707
description: 'Number of records returned or affected',
708708
},
709-
truncated: {
710-
type: 'boolean',
711-
description: 'True when the result hit a row or byte ceiling and records were dropped',
712-
},
713-
truncationReason: {
714-
type: 'string',
715-
description: 'Explanation of the ceiling that truncated the result',
716-
},
717709
summary: {
718710
type: 'json',
719711
description: 'Execution summary with timing and database change counters',

‎apps/sim/blocks/blocks/postgresql.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -427,14 +427,6 @@ Return ONLY the SQL query - no explanations, no markdown, no extra text.`,
427427
type: 'number',
428428
description: 'Number of rows affected by the operation',
429429
},
430-
truncated: {
431-
type: 'boolean',
432-
description: 'True when the result hit a row or byte ceiling and rows were dropped',
433-
},
434-
truncationReason: {
435-
type: 'string',
436-
description: 'Explanation of the ceiling that truncated the result',
437-
},
438430
tables: {
439431
type: 'array',
440432
description: 'Array of table schemas with columns, keys, and indexes (introspect operation)',

‎apps/sim/lib/internal/mongodb/execute-tool.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { getErrorMessage } from '@sim/utils/errors'
22
import type { z } from 'zod'
3+
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
34
import {
45
executeMongodbAggregation,
56
executeMongodbDelete,
@@ -40,6 +41,7 @@ async function executeOperation<TInput>(
4041
return Response.json(result)
4142
} catch (error) {
4243
signal?.throwIfAborted()
44+
if (isPayloadSizeLimitError(error)) throw error
4345
if (error instanceof MongodbOperationInputError) {
4446
return Response.json({ error: error.message }, { status: 400 })
4547
}

‎apps/sim/lib/internal/mongodb/operations.ts‎

Lines changed: 21 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -23,41 +23,23 @@ import type {
2323
MongodbQueryInput,
2424
MongodbUpdateInput,
2525
} from '@/lib/internal/mongodb/schema'
26-
import {
27-
createResultRowsBudget,
28-
resultTruncationReason,
29-
} from '@/lib/internal/tool-operations/response-limits'
26+
import { createResponseRowsBudget } from '@/lib/internal/tool-operations/response-limits'
3027

3128
export class MongodbOperationInputError extends Error {}
3229

3330
/**
34-
* Reads documents until the response budget is spent. Leaving the loop early closes the server
35-
* cursor, so an unbounded find or aggregation never materializes past the ceiling.
31+
* Reads documents one at a time, failing with the tool response size error as soon as they cannot
32+
* fit the response. Leaving the loop early closes the server cursor, so an oversized result is
33+
* never read into memory in full.
3634
*/
37-
async function readCappedDocuments(
38-
cursor: AsyncIterable<Document>,
39-
describe: (count: number) => string,
40-
paginationHint: string
41-
) {
42-
const budget = createResultRowsBudget()
35+
async function readDocuments(cursor: AsyncIterable<Document>, toolId: string) {
36+
const budget = createResponseRowsBudget(toolId)
4337
const documents: Document[] = []
44-
let truncated = false
4538
for await (const document of cursor) {
46-
if (!budget.admit(document)) {
47-
truncated = true
48-
break
49-
}
39+
budget.admit(document)
5040
documents.push(document)
5141
}
52-
const message = describe(documents.length)
53-
if (!truncated) return { message, documents }
54-
const truncationReason = resultTruncationReason(documents.length, paginationHint)
55-
return {
56-
message: `${message} ${truncationReason}`,
57-
documents,
58-
truncated: true,
59-
truncationReason,
60-
}
42+
return documents
6143
}
6244

6345
async function withMongodbClient<TResult>(
@@ -105,13 +87,13 @@ export function executeMongodbQuery(input: MongodbQueryInput, signal?: AbortSign
10587
}
10688

10789
cursor = cursor.limit(input.limit || 100)
108-
const result = await readCappedDocuments(
109-
cursor,
110-
(count) => `Found ${count} documents`,
111-
'Narrow the filter, or page with $skip and $limit in an aggregation pipeline.'
112-
)
90+
const documents = await readDocuments(cursor, 'mongodb_query')
11391
signal?.throwIfAborted()
114-
return { ...result, documentCount: result.documents.length }
92+
return {
93+
message: `Found ${documents.length} documents`,
94+
documents,
95+
documentCount: documents.length,
96+
}
11597
})
11698
}
11799

@@ -126,13 +108,16 @@ export function executeMongodbAggregation(input: MongodbExecuteInput, signal?: A
126108
const pipeline = JSON.parse(input.pipeline) as Document[]
127109

128110
return withMongodbClient(input, signal, async (client) => {
129-
const result = await readCappedDocuments(
111+
const documents = await readDocuments(
130112
client.db(input.database).collection(collectionName).aggregate(pipeline, { signal }),
131-
(count) => `Aggregation completed, returned ${count} documents`,
132-
'Page with $skip and $limit stages to read the rest.'
113+
'mongodb_execute'
133114
)
134115
signal?.throwIfAborted()
135-
return { ...result, documentCount: result.documents.length }
116+
return {
117+
message: `Aggregation completed, returned ${documents.length} documents`,
118+
documents,
119+
documentCount: documents.length,
120+
}
136121
})
137122
}
138123

0 commit comments

Comments
 (0)