Skip to content

Commit f51dad7

Browse files
authored
fix(lint): use structural placeholders for table input references (#10149)
* fix(lint): replace table placeholder search with structural placeholders Quote every bare reference, turn in/nin condition values into one-item lists and drop whole-condition references from all/any groups, then run the Table transform once. Only if that fails are whole-reference fields left out for one final attempt, whose error is reported. This removes the reference-count bound that skipped row data with many references, blames the literal error rather than a deferred field, and stops deferred row data from being called non-static. * fix(lint): track bare reference occurrences and defer reference-only filter groups Give each bare reference occurrence its own placeholder so a quoted value with the same text is left unchanged, rebuild objects with Object.fromEntries so a __proto__ row key survives, and treat a filter whose root group holds only references as runtime-only so the remaining inputs are still checked. * fix(lint): treat only single-key all/any objects as reference-only filter groups
1 parent df44adc commit f51dad7

2 files changed

Lines changed: 197 additions & 66 deletions

File tree

‎apps/sim/lib/workflows/application/read-workflow-lint-tables.test.ts‎

Lines changed: 116 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -307,11 +307,42 @@ describe('standalone table diagnostics against the actual Table block', () => {
307307
).toEqual([expect.objectContaining({ field: 'data', value: ['missing'] })])
308308
})
309309

310-
it('defers a filter with more mixed references than the placeholder search covers', async () => {
311-
const conditions = [
312-
...['a', 'b', 'c', 'd'].map((name) => `{"field":"name","op":"eq","value":<start.${name}>}`),
313-
...['e', 'f', 'g'].map((name) => `{"field":"name","op":"in","value":<start.${name}>}`),
314-
]
310+
const manyReferenceData = `{${['a', 'b', 'c', 'd', 'e', 'f', 'g']
311+
.map((name) => `"name": <start.${name}>`)
312+
.join(',')},"missing":1}`
313+
314+
it.each([
315+
['insert_row', manyReferenceData, undefined],
316+
[
317+
'update_rows_by_filter',
318+
manyReferenceData,
319+
'{"all":[{"field":"name","op":"eq","value":<start.n>},{"field":"name","op":"in","value":<start.names>}]}',
320+
],
321+
])(
322+
'checks literal row keys beside many bare references (%s)',
323+
async (operation, data, filter) => {
324+
setGraph(block('write', { operation, bulkFilterMode: 'json', data, filter }))
325+
const result = await lint()
326+
expect(result.notes.filter((note) => note.includes('could not complete'))).toEqual([])
327+
expect(
328+
result.unresolvedReferences.filter((reference) => reference.kind === 'resource')
329+
).toEqual([expect.objectContaining({ field: 'data', value: ['missing'] })])
330+
}
331+
)
332+
333+
it('checks the literal conditions beside a whole-condition reference', async () => {
334+
setGraph(
335+
block('query', {
336+
filter: '{"all":[<start.condition>,{"field":"other","op":"eq","value":1}]}',
337+
})
338+
)
339+
const result = await lint()
340+
expect(result.notes.filter((note) => note.includes('could not complete'))).toEqual([])
341+
expect(result.tableFieldIssues.map(({ field }) => field)).toEqual(['other'])
342+
})
343+
344+
it('defers a filter group made only of references and still checks row data', async () => {
345+
const conditions = ['a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => `<start.${name}>`)
315346
setGraph(
316347
block('write', {
317348
operation: 'update_rows_by_filter',
@@ -330,6 +361,86 @@ describe('standalone table diagnostics against the actual Table block', () => {
330361
).toEqual([expect.objectContaining({ field: 'data', value: ['missing'] })])
331362
})
332363

364+
it('checks row data whose keys resemble a reference-only filter group', async () => {
365+
setGraph(
366+
block('write', {
367+
operation: 'update_rows_by_filter',
368+
bulkFilterMode: 'json',
369+
filter: '<start.filter>',
370+
data: '{"all":[<start.values>],"missing":1}',
371+
})
372+
)
373+
const result = await lint()
374+
expect(
375+
result.unresolvedReferences.filter((reference) => reference.kind === 'resource')
376+
).toEqual([expect.objectContaining({ field: 'data', value: ['all', 'missing'] })])
377+
})
378+
379+
it('reports a quoted in-list value that matches a bare reference elsewhere', async () => {
380+
setGraph(
381+
block('query', {
382+
filter:
383+
'{"all":[{"field":"name","op":"eq","value":<start.names>},{"field":"name","op":"in","value":"<start.names>"}]}',
384+
})
385+
)
386+
const result = await lint()
387+
expect(result.notes).toEqual(
388+
expect.arrayContaining([
389+
expect.stringContaining(
390+
'Table input checks in block "query" could not complete before runtime:'
391+
),
392+
])
393+
)
394+
})
395+
396+
it('keeps a __proto__ row key beside a bare reference', async () => {
397+
setGraph(
398+
block('write', {
399+
operation: 'insert_row',
400+
data: '{"name":<start.name>,"__proto__":"extra"}',
401+
})
402+
)
403+
const result = await lint()
404+
expect(
405+
result.unresolvedReferences.filter((reference) => reference.kind === 'resource')
406+
).toEqual([expect.objectContaining({ field: 'data', value: ['__proto__'] })])
407+
})
408+
409+
it('blames malformed row data, not a whole-field filter reference', async () => {
410+
setGraph(
411+
block('write', {
412+
operation: 'update_rows_by_filter',
413+
bulkFilterMode: 'json',
414+
filter: '<start.filter>',
415+
data: '{"name":"Ada",}',
416+
})
417+
)
418+
const result = await lint()
419+
expect(result.notes).toEqual(
420+
expect.arrayContaining([
421+
expect.stringContaining(
422+
'Table input checks in block "write" could not complete before runtime: Invalid JSON in Row Data'
423+
),
424+
])
425+
)
426+
})
427+
428+
it('does not call deferred whole-field row data a non-static object', async () => {
429+
setGraph(
430+
block('write', {
431+
operation: 'update_rows_by_filter',
432+
bulkFilterMode: 'json',
433+
filter: '<start.filter>',
434+
data: '<start.row>',
435+
})
436+
)
437+
const result = await lint()
438+
expect(result.notes).toContain(
439+
'Table inputs in block "write" depend on references resolved at runtime and were not checked: data, filterInput.'
440+
)
441+
expect(result.notes.filter((note) => note.includes('not a static object'))).toEqual([])
442+
})
443+
333444
it('still reports an invalid literal condition beside a reference', async () => {
334445
setGraph(
335446
block('query', {

‎apps/sim/lib/workflows/application/read-workflow-lint.ts‎

Lines changed: 81 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { requirePrincipalSubjectUserId } from '@sim/auth/principal'
22
import { getErrorMessage } from '@sim/utils/errors'
3+
import { generateShortId } from '@sim/utils/id'
34
import { isPlainRecord } from '@sim/utils/object'
45
import { findWorkflowReferenceTokens } from '@sim/utils/workflow-references'
56
import type { CursorKey } from '@/lib/api/list-query'
@@ -258,74 +259,89 @@ function hasRuntimeReference(value: string): boolean {
258259
/**
259260
* Outside Function code a reference resolves to JSON text (objects, arrays, numbers, booleans), so
260261
* `{"name": <start.name>}` or a whole-field `<start.row>` is valid JSON at runtime but not before.
261-
* Each reference written outside a JSON string is replaced by a placeholder so the Table block's
262-
* own transform reads the literal parts: a quoted string, or a one-item list for slots such as a
263-
* filter `in`, trying every combination. A field that is a single whole reference has no literal
264-
* part, so if no combination fits it is left out and returned in `runtimeFields`. When the search
265-
* is exhaustive and still fails, the error comes from the literal parts (malformed JSON, an
266-
* invalid literal condition) and is rethrown. Past the search bound that cannot be told apart, so
267-
* every reference-bearing field is left out instead and the other inputs are still checked. Only
268-
* this lint-local copy is rewritten; references to string outputs left unquoted are reported by
269-
* `collectUnquotedJsonStringReferences`.
262+
* Each reference written outside a JSON string becomes a quoted placeholder so the Table block's
263+
* own transform reads the literal parts (see {@link withReferencePlaceholders}). If the transform
264+
* still fails and a field has no literal part, that field is left out, returned in
265+
* `runtimeFields`, and the transform runs once more; an error from that final attempt comes from
266+
* the literal parts and is rethrown. Only this lint-local copy is rewritten; references to string
267+
* outputs left unquoted are reported by `collectUnquotedJsonStringReferences`.
270268
*/
271269
function normalizeTableInputs(
272270
transform: (params: Record<string, unknown>) => Record<string, unknown>,
273271
params: Record<string, unknown>
274272
): { normalized: Record<string, unknown>; runtimeFields: string[] } {
275-
const referenceFields: string[] = []
276-
const wholeReferenceFields: string[] = []
273+
const prepared: Record<string, unknown> = {}
274+
const runtimeFields: string[] = []
277275
for (const [key, value] of Object.entries(params)) {
278-
if (typeof value !== 'string') continue
279-
const trimmed = value.trim()
280-
const tokens = findWorkflowReferenceTokens(trimmed)
281-
if (tokens.length === 0) continue
282-
referenceFields.push(key)
283-
if (tokens.length === 1 && tokens[0].start === 0 && tokens[0].end === trimmed.length) {
284-
wholeReferenceFields.push(key)
285-
}
286-
}
287-
if (referenceFields.length === 0) return { normalized: transform(params), runtimeFields: [] }
288-
const tiers = wholeReferenceFields.length > 0 ? [[], wholeReferenceFields] : [[]]
289-
let searchBounded = false
290-
let firstError: unknown
291-
for (const runtimeFields of tiers) {
292-
const withPlaceholders = (asList: (ordinal: number) => boolean) => {
293-
let ordinal = 0
294-
const candidate: Record<string, unknown> = {}
295-
for (const [key, value] of Object.entries(params)) {
296-
if (runtimeFields.includes(key)) continue
297-
candidate[key] =
298-
typeof value === 'string'
299-
? replaceBareReferences(value, (reference) =>
300-
asList(ordinal++) ? `[${JSON.stringify(reference)}]` : JSON.stringify(reference)
301-
)
302-
: value
303-
}
304-
return { candidate, count: ordinal }
305-
}
306-
const { count } = withPlaceholders(() => false)
307-
if (count > MAX_PLACEHOLDER_SEARCH_REFERENCES) {
308-
searchBounded = true
276+
if (typeof value !== 'string') {
277+
prepared[key] = value
309278
continue
310279
}
311-
for (let mask = 0; mask < 2 ** count; mask++) {
312-
try {
313-
const { candidate } = withPlaceholders((ordinal) => ((mask >> ordinal) & 1) === 1)
314-
return { normalized: transform(candidate), runtimeFields }
315-
} catch (error) {
316-
firstError ??= error
317-
}
318-
}
280+
const input = withReferencePlaceholders(value)
281+
prepared[key] = input.value
282+
if (input.runtimeOnly) runtimeFields.push(key)
283+
}
284+
try {
285+
return { normalized: transform(prepared), runtimeFields: [] }
286+
} catch (error) {
287+
if (runtimeFields.length === 0) throw error
319288
}
320-
if (!searchBounded) throw firstError
321-
const literal = Object.fromEntries(
322-
Object.entries(params).filter(([key]) => !referenceFields.includes(key))
323-
)
324-
return { normalized: transform(literal), runtimeFields: referenceFields }
289+
for (const key of runtimeFields) delete prepared[key]
290+
return { normalized: transform(prepared), runtimeFields }
325291
}
326292

327-
/** Bounds the placeholder search to 2^6 pure transform calls per Table block. */
328-
const MAX_PLACEHOLDER_SEARCH_REFERENCES = 6
293+
/**
294+
* Quotes each bare reference occurrence under its own placeholder, so an already-quoted string
295+
* with the same text is never mistaken for one. A quoted reference fits row values and most
296+
* filter values; the value of an `in`/`nin` condition must be a list, so it becomes a one-item
297+
* list, and a condition or group made only of references has no literal part, so it is left out
298+
* of an `all`/`any` group that keeps a literal member. `runtimeOnly` marks a field whose whole
299+
* value is such a reference or group.
300+
*/
301+
function withReferencePlaceholders(json: string): { value: string; runtimeOnly: boolean } {
302+
const nonce = generateShortId()
303+
const references = new Map<string, string>()
304+
const quoted = replaceBareReferences(json, (reference) => {
305+
const placeholder = `${nonce}:${references.size}`
306+
references.set(placeholder, reference)
307+
return JSON.stringify(placeholder)
308+
})
309+
if (references.size === 0) return { value: json, runtimeOnly: false }
310+
let parsed: unknown
311+
try {
312+
parsed = JSON.parse(quoted)
313+
} catch {
314+
const readable = replaceBareReferences(json, (reference) => JSON.stringify(reference))
315+
return { value: readable, runtimeOnly: false }
316+
}
317+
const isPlaceholder = (node: unknown): node is string =>
318+
typeof node === 'string' && references.has(node)
319+
const isDynamic = (node: unknown): boolean => {
320+
if (isPlaceholder(node)) return true
321+
if (!isPlainRecord(node) || Object.keys(node).length !== 1) return false
322+
const members = node.all ?? node.any
323+
return Array.isArray(members) && members.length > 0 && members.every(isDynamic)
324+
}
325+
const adapt = (node: unknown): unknown => {
326+
if (isPlaceholder(node)) return references.get(node)
327+
if (Array.isArray(node)) return node.map(adapt)
328+
if (!isPlainRecord(node)) return node
329+
return Object.fromEntries(
330+
Object.entries(node).map(([key, value]) => {
331+
const name = references.get(key) ?? key
332+
if ((key === 'all' || key === 'any') && Array.isArray(value)) {
333+
const literal = value.filter((member) => !isDynamic(member))
334+
return [name, adapt(literal.length > 0 ? literal : value)]
335+
}
336+
if (key === 'value' && (node.op === 'in' || node.op === 'nin') && isPlaceholder(value)) {
337+
return [name, [adapt(value)]]
338+
}
339+
return [name, adapt(value)]
340+
})
341+
)
342+
}
343+
return { value: JSON.stringify(adapt(parsed)), runtimeOnly: isDynamic(parsed) }
344+
}
329345

330346
function replaceBareReferences(json: string, placeholder: (reference: string) => string): string {
331347
const tokens = findWorkflowReferenceTokens(json)
@@ -411,12 +427,14 @@ async function readTableDiagnostics(
411427
const transform = getBlock(block.type)?.tools.config?.params
412428
if (!transform) throw new Error('Table input normalization is unavailable')
413429
let normalized: Record<string, unknown>
430+
let runtimeFields: string[]
414431
try {
415432
const inputs = normalizeTableInputs(transform, params)
416433
normalized = inputs.normalized
417-
if (inputs.runtimeFields.length > 0) {
434+
runtimeFields = inputs.runtimeFields
435+
if (runtimeFields.length > 0) {
418436
result.notes.push(
419-
`Table inputs in block "${block.name || blockId}" depend on references resolved at runtime and were not checked: ${inputs.runtimeFields.join(', ')}.`
437+
`Table inputs in block "${block.name || blockId}" depend on references resolved at runtime and were not checked: ${runtimeFields.join(', ')}.`
420438
)
421439
}
422440
} catch (error) {
@@ -464,8 +482,10 @@ async function readTableDiagnostics(
464482
})
465483
}
466484
}
467-
if (Object.hasOwn(normalized, 'data')) inspectRow(normalized.data, 'data')
468-
if (Object.hasOwn(normalized, 'rows')) {
485+
if (Object.hasOwn(normalized, 'data') && !runtimeFields.includes('data')) {
486+
inspectRow(normalized.data, 'data')
487+
}
488+
if (Object.hasOwn(normalized, 'rows') && !runtimeFields.includes('rows')) {
469489
if (Array.isArray(normalized.rows)) {
470490
normalized.rows.forEach((row, index) => inspectRow(row, `rows[${index}]`))
471491
} else {

0 commit comments

Comments
 (0)