diff --git a/lib/documents.js b/lib/documents.js index ebe1ec2b..48aa3272 100644 --- a/lib/documents.js +++ b/lib/documents.js @@ -1,5 +1,5 @@ /* -* Copyright (c) 2015-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +* Copyright (c) 2015-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ 'use strict'; @@ -1302,12 +1302,13 @@ function addWriteConfig(document, hasUri, content, headers, sep) { if (!hasUri) { const extension = document.extension; if (extension != null) { - writeConfig += sep+'extension='+extension; + writeConfig += sep+'extension='+ + (isBody ? extension : validateUnquotedContentDispositionValue(extension)); if (isBody && sep === '?') { sep = '&'; } const directory = document.directory; if (directory != null) { writeConfig += sep+'directory='+ - (isBody ? encodeURIComponent(directory) : directory); + (isBody ? encodeURIComponent(directory) : validateUnquotedContentDispositionValue(directory)); } } } @@ -1317,7 +1318,7 @@ function addWriteConfig(document, hasUri, content, headers, sep) { if (isBody) { headers['If-Match'] = versionId; } else { - writeConfig += sep+'versionId='+versionId; + writeConfig += sep+'versionId='+validateUnquotedContentDispositionValue(versionId); } } @@ -1404,9 +1405,10 @@ function addDocumentParts(operation, partList, document, isContentOptional) { let disposition = ''; if (hasUri) { - disposition = 'attachment; filename="'+uri+'"'; + disposition = 'attachment; filename="'+validateContentDispositionValue(uri)+'"'; if (document.temporalDocument != null) { - disposition += '; temporal-document="'+document.temporalDocument+'"'; + disposition += '; temporal-document="'+ + validateContentDispositionValue(document.temporalDocument)+'"'; } } else { disposition = 'inline'; @@ -1444,6 +1446,24 @@ function addDocumentParts(operation, partList, document, isContentOptional) { } } +function validateContentDispositionValue(value) { + const text = String(value); + // MarkLogic neither honors nor unescapes backslash escapes in quoted parameters, + // so unsafe characters are rejected and all other characters are sent unchanged. + if (/[\r\n\0"]/.test(text)) { + throw new Error('Content-Disposition values must not contain CR, LF, NUL, or double quotes'); + } + return text; +} + +function validateUnquotedContentDispositionValue(value) { + const text = validateContentDispositionValue(value); + if (text.indexOf(';') !== -1) { + throw new Error('Unquoted Content-Disposition values must not contain semicolons'); + } + return text; +} + /** @ignore */ function collectMetadata(document) { let metadata = null; @@ -2373,8 +2393,9 @@ function onWriteAllDocs(jobState, writerId) { } function writeDocs(jobState, val, batchdef, writeBatchArray, writerId){ - writeDocumentsImpl.call(jobState.docInstance, val, [batchdef]) - .result((output) => { + Promise.resolve() + .then(() => writeDocumentsImpl.call(jobState.docInstance, val, [batchdef]).result()) + .then((output) => { jobState.docsWrittenSuccessfully+= writeBatchArray.length; if (jobState.jobOptions.onBatchSuccess) { const progressSoFar = { diff --git a/lib/requester.js b/lib/requester.js index efb13400..d4261f31 100644 --- a/lib/requester.js +++ b/lib/requester.js @@ -145,6 +145,8 @@ function startRequest(operation) { operation.inputSender = singleRequester; break; case 'multipart': + validatePartHeaders(operation.requestPartList); + validateBindingParam(operation.bindingParam); operation.inputSender = multipartRequester; break; case 'chunked': @@ -155,6 +157,7 @@ function startRequest(operation) { operation.inputSender = chunkedRequester; break; case 'chunkedMultipart': + validatePartHeaders(operation.requestDocument); operationResultPromise = responder.operationResultPromise; started = through2(); started.result = mlutil.callbackOn(operation, operationResultPromise); @@ -218,6 +221,51 @@ function startRequest(operation) { return started; } +// Multipart part headers are written into the request body, where Node.js does not +// reject the CR/LF/NUL characters it rejects in real HTTP headers. +function validatePartHeaders(parts) { + if (!Array.isArray(parts)) { + return; + } + for (const part of parts) { + const headers = (part == null) ? null : part.headers; + if (headers == null) { + continue; + } + for (const [name, value] of Object.entries(headers)) { + if (/[\r\n\0]/.test(name) || /[\r\n\0]/.test(String(value))) { + throw new Error('multipart part header "' + JSON.stringify(name).slice(1, -1) + + '" must not contain CR, LF, or NUL characters'); + } + } + } +} +function validateBindingParam(bindingParam) { + if (bindingParam == null) { + return; + } + validateMultipartParameter(bindingParam.key); + + const attachments = bindingParam.attachments; + if (Array.isArray(attachments)) { + for (const attachment of attachments) { + if (attachment != null && typeof attachment === 'object') { + for (const key of Object.keys(attachment)) { + validateMultipartParameter(key); + } + } + } + } else if (attachments != null && typeof attachments === 'object') { + for (const key of Object.keys(attachments)) { + validateMultipartParameter(key); + } + } +} +function validateMultipartParameter(value) { + if (/[\r\n\0"]/.test(String(value))) { + throw new Error('Content-Disposition values must not contain CR, LF, NUL, or double quotes'); + } +} function challengeRequest(operation) { const isRead = (operation.inputSender === null); const options = operation.options; diff --git a/test-basic/documents-multipart-header-test.js b/test-basic/documents-multipart-header-test.js new file mode 100644 index 00000000..718c254c --- /dev/null +++ b/test-basic/documents-multipart-header-test.js @@ -0,0 +1,197 @@ +/* +* Copyright (c) 2015-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +*/ +const http = require('http'); +const assert = require('assert'); +const should = require('should'); +const marklogic = require('../'); +const op = marklogic.planBuilder; + +describe('documents.write multipart headers', function() { + const db = marklogic.createDatabaseClient({ + host: 'localhost', + port: 1, + user: 'u', + password: 'p', + authType: 'basic' + }); + + it('rejects CR, LF, NUL, and double quotes in uri and temporalDocument', function() { + const invalidSuffixes = ['\r', '\n', '\0', '"', '"; filename="y.json']; + for (const suffix of invalidSuffixes) { + for (const field of ['uri', 'temporalDocument']) { + for (const index of [0, 1]) { + const documents = [{ + uri: '/poc/first.json', + contentType: 'application/json', + content: {a: 1} + }, { + uri: '/poc/second.json', + contentType: 'application/json', + content: {b: 2} + }]; + documents[index][field] = '/poc/x.json' + suffix; + assert.throws(function() { + db.documents.write(documents); + }, /Content-Disposition values must not contain/); + } + } + } + }); + + it('rejects CR, LF, and NUL in any multipart part header value', function() { + for (const invalidCharacter of ['\r', '\n', '\0']) { + const injected = invalidCharacter + 'X-Injected: yes'; + const cases = [ + {uri: '/poc/file.json', contentType: 'application/json' + injected, content: {a: 1}}, + {extension: 'json' + injected, contentType: 'application/json', content: {a: 1}}, + {extension: 'json', directory: '/poc/' + injected, contentType: 'application/json', content: {a: 1}}, + {uri: '/poc/file.json', versionId: '1' + injected, contentType: 'application/json', content: {a: 1}} + ]; + for (const document of cases) { + assert.throws(function() { + db.documents.write([document, { + uri: '/poc/second.json', + contentType: 'application/json', + content: {b: 2} + }]); + }, /must not contain/); + } + + for (const method of ['post', 'put']) { + assert.throws(function() { + db.resources[method]({ + name: 'example', + documents: [{contentType: 'application/json' + injected, content: {a: 1}}] + }); + }, /must not contain/); + } + } + }); + + it('rejects quoted content-disposition parameters in multipart writes', function() { + const documents = [ + {extension: 'json"', contentType: 'application/json', content: {a: 1}}, + {extension: 'json', directory: '/poc/"files', contentType: 'application/json', content: {a: 1}}, + {uri: '/poc/file.json', versionId: '"1', contentType: 'application/json', content: {a: 1}} + ]; + for (const document of documents) { + assert.throws(function() { + db.documents.write([document, { + uri: '/poc/second.json', + contentType: 'application/json', + content: {b: 2} + }]); + }, /Content-Disposition values must not contain/); + } + }); + + it('rejects semicolons in unquoted content-disposition parameters', function() { + const documents = [ + {extension: 'json; category=metadata', contentType: 'application/json', content: {a: 1}}, + {extension: 'json', directory: '/poc/; category=metadata', contentType: 'application/json', content: {a: 1}}, + {uri: '/poc/file.json', versionId: '1; category=metadata', contentType: 'application/json', content: {a: 1}} + ]; + for (const document of documents) { + assert.throws(function() { + db.documents.write([document, { + uri: '/poc/second.json', + contentType: 'application/json', + content: {b: 2} + }]); + }, /Unquoted Content-Disposition values must not contain semicolons/); + } + }); + + it('rejects unsafe binding parameter and attachment names', function() { + for (const invalidCharacter of ['\r', '\n', '\0', '"']) { + const bindingKey = 'binding' + invalidCharacter; + assert.throws(function() { + db.rows.query(op.fromParam(bindingKey, null, op.docColTypes()), null, { + [bindingKey]: [] + }); + }, /Content-Disposition values must not contain/); + + const invalidAttachmentKey = 'file' + invalidCharacter; + for (const attachments of [ + {[invalidAttachmentKey]: 'content'}, + [{[invalidAttachmentKey]: 'content'}] + ]) { + assert.throws(function() { + db.rows.query(op.fromParam('bindingParam', null, op.docColTypes()), null, { + bindingParam: [], + attachments: attachments + }); + }, /Content-Disposition values must not contain/); + } + } + }); + + it('rejects unsafe part headers in multipart streaming writes', function() { + for (const invalidCharacter of ['\r', '\n', '\0']) { + assert.throws(function() { + // properties selects the chunkedMultipart request type. + db.documents.createWriteStream({ + uri: '/poc/stream.json', + properties: {}, + contentType: 'application/json' + invalidCharacter + 'X-Injected: yes' + }); + }, /must not contain CR, LF, or NUL/); + } + }); + + it('preserves backslashes and ordinary URI paths unchanged', function(done) { + const uri = '/poc/file\\name.json'; + const temporalDocument = '/poc/temporal\\name.json'; + let body = ''; + let assertionError; + + const server = http.createServer(function(request, response) { + request.setEncoding('utf8'); + request.on('data', function(chunk) { + body += chunk; + }); + request.on('end', function() { + try { + body.should.containEql('filename="/poc/file\\name.json"'); + body.should.containEql('temporal-document="/poc/temporal\\name.json"'); + body.should.containEql('filename="/poc/second; filename=ordinary.json"'); + body.should.not.containEql('%2Fpoc'); + } catch (error) { + assertionError = error; + } + response.writeHead(204); + response.end(); + }); + }); + + server.listen(0, '127.0.0.1', function() { + const db = marklogic.createDatabaseClient({ + host: '127.0.0.1', + port: server.address().port, + user: 'u', + password: 'p', + authType: 'basic' + }); + + db.documents.write([{ + uri, + temporalDocument, + contentType: 'application/json', + content: {a: 1} + }, { + uri: '/poc/second; filename=ordinary.json', + contentType: 'application/json', + content: {b: 2} + }]).result().then(function() { + server.close(function() { + done(assertionError); + }); + }, function(error) { + server.close(function() { + done(assertionError || error); + }); + }); + }); + }); +}); diff --git a/test-basic/documents-write-all-validation-test.js b/test-basic/documents-write-all-validation-test.js new file mode 100644 index 00000000..dba1a4cd --- /dev/null +++ b/test-basic/documents-write-all-validation-test.js @@ -0,0 +1,232 @@ +/* +* Copyright (c) 2015-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +*/ +'use strict'; + +const assert = require('assert'); +const http = require('http'); +const {Readable} = require('stream'); +const marklogic = require('../'); + +describe('documents.writeAll validation errors', function() { + let server; + let db; + let writtenUris; + let rejectWrites; + + beforeEach(async function() { + writtenUris = []; + rejectWrites = false; + server = http.createServer(function(request, response) { + let body = ''; + request.setEncoding('utf8'); + request.on('data', chunk => { body += chunk; }); + request.on('end', function() { + response.setHeader('Content-Type', 'application/json'); + if (request.url === '/v1/internal/forestinfo') { + response.end(JSON.stringify([{host: 'localhost'}])); + } else if (rejectWrites) { + response.writeHead(400); + response.end(JSON.stringify({errorResponse: {message: 'Rejected test batch'}})); + } else { + const uri = new URL(request.url, 'http://localhost').searchParams.get('uri'); + const uris = uri === null ? + Array.from(body.matchAll(/filename="([^"]+)"/g), match => match[1]) : [uri]; + writtenUris.push(...uris); + response.end(JSON.stringify({documents: uris.map(uri => ({uri}))})); + } + }); + }); + await new Promise(function(resolve, reject) { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + db = marklogic.createDatabaseClient({ + host: '127.0.0.1', + port: server.address().port, + user: 'u', + password: 'p', + authType: 'basic' + }); + }); + + afterEach(async function() { + db.release(); + await new Promise(function(resolve, reject) { + server.close(error => error ? reject(error) : resolve()); + }); + }); + + function runWriteAll(onBatchError, { + documentCount = 6, + invalidIndex = 3, + invalidDocument = {uri: '/poc/bad"q.json'} + } = {}) { + const documents = Array.from({length: documentCount}, (_, i) => ({ + uri: '/poc/ok' + i + '.json', + content: {i} + })); + if (invalidDocument !== null) { + Object.assign(documents[invalidIndex], invalidDocument); + } + const errors = []; + const successfulUris = []; + return new Promise(function(resolve) { + db.documents.writeAll(Readable.from(documents), { + batchSize: 2, + onBatchError, + onBatchSuccess: function(progress, batch) { + successfulUris.push(...batch.map(document => document.uri)); + }, + onCompletion: function(summary) { + resolve({summary, errors, successfulUris}); + } + }).on('error', error => errors.push(error)); + }); + } + + it('reports the entire rejected batch and completes when the handler skips it', async function() { + const failures = []; + const result = await runWriteAll(function(progress, documents, error) { + failures.push({progress, documents, error}); + return null; + }); + assert.strictEqual(failures.length, 1); + assert.strictEqual(failures[0].progress.docsFailedToBeWritten, 2); + assert.deepStrictEqual(failures[0].documents.map(document => document.uri).sort(), + ['/poc/bad"q.json', '/poc/ok2.json']); + assert.match(failures[0].error.message, /Content-Disposition values must not contain/); + assert.strictEqual(result.summary.docsWrittenSuccessfully, 4); + assert.strictEqual(result.summary.error, undefined); + assert.deepStrictEqual(result.errors, []); + assert.deepStrictEqual(writtenUris.sort(), + ['/poc/ok0.json', '/poc/ok1.json', '/poc/ok4.json', '/poc/ok5.json']); + assert.deepStrictEqual(result.successfulUris.sort(), writtenUris); + }); + + it('retries a corrected batch and completes without losing its valid document', async function() { + let failures = 0; + const result = await runWriteAll(function(progress, documents) { + failures++; + return documents.map(document => ({ + ...document, + uri: document.uri.replace('"', '') + })); + }); + assert.strictEqual(failures, 1); + assert.strictEqual(result.summary.docsWrittenSuccessfully, 6); + assert.strictEqual(result.summary.docsFailedToBeWritten, 0); + assert.strictEqual(result.summary.error, undefined); + assert.deepStrictEqual(result.errors, []); + assert.deepStrictEqual(writtenUris.sort(), + ['/poc/badq.json', '/poc/ok0.json', '/poc/ok1.json', + '/poc/ok2.json', '/poc/ok4.json', '/poc/ok5.json']); + assert.deepStrictEqual(result.successfulUris.sort(), writtenUris); + }); + + it('emits the handler error and completes when the handler aborts', async function() { + const abortError = new Error('Stop this write job'); + let failures = 0; + const result = await runWriteAll(function() { + failures++; + throw abortError; + }); + assert.strictEqual(failures, 1); + assert.deepStrictEqual(result.errors, [abortError]); + assert.strictEqual(result.summary.docsFailedToBeWritten, 2); + assert.strictEqual(result.summary.error, abortError.toString()); + }); + + it('emits the validation error and completes when no batch-error handler is supplied', async function() { + const result = await runWriteAll(); + assert.strictEqual(result.errors.length, 1); + assert.match(result.errors[0].message, /Content-Disposition values must not contain/); + assert.strictEqual(result.summary.docsFailedToBeWritten, 2); + assert.strictEqual(result.summary.error, result.errors[0].toString()); + }); + + it('limits retries when the replacement batch still fails validation', async function() { + let failures = 0; + const result = await runWriteAll(function(progress, documents) { + failures++; + return documents; + }); + assert.strictEqual(failures, 3); + assert.strictEqual(result.errors.length, 1); + assert.strictEqual(result.summary.docsFailedToBeWritten, 2); + assert.strictEqual(result.summary.error, result.errors[0].toString()); + }); + + it('preserves batch-error handling for asynchronous server rejections', async function() { + rejectWrites = true; + const failures = []; + const result = await runWriteAll(function(progress, documents, error) { + failures.push(error); + return null; + }, {invalidDocument: null}); + assert.strictEqual(failures.length, 3); + assert(failures.every(error => error instanceof Error)); + assert.strictEqual(result.summary.docsWrittenSuccessfully, 0); + assert.strictEqual(result.summary.error, undefined); + assert.deepStrictEqual(result.errors, []); + assert.deepStrictEqual(writtenUris, []); + }); + + for (const [field, invalidDocument, expectedError] of [ + ['temporalDocument', {temporalDocument: '/poc/bad"q.json'}, /Content-Disposition values must not contain/], + ['contentType', {contentType: 'application/json\r\nX-Injected: yes'}, /must not contain CR, LF, or NUL/], + ['extension', {uri: undefined, extension: 'json; category=metadata'}, /must not contain semicolons/], + ['directory', {uri: undefined, extension: 'json', directory: '/poc/; category=metadata'}, /must not contain semicolons/], + ['versionId', {versionId: '1; category=metadata'}, /must not contain semicolons/], + ['repair', {format: 'xml', repair: 'invalid'}, /repair must be/] + ]) { + it('routes synchronous ' + field + ' validation through onBatchError', async function() { + const failures = []; + const result = await runWriteAll(function(progress, documents, error) { + failures.push(error); + return null; + }, {invalidDocument}); + assert.strictEqual(failures.length, 1); + assert.match(failures[0].message, expectedError); + assert.strictEqual(result.summary.docsWrittenSuccessfully, 4); + assert.strictEqual(result.summary.error, undefined); + assert.deepStrictEqual(result.errors, []); + assert.deepStrictEqual(writtenUris.sort(), + ['/poc/ok0.json', '/poc/ok1.json', '/poc/ok4.json', '/poc/ok5.json']); + }); + } + + for (const [documentCount, invalidIndex] of [[2, 0], [3, 0], [14, 8], [14, 13]]) { + it('completes a corrected batch at index ' + invalidIndex + ' of ' + documentCount + ' documents', + async function() { + let failures = 0; + const result = await runWriteAll(function(progress, documents) { + failures++; + return documents.map(document => ({ + ...document, + uri: document.uri.replace('"', '') + })); + }, {documentCount, invalidIndex}); + const expectedUris = Array.from({length: documentCount}, (_, i) => + i === invalidIndex ? '/poc/badq.json' : '/poc/ok' + i + '.json').sort(); + assert.strictEqual(failures, 1); + assert.strictEqual(result.summary.docsWrittenSuccessfully, documentCount); + assert.strictEqual(result.summary.docsFailedToBeWritten, 0); + assert.strictEqual(result.summary.error, undefined); + assert.deepStrictEqual(result.errors, []); + assert.deepStrictEqual(writtenUris.sort(), expectedUris); + assert.deepStrictEqual(result.successfulUris.sort(), expectedUris); + }); + } + + it('preserves successful jobs when workers consume multiple batches', async function() { + const result = await runWriteAll(undefined, {documentCount: 14, invalidDocument: null}); + assert.strictEqual(result.summary.docsWrittenSuccessfully, 14); + assert.strictEqual(result.summary.docsFailedToBeWritten, 0); + assert.strictEqual(result.summary.error, undefined); + assert.deepStrictEqual(result.errors, []); + assert.deepStrictEqual(writtenUris.sort(), + Array.from({length: 14}, (_, i) => '/poc/ok' + i + '.json').sort()); + assert.deepStrictEqual(result.successfulUris.sort(), writtenUris); + }); +});