diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 1de3b0bff..293ce69dd 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -24,7 +24,6 @@ import com.marklogic.client.impl.okhttp.OkHttpUtil; import com.marklogic.client.impl.okhttp.PartIterator; import com.marklogic.client.impl.okhttp.RedactingHttpLogger; -import com.marklogic.client.impl.okhttp.RetryableRequestBody; import com.marklogic.client.io.*; import com.marklogic.client.io.marker.*; import com.marklogic.client.query.*; @@ -300,8 +299,30 @@ private void resetFirstRequestFlag() { } } - private int pingServerBeforeStreaming(int retry) { - return pingServer(baseUri, "ping", retry); + /** + * Called for every request whose body is one-shot (e.g. a Reader), as that body cannot be sent again if the request + * fails. OkHttp does not check a pooled connection before reusing it, and MarkLogic can drop a connection right + * after replying to a gzipped response such as the one for a Row Manager request. The ping has no body, so OkHttp + * can recover from a dead connection by using a new one; the request that follows then gets a live connection. + * If the server is unavailable, this waits and pings again. + */ + private void pingServerBeforeOneShotRequest() { + final int maxPingRetries = 10; + int pingDelay = pingServer(baseUri, "ping", 0); + int pingRetries = 0; + while (pingDelay > 0 && pingRetries < maxPingRetries && !Thread.currentThread().isInterrupted()) { + try { + Thread.sleep(pingDelay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new MarkLogicIOException("Interrupted while waiting to ping server before sending a one-shot request", e); + } + pingRetries++; + pingDelay = pingServer(baseUri, "ping", 0); + } + if (pingDelay > 0) { + logger.warn("Server still unavailable after {} ping attempts before sending a one-shot request", maxPingRetries); + } } private int pingServer(HttpUrl requestUri, String path, int retry) { @@ -501,6 +522,11 @@ private Response sendRequestOnce(Request request) { "You cannot use this connected object anymore--connection has already been released"); } + RequestBody body = request.body(); + if (body != null && body.isOneShot()) { + pingServerBeforeOneShotRequest(); + } + try { return okHttpClient.newCall(request).execute(); } catch (IOException e) { @@ -1218,30 +1244,9 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth "Document write with null value for " + ((uri != null) ? uri : "new document")); } - if (isFirstRequest() && !isResendable && isStreaming(value)) { - int firstRequestDelay = pingServerBeforeStreaming(retryContext.getRetry()); - if (firstRequestDelay != 0) { - retryContext.calculateNextDelay(0, firstRequestDelay); - continue; - } - } - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - if (value instanceof OutputStreamSender) { - StreamingOutputImpl sentStream = - new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType); - requestBldr = - ("put".equals(method)) ? - requestBldr.put(sentStream) : - requestBldr.post(sentStream); - } else { - Object sentObj = (reqlog != null) ? - reqlog.copyContent(value) : value; - requestBldr = - ("put".equals(method)) ? - requestBldr.put(new ObjectRequestBody(sentObj, mediaType)) : - requestBldr.post(new ObjectRequestBody(sentObj, mediaType)); - } + RequestBody body = toRequestBody(value, isResendable, mediaType, reqlog); + requestBldr = ("put".equals(method)) ? requestBldr.put(body) : requestBldr.post(body); response = sendRequestOnce(requestBldr); status = response.code(); @@ -1370,14 +1375,6 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth new String[]{metadataMimetype, contentMimetype}, new AbstractWriteHandle[]{metadataHandle, contentHandle}); - if (isFirstRequest() && hasStreamingPart) { - int firstRequestDelay = pingServerBeforeStreaming(retryContext.getRetry()); - if (firstRequestDelay != 0) { - retryContext.calculateNextDelay(0, firstRequestDelay); - continue; - } - } - requestBldr = ("put".equals(method)) ? requestBldr.put(multiPart.build()) : requestBldr.post(multiPart.build()); response = sendRequestOnce(requestBldr); status = response.code(); @@ -1925,19 +1922,17 @@ Response getResponse() { retryContext.sleepIfNeeded(); if (queryDef instanceof StructuredQueryDefinition && !(queryDef instanceof RawQueryDefinition)) { - response = doPost(reqlog, requestBldr, structure); + response = doPost(requestBldr, toRequestBody(structure, true, contentTypeOf(requestBldr), reqlog)); } else if (queryDef instanceof CombinedQueryDefinition) { - response = doPost(reqlog, requestBldr, structure); + response = doPost(requestBldr, toRequestBody(structure, true, contentTypeOf(requestBldr), reqlog)); } else if (queryDef instanceof DeleteQueryDefinition) { response = doGet(requestBldr); - } else if (queryDef instanceof RawQueryDefinition) { - response = doPost(reqlog, requestBldr, baseHandle.sendContent()); - } else if (queryDef instanceof RawCtsQueryDefinition) { - response = doPost(reqlog, requestBldr, baseHandle.sendContent()); + } else if (queryDef instanceof RawQueryDefinition || queryDef instanceof RawCtsQueryDefinition) { + response = doPost(requestBldr, toRequestBody(baseHandle, contentTypeOf(requestBldr), reqlog)); } else if (queryDef instanceof StringQueryDefinition) { response = doGet(requestBldr); } else if (queryDef instanceof CtsQueryDefinition) { - response = doPost(reqlog, requestBldr, structure); + response = doPost(requestBldr, toRequestBody(structure, true, contentTypeOf(requestBldr), reqlog)); } else { throw new UnsupportedOperationException("Cannot search with " + queryDef.getClass().getName()); @@ -2190,9 +2185,9 @@ public Response apply(Request.Builder funcBuilder) { new Function() { public Response apply(Request.Builder funcBuilder) { String contentType = tempBaseHandle.getMimetype(); - return doPost(null, - (contentType == null) ? funcBuilder : funcBuilder.header(HEADER_CONTENT_TYPE, contentType), - tempBaseHandle.sendContent()); + Request.Builder typedBuilder = + (contentType == null) ? funcBuilder : funcBuilder.header(HEADER_CONTENT_TYPE, contentType); + return doPost(typedBuilder, toRequestBody(tempBaseHandle, contentTypeOf(typedBuilder), null)); } }; @@ -2454,29 +2449,10 @@ private void putPostValueImpl(RequestLogger reqlog, String method, Object nextValue = (handle != null) ? handle.sendContent() : value; - RequestBody sentValue; - if (nextValue instanceof OutputStreamSender) { - sentValue = new StreamingOutputImpl( - (OutputStreamSender) nextValue, reqlog, mediaType); - } else { - if (reqlog != null && retryContext.getRetry() == 0) { - sentValue = new ObjectRequestBody(reqlog.copyContent(nextValue), mediaType); - } else { - sentValue = new ObjectRequestBody(nextValue, mediaType); - } - } - - boolean isStreaming = (isFirstRequest() || handle == null) ? isStreaming(sentValue) : false; - - boolean isResendable = (handle == null) ? !isStreaming : handle.isResendable(); - - if (isFirstRequest() && !isResendable && isStreaming) { - int firstRequestDelay = pingServerBeforeStreaming(retryContext.getRetry()); - if (firstRequestDelay != 0) { - retryContext.calculateNextDelay(0, firstRequestDelay); - continue; - } - } + boolean isResendable = (handle != null) ? handle.isResendable() : !isStreaming(nextValue); + // Non-streaming content is copied to the logger, which is only done for the first attempt + RequestLogger bodyLogger = (nextValue instanceof OutputStreamSender || retryContext.getRetry() == 0) ? reqlog : null; + RequestBody sentValue = toRequestBody(nextValue, isResendable, mediaType, bodyLogger); if ("put".equals(method)) { if (requestBldr == null) { @@ -2831,7 +2807,9 @@ public R putResource(RequestLogger reqlog, Function doPutFunction = new Function() { public Response apply(Request.Builder funcBuilder) { - return doPut(reqlog, funcBuilder, inputBase.sendContent()); + Object value = inputBase.sendContent(); + if (value == null) throw new IllegalArgumentException("Resource write with null value"); + return doPut(funcBuilder, toRequestBody(value, isResendable, contentTypeOf(funcBuilder), reqlog)); } }; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPutFunction, resendableConsumer); @@ -2881,7 +2859,7 @@ public R putResour requestBldr = addTransactionScopedCookies(requestBldr, transaction); requestBldr = addTelemetryAgentId(requestBldr); - response = doPut(requestBldr, multiPart, hasStreamingPart); + response = doPut(requestBldr, multiPart.build()); status = response.code(); if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { @@ -2980,7 +2958,10 @@ public void accept(Boolean resendable) { final Object value = inputBase == null ? null : inputBase.sendContent(); Function doPostFunction = new Function() { public Response apply(Request.Builder funcBuilder) { - return doPost(reqlog, funcBuilder, value); + RequestBody body = (value == null) ? + new ObjectRequestBody(null, null) : + toRequestBody(value, isResendable, contentTypeOf(funcBuilder), reqlog); + return doPost(funcBuilder, body); } }; @@ -3044,7 +3025,7 @@ public R postResou requestBldr = addTransactionScopedCookies(requestBldr, transaction); requestBldr = addTelemetryAgentId(requestBldr); - response = doPost(requestBldr, multiPart, hasStreamingPart); + response = doPost(requestBldr, multiPart.build()); status = response.code(); if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { @@ -3499,7 +3480,7 @@ public RESTServiceResultIterator postMultipartForm( requestBldr = addTrailerHeadersIfNecessary(requestBldr, path); Function doPostFunction = funcBuilder -> - doPost(reqlog, funcBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()), multiBuilder.build()); + doPost(funcBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()), multiBuilder.build()); // The construction of this was based on whether the primary input was resendable or not. We don't have a primary // input with a multipart request. So keeping this as null for now. @@ -3570,11 +3551,10 @@ private U postIteratedResourceImpl( } }; - Function doPostFunction = requestBuilder -> doPost( - reqlog, - requestBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()), - inputBase.sendContent() - ); + Function doPostFunction = requestBuilder -> { + Request.Builder acceptBuilder = requestBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()); + return doPost(acceptBuilder, toRequestBody(inputBase, contentTypeOf(acceptBuilder), reqlog)); + }; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); Objects.requireNonNull(response); @@ -3646,7 +3626,7 @@ private U postIt requestBldr = addTransactionScopedCookies(requestBldr, transaction); requestBldr = addTelemetryAgentId(requestBldr); - response = doPost(requestBldr, multiPart, hasStreamingPart); + response = doPost(requestBldr, multiPart.build()); status = response.code(); if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { @@ -3745,34 +3725,8 @@ private Request.Builder makePutWebResource(String path, return setupRequest(path, params); } - private Response doPut(RequestLogger reqlog, Request.Builder requestBldr, Object value) { - if (value == null) throw new IllegalArgumentException("Resource write with null value"); - - if (isFirstRequest() && isStreaming(value)) pingServerBeforeStreaming(0); - - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - if (value instanceof OutputStreamSender) { - requestBldr = requestBldr.put(new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType)); - } else { - if (reqlog != null) { - requestBldr = requestBldr.put(new ObjectRequestBody(reqlog.copyContent(value), mediaType)); - } else { - requestBldr = requestBldr.put(new ObjectRequestBody(value, mediaType)); - } - } - Response response = sendRequestOnce(requestBldr); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private Response doPut(Request.Builder requestBldr, - MultipartBody.Builder multiPart, boolean hasStreamingPart) { - if (isFirstRequest() && hasStreamingPart) pingServerBeforeStreaming(0); - - requestBldr = requestBldr.put(multiPart.build()); - Response response = sendRequestOnce(requestBldr); + private Response doPut(Request.Builder requestBldr, RequestBody body) { + Response response = sendRequestOnce(requestBldr.put(body)); if (isFirstRequest()) setFirstRequest(false); @@ -3787,38 +3741,8 @@ private Request.Builder makePostWebResource(String path, RequestParameters param return setupRequest(path, params); } - private Response doPost(RequestLogger reqlog, Request.Builder requestBldr, Object value) { - if (isFirstRequest() && isStreaming(value)) { - pingServerBeforeStreaming(0); - } - - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - if (value == null) { - requestBldr = requestBldr.post(new ObjectRequestBody(null, null)); - } else if (value instanceof MultipartBody) { - requestBldr = requestBldr.post((MultipartBody) value); - } else if (value instanceof OutputStreamSender) { - requestBldr = requestBldr - .post(new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType)); - } else { - if (reqlog != null) { - requestBldr = requestBldr.post(new ObjectRequestBody(reqlog.copyContent(value), mediaType)); - } else { - requestBldr = requestBldr.post(new ObjectRequestBody(value, mediaType)); - } - } - Response response = sendRequestOnce(requestBldr); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private Response doPost(Request.Builder requestBldr, - MultipartBody.Builder multiPart, boolean hasStreamingPart) { - if (isFirstRequest() && hasStreamingPart) pingServerBeforeStreaming(0); - - Response response = sendRequestOnce(requestBldr.post(multiPart.build())); + private Response doPost(Request.Builder requestBldr, RequestBody body) { + Response response = sendRequestOnce(requestBldr.post(body)); if (isFirstRequest()) setFirstRequest(false); @@ -4000,19 +3924,8 @@ private boolean addParts( } } - Part bodyPart = null; - if (value instanceof OutputStreamSender) { - bodyPart = Part.create(partHeaders.build(), new StreamingOutputImpl( - (OutputStreamSender) value, reqlog, mediaType)); - } else { - if (reqlog != null) { - bodyPart = Part.create(partHeaders.build(), new ObjectRequestBody(reqlog.copyContent(value), mediaType)); - } else { - bodyPart = Part.create(partHeaders.build(), new ObjectRequestBody(value, mediaType)); - } - } - - multiPart = multiPart.addPart(bodyPart); + RequestBody partBody = toRequestBody(value, handleBase.isResendable(), mediaType, reqlog); + multiPart = multiPart.addPart(Part.create(partHeaders.build(), partBody)); } return hasStreamingPart; @@ -4135,7 +4048,14 @@ private U makeResults(ResultIteratorConstructor String operation, String entityType, Response response) { if (response == null) return null; final List partList = readMultipartBodyParts(response.body()); - throwExceptionIfErrorInTrailers(operation, entityType, response); + try { + throwExceptionIfErrorInTrailers(operation, entityType, response); + } finally { + // Every part has been buffered in memory, so the response can be closed right away instead of waiting for + // the caller to close the iterator. This releases the connection (and its gzip/chunked source) promptly + // even if the caller never closes the iterator or there were no rows. + closeResponse(response); + } return makeResults(constructor, reqlog, operation, entityType, partList, response, response); } @@ -4653,7 +4573,7 @@ public InputStream match(StructureWriteHandle document, Function doPostFunction = new Function() { public Response apply(Request.Builder funcBuilder) { - return doPost(null, funcBuilder, baseHandle.sendContent()); + return doPost(funcBuilder, toRequestBody(baseHandle, contentTypeOf(funcBuilder), null)); } }; Response response = sendRequestWithRetry(requestBldr, doPostFunction, null); @@ -4749,9 +4669,9 @@ public InputStream match(QueryDefinition queryDef, retryContext.sleepIfNeeded(); if (queryDef instanceof StructuredQueryDefinition) { - response = doPost(null, requestBldr, structure); + response = doPost(requestBldr, toRequestBody(structure, true, contentTypeOf(requestBldr), null)); } else if (queryDef instanceof RawQueryDefinition) { - response = doPost(null, requestBldr, baseHandle.sendContent()); + response = doPost(requestBldr, toRequestBody(baseHandle, contentTypeOf(requestBldr), null)); } else if (queryDef instanceof StringQueryDefinition) { response = sendRequestOnce(requestBldr.get()); } else { @@ -5194,10 +5114,10 @@ static private List getPartList(MimeMultipart multipart) { } } - static private class ObjectRequestBody extends RequestBody implements RetryableRequestBody { + static private class ObjectRequestBody extends RequestBody { - private Object obj; - private MediaType contentType; + private final Object obj; + private final MediaType contentType; ObjectRequestBody(Object obj, MediaType contentType) { super(); @@ -5231,10 +5151,10 @@ public void writeTo(BufferedSink sink) throws IOException { } @Override - public boolean isRetryable() { - // Added in 8.0.0 to work with the retry interceptor so it knows whether the body can be retried or not. - // InputStreams cannot be retried as they are consumed on first read. - return !(obj instanceof InputStream); + public boolean isOneShot() { + // An InputStream is consumed by the first write. Using OkHttp's own contract means that a MultipartBody + // containing this body is also reported as one-shot. + return obj instanceof InputStream; } } @@ -5275,15 +5195,36 @@ public static RequestBody makeRequestBodyForContent(AbstractWriteHandle content) if (content == null) { return new EmptyRequestBody(); } - HandleImplementation handleBase = HandleAccessor.as(content); + HandleImplementation handleBase = HandleAccessor.as(content); Format format = handleBase.getFormat(); String mimetype = (format == Format.BINARY) ? null : handleBase.getMimetype(); MediaType mediaType = MediaType.parse( (mimetype != null) ? mimetype : "application/x-unknown-content-type" ); - return (content instanceof OutputStreamSender) ? - new StreamingOutputImpl((OutputStreamSender) content, null, mediaType) : - new ObjectRequestBody(HandleAccessor.sendContent(content), mediaType); + return toRequestBody(handleBase, mediaType, null); + } + + private static MediaType contentTypeOf(Request.Builder requestBldr) { + return makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); + } + + /** + * The one place where a handle's content becomes a request body, so that the handle's resendability is always + * reflected in {@code isOneShot()}. + */ + private static RequestBody toRequestBody(HandleImplementation handle, MediaType mediaType, RequestLogger reqlog) { + return toRequestBody(handle.sendContent(), handle.isResendable(), mediaType, reqlog); + } + + /** + * @param content already obtained from a handle's {@code sendContent()}, or any other content that is sent as is + * @param resendable whether {@code content}, if it is an {@code OutputStreamSender}, can be written again + */ + private static RequestBody toRequestBody(Object content, boolean resendable, MediaType mediaType, RequestLogger reqlog) { + if (content instanceof OutputStreamSender sender) { + return new StreamingOutputImpl(sender, reqlog, mediaType, resendable); + } + return new ObjectRequestBody(reqlog != null ? reqlog.copyContent(content) : content, mediaType); } class CallRequestImpl implements CallRequest { @@ -5389,25 +5330,6 @@ private void executeRequest(CallResponseImpl responseImpl) { }; Function sendRequestFunction = requestBldr -> { - if (isFirstRequest() && hasStreamingPart) { - // Ping the server before streaming; if unavailable, wait and retry the ping - int pingDelay = pingServer(callBaseUri, "", 0); - int pingRetries = 0; - int maxPingRetries = 10; // Prevent infinite loop - while (pingDelay > 0 && pingRetries < maxPingRetries && !Thread.currentThread().isInterrupted()) { - try { - Thread.sleep(pingDelay); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting to ping server before streaming", e); - } - pingRetries++; - pingDelay = pingServer(callBaseUri, "", 0); - } - if (pingRetries >= maxPingRetries) { - logger.warn("Server still unavailable after {} ping attempts before streaming", maxPingRetries); - } - } Response response = sendRequestOnce(requestBldr); if (isFirstRequest()) setFirstRequest(false); return response; @@ -5493,21 +5415,6 @@ private RequestBody makeRequestBody(String value) { return new AtomicRequestBody(value, MediaType.parse("text/plain")); } - private RequestBody makeRequestBody(AbstractWriteHandle document) { - if (document == null) { - return new EmptyRequestBody(); - } - HandleImplementation handleBase = HandleAccessor.as(document); - Format format = handleBase.getFormat(); - String mimetype = (format == Format.BINARY) ? null : handleBase.getMimetype(); - MediaType mediaType = MediaType.parse( - (mimetype != null) ? mimetype : "application/x-unknown-content-type" - ); - return (document instanceof OutputStreamSender) ? - new StreamingOutputImpl((OutputStreamSender) document, null, mediaType) : - new ObjectRequestBody(HandleAccessor.sendContent(document), mediaType); - } - private RequestBody makeRequestBody(CallField[] params) { if (params == null || params.length == 0) { return new EmptyRequestBody(); @@ -5564,7 +5471,7 @@ private RequestBody makeRequestBody(CallField[] params) { hasStreamingPartCondition.set(); } hasValue.set(); - multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); + multiBldr.addFormDataPart(paramName, null, makeRequestBodyForContent(paramValue)); }); } } else if (param instanceof BufferedMultipleNodeCallField) { @@ -5588,7 +5495,7 @@ private RequestBody makeRequestBody(CallField[] params) { paramValues[i] = paramValue; } hasValue.set(); - multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); + multiBldr.addFormDataPart(paramName, null, makeRequestBodyForContent(paramValue)); } } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java index 5fd30e6d0..e87345658 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java @@ -1,9 +1,8 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; -import com.marklogic.client.impl.okhttp.RetryableRequestBody; import com.marklogic.client.io.OutputStreamSender; import com.marklogic.client.util.RequestLogger; import okhttp3.MediaType; @@ -13,17 +12,23 @@ import java.io.IOException; import java.io.OutputStream; -class StreamingOutputImpl extends RequestBody implements RetryableRequestBody { +class StreamingOutputImpl extends RequestBody { - private OutputStreamSender handle; - private RequestLogger logger; - private MediaType contentType; + private final OutputStreamSender handle; + private final RequestLogger logger; + private final MediaType contentType; + private final boolean resendable; - StreamingOutputImpl(OutputStreamSender handle, RequestLogger logger, MediaType contentType) { + /** + * @param resendable the originating handle's {@code isResendable()}; if false, the content is assumed to be + * consumed by the first write and this body is reported to OkHttp as one-shot + */ + StreamingOutputImpl(OutputStreamSender handle, RequestLogger logger, MediaType contentType, boolean resendable) { super(); this.handle = handle; this.logger = logger; this.contentType = contentType; + this.resendable = resendable; } @Override @@ -49,8 +54,9 @@ public void writeTo(BufferedSink sink) throws IOException { } @Override - public boolean isRetryable() { - // Added in 8.0.0; streaming output cannot be retried as the stream is consumed on first write. - return false; + public boolean isOneShot() { + // Using OkHttp's own contract means that a MultipartBody containing this body is also reported as one-shot, which + // OkHttp and RetryIOExceptionInterceptor both honor. + return !resendable; } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java index b2f57e0c3..5b9c46fc9 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java @@ -1,11 +1,12 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl.okhttp; import com.marklogic.client.MarkLogicIOException; import okhttp3.Interceptor; import okhttp3.Request; +import okhttp3.RequestBody; import okhttp3.Response; import org.slf4j.Logger; @@ -40,8 +41,9 @@ public RetryIOExceptionInterceptor(int maxRetries, long initialDelayMs, double b @Override public Response intercept(Chain chain) throws IOException { Request request = chain.request(); + RequestBody requestBody = request.body(); - if (request.body() instanceof RetryableRequestBody body && !body.isRetryable()) { + if (requestBody != null && requestBody.isOneShot()) { return chain.proceed(request); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryableRequestBody.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryableRequestBody.java index ad35a07c3..7dab69752 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryableRequestBody.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryableRequestBody.java @@ -7,7 +7,11 @@ * Interface for RequestBody implementations to signal whether they can be retried after an IOException. * This is used by RetryIOExceptionInterceptor to determine if a failed request can be retried. * Added in 8.0.0. + * + * @deprecated since 8.3.0; no longer used. {@link okhttp3.RequestBody#isOneShot()} is now the only signal that a request + * body cannot be sent again, as it is also honored by OkHttp itself and propagates through a {@code MultipartBody}. */ +@Deprecated public interface RetryableRequestBody { /** * @return false if this request body cannot be retried (e.g., because it consumes a stream that can only be diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/StreamingOutputImplTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/impl/StreamingOutputImplTest.java new file mode 100644 index 000000000..d23ae4451 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/impl/StreamingOutputImplTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.impl.okhttp.RetryIOExceptionInterceptor; +import com.marklogic.client.io.BytesHandle; +import com.marklogic.client.io.FileHandle; +import com.marklogic.client.io.InputStreamHandle; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.ReaderHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.io.marker.AbstractWriteHandle; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies that the request body built for a handle reports {@code isOneShot()} according to whether the handle's + * content can be written more than once, and that OkHttp-based retry honours that. + */ +class StreamingOutputImplTest { + + static Stream handlesAndOneShotFlags() { + return Stream.of( + new Object[]{"StringHandle", new StringHandle(""), false}, + new Object[]{"JacksonHandle", new JacksonHandle(new ObjectMapper().createObjectNode()), false}, + new Object[]{"BytesHandle", new BytesHandle("".getBytes(StandardCharsets.UTF_8)), false}, + new Object[]{"FileHandle", new FileHandle(new File("pom.xml")), false}, + new Object[]{"ReaderHandle", new ReaderHandle(new StringReader("")), true}, + new Object[]{"InputStreamHandle", + new InputStreamHandle(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8))), true} + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("handlesAndOneShotFlags") + void oneShotMatchesWhetherTheHandleCanBeResent(String name, AbstractWriteHandle handle, boolean expectedOneShot) { + assertEquals(expectedOneShot, OkHttpServices.makeRequestBodyForContent(handle).isOneShot()); + } + + @Test + void multipartWithAnyOneShotPartIsOneShot() { + MultipartBody body = multipart( + OkHttpServices.makeRequestBodyForContent(new StringHandle("")), + OkHttpServices.makeRequestBodyForContent(new ReaderHandle(new StringReader(""))) + ); + assertTrue(body.isOneShot()); + } + + @Test + void multipartWithOnlyResendablePartsIsNotOneShot() { + MultipartBody body = multipart( + OkHttpServices.makeRequestBodyForContent(new StringHandle("")), + OkHttpServices.makeRequestBodyForContent(new JacksonHandle(new ObjectMapper().createObjectNode())) + ); + assertFalse(body.isOneShot()); + } + + @Test + void oneShotMultipartIsNotRetried() { + MultipartBody body = multipart( + OkHttpServices.makeRequestBodyForContent(new ReaderHandle(new StringReader(""))) + ); + CountingFailure failure = new CountingFailure(Integer.MAX_VALUE); + + assertThrows(IOException.class, () -> client(failure).newCall(post(body)).execute()); + assertEquals(1, failure.attempts); + } + + @Test + void resendableMultipartIsRetried() throws IOException { + MultipartBody body = multipart( + OkHttpServices.makeRequestBodyForContent(new StringHandle("")) + ); + CountingFailure failure = new CountingFailure(1); + + try (Response response = client(failure).newCall(post(body)).execute()) { + assertEquals(200, response.code()); + } + assertEquals(2, failure.attempts); + } + + private static MultipartBody multipart(RequestBody... parts) { + MultipartBody.Builder builder = new MultipartBody.Builder().setType(MultipartBody.FORM); + for (int i = 0; i < parts.length; i++) { + builder.addFormDataPart("part" + i, null, parts[i]); + } + return builder.build(); + } + + private static Request post(RequestBody body) { + return new Request.Builder().url("http://localhost/").post(body).build(); + } + + private static OkHttpClient client(CountingFailure failure) { + return new OkHttpClient.Builder() + .addInterceptor(new RetryIOExceptionInterceptor(3, 0, 1, 0)) + .addInterceptor(failure) + .build(); + } + + /** + * Fails the first {@code failuresBeforeSuccess} attempts with an IOException, then returns a 200. + */ + private static class CountingFailure implements okhttp3.Interceptor { + private final int failuresBeforeSuccess; + int attempts; + + CountingFailure(int failuresBeforeSuccess) { + this.failuresBeforeSuccess = failuresBeforeSuccess; + } + + @Override + public Response intercept(Chain chain) throws IOException { + if (++attempts <= failuresBeforeSuccess) { + throw new IOException("unexpected end of stream"); + } + return new Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create("", null)) + .build(); + } + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/StreamingPartRequestTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/impl/StreamingPartRequestTest.java new file mode 100644 index 000000000..e4c4e2347 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/impl/StreamingPartRequestTest.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.impl; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.MarkLogicIOException; +import com.marklogic.client.io.ReaderHandle; +import mockwebserver3.Dispatcher; +import mockwebserver3.MockResponse; +import mockwebserver3.MockWebServer; +import mockwebserver3.RecordedRequest; +import mockwebserver3.SocketEffect; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Uses a mock server to verify how requests with non-resendable content (e.g. a Reader) are sent. A real MarkLogic + * instance can drop a pooled connection right after a response, such as a compressed rows response. If a one-shot + * request is the next one on that connection, it fails and cannot be resent, because its stream has been consumed. + */ +class StreamingPartRequestTest { + + private static final String PAYLOAD = "{\"a\":1}"; + + private MockWebServer mockWebServer; + private final List requests = new CopyOnWriteArrayList<>(); + private final AtomicBoolean dropConnectionOnce = new AtomicBoolean(); + private DatabaseClient client; + + @BeforeEach + void beforeEach() throws IOException { + mockWebServer = new MockWebServer(); + mockWebServer.start(); + client = DatabaseClientFactory.newClient("localhost", mockWebServer.getPort(), + new DatabaseClientFactory.BasicAuthContext("user", "password")); + } + + @AfterEach + void afterEach() { + client.release(); + mockWebServer.close(); + } + + @Test + void streamingRequestSucceedsWhenServerDropsThePooledConnection() { + // The server answers the first call, then drops the connection when it answers the next request on it. + serve(request -> dropConnectionOnce.compareAndSet(true, false) ? + new MockResponse.Builder().onResponseStart(new SocketEffect.CloseSocket()).build() : + new MockResponse.Builder().code(200).build()); + + callWithStreamingPart(); + dropConnectionOnce.set(true); + callWithStreamingPart(); + + List posts = requests.stream().filter(r -> "POST".equals(r.getMethod())).toList(); + assertEquals(2, posts.size(), "Each call should have sent its POST exactly once"); + assertEquals(1, countOccurrences(posts.get(1).getBody().utf8(), PAYLOAD), + "The body must arrive once and intact: " + posts.get(1).getBody().utf8()); + } + + @Test + void documentWriteWithReaderSucceedsWhenServerDropsThePooledConnection() { + serve(request -> dropConnectionOnce.compareAndSet(true, false) ? + new MockResponse.Builder().onResponseStart(new SocketEffect.CloseSocket()).build() : + new MockResponse.Builder().code("HEAD".equals(request.getMethod()) ? 200 : 204).build()); + + client.newJSONDocumentManager().write("/a.json", new ReaderHandle(new StringReader(PAYLOAD))); + dropConnectionOnce.set(true); + client.newJSONDocumentManager().write("/b.json", new ReaderHandle(new StringReader(PAYLOAD))); + + List writes = requests.stream().filter(r -> !"HEAD".equals(r.getMethod())).toList(); + assertEquals(2, writes.size()); + assertEquals(PAYLOAD, writes.get(1).getBody().utf8()); + } + + @Test + void failedStreamingRequestIsNotResent() { + serve(request -> "POST".equals(request.getMethod()) ? + new MockResponse.Builder().onResponseStart(new SocketEffect.CloseSocket()).build() : + new MockResponse.Builder().code(200).build()); + + MarkLogicIOException ex = assertThrows(MarkLogicIOException.class, this::callWithStreamingPart); + + assertFalse(ex.getMessage().contains("Stream closed"), + "The stream cannot be resent, so the original failure should be reported; message: " + ex.getMessage()); + assertEquals(1, requests.stream().filter(r -> "POST".equals(r.getMethod())).count(), + "The streaming request must not be sent a second time"); + } + + @Test + void failedStreamingRequestIsNotResentWithDigestAuthentication() { + client.release(); + client = DatabaseClientFactory.newClient("localhost", mockWebServer.getPort(), + new DatabaseClientFactory.DigestAuthContext("user", "password")); + serve(request -> { + if (request.getHeaders().get("Authorization") == null) { + return new MockResponse.Builder().code(401) + .addHeader("WWW-Authenticate", "Digest realm=\"public\", qop=\"auth\", nonce=\"abc\", opaque=\"xyz\"") + .build(); + } + return "POST".equals(request.getMethod()) ? + new MockResponse.Builder().onResponseStart(new SocketEffect.CloseSocket()).build() : + new MockResponse.Builder().code(200).build(); + }); + + assertThrows(MarkLogicIOException.class, this::callWithStreamingPart); + + assertEquals(1, requests.stream() + .filter(r -> "POST".equals(r.getMethod()) && r.getHeaders().get("Authorization") != null).count(), + "The authenticated streaming request must not be sent a second time"); + } + + private void serve(Function responder) { + mockWebServer.setDispatcher(new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + requests.add(request); + return responder.apply(request); + } + }); + } + + private void callWithStreamingPart() { + BaseProxy.request("/ds/custom-content/", "importContent.xqy", BaseProxy.ParameterValuesKind.MULTIPLE_NODES) + .on(client) + .withSession() + .withParams(BaseProxy.documentParam("content", false, + Stream.of(new StringReader(PAYLOAD)).map(BaseProxy.JsonDocumentType::fromReader))) + .responseNone(); + } + + private static int countOccurrences(String text, String token) { + int count = 0; + for (int i = text.indexOf(token); i >= 0; i = text.indexOf(token, i + token.length())) { + count++; + } + return count; + } +}